regex.h 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753
  1. // Tencent is pleased to support the open source community by making RapidJSON
  2. // available.
  3. //
  4. // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All
  5. // rights reserved.
  6. //
  7. // Licensed under the MIT License (the "License"); you may not use this file
  8. // except in compliance with the License. You may obtain a copy of the License
  9. // at
  10. //
  11. // http://opensource.org/licenses/MIT
  12. //
  13. // Unless required by applicable law or agreed to in writing, software
  14. // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  15. // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  16. // License for the specific language governing permissions and limitations under
  17. // the License.
  18. #ifndef RAPIDJSON_INTERNAL_REGEX_H_
  19. #define RAPIDJSON_INTERNAL_REGEX_H_
  20. #include "../allocators.h"
  21. #include "../stream.h"
  22. #include "stack.h"
  23. #ifdef __clang__
  24. RAPIDJSON_DIAG_PUSH
  25. RAPIDJSON_DIAG_OFF(padded)
  26. RAPIDJSON_DIAG_OFF(switch - enum)
  27. #elif defined(_MSC_VER)
  28. RAPIDJSON_DIAG_PUSH
  29. RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated
  30. #endif
  31. #ifdef __GNUC__
  32. RAPIDJSON_DIAG_PUSH
  33. RAPIDJSON_DIAG_OFF(effc++)
  34. #endif
  35. #ifndef RAPIDJSON_REGEX_VERBOSE
  36. #define RAPIDJSON_REGEX_VERBOSE 0
  37. #endif
  38. RAPIDJSON_NAMESPACE_BEGIN
  39. namespace internal {
  40. ///////////////////////////////////////////////////////////////////////////////
  41. // DecodedStream
  42. template <typename SourceStream, typename Encoding>
  43. class DecodedStream {
  44. public:
  45. DecodedStream(SourceStream &ss) : ss_(ss), codepoint_() { Decode(); }
  46. unsigned Peek() { return codepoint_; }
  47. unsigned Take() {
  48. unsigned c = codepoint_;
  49. if (c) // No further decoding when '\0'
  50. Decode();
  51. return c;
  52. }
  53. private:
  54. void Decode() {
  55. if (!Encoding::Decode(ss_, &codepoint_)) codepoint_ = 0;
  56. }
  57. SourceStream &ss_;
  58. unsigned codepoint_;
  59. };
  60. ///////////////////////////////////////////////////////////////////////////////
  61. // GenericRegex
  62. static const SizeType kRegexInvalidState = ~SizeType(
  63. 0); //!< Represents an invalid index in GenericRegex::State::out, out1
  64. static const SizeType kRegexInvalidRange = ~SizeType(0);
  65. template <typename Encoding, typename Allocator>
  66. class GenericRegexSearch;
  67. //! Regular expression engine with subset of ECMAscript grammar.
  68. /*!
  69. Supported regular expression syntax:
  70. - \c ab Concatenation
  71. - \c a|b Alternation
  72. - \c a? Zero or one
  73. - \c a* Zero or more
  74. - \c a+ One or more
  75. - \c a{3} Exactly 3 times
  76. - \c a{3,} At least 3 times
  77. - \c a{3,5} 3 to 5 times
  78. - \c (ab) Grouping
  79. - \c ^a At the beginning
  80. - \c a$ At the end
  81. - \c . Any character
  82. - \c [abc] Character classes
  83. - \c [a-c] Character class range
  84. - \c [a-z0-9_] Character class combination
  85. - \c [^abc] Negated character classes
  86. - \c [^a-c] Negated character class range
  87. - \c [\b] Backspace (U+0008)
  88. - \c \\| \\\\ ... Escape characters
  89. - \c \\f Form feed (U+000C)
  90. - \c \\n Line feed (U+000A)
  91. - \c \\r Carriage return (U+000D)
  92. - \c \\t Tab (U+0009)
  93. - \c \\v Vertical tab (U+000B)
  94. \note This is a Thompson NFA engine, implemented with reference to
  95. Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is
  96. slow in Java, Perl, PHP, Python, Ruby,...).",
  97. https://swtch.com/~rsc/regexp/regexp1.html
  98. */
  99. template <typename Encoding, typename Allocator = CrtAllocator>
  100. class GenericRegex {
  101. public:
  102. typedef Encoding EncodingType;
  103. typedef typename Encoding::Ch Ch;
  104. template <typename, typename>
  105. friend class GenericRegexSearch;
  106. GenericRegex(const Ch *source, Allocator *allocator = 0)
  107. : ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()),
  108. allocator_(allocator ? allocator : ownAllocator_),
  109. states_(allocator_, 256),
  110. ranges_(allocator_, 256),
  111. root_(kRegexInvalidState),
  112. stateCount_(),
  113. rangeCount_(),
  114. anchorBegin_(),
  115. anchorEnd_() {
  116. GenericStringStream<Encoding> ss(source);
  117. DecodedStream<GenericStringStream<Encoding>, Encoding> ds(ss);
  118. Parse(ds);
  119. }
  120. ~GenericRegex() { RAPIDJSON_DELETE(ownAllocator_); }
  121. bool IsValid() const { return root_ != kRegexInvalidState; }
  122. private:
  123. enum Operator {
  124. kZeroOrOne,
  125. kZeroOrMore,
  126. kOneOrMore,
  127. kConcatenation,
  128. kAlternation,
  129. kLeftParenthesis
  130. };
  131. static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.'
  132. static const unsigned kRangeCharacterClass = 0xFFFFFFFE;
  133. static const unsigned kRangeNegationFlag = 0x80000000;
  134. struct Range {
  135. unsigned start; //
  136. unsigned end;
  137. SizeType next;
  138. };
  139. struct State {
  140. SizeType out; //!< Equals to kInvalid for matching state
  141. SizeType out1; //!< Equals to non-kInvalid for split
  142. SizeType rangeStart;
  143. unsigned codepoint;
  144. };
  145. struct Frag {
  146. Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {}
  147. SizeType start;
  148. SizeType out; //!< link-list of all output states
  149. SizeType minIndex;
  150. };
  151. State &GetState(SizeType index) {
  152. RAPIDJSON_ASSERT(index < stateCount_);
  153. return states_.template Bottom<State>()[index];
  154. }
  155. const State &GetState(SizeType index) const {
  156. RAPIDJSON_ASSERT(index < stateCount_);
  157. return states_.template Bottom<State>()[index];
  158. }
  159. Range &GetRange(SizeType index) {
  160. RAPIDJSON_ASSERT(index < rangeCount_);
  161. return ranges_.template Bottom<Range>()[index];
  162. }
  163. const Range &GetRange(SizeType index) const {
  164. RAPIDJSON_ASSERT(index < rangeCount_);
  165. return ranges_.template Bottom<Range>()[index];
  166. }
  167. template <typename InputStream>
  168. void Parse(DecodedStream<InputStream, Encoding> &ds) {
  169. Stack<Allocator> operandStack(allocator_, 256); // Frag
  170. Stack<Allocator> operatorStack(allocator_, 256); // Operator
  171. Stack<Allocator> atomCountStack(allocator_,
  172. 256); // unsigned (Atom per parenthesis)
  173. *atomCountStack.template Push<unsigned>() = 0;
  174. unsigned codepoint;
  175. while (ds.Peek() != 0) {
  176. switch (codepoint = ds.Take()) {
  177. case '^':
  178. anchorBegin_ = true;
  179. break;
  180. case '$':
  181. anchorEnd_ = true;
  182. break;
  183. case '|':
  184. while (!operatorStack.Empty() &&
  185. *operatorStack.template Top<Operator>() < kAlternation)
  186. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
  187. return;
  188. *operatorStack.template Push<Operator>() = kAlternation;
  189. *atomCountStack.template Top<unsigned>() = 0;
  190. break;
  191. case '(':
  192. *operatorStack.template Push<Operator>() = kLeftParenthesis;
  193. *atomCountStack.template Push<unsigned>() = 0;
  194. break;
  195. case ')':
  196. while (!operatorStack.Empty() &&
  197. *operatorStack.template Top<Operator>() != kLeftParenthesis)
  198. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
  199. return;
  200. if (operatorStack.Empty()) return;
  201. operatorStack.template Pop<Operator>(1);
  202. atomCountStack.template Pop<unsigned>(1);
  203. ImplicitConcatenation(atomCountStack, operatorStack);
  204. break;
  205. case '?':
  206. if (!Eval(operandStack, kZeroOrOne)) return;
  207. break;
  208. case '*':
  209. if (!Eval(operandStack, kZeroOrMore)) return;
  210. break;
  211. case '+':
  212. if (!Eval(operandStack, kOneOrMore)) return;
  213. break;
  214. case '{': {
  215. unsigned n, m;
  216. if (!ParseUnsigned(ds, &n)) return;
  217. if (ds.Peek() == ',') {
  218. ds.Take();
  219. if (ds.Peek() == '}')
  220. m = kInfinityQuantifier;
  221. else if (!ParseUnsigned(ds, &m) || m < n)
  222. return;
  223. } else
  224. m = n;
  225. if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') return;
  226. ds.Take();
  227. } break;
  228. case '.':
  229. PushOperand(operandStack, kAnyCharacterClass);
  230. ImplicitConcatenation(atomCountStack, operatorStack);
  231. break;
  232. case '[': {
  233. SizeType range;
  234. if (!ParseRange(ds, &range)) return;
  235. SizeType s = NewState(kRegexInvalidState, kRegexInvalidState,
  236. kRangeCharacterClass);
  237. GetState(s).rangeStart = range;
  238. *operandStack.template Push<Frag>() = Frag(s, s, s);
  239. }
  240. ImplicitConcatenation(atomCountStack, operatorStack);
  241. break;
  242. case '\\': // Escape character
  243. if (!CharacterEscape(ds, &codepoint))
  244. return; // Unsupported escape character
  245. // fall through to default
  246. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  247. default: // Pattern character
  248. PushOperand(operandStack, codepoint);
  249. ImplicitConcatenation(atomCountStack, operatorStack);
  250. }
  251. }
  252. while (!operatorStack.Empty())
  253. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1))) return;
  254. // Link the operand to matching state.
  255. if (operandStack.GetSize() == sizeof(Frag)) {
  256. Frag *e = operandStack.template Pop<Frag>(1);
  257. Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0));
  258. root_ = e->start;
  259. #if RAPIDJSON_REGEX_VERBOSE
  260. printf("root: %d\n", root_);
  261. for (SizeType i = 0; i < stateCount_; i++) {
  262. State &s = GetState(i);
  263. printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1,
  264. (char)s.codepoint);
  265. }
  266. printf("\n");
  267. #endif
  268. }
  269. }
  270. SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) {
  271. State *s = states_.template Push<State>();
  272. s->out = out;
  273. s->out1 = out1;
  274. s->codepoint = codepoint;
  275. s->rangeStart = kRegexInvalidRange;
  276. return stateCount_++;
  277. }
  278. void PushOperand(Stack<Allocator> &operandStack, unsigned codepoint) {
  279. SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint);
  280. *operandStack.template Push<Frag>() = Frag(s, s, s);
  281. }
  282. void ImplicitConcatenation(Stack<Allocator> &atomCountStack,
  283. Stack<Allocator> &operatorStack) {
  284. if (*atomCountStack.template Top<unsigned>())
  285. *operatorStack.template Push<Operator>() = kConcatenation;
  286. (*atomCountStack.template Top<unsigned>())++;
  287. }
  288. SizeType Append(SizeType l1, SizeType l2) {
  289. SizeType old = l1;
  290. while (GetState(l1).out != kRegexInvalidState) l1 = GetState(l1).out;
  291. GetState(l1).out = l2;
  292. return old;
  293. }
  294. void Patch(SizeType l, SizeType s) {
  295. for (SizeType next; l != kRegexInvalidState; l = next) {
  296. next = GetState(l).out;
  297. GetState(l).out = s;
  298. }
  299. }
  300. bool Eval(Stack<Allocator> &operandStack, Operator op) {
  301. switch (op) {
  302. case kConcatenation:
  303. RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2);
  304. {
  305. Frag e2 = *operandStack.template Pop<Frag>(1);
  306. Frag e1 = *operandStack.template Pop<Frag>(1);
  307. Patch(e1.out, e2.start);
  308. *operandStack.template Push<Frag>() =
  309. Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex));
  310. }
  311. return true;
  312. case kAlternation:
  313. if (operandStack.GetSize() >= sizeof(Frag) * 2) {
  314. Frag e2 = *operandStack.template Pop<Frag>(1);
  315. Frag e1 = *operandStack.template Pop<Frag>(1);
  316. SizeType s = NewState(e1.start, e2.start, 0);
  317. *operandStack.template Push<Frag>() =
  318. Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex));
  319. return true;
  320. }
  321. return false;
  322. case kZeroOrOne:
  323. if (operandStack.GetSize() >= sizeof(Frag)) {
  324. Frag e = *operandStack.template Pop<Frag>(1);
  325. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  326. *operandStack.template Push<Frag>() =
  327. Frag(s, Append(e.out, s), e.minIndex);
  328. return true;
  329. }
  330. return false;
  331. case kZeroOrMore:
  332. if (operandStack.GetSize() >= sizeof(Frag)) {
  333. Frag e = *operandStack.template Pop<Frag>(1);
  334. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  335. Patch(e.out, s);
  336. *operandStack.template Push<Frag>() = Frag(s, s, e.minIndex);
  337. return true;
  338. }
  339. return false;
  340. case kOneOrMore:
  341. if (operandStack.GetSize() >= sizeof(Frag)) {
  342. Frag e = *operandStack.template Pop<Frag>(1);
  343. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  344. Patch(e.out, s);
  345. *operandStack.template Push<Frag>() = Frag(e.start, s, e.minIndex);
  346. return true;
  347. }
  348. return false;
  349. default:
  350. // syntax error (e.g. unclosed kLeftParenthesis)
  351. return false;
  352. }
  353. }
  354. bool EvalQuantifier(Stack<Allocator> &operandStack, unsigned n, unsigned m) {
  355. RAPIDJSON_ASSERT(n <= m);
  356. RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag));
  357. if (n == 0) {
  358. if (m == 0) // a{0} not support
  359. return false;
  360. else if (m == kInfinityQuantifier)
  361. Eval(operandStack, kZeroOrMore); // a{0,} -> a*
  362. else {
  363. Eval(operandStack, kZeroOrOne); // a{0,5} -> a?
  364. for (unsigned i = 0; i < m - 1; i++)
  365. CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a?
  366. for (unsigned i = 0; i < m - 1; i++)
  367. Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a?
  368. }
  369. return true;
  370. }
  371. for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a
  372. CloneTopOperand(operandStack);
  373. if (m == kInfinityQuantifier)
  374. Eval(operandStack, kOneOrMore); // a{3,} -> a a a+
  375. else if (m > n) {
  376. CloneTopOperand(operandStack); // a{3,5} -> a a a a
  377. Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a?
  378. for (unsigned i = n; i < m - 1; i++)
  379. CloneTopOperand(operandStack); // a{3,5} -> a a a a? a?
  380. for (unsigned i = n; i < m; i++)
  381. Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a?
  382. }
  383. for (unsigned i = 0; i < n - 1; i++)
  384. Eval(operandStack,
  385. kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a?
  386. return true;
  387. }
  388. static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; }
  389. void CloneTopOperand(Stack<Allocator> &operandStack) {
  390. const Frag src =
  391. *operandStack
  392. .template Top<Frag>(); // Copy constructor to prevent invalidation
  393. SizeType count =
  394. stateCount_ - src.minIndex; // Assumes top operand contains states in
  395. // [src->minIndex, stateCount_)
  396. State *s = states_.template Push<State>(count);
  397. memcpy(s, &GetState(src.minIndex), count * sizeof(State));
  398. for (SizeType j = 0; j < count; j++) {
  399. if (s[j].out != kRegexInvalidState) s[j].out += count;
  400. if (s[j].out1 != kRegexInvalidState) s[j].out1 += count;
  401. }
  402. *operandStack.template Push<Frag>() =
  403. Frag(src.start + count, src.out + count, src.minIndex + count);
  404. stateCount_ += count;
  405. }
  406. template <typename InputStream>
  407. bool ParseUnsigned(DecodedStream<InputStream, Encoding> &ds, unsigned *u) {
  408. unsigned r = 0;
  409. if (ds.Peek() < '0' || ds.Peek() > '9') return false;
  410. while (ds.Peek() >= '0' && ds.Peek() <= '9') {
  411. if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295
  412. return false; // overflow
  413. r = r * 10 + (ds.Take() - '0');
  414. }
  415. *u = r;
  416. return true;
  417. }
  418. template <typename InputStream>
  419. bool ParseRange(DecodedStream<InputStream, Encoding> &ds, SizeType *range) {
  420. bool isBegin = true;
  421. bool negate = false;
  422. int step = 0;
  423. SizeType start = kRegexInvalidRange;
  424. SizeType current = kRegexInvalidRange;
  425. unsigned codepoint;
  426. while ((codepoint = ds.Take()) != 0) {
  427. if (isBegin) {
  428. isBegin = false;
  429. if (codepoint == '^') {
  430. negate = true;
  431. continue;
  432. }
  433. }
  434. switch (codepoint) {
  435. case ']':
  436. if (start == kRegexInvalidRange)
  437. return false; // Error: nothing inside []
  438. if (step == 2) { // Add trailing '-'
  439. SizeType r = NewRange('-');
  440. RAPIDJSON_ASSERT(current != kRegexInvalidRange);
  441. GetRange(current).next = r;
  442. }
  443. if (negate) GetRange(start).start |= kRangeNegationFlag;
  444. *range = start;
  445. return true;
  446. case '\\':
  447. if (ds.Peek() == 'b') {
  448. ds.Take();
  449. codepoint = 0x0008; // Escape backspace character
  450. } else if (!CharacterEscape(ds, &codepoint))
  451. return false;
  452. // fall through to default
  453. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  454. default:
  455. switch (step) {
  456. case 1:
  457. if (codepoint == '-') {
  458. step++;
  459. break;
  460. }
  461. // fall through to step 0 for other characters
  462. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  463. case 0: {
  464. SizeType r = NewRange(codepoint);
  465. if (current != kRegexInvalidRange) GetRange(current).next = r;
  466. if (start == kRegexInvalidRange) start = r;
  467. current = r;
  468. }
  469. step = 1;
  470. break;
  471. default:
  472. RAPIDJSON_ASSERT(step == 2);
  473. GetRange(current).end = codepoint;
  474. step = 0;
  475. }
  476. }
  477. }
  478. return false;
  479. }
  480. SizeType NewRange(unsigned codepoint) {
  481. Range *r = ranges_.template Push<Range>();
  482. r->start = r->end = codepoint;
  483. r->next = kRegexInvalidRange;
  484. return rangeCount_++;
  485. }
  486. template <typename InputStream>
  487. bool CharacterEscape(DecodedStream<InputStream, Encoding> &ds,
  488. unsigned *escapedCodepoint) {
  489. unsigned codepoint;
  490. switch (codepoint = ds.Take()) {
  491. case '^':
  492. case '$':
  493. case '|':
  494. case '(':
  495. case ')':
  496. case '?':
  497. case '*':
  498. case '+':
  499. case '.':
  500. case '[':
  501. case ']':
  502. case '{':
  503. case '}':
  504. case '\\':
  505. *escapedCodepoint = codepoint;
  506. return true;
  507. case 'f':
  508. *escapedCodepoint = 0x000C;
  509. return true;
  510. case 'n':
  511. *escapedCodepoint = 0x000A;
  512. return true;
  513. case 'r':
  514. *escapedCodepoint = 0x000D;
  515. return true;
  516. case 't':
  517. *escapedCodepoint = 0x0009;
  518. return true;
  519. case 'v':
  520. *escapedCodepoint = 0x000B;
  521. return true;
  522. default:
  523. return false; // Unsupported escape character
  524. }
  525. }
  526. Allocator *ownAllocator_;
  527. Allocator *allocator_;
  528. Stack<Allocator> states_;
  529. Stack<Allocator> ranges_;
  530. SizeType root_;
  531. SizeType stateCount_;
  532. SizeType rangeCount_;
  533. static const unsigned kInfinityQuantifier = ~0u;
  534. // For SearchWithAnchoring()
  535. bool anchorBegin_;
  536. bool anchorEnd_;
  537. };
  538. template <typename RegexType, typename Allocator = CrtAllocator>
  539. class GenericRegexSearch {
  540. public:
  541. typedef typename RegexType::EncodingType Encoding;
  542. typedef typename Encoding::Ch Ch;
  543. GenericRegexSearch(const RegexType &regex, Allocator *allocator = 0)
  544. : regex_(regex),
  545. allocator_(allocator),
  546. ownAllocator_(0),
  547. state0_(allocator, 0),
  548. state1_(allocator, 0),
  549. stateSet_() {
  550. RAPIDJSON_ASSERT(regex_.IsValid());
  551. if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)();
  552. stateSet_ = static_cast<unsigned *>(allocator_->Malloc(GetStateSetSize()));
  553. state0_.template Reserve<SizeType>(regex_.stateCount_);
  554. state1_.template Reserve<SizeType>(regex_.stateCount_);
  555. }
  556. ~GenericRegexSearch() {
  557. Allocator::Free(stateSet_);
  558. RAPIDJSON_DELETE(ownAllocator_);
  559. }
  560. template <typename InputStream>
  561. bool Match(InputStream &is) {
  562. return SearchWithAnchoring(is, true, true);
  563. }
  564. bool Match(const Ch *s) {
  565. GenericStringStream<Encoding> is(s);
  566. return Match(is);
  567. }
  568. template <typename InputStream>
  569. bool Search(InputStream &is) {
  570. return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_);
  571. }
  572. bool Search(const Ch *s) {
  573. GenericStringStream<Encoding> is(s);
  574. return Search(is);
  575. }
  576. private:
  577. typedef typename RegexType::State State;
  578. typedef typename RegexType::Range Range;
  579. template <typename InputStream>
  580. bool SearchWithAnchoring(InputStream &is, bool anchorBegin, bool anchorEnd) {
  581. DecodedStream<InputStream, Encoding> ds(is);
  582. state0_.Clear();
  583. Stack<Allocator> *current = &state0_, *next = &state1_;
  584. const size_t stateSetSize = GetStateSetSize();
  585. std::memset(stateSet_, 0, stateSetSize);
  586. bool matched = AddState(*current, regex_.root_);
  587. unsigned codepoint;
  588. while (!current->Empty() && (codepoint = ds.Take()) != 0) {
  589. std::memset(stateSet_, 0, stateSetSize);
  590. next->Clear();
  591. matched = false;
  592. for (const SizeType *s = current->template Bottom<SizeType>();
  593. s != current->template End<SizeType>(); ++s) {
  594. const State &sr = regex_.GetState(*s);
  595. if (sr.codepoint == codepoint ||
  596. sr.codepoint == RegexType::kAnyCharacterClass ||
  597. (sr.codepoint == RegexType::kRangeCharacterClass &&
  598. MatchRange(sr.rangeStart, codepoint))) {
  599. matched = AddState(*next, sr.out) || matched;
  600. if (!anchorEnd && matched) return true;
  601. }
  602. if (!anchorBegin) AddState(*next, regex_.root_);
  603. }
  604. internal::Swap(current, next);
  605. }
  606. return matched;
  607. }
  608. size_t GetStateSetSize() const { return (regex_.stateCount_ + 31) / 32 * 4; }
  609. // Return whether the added states is a match state
  610. bool AddState(Stack<Allocator> &l, SizeType index) {
  611. RAPIDJSON_ASSERT(index != kRegexInvalidState);
  612. const State &s = regex_.GetState(index);
  613. if (s.out1 != kRegexInvalidState) { // Split
  614. bool matched = AddState(l, s.out);
  615. return AddState(l, s.out1) || matched;
  616. } else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) {
  617. stateSet_[index >> 5] |= (1u << (index & 31));
  618. *l.template PushUnsafe<SizeType>() = index;
  619. }
  620. return s.out ==
  621. kRegexInvalidState; // by using PushUnsafe() above, we can ensure s
  622. // is not validated due to reallocation.
  623. }
  624. bool MatchRange(SizeType rangeIndex, unsigned codepoint) const {
  625. bool yes = (regex_.GetRange(rangeIndex).start &
  626. RegexType::kRangeNegationFlag) == 0;
  627. while (rangeIndex != kRegexInvalidRange) {
  628. const Range &r = regex_.GetRange(rangeIndex);
  629. if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) &&
  630. codepoint <= r.end)
  631. return yes;
  632. rangeIndex = r.next;
  633. }
  634. return !yes;
  635. }
  636. const RegexType &regex_;
  637. Allocator *allocator_;
  638. Allocator *ownAllocator_;
  639. Stack<Allocator> state0_;
  640. Stack<Allocator> state1_;
  641. uint32_t *stateSet_;
  642. };
  643. typedef GenericRegex<UTF8<>> Regex;
  644. typedef GenericRegexSearch<Regex> RegexSearch;
  645. } // namespace internal
  646. RAPIDJSON_NAMESPACE_END
  647. #ifdef __GNUC__
  648. RAPIDJSON_DIAG_POP
  649. #endif
  650. #if defined(__clang__) || defined(_MSC_VER)
  651. RAPIDJSON_DIAG_POP
  652. #endif
  653. #endif // RAPIDJSON_INTERNAL_REGEX_H_