Reject overflowed repeat/null-check IDs in huge regexps (ruby#16801)

Add explicit errors for excessive range-repeat and null-check IDs;
map new errors in regerror; add regression tests for oversized generated
patterns.

Author: makenowjust

include/ruby/onigmo.h

@@ -688,6 +688,8 @@ ONIG_EXTERN const OnigSyntaxType*   OnigDefaultSyntax;
 #define ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY        -222
 #define ONIGERR_INVALID_CHAR_PROPERTY_NAME                   -223
 #define ONIGERR_TOO_BIG_COMPILED_PROGRAM                     -226
+#define ONIGERR_TOO_MANY_RANGE_REPEAT                        -224
+#define ONIGERR_TOO_MANY_NULL_CHECK                          -225
 #define ONIGERR_INVALID_CODE_POINT_VALUE                     -400
 #define ONIGERR_INVALID_WIDE_CHAR_VALUE                      -400
 #define ONIGERR_TOO_BIG_WIDE_CHAR_VALUE                      -401

regcomp.c

@@ -398,6 +398,7 @@ compile_tree_empty_check(Node* node, regex_t* reg, int empty_info)
     r = add_mem_num(reg, reg->num_null_check); /* NULL CHECK ID */
     if (r) return r;
     reg->num_null_check++;
+    if ((MemNumType)reg->num_null_check <= 0) return ONIGERR_TOO_MANY_NULL_CHECK;
   }
 
   r = compile_tree(node, reg);
@@ -707,6 +708,7 @@ compile_range_repeat_node(QtfrNode* qn, int target_len, int empty_info,
   if (r) return r;
   r = add_mem_num(reg, num_repeat); /* OP_REPEAT ID */
   reg->num_repeat++;
+  if ((MemNumType)reg->num_repeat <= 0) return ONIGERR_TOO_MANY_RANGE_REPEAT;
   if (r) return r;
   r = add_rel_addr(reg, target_len + SIZE_OP_REPEAT_INC);
   if (r) return r;

regerror.c

@@ -167,6 +167,10 @@ onig_error_code_to_format(OnigPosition code)
     p = "invalid character property name {%n}"; break;
   case ONIGERR_TOO_BIG_COMPILED_PROGRAM:
     p = "too big compiled program"; break;
+  case ONIGERR_TOO_MANY_RANGE_REPEAT:
+    p = "too many range repeat"; break;
+  case ONIGERR_TOO_MANY_NULL_CHECK:
+    p = "too many null check"; break;
   case ONIGERR_TOO_MANY_CAPTURE_GROUPS:
     p = "too many capture groups are specified"; break;
   case ONIGERR_INVALID_CODE_POINT_VALUE:

test/ruby/test_regexp.rb

@@ -2348,4 +2348,16 @@ def test_nested_repeat_expansion_overflow
       end
     end;
   end
+
+  def test_too_many_range_repeat
+    source = '(?:foobar){0,100}' * 100000
+    assert_raise(RegexpError) { Regexp.new(source) }
+    assert_raise(SyntaxError) { eval("/#{source}/") }
+  end
+
+  def test_too_many_null_check
+    source = '(?:(?:foo)?|(?:bar)?)*' * 100000
+    assert_raise(RegexpError) { Regexp.new(source) }
+    assert_raise(SyntaxError) { eval("/#{source}/") }
+  end
 end