diff --git a/.github/actions/setup/directories/action.yml b/.github/actions/setup/directories/action.yml index 3890c4d9dc1b26..ea5f2720ef343e 100644 --- a/.github/actions/setup/directories/action.yml +++ b/.github/actions/setup/directories/action.yml @@ -94,7 +94,8 @@ runs: run: | touch config.status touch .rbconfig.time - sed -f tool/prereq.status template/Makefile.in common.mk > Makefile + sed -f tool/prereq.status template/Makefile.in > Makefile + sed -f tool/prereq.status template/GNUmakefile.in > GNUmakefile make up # Cleanup, runs even on failure diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 04edf2364101a7..dcf3ee87062c54 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -78,6 +78,7 @@ jobs: category: '/language:${{ matrix.language }}' upload: False output: sarif-results + ram: 8192 - name: filter-sarif uses: advanced-security/filter-sarif@f3b8118a9349d88f7b1c0c488476411145b6270d # v1.0 diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index eea20d6be95aa0..66a6805abce2b1 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -95,7 +95,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' + RUBY_TESTOPTS: '-q --tty=no' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-spec diff --git a/.gitignore b/.gitignore index 9218a84c4ddacd..631a489eeff1f8 100644 --- a/.gitignore +++ b/.gitignore @@ -256,6 +256,7 @@ lcov*.info /wasm/tests/*.wasm # YARP +/lib/yarp/mutation_visitor.rb /lib/yarp/node.rb /lib/yarp/serialize.rb /yarp/api_node.c diff --git a/NEWS.md b/NEWS.md index edad5de51e6ca1..3764db7df10cc1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,9 @@ Note that each entry is kept to a minimum, see links for details. * A new `performance` warning category was introduced. They are not displayed by default even in verbose mode. Turn them on with `-W:performance` or `Warning[:performance] = true`. [[Feature #19538]] +* The `RUBY_GC_HEAP_INIT_SLOTS` environment variable has been deprecated and + removed. Environment variables `RUBY_GC_HEAP_%d_INIT_SLOTS` should be + used instead. [[Feature #19785]] ## Core classes updates @@ -19,25 +22,27 @@ Note: We're only listing outstanding class updates. * Array - * `Array#pack` now raises ArgumentError for unknown directives. [[Bug #19150]] + * Array#pack now raises ArgumentError for unknown directives. [[Bug #19150]] * Dir - * `Dir.for_fd` added for returning a Dir object for the directory specified + * Dir.for_fd added for returning a Dir object for the directory specified by the provided directory file descriptor. [[Feature #19347]] - * `Dir.fchdir` added for changing the directory to the directory specified + * Dir.fchdir added for changing the directory to the directory specified by the provided directory file descriptor. [[Feature #19347]] - * `Dir#chdir` added for changing the directory to the directory specified - by the provided `Dir` object. [[Feature #19347]] + * Dir#chdir added for changing the directory to the directory specified by + the provided `Dir` object. [[Feature #19347]] * MatchData - * MatchData#named_captures now accepts optional `symbolize_names` keyword. [[Feature #19591]] + * MatchData#named_captures now accepts optional `symbolize_names` + keyword. [[Feature #19591]] * String - * `String#unpack` now raises ArgumentError for unknown directives. [[Bug #19150]] - * `String#bytesplice` now accepts new arguments index/length or range of the source string to be copied. [[Feature #19314]] + * String#unpack now raises ArgumentError for unknown directives. [[Bug #19150]] + * String#bytesplice now accepts new arguments index/length or range of the + source string to be copied. [[Feature #19314]] * ObjectSpace::WeakKeyMap @@ -47,7 +52,8 @@ Note: We're only listing outstanding class updates. * Module - * `Module#set_temporary_name` added for setting a temporary name for a module. [[Feature #19521]] + * Module#set_temporary_name added for setting a temporary name for a + module. [[Feature #19521]] * Process.warmup @@ -59,10 +65,17 @@ Note: We're only listing outstanding class updates. * Refinement * Add Refinement#target as an alternative of Refinement#refined_class. - Refinement#refined_class is deprecated and will be removed in Ruby 3.4. [[Feature #19714]] + Refinement#refined_class is deprecated and will be removed in Ruby + 3.4. [[Feature #19714]] ## Stdlib updates +* RubyGems and Bundler warn if users require gem that is scheduled to become the bundled gems + in the future version of Ruby. [[Feature #19351]] [[Feature #19776]] [[Feature #19843]] + +* Random::Formatter#alphanumeric is extended to accept optional `chars` + keyword argument. [[Feature #18183]] + The following default gems are updated. * RubyGems 3.5.0.dev @@ -72,18 +85,18 @@ The following default gems are updated. * erb 4.0.3 * fiddle 1.1.2 * fileutils 1.7.1 -* irb 1.7.4 +* irb 1.8.0 * nkf 0.1.3 * optparse 0.4.0.pre.1 * psych 5.1.0 * reline 0.3.8 -* stringio 3.0.8 +* stringio 3.0.9 * strscan 3.0.7 * syntax_suggest 1.1.0 * time 0.2.2 * timeout 0.4.0 * uri 0.12.2 -* yarp 0.8.0 +* yarp 0.9.0 The following bundled gems are updated. @@ -92,8 +105,8 @@ The following bundled gems are updated. * rexml 3.2.6 * rss 0.3.0 * net-imap 0.3.7 -* rbs 3.1.3 -* typeprof 0.21.7 +* rbs 3.2.1 +* typeprof 0.21.8 * debug 1.8.0 The following default gem is now bundled. @@ -131,13 +144,13 @@ changelog for details of the default gems or bundled gems. * Instance variables no longer exit to the interpreter with megamorphic Object Shapes. * Unsupported call types no longer exit to the interpreter. - * `Integer#!=`, `String#!=`, `Kernel#block_given?`, `Kernel#is_a?`, - `Kernel#instance_of?`, `Module#===` are specially optimized. + * Integer#!=, String#!=, Kernel#block_given?, Kernel#is_a?, + Kernel#instance_of?, Module#=== are specially optimized. * Now more than 3x faster than the interpreter on optcarrot! * Metadata for compiled code uses a lot less memory. * Generate more compact code on ARM64 * Option to start YJIT in paused mode and then later enable it manually - * `--yjit-pause` and `RubyVM::YJIT.resume` + * `--yjit-pause` and RubyVM::YJIT.resume * This can be used to enable YJIT only once your application is done booting * `ratio_in_yjit` stat produced by `--yjit-stats` is now avaiable in release builds, a special stats or dev build is no longer required. @@ -155,12 +168,17 @@ changelog for details of the default gems or bundled gems. * RJIT exists only for experimental purposes. * You should keep using YJIT in production. +[Feature #18183]: https://bugs.ruby-lang.org/issues/18183 [Feature #18498]: https://bugs.ruby-lang.org/issues/18498 [Feature #18885]: https://bugs.ruby-lang.org/issues/18885 [Bug #19150]: https://bugs.ruby-lang.org/issues/19150 [Feature #19314]: https://bugs.ruby-lang.org/issues/19314 [Feature #19347]: https://bugs.ruby-lang.org/issues/19347 +[Feature #19351]: https://bugs.ruby-lang.org/issues/19351 [Feature #19521]: https://bugs.ruby-lang.org/issues/19521 [Feature #19538]: https://bugs.ruby-lang.org/issues/19538 [Feature #19591]: https://bugs.ruby-lang.org/issues/19591 [Feature #19714]: https://bugs.ruby-lang.org/issues/19714 +[Feature #19776]: https://bugs.ruby-lang.org/issues/19776 +[Feature #19785]: https://bugs.ruby-lang.org/issues/19785 +[Feature #19843]: https://bugs.ruby-lang.org/issues/19843 diff --git a/array.c b/array.c index b4d4cd5bff1410..65041c9365f359 100644 --- a/array.c +++ b/array.c @@ -3522,8 +3522,8 @@ rb_ary_bsearch_index(VALUE ary) const VALUE zero = INT2FIX(0); switch (rb_cmpint(rb_funcallv(v, id_cmp, 1, &zero), v, zero)) { case 0: return INT2FIX(mid); - case 1: smaller = 1; break; - case -1: smaller = 0; + case 1: smaller = 0; break; + case -1: smaller = 1; } } else { @@ -4310,7 +4310,7 @@ rb_ary_reject(VALUE ary) * a = [:foo, 'bar', 2] * a.delete_if # => # * -3 */ + */ static VALUE rb_ary_delete_if(VALUE ary) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 4f73af89e8507d..3c53641f91e2fb 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -1,3 +1,28 @@ +# regression test for overly generous guard elision +assert_equal '[0, :sum, 0, :sum]', %q{ + # In faulty versions, the following happens: + # 1. YJIT puts object on the temp stack with type knowledge + # (CArray or CString) about RBASIC_CLASS(object). + # 2. In iter=0, due to the type knowledge, YJIT generates + # a call to sum() without any guard on RBASIC_CLASS(object). + # 3. In iter=1, a singleton class is added to the object, + # changing RBASIC_CLASS(object), falsifying the type knowledge. + # 4. Because the code from (1) has no class guard, it is incorrectly + # reused and the wrong method is invoked. + # Putting a literal is important for gaining type knowledge. + def carray(iter) + array = [] + array.sum(iter.times { def array.sum(_) = :sum }) + end + + def cstring(iter) + string = "" + string.sum(iter.times { def string.sum(_) = :sum }) + end + + [carray(0), carray(1), cstring(0), cstring(1)] +} + # regression test for return type of Integer#/ # It can return a T_BIGNUM when inputs are T_FIXNUM. assert_equal 0x3fffffffffffffff.to_s, %q{ diff --git a/common.mk b/common.mk index b0f9369fbc8682..d55d1788aa8e01 100644 --- a/common.mk +++ b/common.mk @@ -45,7 +45,8 @@ RUN_OPTS = --disable-gems # GITPULLOPTIONS = --no-tags -INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(srcdir)/yarp -I$(UNICODE_HDR_DIR) +YARP_SRCDIR = $(srcdir)/yarp +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(YARP_SRCDIR) -I$(UNICODE_HDR_DIR) GEM_HOME = GEM_PATH = @@ -202,39 +203,44 @@ $(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time $(Q) $(MAKEDIRS) $(@D) @$(NULLCMD) > $@ +main: $(srcdir)/lib/yarp/mutation_visitor.rb +srcs: $(srcdir)/lib/yarp/mutation_visitor.rb +$(srcdir)/lib/yarp/mutation_visitor.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/mutation_visitor.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/mutation_visitor.rb $(srcdir)/lib/yarp/mutation_visitor.rb + main: $(srcdir)/lib/yarp/node.rb srcs: $(srcdir)/lib/yarp/node.rb -$(srcdir)/lib/yarp/node.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/node.rb.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb +$(srcdir)/lib/yarp/node.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/node.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb main: $(srcdir)/lib/yarp/serialize.rb srcs: $(srcdir)/lib/yarp/serialize.rb -$(srcdir)/lib/yarp/serialize.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/serialize.rb.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/serialize.rb $(srcdir)/lib/yarp/serialize.rb +$(srcdir)/lib/yarp/serialize.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/serialize.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/serialize.rb $(srcdir)/lib/yarp/serialize.rb srcs: yarp/api_node.c -yarp/api_node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/ext/yarp/api_node.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb ext/yarp/api_node.c $@ +yarp/api_node.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/ext/yarp/api_node.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb ext/yarp/api_node.c $@ srcs: yarp/ast.h -yarp/ast.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/ast.h.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/ast.h $@ +yarp/ast.h: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/include/yarp/ast.h.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb include/yarp/ast.h $@ srcs: yarp/node.c -yarp/node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/node.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/node.c $@ +yarp/node.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/node.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/node.c $@ srcs: yarp/prettyprint.c -yarp/prettyprint.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/prettyprint.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/prettyprint.c $@ +yarp/prettyprint.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/prettyprint.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/prettyprint.c $@ srcs: yarp/serialize.c -yarp/serialize.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/serialize.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/serialize.c $@ +yarp/serialize.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/serialize.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/serialize.c $@ srcs: yarp/token_type.c -yarp/token_type.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/token_type.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/token_type.c $@ +yarp/token_type.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/token_type.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/token_type.c $@ EXPORTOBJS = $(DLNOBJ) \ localeinit.$(OBJEXT) \ @@ -3176,6 +3182,26 @@ compile.$(OBJEXT): $(top_srcdir)/internal/thread.h compile.$(OBJEXT): $(top_srcdir)/internal/variable.h compile.$(OBJEXT): $(top_srcdir)/internal/vm.h compile.$(OBJEXT): $(top_srcdir)/internal/warnings.h +compile.$(OBJEXT): $(top_srcdir)/yarp/defines.h +compile.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h +compile.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h +compile.$(OBJEXT): $(top_srcdir)/yarp/node.h +compile.$(OBJEXT): $(top_srcdir)/yarp/pack.h +compile.$(OBJEXT): $(top_srcdir)/yarp/parser.h +compile.$(OBJEXT): $(top_srcdir)/yarp/regexp.h +compile.$(OBJEXT): $(top_srcdir)/yarp/unescape.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_memchr.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h +compile.$(OBJEXT): $(top_srcdir)/yarp/yarp.h +compile.$(OBJEXT): $(top_srcdir)/yarp/yarp_compiler.c compile.$(OBJEXT): {$(VPATH)}assert.h compile.$(OBJEXT): {$(VPATH)}atomic.h compile.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -3373,6 +3399,9 @@ compile.$(OBJEXT): {$(VPATH)}vm_callinfo.h compile.$(OBJEXT): {$(VPATH)}vm_core.h compile.$(OBJEXT): {$(VPATH)}vm_debug.h compile.$(OBJEXT): {$(VPATH)}vm_opts.h +compile.$(OBJEXT): {$(VPATH)}yarp/ast.h +compile.$(OBJEXT): {$(VPATH)}yarp/version.h +compile.$(OBJEXT): {$(VPATH)}yarp/yarp.h complex.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h complex.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h complex.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -8200,6 +8229,25 @@ iseq.$(OBJEXT): $(top_srcdir)/internal/thread.h iseq.$(OBJEXT): $(top_srcdir)/internal/variable.h iseq.$(OBJEXT): $(top_srcdir)/internal/vm.h iseq.$(OBJEXT): $(top_srcdir)/internal/warnings.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/defines.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/node.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/pack.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/parser.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/regexp.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/unescape.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_memchr.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/yarp.h iseq.$(OBJEXT): {$(VPATH)}assert.h iseq.$(OBJEXT): {$(VPATH)}atomic.h iseq.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -8394,6 +8442,9 @@ iseq.$(OBJEXT): {$(VPATH)}util.h iseq.$(OBJEXT): {$(VPATH)}vm_callinfo.h iseq.$(OBJEXT): {$(VPATH)}vm_core.h iseq.$(OBJEXT): {$(VPATH)}vm_opts.h +iseq.$(OBJEXT): {$(VPATH)}yarp/ast.h +iseq.$(OBJEXT): {$(VPATH)}yarp/version.h +iseq.$(OBJEXT): {$(VPATH)}yarp/yarp.h iseq.$(OBJEXT): {$(VPATH)}yjit.h load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h @@ -18851,9 +18902,6 @@ weakmap.$(OBJEXT): {$(VPATH)}vm_opts.h weakmap.$(OBJEXT): {$(VPATH)}weakmap.c yarp/api_node.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_node.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c -yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19040,11 +19088,12 @@ yarp/api_node.$(OBJEXT): {$(VPATH)}onigmo.h yarp/api_node.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/api_node.$(OBJEXT): {$(VPATH)}st.h yarp/api_node.$(OBJEXT): {$(VPATH)}subst.h +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/api_pack.c -yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19231,151 +19280,123 @@ yarp/api_pack.$(OBJEXT): {$(VPATH)}onigmo.h yarp/api_pack.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/api_pack.$(OBJEXT): {$(VPATH)}st.h yarp/api_pack.$(OBJEXT): {$(VPATH)}subst.h -yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.c yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/diagnostic.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_ascii.c yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_ascii.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_big5.c yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_big5.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_euc_jp.c yarp/enc/yp_euc_jp.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_gbk.c yarp/enc/yp_gbk.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_1.c yarp/enc/yp_iso_8859_1.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_10.c yarp/enc/yp_iso_8859_10.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_11.c yarp/enc/yp_iso_8859_11.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_13.c yarp/enc/yp_iso_8859_13.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_14.c yarp/enc/yp_iso_8859_14.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_15.c yarp/enc/yp_iso_8859_15.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_16.c yarp/enc/yp_iso_8859_16.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_2.c yarp/enc/yp_iso_8859_2.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_3.c yarp/enc/yp_iso_8859_3.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_4.c yarp/enc/yp_iso_8859_4.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_5.c yarp/enc/yp_iso_8859_5.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_6.c yarp/enc/yp_iso_8859_6.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_7.c yarp/enc/yp_iso_8859_7.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_8.c yarp/enc/yp_iso_8859_8.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_9.c yarp/enc/yp_iso_8859_9.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_koi8_r.c yarp/enc/yp_koi8_r.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_shared.c yarp/enc/yp_shared.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_shift_jis.c yarp/enc/yp_shift_jis.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_tables.c yarp/enc/yp_tables.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_unicode.c yarp/enc/yp_unicode.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_1251.c yarp/enc/yp_windows_1251.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_1252.c yarp/enc/yp_windows_1252.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_31j.c yarp/enc/yp_windows_31j.$(OBJEXT): {$(VPATH)}config.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19563,12 +19584,11 @@ yarp/extension.$(OBJEXT): {$(VPATH)}onigmo.h yarp/extension.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/extension.$(OBJEXT): {$(VPATH)}st.h yarp/extension.$(OBJEXT): {$(VPATH)}subst.h -yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/node.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/extension.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h -yarp/node.$(OBJEXT): {$(VPATH)}yarp/node.c yarp/node.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19585,17 +19605,15 @@ yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/node.$(OBJEXT): {$(VPATH)}config.h -yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/node.$(OBJEXT): {$(VPATH)}yarp/node.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/pack.$(OBJEXT): {$(VPATH)}config.h -yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/parser.h -yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/prettyprint.c yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h @@ -19603,8 +19621,8 @@ yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/prettyprint.$(OBJEXT): {$(VPATH)}config.h -yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/prettyprint.c yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19618,8 +19636,7 @@ yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/regexp.$(OBJEXT): {$(VPATH)}config.h -yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19627,7 +19644,6 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/parser.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/regexp.h -yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/serialize.c yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/unescape.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h @@ -19641,15 +19657,15 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/serialize.$(OBJEXT): {$(VPATH)}config.h -yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/serialize.c +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/defines.h -yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/token_type.$(OBJEXT): {$(VPATH)}config.h -yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19671,29 +19687,25 @@ yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/unescape.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.c yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/util/yp_buffer.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.c yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_char.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.c yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/util/yp_constant_pool.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.c yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/util/yp_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19705,32 +19717,26 @@ yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.c yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_newline_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.c yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/util/yp_state_stack.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.c yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_string.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.c yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/util/yp_string_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strncasecmp.c -yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19742,8 +19748,7 @@ yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.c yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}config.h -yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/config.h +yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19766,10 +19771,10 @@ yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/version.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.c yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/yarp.$(OBJEXT): {$(VPATH)}config.h +yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19957,6 +19962,8 @@ yarp/yarp_init.$(OBJEXT): {$(VPATH)}onigmo.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}st.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}subst.h +yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/version.h yjit.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h yjit.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h yjit.$(OBJEXT): $(CCAN_DIR)/list/list.h diff --git a/compile.c b/compile.c index ec7094cbd3be54..1c220bdbb2a887 100644 --- a/compile.c +++ b/compile.c @@ -44,6 +44,7 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" +#include "yarp/yarp.h" #undef RUBY_UNTYPED_DATA_WARNING #define RUBY_UNTYPED_DATA_WARNING 0 @@ -856,6 +857,45 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) return iseq_setup(iseq, ret); } +typedef struct yp_compile_context { + yp_parser_t *parser; + struct yp_compile_context *previous; + ID *constants; + st_table *index_lookup_table; +} yp_compile_context_t; + +static VALUE rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_compile_context_t *compile_context); + +VALUE +rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t *yarp_pointer, yp_parser_t *parser) +{ + DECL_ANCHOR(ret); + INIT_ANCHOR(ret); + + ID *constants = calloc(parser->constant_pool.size, sizeof(ID)); + rb_encoding *encoding = rb_enc_find(parser->encoding.name); + + for (size_t index = 0; index < parser->constant_pool.capacity; index++) { + yp_constant_t constant = parser->constant_pool.constants[index]; + + if (constant.id != 0) { + constants[constant.id - 1] = rb_intern3((const char *) constant.start, constant.length, encoding); + } + } + + yp_compile_context_t compile_context = { + .parser = parser, + .previous = NULL, + .constants = constants + }; + + CHECK(rb_translate_yarp(iseq, yarp_pointer, ret, &compile_context)); + free(constants); + + CHECK(iseq_setup_insn(iseq, ret)); + return iseq_setup(iseq, ret); +} + static int rb_iseq_translate_threaded_code(rb_iseq_t *iseq) { @@ -13294,3 +13334,5 @@ rb_iseq_ibf_load_extra_data(VALUE str) RB_GC_GUARD(loader_obj); return extra_str; } + +#include "yarp/yarp_compiler.c" diff --git a/configure.ac b/configure.ac index 05dd41e79c9f69..8447eba1051b09 100644 --- a/configure.ac +++ b/configure.ac @@ -2153,9 +2153,6 @@ AC_CHECK_FUNCS(__sinpi) AS_IF([test "x$ac_cv_member_struct_statx_stx_btime" = xyes], [AC_CHECK_FUNCS(statx)]) -AS_CASE(["$ac_cv_func_memset_s:$ac_cv_func_qsort_s"], [*yes*], - [RUBY_DEFINE_IF([!defined __STDC_WANT_LIB_EXT1__], [__STDC_WANT_LIB_EXT1__], 1)]) - AS_IF([test "$ac_cv_func_getcwd" = yes], [ AC_CACHE_CHECK(if getcwd allocates buffer if NULL is given, [rb_cv_getcwd_malloc], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ diff --git a/debug_counter.h b/debug_counter.h index 01b1a63f86312d..a8b95edded258b 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -100,6 +100,13 @@ RB_DEBUG_COUNTER(ccf_opt_block_call) RB_DEBUG_COUNTER(ccf_opt_struct_aref) RB_DEBUG_COUNTER(ccf_opt_struct_aset) RB_DEBUG_COUNTER(ccf_super_method) +RB_DEBUG_COUNTER(ccf_cfunc_other) +RB_DEBUG_COUNTER(ccf_cfunc_only_splat) +RB_DEBUG_COUNTER(ccf_cfunc_only_splat_kw) +RB_DEBUG_COUNTER(ccf_iseq_bmethod) +RB_DEBUG_COUNTER(ccf_noniseq_bmethod) +RB_DEBUG_COUNTER(ccf_opt_send_complex) +RB_DEBUG_COUNTER(ccf_opt_send_simple) /* * control frame push counts. diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 02efb2384fc47a..b96fa4bc0bd921 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -157,6 +157,14 @@ with the Ruby script you'd like to run. You can use the following make targets: * `make lldb-ruby`: Runs `test.rb` using Ruby in lldb * `make gdb-ruby`: Runs `test.rb` using Ruby in gdb +### Compiling for Debugging + +You should configure Ruby without optimization and other flags that may interfere with debugging: + +``` shell +./configure --enable-debug-env optflags="-O0 -fno-omit-frame-pointer" +``` + ### Building with Address Sanitizer Using the address sanitizer is a great way to detect memory issues. diff --git a/doc/irb/irb.rd.ja b/doc/irb/irb.rd.ja index 633c08cbd4c846..c51e0bd60d566b 100644 --- a/doc/irb/irb.rd.ja +++ b/doc/irb/irb.rd.ja @@ -125,7 +125,6 @@ irb起動時に``~/.irbrc''を読み込みます. もし存在しない場合は IRB.conf[:PROMPT][:MY_PROMPT] = { # プロンプトモードの名前 :PROMPT_I => nil, # 通常のプロンプト - :PROMPT_N => nil, # 継続行のプロンプト :PROMPT_S => nil, # 文字列などの継続行のプロンプト :PROMPT_C => nil, # 式が継続している時のプロンプト :RETURN => " ==>%s\n" # リターン時のプロンプト @@ -140,7 +139,7 @@ OKです. IRB.conf[:PROMPT_MODE] = :MY_PROMPT -PROMPT_I, PROMPT_N, PROMPT_S, PROMPT_Cは, フォーマットを指定します. +PROMPT_I, PROMPT_S, PROMPT_Cは, フォーマットを指定します. %N 起動しているコマンド名が出力される. %m mainオブジェクト(self)がto_sで出力される. @@ -155,7 +154,6 @@ PROMPT_I, PROMPT_N, PROMPT_S, PROMPT_Cは, フォーマットを指定します. IRB.conf[:PROMPT][:DEFAULT] = { :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", :PROMPT_S => "%N(%m):%03n:%i%l ", :PROMPT_C => "%N(%m):%03n:%i* ", :RETURN => "=> %s\n" diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index 6d73d259c49ac5..013412c27f5329 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -82,30 +82,62 @@ ossl_pkey_new(EVP_PKEY *pkey) #if OSSL_OPENSSL_PREREQ(3, 0, 0) # include -EVP_PKEY * -ossl_pkey_read_generic(BIO *bio, VALUE pass) +static EVP_PKEY * +ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass) { void *ppass = (void *)pass; OSSL_DECODER_CTX *dctx; EVP_PKEY *pkey = NULL; int pos = 0, pos2; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, NULL, 0, NULL, NULL); + dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL, + selection, NULL, NULL); if (!dctx) goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - - /* First check DER */ - if (OSSL_DECODER_from_bio(dctx, bio) == 1) + if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, + ppass) != 1) goto out; + while (1) { + if (OSSL_DECODER_from_bio(dctx, bio) == 1) + goto out; + if (BIO_eof(bio)) + break; + pos2 = BIO_tell(bio); + if (pos2 < 0 || pos2 <= pos) + break; + ossl_clear_error(); + pos = pos2; + } + out: OSSL_BIO_reset(bio); + OSSL_DECODER_CTX_free(dctx); + return pkey; +} +EVP_PKEY * +ossl_pkey_read_generic(BIO *bio, VALUE pass) +{ + EVP_PKEY *pkey = NULL; + /* First check DER, then check PEM. */ + const char *input_types[] = {"DER", "PEM"}; + int input_type_num = (int)(sizeof(input_types) / sizeof(char *)); /* - * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed. + * Non-zero selections to try to decode. + * + * See EVP_PKEY_fromdata(3) - Selections to see all the selections. * - * First check for private key formats. This is to keep compatibility with - * ruby/openssl < 3.0 which decoded the following as a private key. + * This is a workaround for the decoder failing to decode or returning + * bogus keys with selection 0, if a key management provider is different + * from a decoder provider. The workaround is to avoid using selection 0. + * + * Affected OpenSSL versions: >= 3.1.0, <= 3.1.2, or >= 3.0.0, <= 3.0.10 + * Fixed OpenSSL versions: 3.2, next release of the 3.1.z and 3.0.z + * + * See https://github.com/openssl/openssl/pull/21519 for details. + * + * First check for private key formats (EVP_PKEY_KEYPAIR). This is to keep + * compatibility with ruby/openssl < 3.0 which decoded the following as a + * private key. * * $ openssl ecparam -name prime256v1 -genkey -outform PEM * -----BEGIN EC PARAMETERS----- @@ -126,50 +158,25 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass) * * Note that we need to create the OSSL_DECODER_CTX variable each time when * we use the different selection as a workaround. - * https://github.com/openssl/openssl/issues/20657 + * See https://github.com/openssl/openssl/issues/20657 for details. */ - OSSL_DECODER_CTX_free(dctx); - dctx = NULL; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, - EVP_PKEY_KEYPAIR, NULL, NULL); - if (!dctx) - goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - while (1) { - if (OSSL_DECODER_from_bio(dctx, bio) == 1) - goto out; - if (BIO_eof(bio)) - break; - pos2 = BIO_tell(bio); - if (pos2 < 0 || pos2 <= pos) - break; - ossl_clear_error(); - pos = pos2; - } - - OSSL_BIO_reset(bio); - OSSL_DECODER_CTX_free(dctx); - dctx = NULL; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, 0, NULL, NULL); - if (!dctx) - goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - while (1) { - if (OSSL_DECODER_from_bio(dctx, bio) == 1) - goto out; - if (BIO_eof(bio)) - break; - pos2 = BIO_tell(bio); - if (pos2 < 0 || pos2 <= pos) - break; - ossl_clear_error(); - pos = pos2; + int selections[] = { + EVP_PKEY_KEYPAIR, + EVP_PKEY_KEY_PARAMETERS, + EVP_PKEY_PUBLIC_KEY + }; + int selection_num = (int)(sizeof(selections) / sizeof(int)); + int i, j; + + for (i = 0; i < input_type_num; i++) { + for (j = 0; j < selection_num; j++) { + pkey = ossl_pkey_read(bio, input_types[i], selections[j], pass); + if (pkey) { + goto out; + } + } } - out: - OSSL_DECODER_CTX_free(dctx); return pkey; } #else diff --git a/ext/socket/ancdata.c b/ext/socket/ancdata.c index 7406177de2cb6a..6ef040b692f338 100644 --- a/ext/socket/ancdata.c +++ b/ext/socket/ancdata.c @@ -1555,6 +1555,10 @@ bsock_recvmsg_internal(VALUE sock, ss = rb_recvmsg(fptr->fd, &mh, flags); + if (ss == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } + if (ss == -1) { int e; if (!nonblock && rb_io_maybe_wait_readable(errno, fptr->self, RUBY_IO_TIMEOUT_DEFAULT)) { diff --git a/ext/socket/init.c b/ext/socket/init.c index 557d4374a51e0c..e9dc6f84839f25 100644 --- a/ext/socket/init.c +++ b/ext/socket/init.c @@ -116,6 +116,7 @@ recvfrom_blocking(void *data) ssize_t ret; ret = recvfrom(arg->fd, RSTRING_PTR(arg->str), arg->length, arg->flags, &arg->buf.addr, &arg->alen); + if (ret != -1 && len0 < arg->alen) arg->alen = len0; @@ -147,6 +148,18 @@ recvfrom_locktmp(VALUE v) return rb_thread_io_blocking_region(recvfrom_blocking, arg, arg->fd); } +int +rsock_is_dgram(rb_io_t *fptr) +{ + int socktype; + socklen_t optlen = (socklen_t)sizeof(socktype); + int ret = getsockopt(fptr->fd, SOL_SOCKET, SO_TYPE, (void*)&socktype, &optlen); + if (ret == -1) { + rb_sys_fail("getsockopt(SO_TYPE)"); + } + return socktype == SOCK_DGRAM; +} + VALUE rsock_s_recvfrom(VALUE socket, int argc, VALUE *argv, enum sock_recv_type from) { @@ -187,6 +200,9 @@ rsock_s_recvfrom(VALUE socket, int argc, VALUE *argv, enum sock_recv_type from) slen = (long)rb_str_locktmp_ensure(str, recvfrom_locktmp, (VALUE)&arg); + if (slen == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } if (slen >= 0) break; if (!rb_io_maybe_wait_readable(errno, socket, RUBY_IO_TIMEOUT_DEFAULT)) @@ -259,6 +275,10 @@ rsock_s_recvfrom_nonblock(VALUE sock, VALUE len, VALUE flg, VALUE str, if (slen != -1 && len0 < alen) alen = len0; + if (slen == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } + if (slen < 0) { int e = errno; switch (e) { diff --git a/ext/socket/rubysocket.h b/ext/socket/rubysocket.h index 5f803ba0da3c6f..7c5739808d1618 100644 --- a/ext/socket/rubysocket.h +++ b/ext/socket/rubysocket.h @@ -459,6 +459,8 @@ VALUE rsock_write_nonblock(VALUE sock, VALUE buf, VALUE ex); void rsock_make_fd_nonblock(int fd); +int rsock_is_dgram(rb_io_t *fptr); + #if !defined HAVE_INET_NTOP && ! defined _WIN32 const char *inet_ntop(int, const void *, char *, size_t); #elif defined __MINGW32__ diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index f5213f2fc0fc92..e89a1d780cf996 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -12,7 +12,7 @@ **********************************************************************/ -#define STRINGIO_VERSION "3.0.8" +#define STRINGIO_VERSION "3.0.9" #include "ruby.h" #include "ruby/io.h" diff --git a/file.c b/file.c index e753e116de68f7..4a5c8a8f8aa965 100644 --- a/file.c +++ b/file.c @@ -4610,7 +4610,7 @@ rmext(const char *p, long l0, long l1, const char *e, long l2, rb_encoding *enc) if (l1 < l2) return l1; s = p+l1-l2; - if (rb_enc_left_char_head(p, s, p+l1, enc) != s) return 0; + if (!at_char_boundary(p, s, p+l1, enc)) return 0; #if CASEFOLD_FILESYSTEM #define fncomp strncasecmp #else @@ -7436,7 +7436,7 @@ Init_File(void) * There are two families of constants here: * * - Those having to do with {file access}[rdoc-ref:File::Constants@File+Access]. - * - Those having to do with {file globbing}[rdoc-ref:File::Constants@File+Globbing]. + * - Those having to do with {filename globbing}[rdoc-ref:File::Constants@Filename+Globbing+Constants+-28File-3A-3AFNM_-2A-29]. * * \File constants defined for the local process may be retrieved * with method File::Constants.constants: diff --git a/gc.c b/gc.c index e7f7244531efd7..d44ebff5b45145 100644 --- a/gc.c +++ b/gc.c @@ -2745,11 +2745,22 @@ heap_add_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *he #endif static size_t -minimum_pages_for_size_pool(rb_objspace_t *objspace, int size_pool_idx) +slots_to_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t slots) { - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; - int multiple = size_pool->slot_size / BASE_SLOT_SIZE; - return gc_params.size_pool_init_slots[size_pool_idx] * multiple / HEAP_PAGE_OBJ_LIMIT; + size_t multiple = size_pool->slot_size / BASE_SLOT_SIZE; + /* Due to alignment, heap pages may have one less slot. We should + * ensure there is enough pages to guarantee that we will have at + * least the required number of slots after allocating all the pages. */ + size_t slots_per_page = (HEAP_PAGE_OBJ_LIMIT / multiple) - 1; + return CEILDIV(slots, slots_per_page); +} + +static size_t +minimum_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) +{ + size_t size_pool_idx = size_pool - size_pools; + size_t init_slots = gc_params.size_pool_init_slots[size_pool_idx]; + return slots_to_pages_for_size_pool(objspace, size_pool, init_slots); } static size_t @@ -2762,7 +2773,7 @@ heap_extend_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t fre next_used = (size_t)(used * gc_params.growth_factor); } else if (total_slots == 0) { - next_used = minimum_pages_for_size_pool(objspace, (int)(size_pool - size_pools)); + next_used = minimum_pages_for_size_pool(objspace, size_pool); } else { /* Find `f' where free_slots = f * total_slots * goal_ratio @@ -2822,7 +2833,7 @@ gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) gc_enter(objspace, gc_enter_event_continue, false); /* Continue marking if in incremental marking. */ - if (heap->free_pages == NULL && is_incremental_marking(objspace)) { + if (is_incremental_marking(objspace)) { if (gc_marks_continue(objspace, size_pool, heap)) { gc_sweep(objspace); } @@ -4418,10 +4429,12 @@ Init_heap(rb_objspace_t *objspace) /* Set size pools allocatable pages. */ for (int i = 0; i < SIZE_POOL_COUNT; i++) { + rb_size_pool_t *size_pool = &size_pools[i]; + /* Set the default value of size_pool_init_slots. */ gc_params.size_pool_init_slots[i] = GC_HEAP_INIT_SLOTS; - size_pools[i].allocatable_pages = minimum_pages_for_size_pool(objspace, i); + size_pool->allocatable_pages = minimum_pages_for_size_pool(objspace, size_pool); } heap_pages_expand_sorted(objspace); @@ -6612,12 +6625,11 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) if (swept_slots < min_free_slots) { bool grow_heap = is_full_marking(objspace); - if (!is_full_marking(objspace)) { - /* The heap is a growth heap if it freed more slots than had empty - * slots and used up all of its allocatable pages. */ - bool is_growth_heap = (size_pool->empty_slots == 0 || - size_pool->freed_slots > size_pool->empty_slots) && - size_pool->allocatable_pages == 0; + /* Consider growing or starting a major GC if we are not currently in a + * major GC and we can't allocate any more pages. */ + if (!is_full_marking(objspace) && size_pool->allocatable_pages == 0) { + /* The heap is a growth heap if it freed more slots than had empty slots. */ + bool is_growth_heap = size_pool->empty_slots == 0 || size_pool->freed_slots > size_pool->empty_slots; /* Grow this heap if we haven't run at least RVALUE_OLD_AGE minor * GC since the last major GC or if this heap is smaller than the @@ -7242,16 +7254,6 @@ rb_gc_mark_locations(const VALUE *start, const VALUE *end) gc_mark_locations(&rb_objspace, start, end, gc_mark_maybe); } -static void -gc_mark_values(rb_objspace_t *objspace, long n, const VALUE *values) -{ - long i; - - for (i=0; irgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED)); + /* If we are in a minor GC and the other object is old, then obj should + * already be marked and cannot be reclaimed in this GC cycle so we don't + * need to add it to the weak refences list. */ + if (!is_full_marking(objspace) && RVALUE_OLD_P(obj)) { + GC_ASSERT(RVALUE_MARKED(obj)); + GC_ASSERT(!objspace->flags.during_compacting); + + return; + } + rgengc_check_relation(objspace, obj); rb_darray_append_without_gc(&objspace->weak_references, ptr); @@ -7942,7 +7954,7 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj) // just after newobj() can be NULL here. GC_ASSERT(env->ep[VM_ENV_DATA_INDEX_ENV] == obj); GC_ASSERT(VM_ENV_ESCAPED_P(env->ep)); - gc_mark_values(objspace, (long)env->env_size, env->env); + rb_gc_mark_values((long)env->env_size, env->env); VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED); gc_mark(objspace, (VALUE)rb_vm_env_prev_env(env)); gc_mark(objspace, (VALUE)env->iseq); @@ -11027,11 +11039,55 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE return Qnil; } +static void +free_empty_pages(void) +{ + rb_objspace_t *objspace = &rb_objspace; + + for (int i = 0; i < SIZE_POOL_COUNT; i++) { + /* Move all empty pages to the tomb heap for freeing. */ + rb_size_pool_t *size_pool = &size_pools[i]; + rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + rb_heap_t *tomb_heap = SIZE_POOL_TOMB_HEAP(size_pool); + + size_t freed_pages = 0; + + struct heap_page **next_page_ptr = &heap->free_pages; + struct heap_page *page = heap->free_pages; + while (page) { + /* All finalizers should have been ran in gc_start_internal, so there + * should be no objects that require finalization. */ + GC_ASSERT(page->final_slots == 0); + + struct heap_page *next_page = page->free_next; + + if (page->free_slots == page->total_slots) { + heap_unlink_page(objspace, heap, page); + heap_add_page(objspace, size_pool, tomb_heap, page); + freed_pages++; + } + else { + *next_page_ptr = page; + next_page_ptr = &page->free_next; + } + + page = next_page; + } + + *next_page_ptr = NULL; + + size_pool_allocatable_pages_set(objspace, size_pool, size_pool->allocatable_pages + freed_pages); + } + + heap_pages_free_unused_pages(objspace); +} + void rb_gc_prepare_heap(void) { rb_objspace_each_objects(gc_set_candidate_object_i, NULL); gc_start_internal(NULL, Qtrue, Qtrue, Qtrue, Qtrue, Qfalse, Qtrue); + free_empty_pages(); } static int @@ -11486,6 +11542,12 @@ gc_update_values(rb_objspace_t *objspace, long n, VALUE *values) } } +void +rb_gc_update_values(long n, VALUE *values) +{ + gc_update_values(&rb_objspace, n, values); +} + static bool moved_or_living_object_strictly_p(rb_objspace_t *objspace, VALUE obj) { @@ -12207,7 +12269,7 @@ gc_verify_compaction_references(rb_execution_context_t *ec, VALUE self, VALUE do size_t minimum_pages = 0; if (RTEST(expand_heap)) { - minimum_pages = minimum_pages_for_size_pool(objspace, i); + minimum_pages = minimum_pages_for_size_pool(objspace, size_pool); } heap_add_pages(objspace, size_pool, heap, MAX(minimum_pages, heap->total_pages)); @@ -13080,8 +13142,8 @@ gc_set_initial_pages(rb_objspace_t *objspace) for (int i = 0; i < SIZE_POOL_COUNT; i++) { rb_size_pool_t *size_pool = &size_pools[i]; - char env_key[sizeof("RUBY_GC_HEAP_INIT_SIZE_" "_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(size_pool->slot_size) * CHAR_BIT)]; - snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS", size_pool->slot_size); + char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)]; + snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i); size_t size_pool_init_slots = gc_params.size_pool_init_slots[i]; if (get_envparam_size(env_key, &size_pool_init_slots, 0)) { @@ -13090,8 +13152,7 @@ gc_set_initial_pages(rb_objspace_t *objspace) if (size_pool_init_slots > size_pool->eden_heap.total_slots) { size_t slots = size_pool_init_slots - size_pool->eden_heap.total_slots; - int multiple = size_pool->slot_size / BASE_SLOT_SIZE; - size_pool->allocatable_pages = slots * multiple / HEAP_PAGE_OBJ_LIMIT; + size_pool->allocatable_pages = slots_to_pages_for_size_pool(objspace, size_pool, slots); } else { /* We already have more slots than size_pool_init_slots allows, so @@ -13105,8 +13166,6 @@ gc_set_initial_pages(rb_objspace_t *objspace) /* * GC tuning environment variables * - * * RUBY_GC_HEAP_INIT_SLOTS - * - Initial allocation slots. * * RUBY_GC_HEAP_FREE_SLOTS * - Prepare at least this amount of slots after GC. * - Allocate slots if there are not enough slots. @@ -13153,13 +13212,6 @@ ruby_gc_set_params(void) /* ok */ } - /* RUBY_GC_HEAP_INIT_SLOTS */ - size_t global_init_slots = GC_HEAP_INIT_SLOTS; - if (get_envparam_size("RUBY_GC_HEAP_INIT_SLOTS", &global_init_slots, 0)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - gc_params.size_pool_init_slots[i] = global_init_slots; - } - } gc_set_initial_pages(objspace); get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); @@ -15363,6 +15415,7 @@ Init_GC(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE)); if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) { rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue); } diff --git a/gems/bundled_gems b/gems/bundled_gems index af08c6c82d1c3e..f11380027ac77c 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -17,7 +17,7 @@ net-pop 0.1.2 https://github.com/ruby/net-pop net-smtp 0.3.3 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime -rbs 3.1.3 https://github.com/ruby/rbs -typeprof 0.21.7 https://github.com/ruby/typeprof aabc019684d8b4a1ed66c2a1ca48da7bbb18dcc0 +rbs 3.2.1 https://github.com/ruby/rbs +typeprof 0.21.8 https://github.com/ruby/typeprof debug 1.8.0 https://github.com/ruby/debug racc 1.7.1 https://github.com/ruby/racc diff --git a/include/ruby/internal/intern/process.h b/include/ruby/internal/intern/process.h index 7a7b24ed4b0ae0..5dcf85e80c04e1 100644 --- a/include/ruby/internal/intern/process.h +++ b/include/ruby/internal/intern/process.h @@ -30,6 +30,15 @@ RBIMPL_SYMBOL_EXPORT_BEGIN() /* process.c */ +/** + * Wait for the specified process to terminate, reap it, and return its status. + * + * @param[in] pid The process ID to wait for. + * @param[in] flags The flags to pass to waitpid(2). + * @return VALUE An instance of Process::Status. + */ +VALUE rb_process_status_wait(rb_pid_t pid, int flags); + /** * Sets the "last status", or the `$?`. * diff --git a/include/ruby/io.h b/include/ruby/io.h index 60029fedb6f0d6..e9dfeda5b1214b 100644 --- a/include/ruby/io.h +++ b/include/ruby/io.h @@ -138,6 +138,7 @@ struct rb_io_encoding { }; #ifndef HAVE_RB_IO_T +#define HAVE_RB_IO_T 1 /** Ruby's IO, metadata and buffers. */ struct rb_io { /** The IO's Ruby level counterpart. */ diff --git a/internal/gc.h b/internal/gc.h index 7195b9caf6b8dd..57aca2b0152fc6 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -309,6 +309,7 @@ void rb_gc_verify_internal_consistency(void); size_t rb_obj_gc_flags(VALUE, ID[], size_t); void rb_gc_mark_values(long n, const VALUE *values); void rb_gc_mark_vm_stack_values(long n, const VALUE *values); +void rb_gc_update_values(long n, VALUE *values); void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2)); void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); void ruby_sized_xfree(void *x, size_t size); diff --git a/internal/string.h b/internal/string.h index 5f59d9621b6b81..cfaf628e02140d 100644 --- a/internal/string.h +++ b/internal/string.h @@ -119,6 +119,12 @@ is_broken_string(VALUE str) return rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN; } +static inline bool +at_char_boundary(const char *s, const char *p, const char *e, rb_encoding *enc) +{ + return rb_enc_left_char_head(s, p, e, enc) == p; +} + /* expect tail call optimization */ // YJIT needs this function to never allocate and never raise static inline VALUE diff --git a/io.c b/io.c index 6e27ed6273ae35..433ec75b27f5e3 100644 --- a/io.c +++ b/io.c @@ -4144,8 +4144,7 @@ rb_io_getline_0(VALUE rs, long limit, int chomp, rb_io_t *fptr) s = RSTRING_PTR(str); e = RSTRING_END(str); p = e - rslen; - pp = rb_enc_left_char_head(s, p, e, enc); - if (pp != p) continue; + if (!at_char_boundary(s, p, e, enc)) continue; if (!rspara) rscheck(rsptr, rslen, rs); if (memcmp(p, rsptr, rslen) == 0) { if (chomp) { @@ -8945,6 +8944,10 @@ rb_p_result(int argc, const VALUE *argv) * 0..4 * [0..4, 0..4, 0..4] * + * Kernel#p is designed for debugging purposes. + * Ruby implementations may define Kernel#p to be uninterruptible + * in whole or in part. + * On CRuby, Kernel#p's writing of data is uninterruptible. */ static VALUE diff --git a/io_buffer.c b/io_buffer.c index 8d09e09a894f74..d987b8fa38e3cf 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -2181,8 +2181,8 @@ rb_io_buffer_initialize_copy(VALUE self, VALUE source) * call-seq: * copy(source, [offset, [length, [source_offset]]]) -> size * - * Efficiently copy buffer from a source IO::Buffer into the buffer, - * at +offset+ using +memcpy+. For copying String instances, see #set_string. + * Efficiently copy from a source IO::Buffer into the buffer, at +offset+ + * using +memcpy+. For copying String instances, see #set_string. * * buffer = IO::Buffer.new(32) * # => diff --git a/iseq.c b/iseq.c index 80bd280a14b25e..db7303aac5bd96 100644 --- a/iseq.c +++ b/iseq.c @@ -43,6 +43,7 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" +#include "yarp/yarp.h" VALUE rb_cISeq; static VALUE iseqw_new(const rb_iseq_t *iseq); @@ -729,7 +730,7 @@ set_compile_option_from_hash(rb_compile_option_t *option, VALUE opt) else if (flag == Qfalse) { (o)->mem = 0; } \ } #define SET_COMPILE_OPTION_NUM(o, h, mem) \ - { VALUE num = rb_hash_aref(opt, ID2SYM(rb_intern(#mem))); \ + { VALUE num = rb_hash_aref((h), ID2SYM(rb_intern(#mem))); \ if (!NIL_P(num)) (o)->mem = NUM2INT(num); \ } SET_COMPILE_OPTION(option, opt, inline_const_cache); @@ -746,20 +747,15 @@ set_compile_option_from_hash(rb_compile_option_t *option, VALUE opt) #undef SET_COMPILE_OPTION_NUM } -static VALUE -make_compile_option_from_ast(const rb_ast_body_t *ast) +static rb_compile_option_t * +set_compile_option_from_ast(rb_compile_option_t *option, const rb_ast_body_t *ast) { - VALUE opt = rb_obj_hide(rb_ident_hash_new()); - if (ast->frozen_string_literal >= 0) rb_hash_aset(opt, rb_sym_intern_ascii_cstr("frozen_string_literal"), RBOOL(ast->frozen_string_literal)); - if (ast->coverage_enabled >= 0) rb_hash_aset(opt, rb_sym_intern_ascii_cstr("coverage_enabled"), RBOOL(ast->coverage_enabled)); - return opt; -} - -static void -rb_iseq_make_compile_option(rb_compile_option_t *option, VALUE opt) -{ - Check_Type(opt, T_HASH); - set_compile_option_from_hash(option, opt); +#define SET_COMPILE_OPTION(o, a, mem) \ + ((a)->mem < 0 ? 0 : ((o)->mem = (a)->mem > 0)) + SET_COMPILE_OPTION(option, ast, frozen_string_literal); + SET_COMPILE_OPTION(option, ast, coverage_enabled); +#undef SET_COMPILE_OPTION + return option; } static void @@ -915,13 +911,11 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea rb_iseq_t *iseq = iseq_alloc(); rb_compile_option_t new_opt; - if (option) { + if (!option) option = &COMPILE_OPTION_DEFAULT; + if (ast) { new_opt = *option; + option = set_compile_option_from_ast(&new_opt, ast); } - else { - new_opt = COMPILE_OPTION_DEFAULT; - } - if (ast) rb_iseq_make_compile_option(&new_opt, make_compile_option_from_ast(ast)); VALUE script_lines = Qnil; @@ -933,7 +927,7 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea } prepare_iseq_build(iseq, name, path, realpath, first_lineno, node ? &node->nd_loc : NULL, node ? nd_node_id(node) : -1, - parent, isolated_depth, type, script_lines, &new_opt); + parent, isolated_depth, type, script_lines, option); rb_iseq_compile_node(iseq, node); finish_iseq_build(iseq); @@ -941,6 +935,46 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea return iseq_translate(iseq); } +VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer, yp_parser_t *parser); + +rb_iseq_t * +yp_iseq_new_with_opt(yp_node_t *node, yp_parser_t *parser, VALUE name, VALUE path, VALUE realpath, + int first_lineno, const rb_iseq_t *parent, int isolated_depth, + enum rb_iseq_type type, const rb_compile_option_t *option) +{ + rb_iseq_t *iseq = iseq_alloc(); + VALUE script_lines = Qnil; + rb_code_location_t code_loc; + + if (!option) option = &COMPILE_OPTION_DEFAULT; + + if (node) { + yp_line_column_t start_line_col = yp_newline_list_line_column(&(parser->newline_list), node->location.start); + yp_line_column_t end_line_col= yp_newline_list_line_column(&(parser->newline_list), node->location.end); + code_loc = (rb_code_location_t) { + .beg_pos = { + .lineno = (int)start_line_col.line, + .column = (int)start_line_col.column + }, + .end_pos = { + .lineno = (int)end_line_col.line, + .column = (int)end_line_col.column + }, + }; + } + + // TODO: node_id + int node_id = -1; + prepare_iseq_build(iseq, name, path, realpath, first_lineno, &code_loc, node_id, + parent, isolated_depth, type, script_lines, option); + + rb_iseq_compile_yarp_node(iseq, node, parser); + + finish_iseq_build(iseq); + + return iseq_translate(iseq); +} + rb_iseq_t * rb_iseq_new_with_callback( const struct rb_iseq_new_with_callback_callback_func * ifunc, @@ -1326,7 +1360,7 @@ rb_iseqw_new(const rb_iseq_t *iseq) static VALUE iseqw_s_compile(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = INT2FIX(1), opt = Qnil; + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; int i; i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); @@ -1348,6 +1382,68 @@ iseqw_s_compile(int argc, VALUE *argv, VALUE self) return iseqw_new(rb_iseq_compile_with_option(src, file, path, line, opt)); } +static VALUE +iseqw_s_compile_yarp(int argc, VALUE *argv, VALUE self) +{ + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; + int i; + + i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); + if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); + switch (i) { + case 5: opt = argv[--i]; + case 4: line = argv[--i]; + case 3: path = argv[--i]; + case 2: file = argv[--i]; + } + + if (NIL_P(file)) file = rb_fstring_lit(""); + if (NIL_P(path)) path = file; + if (NIL_P(line)) line = INT2FIX(1); + + Check_Type(path, T_STRING); + Check_Type(file, T_STRING); + + rb_iseq_t *iseq = iseq_alloc(); + + yp_parser_t parser; + size_t len = RSTRING_LEN(src); + VALUE name = rb_fstring_lit(""); + + yp_parser_init(&parser, (const uint8_t *) RSTRING_PTR(src), len, ""); + + yp_node_t *node = yp_parse(&parser); + + int first_lineno = NUM2INT(line); + yp_line_column_t start_loc = yp_newline_list_line_column(&parser.newline_list, node->location.start); + yp_line_column_t end_loc = yp_newline_list_line_column(&parser.newline_list, node->location.end); + + rb_code_location_t node_location; + node_location.beg_pos.lineno = (int)start_loc.line; + node_location.beg_pos.column = (int)start_loc.column; + node_location.end_pos.lineno = (int)end_loc.line; + node_location.end_pos.column = (int)end_loc.column; + + int node_id = 0; + + rb_iseq_t *parent = NULL; + enum rb_iseq_type iseq_type = ISEQ_TYPE_TOP; + rb_compile_option_t option; + + make_compile_option(&option, opt); + + prepare_iseq_build(iseq, name, file, path, first_lineno, &node_location, node_id, + parent, 0, (enum rb_iseq_type)iseq_type, Qnil, &option); + + rb_iseq_compile_yarp_node(iseq, node, &parser); + + finish_iseq_build(iseq); + yp_node_destroy(&parser, node); + yp_parser_free(&parser); + + return iseqw_new(iseq); +} + /* * call-seq: * InstructionSequence.compile_file(file[, options]) -> iseq @@ -3920,6 +4016,7 @@ Init_ISeq(void) (void)iseq_s_load; rb_define_singleton_method(rb_cISeq, "compile", iseqw_s_compile, -1); + rb_define_singleton_method(rb_cISeq, "compile_yarp", iseqw_s_compile_yarp, -1); rb_define_singleton_method(rb_cISeq, "new", iseqw_s_compile, -1); rb_define_singleton_method(rb_cISeq, "compile_file", iseqw_s_compile_file, -1); rb_define_singleton_method(rb_cISeq, "compile_option", iseqw_s_compile_option_get, 0); diff --git a/kernel.rb b/kernel.rb index c8cbc991757fab..43d1abec31118e 100644 --- a/kernel.rb +++ b/kernel.rb @@ -216,4 +216,97 @@ def Float(arg, exception: true) Primitive.rb_f_float(arg, exception) end end + + # call-seq: + # Integer(object, base = 0, exception: true) -> integer or nil + # + # Returns an integer converted from +object+. + # + # Tries to convert +object+ to an integer + # using +to_int+ first and +to_i+ second; + # see below for exceptions. + # + # With a non-zero +base+, +object+ must be a string or convertible + # to a string. + # + # ==== numeric objects + # + # With integer argument +object+ given, returns +object+: + # + # Integer(1) # => 1 + # Integer(-1) # => -1 + # + # With floating-point argument +object+ given, + # returns +object+ truncated to an integer: + # + # Integer(1.9) # => 1 # Rounds toward zero. + # Integer(-1.9) # => -1 # Rounds toward zero. + # + # ==== string objects + # + # With string argument +object+ and zero +base+ given, + # returns +object+ converted to an integer in base 10: + # + # Integer('100') # => 100 + # Integer('-100') # => -100 + # + # With +base+ zero, string +object+ may contain leading characters + # to specify the actual base (radix indicator): + # + # Integer('0100') # => 64 # Leading '0' specifies base 8. + # Integer('0b100') # => 4 # Leading '0b', specifies base 2. + # Integer('0x100') # => 256 # Leading '0x' specifies base 16. + # + # With a positive +base+ (in range 2..36) given, returns +object+ + # converted to an integer in the given base: + # + # Integer('100', 2) # => 4 + # Integer('100', 8) # => 64 + # Integer('-100', 16) # => -256 + # + # With a negative +base+ (in range -36..-2) given, returns +object+ + # converted to an integer in the radix indicator if exists or + # +-base+: + # + # Integer('0x100', -2) # => 256 + # Integer('100', -2) # => 4 + # Integer('0b100', -8) # => 4 + # Integer('100', -8) # => 64 + # Integer('0o100', -10) # => 64 + # Integer('100', -10) # => 100 + # + # +base+ -1 is equal the -10 case. + # + # When converting strings, surrounding whitespace and embedded underscores + # are allowed and ignored: + # + # Integer(' 100 ') # => 100 + # Integer('-1_0_0', 16) # => -256 + # + # ==== other classes + # + # Examples with +object+ of various other classes: + # + # Integer(Rational(9, 10)) # => 0 # Rounds toward zero. + # Integer(Complex(2, 0)) # => 2 # Imaginary part must be zero. + # Integer(Time.now) # => 1650974042 + # + # ==== keywords + # + # With optional keyword argument +exception+ given as +true+ (the default): + # + # - Raises TypeError if +object+ does not respond to +to_int+ or +to_i+. + # - Raises TypeError if +object+ is +nil+. + # - Raise ArgumentError if +object+ is an invalid string. + # + # With +exception+ given as +false+, an exception of any kind is suppressed + # and +nil+ is returned. + + def Integer(arg, base = 0, exception: true) + if Primitive.mandatory_only? + Primitive.rb_f_integer1(arg) + else + Primitive.rb_f_integer(arg, base, exception); + end + end end diff --git a/lib/bundler/cli/update.rb b/lib/bundler/cli/update.rb index b49182655bf9d7..22dd1a78dd1448 100644 --- a/lib/bundler/cli/update.rb +++ b/lib/bundler/cli/update.rb @@ -63,6 +63,7 @@ def run opts = options.dup opts["update"] = true opts["local"] = options[:local] + opts["force"] = options[:redownload] Bundler.settings.set_command_option_if_given :jobs, opts["jobs"] diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index 2119799f688437..9b64a3c771b408 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -9,6 +9,7 @@ module Bundler # Handles all the fetching with the rubygems server class Fetcher + autoload :Base, File.expand_path("fetcher/base", __dir__) autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__) autoload :Downloader, File.expand_path("fetcher/downloader", __dir__) autoload :Dependency, File.expand_path("fetcher/dependency", __dir__) @@ -134,18 +135,7 @@ def specs_with_retry(gem_names, source) def specs(gem_names, source) index = Bundler::Index.new - if Bundler::Fetcher.disable_endpoint - @use_api = false - specs = fetchers.last.specs(gem_names) - else - specs = [] - @fetchers = fetchers.drop_while do |f| - !f.available? || (f.api_fetcher? && !gem_names) || !specs = f.specs(gem_names) - end - @use_api = false if fetchers.none?(&:api_fetcher?) - end - - specs.each do |name, version, platform, dependencies, metadata| + fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata| spec = if dependencies EndpointSpecification.new(name, version, platform, self, dependencies, metadata) else @@ -158,22 +148,10 @@ def specs(gem_names, source) index rescue CertificateFailureError - Bundler.ui.info "" if gem_names && use_api # newline after dots + Bundler.ui.info "" if gem_names && api_fetcher? # newline after dots raise end - def use_api - return @use_api if defined?(@use_api) - - fetchers.shift until fetchers.first.available? - - @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint - false - else - fetchers.first.api_fetcher? - end - end - def user_agent @user_agent ||= begin ruby = Bundler::RubyVersion.system @@ -209,10 +187,6 @@ def user_agent end end - def fetchers - @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) } - end - def http_proxy return unless uri = connection.proxy_uri uri.to_s @@ -222,9 +196,36 @@ def inspect "#<#{self.class}:0x#{object_id} uri=#{uri}>" end + def api_fetcher? + fetchers.first.api_fetcher? + end + private - FETCHERS = [CompactIndex, Dependency, Index].freeze + def available_fetchers + if Bundler::Fetcher.disable_endpoint + [Index] + elsif remote_uri.scheme == "file" + Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") + [Index] + else + [CompactIndex, Dependency, Index] + end + end + + def fetchers + @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri) }.drop_while {|f| !f.available? } + end + + def fetch_specs(gem_names) + fetchers.reject!(&:api_fetcher?) unless gem_names + fetchers.reject! do |f| + specs = f.specs(gem_names) + return specs if specs + true + end + [] + end def cis env_cis = { diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index d3160251cb0056..6786a841f56b02 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -60,10 +60,6 @@ def available? Bundler.ui.debug("FIPS mode is enabled, bundler can't use the CompactIndex API") return nil end - if fetch_uri.scheme == "file" - Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") - return false - end # Read info file checksums out of /versions, so we can know if gems are up to date compact_index_client.update_and_parse_checksums! rescue CompactIndexClient::Updater::MisMatchedChecksumError => e diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 2ad35bc931a136..3263913b7ff9d4 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -37,9 +37,17 @@ def setup_solver root_version = Resolver::Candidate.new(0) @all_specs = Hash.new do |specs, name| - specs[name] = source_for(name).specs.search(name).reject do |s| - s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } # ignore versions that depend on themselves incorrectly - end.sort_by {|s| [s.version, s.platform.to_s] } + source = source_for(name) + matches = source.specs.search(name) + + # Don't bother to check for circular deps when no dependency API are + # available, since it's too slow to be usable. That edge case won't work + # but resolution other than that should work fine and reasonably fast. + if source.respond_to?(:dependency_api_available?) && source.dependency_api_available? + matches = filter_invalid_self_dependencies(matches, name) + end + + specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } end @sorted_versions = Hash.new do |candidates, package| @@ -318,6 +326,13 @@ def filter_prereleases(specs, package) specs.reject {|s| s.version.prerelease? } end + # Ignore versions that depend on themselves incorrectly + def filter_invalid_self_dependencies(specs, name) + specs.reject do |s| + s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } + end + end + def requirement_satisfied_by?(requirement, spec) requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec) end diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 060be66c8c3e15..d6f179cc96f6dc 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -253,21 +253,21 @@ def replace_require(specs) rescue GemfileNotFound "inline Gemfile" end - be = RUBY_VERSION < ::Gem::BUNDLED_GEMS::SINCE[name] ? "will be" : "is" + be = ::Gem::BUNDLED_GEMS::SINCE[name] > RUBY_VERSION ? "will be" : "is" message = "#{name} #{be} not part of the default gems since Ruby #{::Gem::BUNDLED_GEMS::SINCE[name]}." \ " Add #{name} to your #{target_file}." location = caller_locations(1,1)[0]&.path if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) caller_gem = nil Gem.path.each do |path| - if location =~ /#{path}\/gems\/([\w\-\.]+)/ + if location =~ %r{#{path}/gems/([\w\-\.]+)} caller_gem = $1 break end end message += " Also contact author of #{caller_gem} to add #{name} into its gemspec." end - warn message, uplevel: 1 + warn message, :uplevel => 1 end end kernel_class.send(:no_warning_require, file) diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index 5d7546e9262965..1925a266d977db 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -170,6 +170,8 @@ def lockfile_version parsed_version = Bundler::LockfileParser.bundled_with @lockfile_version = parsed_version ? Gem::Version.new(parsed_version) : nil + rescue ArgumentError + @lockfile_version = nil end def restart_version diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index f5999481a8f73b..92b4a6a4e37787 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -102,10 +102,12 @@ def initialize(root = nil) def [](name) key = key_for(name) - values = configs.values - values.map! {|config| config[key] } - values.compact! - value = values.first + value = nil + configs.each do |_, config| + value = config[key] + next if value.nil? + break + end converted_value(value, name) end @@ -316,7 +318,7 @@ def key_for(key) private def configs - { + @configs ||= { :temporary => @temporary, :local => @local_config, :env => @env_config, diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 9790753204b91e..7cd24092c93112 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -237,7 +237,7 @@ def add_remote(source) end def spec_names - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.spec_names else [] @@ -245,7 +245,7 @@ def spec_names end def unmet_deps - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.unmet_dependency_names else [] @@ -260,7 +260,6 @@ def fetchers end def double_check_for(unmet_dependency_names) - return unless @allow_remote return unless dependency_api_available? unmet_dependency_names = unmet_dependency_names.call @@ -275,7 +274,9 @@ def double_check_for(unmet_dependency_names) Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}" - fetch_names(api_fetchers, unmet_dependency_names, specs, false) + fetch_names(api_fetchers, unmet_dependency_names, remote_specs, false) + + specs.use(remote_specs, false) end def dependency_names_to_double_check @@ -392,7 +393,7 @@ def cached_specs end def api_fetchers - fetchers.select {|f| f.use_api && f.fetchers.first.api_fetcher? } + fetchers.select(&:api_fetcher?) end def remote_specs diff --git a/lib/irb.rb b/lib/irb.rb index aa412583056b3c..57ec911f344b3e 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -197,7 +197,6 @@ # # IRB.conf[:PROMPT_MODE][:DEFAULT] = { # :PROMPT_I => "%N(%m):%03n> ", -# :PROMPT_N => "%N(%m):%03n> ", # :PROMPT_S => "%N(%m):%03n%l ", # :PROMPT_C => "%N(%m):%03n* ", # :RETURN => "%s\n" # used to printf @@ -207,35 +206,30 @@ # # # :NULL: # # :PROMPT_I: -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: | # # %s # # :DEFAULT: # # :PROMPT_I: ! '%N(%m):%03n> ' -# # :PROMPT_N: ! '%N(%m):%03n> ' # # :PROMPT_S: ! '%N(%m):%03n%l ' # # :PROMPT_C: ! '%N(%m):%03n* ' # # :RETURN: | # # => %s # # :CLASSIC: # # :PROMPT_I: ! '%N(%m):%03n:%i> ' -# # :PROMPT_N: ! '%N(%m):%03n:%i> ' # # :PROMPT_S: ! '%N(%m):%03n:%i%l ' # # :PROMPT_C: ! '%N(%m):%03n:%i* ' # # :RETURN: | # # %s # # :SIMPLE: # # :PROMPT_I: ! '>> ' -# # :PROMPT_N: ! '>> ' # # :PROMPT_S: # # :PROMPT_C: ! '?> ' # # :RETURN: | # # => %s # # :INF_RUBY: # # :PROMPT_I: ! '%N(%m):%03n> ' -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: | @@ -243,7 +237,6 @@ # # :AUTO_INDENT: true # # :XMP: # # :PROMPT_I: -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: |2 @@ -446,9 +439,9 @@ def initialize(workspace = nil, input_method = nil) @scanner = RubyLex.new(@context) end - # A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up + # A hook point for `debug` command's breakpoint after :IRB_EXIT as well as its clean-up def debug_break - # it means the debug command is executed + # it means the debug integration has been activated if defined?(DEBUGGER__) && DEBUGGER__.respond_to?(:capture_frames_without_irb) # after leaving this initial breakpoint, revert the capture_frames patch DEBUGGER__.singleton_class.send(:alias_method, :capture_frames, :capture_frames_without_irb) @@ -528,8 +521,6 @@ def eval_input f = @context.prompt_s elsif continue f = @context.prompt_c - elsif indent > 0 - f = @context.prompt_n else f = @context.prompt_i end @@ -544,7 +535,6 @@ def eval_input prompt_i = @context.prompt_i.nil? ? "" : @context.prompt_i ind = prompt(prompt_i, ltype, indent, line_no)[/.*\z/].size + indent * 2 - p.size - ind += 2 if continue @context.io.prompt = p + " " * ind if ind > 0 end end @@ -556,14 +546,13 @@ def eval_input each_top_level_statement do |statement, line_no| signal_status(:IN_EVAL) do begin - # If the integration with debugger is activated, we need to handle certain input differently + # If the integration with debugger is activated, we return certain input if it should be dealt with by debugger if @context.with_debugger && statement.should_be_handled_by_debugger? return statement.code end @context.evaluate(statement.evaluable_code, line_no) - # Don't echo if the line ends with a semicolon if @context.echo? && !statement.suppresses_echo? if statement.is_assignment? if @context.echo_on_assignment? diff --git a/lib/irb/cmd/show_cmds.rb b/lib/irb/cmd/show_cmds.rb index cef6254ca039de..7d6b3ec2668b6f 100644 --- a/lib/irb/cmd/show_cmds.rb +++ b/lib/irb/cmd/show_cmds.rb @@ -15,6 +15,16 @@ class ShowCmds < Nop def execute(*args) commands_info = IRB::ExtendCommandBundle.all_commands_info commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } + + if irb_context.with_debugger + # Remove the original "Debugging" category + commands_grouped_by_categories.delete("Debugging") + # Remove the `help` command as it's delegated to the debugger + commands_grouped_by_categories["Context"].delete_if { |cmd| cmd[:display_name] == :help } + # Add an empty "Debugging (from debug.gem)" category at the end + commands_grouped_by_categories["Debugging (from debug.gem)"] = [] + end + longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max output = StringIO.new @@ -29,6 +39,11 @@ def execute(*args) output.puts end + # Append the debugger help at the end + if irb_context.with_debugger + output.puts DEBUGGER__.help + end + Pager.page_content(output.string) end end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 43d9b53435df98..a20510d73c3ee8 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -229,8 +229,19 @@ def main # # See IRB@Customizing+the+IRB+Prompt for more information. attr_accessor :prompt_c - # See IRB@Customizing+the+IRB+Prompt for more information. - attr_accessor :prompt_n + + # TODO: Remove this when developing v2.0 + def prompt_n + warn "IRB::Context#prompt_n is deprecated and will be removed in the next major release." + "" + end + + # TODO: Remove this when developing v2.0 + def prompt_n=(_) + warn "IRB::Context#prompt_n= is deprecated and will be removed in the next major release." + "" + end + # Can be either the default IRB.conf[:AUTO_INDENT], or the # mode set by #prompt_mode= # @@ -414,7 +425,6 @@ def prompt_mode=(mode) @prompt_i = pconf[:PROMPT_I] @prompt_s = pconf[:PROMPT_S] @prompt_c = pconf[:PROMPT_C] - @prompt_n = pconf[:PROMPT_N] @return_format = pconf[:RETURN] @return_format = "%s\n" if @return_format == nil if ai = pconf.include?(:AUTO_INDENT) diff --git a/lib/irb/debug.rb b/lib/irb/debug.rb index f72282299322c0..f819f850b10abf 100644 --- a/lib/irb/debug.rb +++ b/lib/irb/debug.rb @@ -2,10 +2,6 @@ module IRB module Debug - BINDING_IRB_FRAME_REGEXPS = [ - '', - binding.method(:irb).source_location.first, - ].map { |file| /\A#{Regexp.escape(file)}:\d+:in `irb'\z/ } IRB_DIR = File.expand_path('..', __dir__) class << self @@ -76,14 +72,6 @@ def configure_irb_for_debugger(irb) irb.context.irb_name += ":rdbg" end - def binding_irb? - caller.any? do |frame| - BINDING_IRB_FRAME_REGEXPS.any? do |regexp| - frame.match?(regexp) - end - end - end - module SkipPathHelperForIRB def skip_internal_path?(path) # The latter can be removed once https://github.com/ruby/debug/issues/866 is resolved diff --git a/lib/irb/init.rb b/lib/irb/init.rb index ef07a5f1e6b3f9..d1097b573854ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -58,35 +58,30 @@ def IRB.init_config(ap_path) @CONF[:PROMPT] = { :NULL => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n" }, :DEFAULT => { :PROMPT_I => "%N(%m):%03n> ", - :PROMPT_N => "%N(%m):%03n> ", :PROMPT_S => "%N(%m):%03n%l ", :PROMPT_C => "%N(%m):%03n* ", :RETURN => "=> %s\n" }, :CLASSIC => { :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", :PROMPT_S => "%N(%m):%03n:%i%l ", :PROMPT_C => "%N(%m):%03n:%i* ", :RETURN => "%s\n" }, :SIMPLE => { :PROMPT_I => ">> ", - :PROMPT_N => ">> ", :PROMPT_S => "%l> ", :PROMPT_C => "?> ", :RETURN => "=> %s\n" }, :INF_RUBY => { :PROMPT_I => "%N(%m):%03n> ", - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n", @@ -94,7 +89,6 @@ def IRB.init_config(ap_path) }, :XMP => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => " ==>%s\n" diff --git a/lib/irb/irb.gemspec b/lib/irb/irb.gemspec index 4d47481d8794ec..a008a39f9d5d81 100644 --- a/lib/irb/irb.gemspec +++ b/lib/irb/irb.gemspec @@ -41,6 +41,6 @@ Gem::Specification.new do |spec| spec.required_ruby_version = Gem::Requirement.new(">= 2.7") - spec.add_dependency "reline", ">= 0.3.6" - spec.add_dependency "rdoc", "~> 6.5" + spec.add_dependency "reline", ">= 0.3.8" + spec.add_dependency "rdoc" end diff --git a/lib/irb/statement.rb b/lib/irb/statement.rb index 9493c3ffb1277a..b12110600cc728 100644 --- a/lib/irb/statement.rb +++ b/lib/irb/statement.rb @@ -60,7 +60,9 @@ def suppresses_echo? end def should_be_handled_by_debugger? - IRB::ExtendCommand::DebugCommand > @command_class + require_relative 'cmd/help' + require_relative 'cmd/debug' + IRB::ExtendCommand::DebugCommand > @command_class || IRB::ExtendCommand::Help == @command_class end def evaluable_code diff --git a/lib/irb/version.rb b/lib/irb/version.rb index 27103fe0bd6eb7..04a9ccfe292c12 100644 --- a/lib/irb/version.rb +++ b/lib/irb/version.rb @@ -5,7 +5,7 @@ # module IRB # :nodoc: - VERSION = "1.7.4" + VERSION = "1.8.0" @RELEASE_VERSION = VERSION - @LAST_UPDATE_DATE = "2023-07-15" + @LAST_UPDATE_DATE = "2023-08-30" end diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 4dea61c16c24af..45b53a7b6e620e 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -226,11 +226,13 @@ def uuid # # The argument _n_ specifies the length, in characters, of the alphanumeric # string to be generated. + # The argument _chars_ specifies the character list which the result is + # consist of. # # If _n_ is not specified or is nil, 16 is assumed. # It may be larger in the future. # - # The result may contain A-Z, a-z and 0-9. + # The result may contain A-Z, a-z and 0-9, unless _chars_ is specified. # # require 'random/formatter' # @@ -238,8 +240,13 @@ def uuid # # or # prng = Random.new # prng.alphanumeric(10) #=> "i6K93NdqiH" - def alphanumeric(n=nil) + # + # Random.alphanumeric(4, chars: [*"0".."9"]) #=> "2952" + # # or + # prng = Random.new + # prng.alphanumeric(10, chars: [*"!".."/"]) #=> ",.,++%/''." + def alphanumeric(n = nil, chars: ALPHANUMERIC) n = 16 if n.nil? - choose(ALPHANUMERIC, n) + choose(chars, n) end end diff --git a/lib/reline/unicode/east_asian_width.rb b/lib/reline/unicode/east_asian_width.rb index 97dfec4c52f8f8..c1c941f7993cc9 100644 --- a/lib/reline/unicode/east_asian_width.rb +++ b/lib/reline/unicode/east_asian_width.rb @@ -1,6 +1,6 @@ class Reline::Unicode::EastAsianWidth # This is based on EastAsianWidth.txt - # EastAsianWidth.txt + # UNICODE_VERSION = '15.0.0' # Fullwidth TYPE_F = /^[#{ %W( diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index 5b2beb68edb74e..96dfa55c69f383 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -794,7 +794,7 @@ def putstring(jit, ctx, asm) asm.mov(C_ARGS[1], to_value(put_val)) asm.call(C.rb_ec_str_resurrect) - stack_top = ctx.stack_push(Type::CString) + stack_top = ctx.stack_push(Type::TString) asm.mov(stack_top, C_RET) KeepCompiling @@ -817,7 +817,7 @@ def concatstrings(jit, ctx, asm) asm.call(C.rb_str_concat_literals) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) asm.mov(stack_ret, C_RET) KeepCompiling @@ -932,7 +932,7 @@ def newarray(jit, ctx, asm) asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -954,7 +954,7 @@ def duparray(jit, ctx, asm) asm.mov(C_ARGS[0], ary) asm.call(C.rb_ary_resurrect) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -3082,7 +3082,7 @@ def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class) asm.test(recv_reg, C::RUBY_ENCODING_MASK) # Push once, use the resulting operand in both branches below. - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) enc_mismatch = asm.new_label('enc_mismatch') asm.jnz(enc_mismatch) @@ -3779,9 +3779,14 @@ def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, insn_opnd, compt jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) if known_klass == C.rb_cString - ctx.upgrade_opnd_type(insn_opnd, Type::CString) + # Upgrading to Type::CString here is incorrect. + # The guard we put only checks RBASIC_CLASS(obj), + # which adding a singleton class can change. We + # additionally need to know the string is frozen + # to claim Type::CString. + ctx.upgrade_opnd_type(insn_opnd, Type::TString) elsif known_klass == C.rb_cArray - ctx.upgrade_opnd_type(insn_opnd, Type::CArray) + ctx.upgrade_opnd_type(insn_opnd, Type::TArray) end end end @@ -4723,7 +4728,7 @@ def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: n asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) end end diff --git a/lib/ruby_vm/rjit/type.rb b/lib/ruby_vm/rjit/type.rb index 155e141d63c8b8..119692014bfe02 100644 --- a/lib/ruby_vm/rjit/type.rb +++ b/lib/ruby_vm/rjit/type.rb @@ -35,7 +35,6 @@ def heap? case self in Type::UnknownHeap then true in Type::TArray then true - in Type::CArray then true in Type::Hash then true in Type::HeapSymbol then true in Type::TString then true @@ -45,11 +44,10 @@ def heap? end end - # Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) + # Check if it's a T_ARRAY object def array? case self in Type::TArray then true - in Type::CArray then true else false end end @@ -73,7 +71,6 @@ def known_class in Type::Flonum then C.rb_cFloat in Type::ImmSymbol | Type::HeapSymbol then C.rb_cSymbol in Type::CString then C.rb_cString - in Type::CArray then C.rb_cArray else nil end end @@ -115,11 +112,6 @@ def diff(dst) return TypeDiff::Compatible[1] end - # A CArray is also a TArray. - if self == Type::CArray && dst == Type::TArray - return TypeDiff::Compatible[1] - end - # Specific heap type into unknown heap type is imperfect but valid if self.heap? && dst == Type::UnknownHeap return TypeDiff::Compatible[1] @@ -169,12 +161,9 @@ def from(val) end else val_class = C.to_value(C.rb_class_of(val)) - if val_class == C.rb_cString + if val_class == C.rb_cString && C.rb_obj_frozen_p(val) return Type::CString end - if val_class == C.rb_cArray - return Type::CArray - end if C.to_value(val) == C.rb_block_param_proxy return Type::BlockParamProxy end @@ -222,7 +211,6 @@ def static_symbol?(obj) Type::TString = Type[:TString] # An object with the T_STRING flag set, possibly an rb_cString Type::CString = Type[:CString] # An un-subclassed string of type rb_cString (can have instance vars in some cases) Type::TArray = Type[:TArray] # An object with the T_ARRAY flag set, possibly an rb_cArray - Type::CArray = Type[:CArray] # An un-subclassed string of type rb_cArray (can have instance vars in some cases) Type::BlockParamProxy = Type[:BlockParamProxy] # A special sentinel value indicating the block parameter should be read from diff --git a/lib/shellwords.rb b/lib/shellwords.rb index 5866601b1792e5..58f4af26ae56cd 100644 --- a/lib/shellwords.rb +++ b/lib/shellwords.rb @@ -68,7 +68,6 @@ # 1: {IEEE Std 1003.1-2008, 2016 Edition, the Shell & Utilities volume}[http://pubs.opengroup.org/onlinepubs/9699919799/utilities/contents.html] module Shellwords - VERSION = "0.1.0" # Splits a string into an array of tokens in the same way the UNIX diff --git a/lib/yarp.rb b/lib/yarp.rb index 15217e80f490a7..316918b2db406d 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -37,7 +37,7 @@ def compute_offsets(code) class Location # A Source object that is used to determine more information from the given # offset and length. - private attr_reader :source + protected attr_reader :source # The byte offset from the beginning of the source where this location # starts. @@ -52,6 +52,16 @@ def initialize(source, start_offset, length) @length = length end + # Create a new location object with the given options. + def copy(**options) + Location.new( + options.fetch(:source) { source }, + options.fetch(:start_offset) { start_offset }, + options.fetch(:length) { length } + ) + end + + # Returns a string representation of this location. def inspect "#" end @@ -102,6 +112,16 @@ def ==(other) other.end_offset == end_offset end + # Returns a new location that stretches from this location to the given + # other location. Raises an error if this location is not before the other + # location or if they don't share the same source. + def join(other) + raise "Incompatible sources" if source != other.source + raise "Incompatible locations" if start_offset > other.start_offset + + Location.new(source, start_offset, other.end_offset - start_offset) + end + def self.null new(0, 0) end @@ -333,7 +353,7 @@ def value class RationalNode < Node def value - Rational(numeric.value) + Rational(slice.chomp("r")) end end @@ -442,10 +462,10 @@ def self.yarp_locals(source) # order here so that we can compare properly. if params sorted = [ - *params.requireds.grep(RequiredParameterNode).map(&:constant_id), - *params.optionals.map(&:constant_id), + *params.requireds.grep(RequiredParameterNode).map(&:name), + *params.optionals.map(&:name), *((params.rest.name ? params.rest.name.to_sym : :*) if params.rest && params.rest.operator != ","), - *params.posts.grep(RequiredParameterNode).map(&:constant_id), + *params.posts.grep(RequiredParameterNode).map(&:name), *params.keywords.reject(&:value).map { |param| param.name.chomp(":").to_sym }, *params.keywords.select(&:value).map { |param| param.name.chomp(":").to_sym } ] @@ -465,9 +485,9 @@ def self.yarp_locals(source) when RequiredDestructuredParameterNode param_stack.concat(param.parameters.reverse) when RequiredParameterNode - sorted << param.constant_id + sorted << param.name when SplatNode - sorted << param.expression.constant_id if param.expression + sorted << param.expression.name if param.expression end end @@ -507,6 +527,8 @@ def self.parse_serialize_file(filepath) end require_relative "yarp/lex_compat" +require_relative "yarp/mutation_visitor" +require_relative "yarp/desugar_visitor" require_relative "yarp/node" require_relative "yarp/ripper_compat" require_relative "yarp/serialize" diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb new file mode 100644 index 00000000000000..a988449dc07a78 --- /dev/null +++ b/lib/yarp/desugar_visitor.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +module YARP + class DesugarVisitor < MutationVisitor + # @@foo &&= bar + # + # becomes + # + # @@foo && @@foo = bar + def visit_class_variable_and_write_node(node) + desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) + end + + # @@foo ||= bar + # + # becomes + # + # defined?(@@foo) ? @@foo : @@foo = bar + def visit_class_variable_or_write_node(node) + desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) + end + + # @@foo += bar + # + # becomes + # + # @@foo = @@foo + bar + def visit_class_variable_operator_write_node(node) + desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) + end + + # Foo &&= bar + # + # becomes + # + # Foo && Foo = bar + def visit_constant_and_write_node(node) + desugar_and_write_node(node, ConstantReadNode, ConstantWriteNode) + end + + # Foo ||= bar + # + # becomes + # + # defined?(Foo) ? Foo : Foo = bar + def visit_constant_or_write_node(node) + desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode) + end + + # Foo += bar + # + # becomes + # + # Foo = Foo + bar + def visit_constant_operator_write_node(node) + desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode) + end + + # $foo &&= bar + # + # becomes + # + # $foo && $foo = bar + def visit_global_variable_and_write_node(node) + desugar_and_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) + end + + # $foo ||= bar + # + # becomes + # + # defined?($foo) ? $foo : $foo = bar + def visit_global_variable_or_write_node(node) + desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) + end + + # $foo += bar + # + # becomes + # + # $foo = $foo + bar + def visit_global_variable_operator_write_node(node) + desugar_operator_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) + end + + # @foo &&= bar + # + # becomes + # + # @foo && @foo = bar + def visit_instance_variable_and_write_node(node) + desugar_and_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) + end + + # @foo ||= bar + # + # becomes + # + # @foo || @foo = bar + def visit_instance_variable_or_write_node(node) + desugar_or_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) + end + + # @foo += bar + # + # becomes + # + # @foo = @foo + bar + def visit_instance_variable_operator_write_node(node) + desugar_operator_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) + end + + # foo &&= bar + # + # becomes + # + # foo && foo = bar + def visit_local_variable_and_write_node(node) + desugar_and_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) + end + + # foo ||= bar + # + # becomes + # + # foo || foo = bar + def visit_local_variable_or_write_node(node) + desugar_or_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) + end + + # foo += bar + # + # becomes + # + # foo = foo + bar + def visit_local_variable_operator_write_node(node) + desugar_operator_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) + end + + private + + # Desugar `x &&= y` to `x && x = y` + def desugar_and_write_node(node, read_class, write_class, arguments: []) + AndNode.new( + read_class.new(*arguments, node.name_loc), + write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Desugar `x += y` to `x = x + y` + def desugar_operator_write_node(node, read_class, write_class, arguments: []) + write_class.new( + *arguments, + node.name_loc, + CallNode.new( + read_class.new(*arguments, node.name_loc), + nil, + node.operator_loc.copy(length: node.operator_loc.length - 1), + nil, + ArgumentsNode.new([node.value], node.value.location), + nil, + nil, + 0, + node.operator_loc.slice.chomp("="), + node.location + ), + node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.location + ) + end + + # Desugar `x ||= y` to `x || x = y` + def desugar_or_write_node(node, read_class, write_class, arguments: []) + OrNode.new( + read_class.new(*arguments, node.name_loc), + write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Desugar `x ||= y` to `defined?(x) ? x : x = y` + def desugar_or_write_defined_node(node, read_class, write_class, arguments: []) + IfNode.new( + node.operator_loc, + DefinedNode.new(nil, read_class.new(*arguments, node.name_loc), nil, node.operator_loc, node.name_loc), + StatementsNode.new([read_class.new(*arguments, node.name_loc)], node.location), + ElseNode.new( + node.operator_loc, + StatementsNode.new( + [write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location)], + node.location + ), + node.operator_loc, + node.location + ), + node.operator_loc, + node.location + ) + end + end +end diff --git a/lib/yarp/ffi.rb b/lib/yarp/ffi.rb index a547d506063f42..26b6019b27f894 100644 --- a/lib/yarp/ffi.rb +++ b/lib/yarp/ffi.rb @@ -70,7 +70,8 @@ def self.load_exported_functions_from(header, *functions) "yarp.h", "yp_version", "yp_parse_serialize", - "yp_lex_serialize" + "yp_lex_serialize", + "yp_parse_lex_serialize" ) load_exported_functions_from( @@ -223,4 +224,29 @@ def self.parse_file(filepath) parse(string.read, filepath) end end + + # Mirror the YARP.parse_lex API by using the serialization API. + def self.parse_lex(code, filepath = nil) + LibRubyParser::YPBuffer.with do |buffer| + metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath + LibRubyParser.yp_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata) + + source = Source.new(code) + loader = Serialize::Loader.new(source, buffer.read) + + tokens = loader.load_tokens + node, comments, errors, warnings = loader.load_nodes + + tokens.each { |token,| token.value.force_encoding(loader.encoding) } + + ParseResult.new([node, tokens], comments, errors, warnings, source) + end + end + + # Mirror the YARP.parse_lex_file API by using the serialization API. + def self.parse_lex_file(filepath) + LibRubyParser::YPString.with(filepath) do |string| + parse_lex(string.read, filepath) + end + end end diff --git a/lib/yarp/version.rb b/lib/yarp/version.rb new file mode 100644 index 00000000000000..e450bfb526f866 --- /dev/null +++ b/lib/yarp/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module YARP + VERSION = "0.8.0" +end diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index db69bcb451b8d7..ceea5e8eb6e10b 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "yarp" - spec.version = "0.8.0" + spec.version = "0.9.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -59,8 +59,10 @@ Gem::Specification.new do |spec| "include/yarp/util/yp_strpbrk.h", "include/yarp/version.h", "lib/yarp.rb", + "lib/yarp/desugar_visitor.rb", "lib/yarp/ffi.rb", "lib/yarp/lex_compat.rb", + "lib/yarp/mutation_visitor.rb", "lib/yarp/node.rb", "lib/yarp/pack.rb", "lib/yarp/ripper_compat.rb", diff --git a/man/ruby.1 b/man/ruby.1 index cbd14e0ecce2d1..e62decbf56d752 100644 --- a/man/ruby.1 +++ b/man/ruby.1 @@ -557,9 +557,9 @@ the following 11 environment variables: .It Ev RUBY_GC_HEAP_INIT_SLOTS Initial allocation slots. Applies to all slot sizes. Introduced in Ruby 2.1, default: 10000. .Pp -.It Ev RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS -Initial allocation of slots in a specific size pool. -The available size pools can be found in `GC.stat_heap`. +.It Ev RUBY_GC_HEAP_%d_INIT_SLOTS +Initial allocation of slots in a specific heap. +The available heaps can be found in the keys of `GC.stat_heap`. Introduced in Ruby 3.3. .Pp .It Ev RUBY_GC_HEAP_FREE_SLOTS diff --git a/misc/lldb_cruby.py b/misc/lldb_cruby.py index 95e03c620901b0..e667a466287ddf 100755 --- a/misc/lldb_cruby.py +++ b/misc/lldb_cruby.py @@ -197,12 +197,11 @@ def string2cstr(rstring): flags = rstring.GetValueForExpressionPath(".basic->flags").unsigned if flags & RUBY_T_MASK != RUBY_T_STRING: raise TypeError("not a string") + clen = int(rstring.GetValueForExpressionPath(".len").value, 0) if flags & RUBY_FL_USER1: cptr = int(rstring.GetValueForExpressionPath(".as.heap.ptr").value, 0) - clen = int(rstring.GetValueForExpressionPath(".as.heap.len").value, 0) else: cptr = int(rstring.GetValueForExpressionPath(".as.embed.ary").location, 0) - clen = int(rstring.GetValueForExpressionPath(".as.embed.len").value, 0) return cptr, clen def output_string(debugger, result, rstring): diff --git a/missing/explicit_bzero.c b/missing/explicit_bzero.c index 1220e5f9ad7c23..59417e158ebf10 100644 --- a/missing/explicit_bzero.c +++ b/missing/explicit_bzero.c @@ -1,12 +1,9 @@ #ifndef __STDC_WANT_LIB_EXT1__ -#define __STDC_WANT_LIB_EXT1__ 1 +#define __STDC_WANT_LIB_EXT1__ 1 /* for memset_s() */ #endif #include "ruby/missing.h" #include -#ifdef HAVE_MEMSET_S -# include -#endif #ifdef _WIN32 #include diff --git a/object.c b/object.c index 70cb64eb850d17..c32cb82dd991e1 100644 --- a/object.c +++ b/object.c @@ -3312,121 +3312,17 @@ rb_opts_exception_p(VALUE opts, int default_value) return default_value; } -#define opts_exception_p(opts) rb_opts_exception_p((opts), TRUE) - -/* - * call-seq: - * Integer(object, base = 0, exception: true) -> integer or nil - * - * Returns an integer converted from +object+. - * - * Tries to convert +object+ to an integer - * using +to_int+ first and +to_i+ second; - * see below for exceptions. - * - * With a non-zero +base+, +object+ must be a string or convertible - * to a string. - * - * ==== numeric objects - * - * With integer argument +object+ given, returns +object+: - * - * Integer(1) # => 1 - * Integer(-1) # => -1 - * - * With floating-point argument +object+ given, - * returns +object+ truncated to an integer: - * - * Integer(1.9) # => 1 # Rounds toward zero. - * Integer(-1.9) # => -1 # Rounds toward zero. - * - * ==== string objects - * - * With string argument +object+ and zero +base+ given, - * returns +object+ converted to an integer in base 10: - * - * Integer('100') # => 100 - * Integer('-100') # => -100 - * - * With +base+ zero, string +object+ may contain leading characters - * to specify the actual base (radix indicator): - * - * Integer('0100') # => 64 # Leading '0' specifies base 8. - * Integer('0b100') # => 4 # Leading '0b', specifies base 2. - * Integer('0x100') # => 256 # Leading '0x' specifies base 16. - * - * With a positive +base+ (in range 2..36) given, returns +object+ - * converted to an integer in the given base: - * - * Integer('100', 2) # => 4 - * Integer('100', 8) # => 64 - * Integer('-100', 16) # => -256 - * - * With a negative +base+ (in range -36..-2) given, returns +object+ - * converted to an integer in the radix indicator if exists or - * +-base+: - * - * Integer('0x100', -2) # => 256 - * Integer('100', -2) # => 4 - * Integer('0b100', -8) # => 4 - * Integer('100', -8) # => 64 - * Integer('0o100', -10) # => 64 - * Integer('100', -10) # => 100 - * - * +base+ -1 is equal the -10 case. - * - * When converting strings, surrounding whitespace and embedded underscores - * are allowed and ignored: - * - * Integer(' 100 ') # => 100 - * Integer('-1_0_0', 16) # => -256 - * - * ==== other classes - * - * Examples with +object+ of various other classes: - * - * Integer(Rational(9, 10)) # => 0 # Rounds toward zero. - * Integer(Complex(2, 0)) # => 2 # Imaginary part must be zero. - * Integer(Time.now) # => 1650974042 - * - * ==== keywords - * - * With optional keyword argument +exception+ given as +true+ (the default): - * - * - Raises TypeError if +object+ does not respond to +to_int+ or +to_i+. - * - Raises TypeError if +object+ is +nil+. - * - Raise ArgumentError if +object+ is an invalid string. - * - * With +exception+ given as +false+, an exception of any kind is suppressed - * and +nil+ is returned. - * - */ - static VALUE -rb_f_integer(int argc, VALUE *argv, VALUE obj) +rb_f_integer1(rb_execution_context_t *ec, VALUE obj, VALUE arg) { - VALUE arg = Qnil, opts = Qnil; - int base = 0; - - if (argc > 1) { - int narg = 1; - VALUE vbase = rb_check_to_int(argv[1]); - if (!NIL_P(vbase)) { - base = NUM2INT(vbase); - narg = 2; - } - if (argc > narg) { - VALUE hash = rb_check_hash_type(argv[argc-1]); - if (!NIL_P(hash)) { - opts = rb_extract_keywords(&hash); - if (!hash) --argc; - } - } - } - rb_check_arity(argc, 1, 2); - arg = argv[0]; + return rb_convert_to_integer(arg, 0, TRUE); +} - return rb_convert_to_integer(arg, base, opts_exception_p(opts)); +static VALUE +rb_f_integer(rb_execution_context_t *ec, VALUE obj, VALUE arg, VALUE base, VALUE exception) +{ + int exc = rb_bool_expected(exception, "exception", TRUE); + return rb_convert_to_integer(arg, NUM2INT(base), exc); } static double @@ -4475,8 +4371,6 @@ InitVM_Object(void) rb_define_global_function("sprintf", f_sprintf, -1); rb_define_global_function("format", f_sprintf, -1); - rb_define_global_function("Integer", rb_f_integer, -1); - rb_define_global_function("String", rb_f_string, 1); rb_define_global_function("Array", rb_f_array, 1); rb_define_global_function("Hash", rb_f_hash, 1); diff --git a/parse.y b/parse.y index ba146b7439ece5..8b88b1b38df6a8 100644 --- a/parse.y +++ b/parse.y @@ -910,6 +910,13 @@ set_line_body(NODE *body, int line) } } +static void +set_embraced_location(NODE *node, const rb_code_location_t *beg, const rb_code_location_t *end) +{ + node->nd_body->nd_loc = code_loc_gen(beg, end); + nd_set_line(node, beg->end_pos.lineno); +} + #define yyparse ruby_yyparse static NODE* cond(struct parser_params *p, NODE *node, const YYLTYPE *loc); @@ -2250,8 +2257,7 @@ cmd_brace_block : tLBRACE_ARG brace_body '}' { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; @@ -2314,6 +2320,14 @@ command : fcall command_args %prec tLOWEST /*% %*/ /*% ripper: method_add_block!(command_call!($1, $2, $3, $4), $5) %*/ } + | primary_value tCOLON2 tCONSTANT '{' brace_body '}' + { + /*%%%*/ + set_embraced_location($5, &@4, &@6); + $$ = new_command_qcall(p, ID2VAL(idCOLON2), $1, $3, Qnull, $5, &@3, &@$); + /*% %*/ + /*% ripper: method_add_block!(command_call!($1, $2, $3, Qnull), $5) %*/ + } | keyword_super command_args { /*%%%*/ @@ -3348,14 +3362,7 @@ primary : literal /*% %*/ /*% ripper: begin!($3) %*/ } - | tLPAREN_ARG {SET_LEX_STATE(EXPR_ENDARG);} rparen - { - /*%%%*/ - $$ = NEW_BEGIN(0, &@$); - /*% %*/ - /*% ripper: paren!(0) %*/ - } - | tLPAREN_ARG stmt {SET_LEX_STATE(EXPR_ENDARG);} rparen + | tLPAREN_ARG compstmt {SET_LEX_STATE(EXPR_ENDARG);} ')' { /*%%%*/ if (nd_type_p($2, NODE_SELF)) $2->nd_state = 0; @@ -4274,8 +4281,7 @@ do_block : k_do_block do_body k_end { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; @@ -4393,16 +4399,14 @@ brace_block : '{' brace_body '}' { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } | k_do do_body k_end { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; @@ -12114,7 +12118,7 @@ value_expr_check(struct parser_params *p, NODE *node) if (node->nd_body->nd_body) { return NULL; } - /* single line pattern matching */ + /* single line pattern matching with "=>" operator */ return void_node ? void_node : node; case NODE_BLOCK: @@ -12737,7 +12741,6 @@ new_args_tail(struct parser_params *p, NODE *kw_args, ID kw_rest_arg, ID block, if (block) arg_var(p, block); args->kw_rest_arg = NEW_DVAR(kw_rest_arg, kw_rest_loc); - args->kw_rest_arg->nd_cflag = kw_bits; } else if (kw_rest_arg == idNil) { args->no_kwarg = 1; diff --git a/process.c b/process.c index fa2ae7344ab20c..47db4fa6265341 100644 --- a/process.c +++ b/process.c @@ -1197,7 +1197,7 @@ rb_process_status_wait(rb_pid_t pid, int flags) * This is an EXPERIMENTAL FEATURE. */ -VALUE +static VALUE rb_process_status_waitv(int argc, VALUE *argv, VALUE _) { rb_check_arity(argc, 0, 2); @@ -1222,7 +1222,7 @@ rb_waitpid(rb_pid_t pid, int *st, int flags) VALUE status = rb_process_status_wait(pid, flags); if (NIL_P(status)) return 0; - struct rb_process_status *data = RTYPEDDATA_DATA(status); + struct rb_process_status *data = rb_check_typeddata(status, &rb_process_status_type); pid = data->pid; if (st) *st = data->status; @@ -3013,6 +3013,9 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * - Passing string +command_line+ to the shell. * - Invoking the executable at +exe_path+. * + * This method has potential security vulnerabilities if called with untrusted input; + * see {Command Injection}[rdoc-ref:command_injection.rdoc]. + * * The new process is created using the * {exec system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/execve.html]; * it may inherit some of its environment from the calling program @@ -3035,8 +3038,19 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * * \String argument +command_line+ is a command line to be passed to a shell; * it must begin with a shell reserved word, begin with a special built-in, - * or contain meta characters. - * It may also contain arguments and options for that command. + * or contain meta characters: + * + * exec('echo') # Built-in. + * exec('if true; then echo "Foo"; fi') # Shell reserved word. + * exec('date > date.tmp') # Contains meta character. + * + * The command line may also contain arguments and options for the command: + * + * exec('echo "Foo"') + * + * Output: + * + * Foo * * On a Unix-like system, the shell is /bin/sh; * otherwise the shell is determined by environment variable @@ -3046,9 +3060,15 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * the entire string +command_line+ is passed as an argument * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * The shell performs normal shell expansion on the command line. + * The shell performs normal shell expansion on the command line: + * + * exec('echo C*') + * + * Output: * - * Raises an exception if the new process fails to execute. + * CONTRIBUTING.md COPYING COPYING.ja + * + * Raises an exception if the new process could not execute. * * Argument +exe_path+ * @@ -3058,12 +3078,30 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * - A 2-element array containing the path to an executable * and the string to be used as the name of the executing process. * - * Ruby invokes the executable directly, with no shell and no shell expansion. + * Example: + * + * exec('/usr/bin/date') + * + * Output: + * + * Sat Aug 26 09:38:00 AM CDT 2023 + * + * Ruby invokes the executable directly, with no shell and no shell expansion: + * + * exec('doesnt_exist') # Raises Errno::ENOENT * * If one or more +args+ is given, each is an argument or option - * to be passed to the executable. + * to be passed to the executable: + * + * exec('echo', 'C*') + * exec('echo', 'hello', 'world') + * + * Output: + * + * C* + * hello world * - * Raises an exception if the new process fails to execute. + * Raises an exception if the new process could not execute. */ static VALUE @@ -4641,56 +4679,132 @@ rb_spawn(int argc, const VALUE *argv) /* * call-seq: - * system([env,] command... [,options], exception: false) -> true, false or nil + * system([env, ] command_line, options = {}, exception: false) -> true, false, or nil + * system([env, ] exe_path, *args, options = {}, exception: false) -> true, false, or nil * - * Executes _command..._ in a subshell. - * _command..._ is one of following forms. + * Creates a new child process by doing one of the following + * in that process: + * + * - Passing string +command_line+ to the shell. + * - Invoking the executable at +exe_path+. * * This method has potential security vulnerabilities if called with untrusted input; * see {Command Injection}[rdoc-ref:command_injection.rdoc]. * - * [commandline] - * command line string which is passed to the standard shell - * [cmdname, arg1, ...] - * command name and one or more arguments (no shell) - * [[cmdname, argv0], arg1, ...] - * command name, argv[0] and zero or more arguments (no shell) + * Returns: * - * system returns +true+ if the command gives zero exit status, - * +false+ for non zero exit status. - * Returns +nil+ if command execution fails. - * An error status is available in $?. + * - +true+ if the command exits with status zero. + * - +false+ if the exit status is a non-zero integer. + * - +nil+ if the command could not execute. * - * If the exception: true argument is passed, the method - * raises an exception instead of returning +false+ or +nil+. + * Raises an exception (instead of returning +false+ or +nil+) + * if keyword argument +exception+ is set to +true+. * - * The arguments are processed in the same way as - * for Kernel#spawn. + * Assigns the command's error status to $?. * - * The hash arguments, env and options, are same as #exec and #spawn. - * See Kernel#spawn for details. + * The new process is created using the + * {system system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/system.html]; + * it may inherit some of its environment from the calling program + * (possibly including open file descriptors). * - * system("echo *") - * system("echo", "*") + * Argument +env+, if given, is a hash that affects +ENV+ for the new process; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. * - * produces: + * Argument +options+ is a hash of options for the new process; + * see {Execution Options}[rdoc-ref:Process@Execution+Options]. * - * config.h main.rb - * * + * The first required argument is one of the following: * - * Error handling: + * - +command_line+ if it is a string, + * and if it begins with a shell reserved word or special built-in, + * or if it contains one or more metacharacters. + * - +exe_path+ otherwise. * - * system("cat nonexistent.txt") - * # => false - * system("catt nonexistent.txt") - * # => nil + * Argument +command_line+ * - * system("cat nonexistent.txt", exception: true) - * # RuntimeError (Command failed with exit 1: cat) - * system("catt nonexistent.txt", exception: true) - * # Errno::ENOENT (No such file or directory - catt) + * \String argument +command_line+ is a command line to be passed to a shell; + * it must begin with a shell reserved word, begin with a special built-in, + * or contain meta characters: * - * See Kernel#exec for the standard shell. + * system('echo') # => true # Built-in. + * system('if true; then echo "Foo"; fi') # => true # Shell reserved word. + * system('date > /tmp/date.tmp') # => true # Contains meta character. + * system('date > /nop/date.tmp') # => false + * system('date > /nop/date.tmp', exception: true) # Raises RuntimeError. + * + * Assigns the command's error status to $?: + * + * system('echo') # => true # Built-in. + * $? # => # + * system('date > /nop/date.tmp') # => false + * $? # => # + * + * The command line may also contain arguments and options for the command: + * + * system('echo "Foo"') # => true + * + * Output: + * + * Foo + * + * On a Unix-like system, the shell is /bin/sh; + * otherwise the shell is determined by environment variable + * ENV['RUBYSHELL'], if defined, or ENV['COMSPEC'] otherwise. + * + * Except for the +COMSPEC+ case, + * the entire string +command_line+ is passed as an argument + * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. + * + * The shell performs normal shell expansion on the command line: + * + * system('echo C*') # => true + * + * Output: + * + * CONTRIBUTING.md COPYING COPYING.ja + * + * Raises an exception if the new process could not execute. + * + * Argument +exe_path+ + * + * Argument +exe_path+ is one of the following: + * + * - The string path to an executable to be called. + * - A 2-element array containing the path to an executable + * and the string to be used as the name of the executing process. + * + * Example: + * + * system('/usr/bin/date') # => true # Path to date on Unix-style system. + * system('foo') # => nil # Command failed. + * + * Output: + * + * Mon Aug 28 11:43:10 AM CDT 2023 + * + * Assigns the command's error status to $?: + * + * system('/usr/bin/date') # => true + * $? # => # + * system('foo') # => nil + * $? # => # + * + * Ruby invokes the executable directly, with no shell and no shell expansion: + * + * system('doesnt_exist') # => nil + * + * If one or more +args+ is given, each is an argument or option + * to be passed to the executable: + * + * system('echo', 'C*') # => true + * system('echo', 'hello', 'world') # => true + * + * Output: + * + * C* + * hello world + * + * Raises an exception if the new process could not execute. */ static VALUE @@ -4710,7 +4824,8 @@ rb_f_system(int argc, VALUE *argv, VALUE _) if (pid > 0) { VALUE status = rb_process_status_wait(pid, 0); - struct rb_process_status *data = RTYPEDDATA_DATA(status); + + struct rb_process_status *data = rb_check_typeddata(status, &rb_process_status_type); // Set the last status: rb_obj_freeze(status); @@ -4757,271 +4872,120 @@ rb_f_system(int argc, VALUE *argv, VALUE _) /* * call-seq: - * spawn([env,] command... [,options]) -> pid - * Process.spawn([env,] command... [,options]) -> pid - * - * spawn executes specified command and return its pid. - * - * pid = spawn("tar xf ruby-2.0.0-p195.tar.bz2") - * Process.wait pid - * - * pid = spawn(RbConfig.ruby, "-eputs'Hello, world!'") - * Process.wait pid - * - * This method is similar to Kernel#system but it doesn't wait for the command - * to finish. - * - * The parent process should - * use Process.wait to collect - * the termination status of its child or - * use Process.detach to register - * disinterest in their status; - * otherwise, the operating system may accumulate zombie processes. - * - * spawn has bunch of options to specify process attributes: - * - * env: hash - * name => val : set the environment variable - * name => nil : unset the environment variable - * - * the keys and the values except for +nil+ must be strings. - * command...: - * commandline : command line string which is passed to the standard shell - * cmdname, arg1, ... : command name and one or more arguments (This form does not use the shell. See below for caveats.) - * [cmdname, argv0], arg1, ... : command name, argv[0] and zero or more arguments (no shell) - * options: hash - * clearing environment variables: - * :unsetenv_others => true : clear environment variables except specified by env - * :unsetenv_others => false : don't clear (default) - * process group: - * :pgroup => true or 0 : make a new process group - * :pgroup => pgid : join the specified process group - * :pgroup => nil : don't change the process group (default) - * create new process group: Windows only - * :new_pgroup => true : the new process is the root process of a new process group - * :new_pgroup => false : don't create a new process group (default) - * resource limit: resourcename is core, cpu, data, etc. See Process.setrlimit. - * :rlimit_resourcename => limit - * :rlimit_resourcename => [cur_limit, max_limit] - * umask: - * :umask => int - * redirection: - * key: - * FD : single file descriptor in child process - * [FD, FD, ...] : multiple file descriptor in child process - * value: - * FD : redirect to the file descriptor in parent process - * string : redirect to file with open(string, "r" or "w") - * [string] : redirect to file with open(string, File::RDONLY) - * [string, open_mode] : redirect to file with open(string, open_mode, 0644) - * [string, open_mode, perm] : redirect to file with open(string, open_mode, perm) - * [:child, FD] : redirect to the redirected file descriptor - * :close : close the file descriptor in child process - * FD is one of follows - * :in : the file descriptor 0 which is the standard input - * :out : the file descriptor 1 which is the standard output - * :err : the file descriptor 2 which is the standard error - * integer : the file descriptor of specified the integer - * io : the file descriptor specified as io.fileno - * file descriptor inheritance: close non-redirected non-standard fds (3, 4, 5, ...) or not - * :close_others => false : inherit - * current directory: - * :chdir => str - * - * The cmdname, arg1, ... form does not use the shell. - * However, on different OSes, different things are provided as - * built-in commands. An example of this is +'echo'+, which is a - * built-in on Windows, but is a normal program on Linux and Mac OS X. - * This means that Process.spawn 'echo', '%Path%' will - * display the contents of the %Path% environment variable - * on Windows, but Process.spawn 'echo', '$PATH' prints - * the literal $PATH. - * - * If a hash is given as +env+, the environment is - * updated by +env+ before exec(2) in the child process. - * If a pair in +env+ has nil as the value, the variable is deleted. - * - * # set FOO as BAR and unset BAZ. - * pid = spawn({"FOO"=>"BAR", "BAZ"=>nil}, command) - * - * If a hash is given as +options+, - * it specifies - * process group, - * create new process group, - * resource limit, - * current directory, - * umask and - * redirects for the child process. - * Also, it can be specified to clear environment variables. - * - * The :unsetenv_others key in +options+ specifies - * to clear environment variables, other than specified by +env+. - * - * pid = spawn(command, :unsetenv_others=>true) # no environment variable - * pid = spawn({"FOO"=>"BAR"}, command, :unsetenv_others=>true) # FOO only - * - * The :pgroup key in +options+ specifies a process group. - * The corresponding value should be true, zero, a positive integer, or nil. - * true and zero cause the process to be a process leader of a new process group. - * A non-zero positive integer causes the process to join the provided process group. - * The default value, nil, causes the process to remain in the same process group. - * - * pid = spawn(command, :pgroup=>true) # process leader - * pid = spawn(command, :pgroup=>10) # belongs to the process group 10 - * - * The :new_pgroup key in +options+ specifies to pass - * +CREATE_NEW_PROCESS_GROUP+ flag to CreateProcessW() that is - * Windows API. This option is only for Windows. - * true means the new process is the root process of the new process group. - * The new process has CTRL+C disabled. This flag is necessary for - * Process.kill(:SIGINT, pid) on the subprocess. - * :new_pgroup is false by default. - * - * pid = spawn(command, :new_pgroup=>true) # new process group - * pid = spawn(command, :new_pgroup=>false) # same process group - * - * The :rlimit_foo key specifies a resource limit. - * foo should be one of resource types such as core. - * The corresponding value should be an integer or an array which have one or - * two integers: same as cur_limit and max_limit arguments for - * Process.setrlimit. - * - * cur, max = Process.getrlimit(:CORE) - * pid = spawn(command, :rlimit_core=>[0,max]) # disable core temporary. - * pid = spawn(command, :rlimit_core=>max) # enable core dump - * pid = spawn(command, :rlimit_core=>0) # never dump core. - * - * The :umask key in +options+ specifies the umask. - * - * pid = spawn(command, :umask=>077) - * - * The :in, :out, :err, an integer, an IO and an array key specifies a redirection. - * The redirection maps a file descriptor in the child process. - * - * For example, stderr can be merged into stdout as follows: - * - * pid = spawn(command, :err=>:out) - * pid = spawn(command, 2=>1) - * pid = spawn(command, STDERR=>:out) - * pid = spawn(command, STDERR=>STDOUT) - * - * The hash keys specifies a file descriptor in the child process - * started by #spawn. - * :err, 2 and STDERR specifies the standard error stream (stderr). - * - * The hash values specifies a file descriptor in the parent process - * which invokes #spawn. - * :out, 1 and STDOUT specifies the standard output stream (stdout). - * - * In the above example, - * the standard output in the child process is not specified. - * So it is inherited from the parent process. - * - * The standard input stream (stdin) can be specified by :in, 0 and STDIN. - * - * A filename can be specified as a hash value. - * - * pid = spawn(command, :in=>File::NULL) # read mode - * pid = spawn(command, :out=>File::NULL) # write mode - * pid = spawn(command, :err=>"log") # write mode - * pid = spawn(command, [:out, :err]=>File::NULL) # write mode - * pid = spawn(command, 3=>File::NULL) # read mode + * spawn([env, ] command_line, options = {}) -> pid + * spawn([env, ] exe_path, *args, options = {}) -> pid * - * For stdout and stderr (and combination of them), - * it is opened in write mode. - * Otherwise read mode is used. + * Creates a new child process by doing one of the following + * in that process: * - * For specifying flags and permission of file creation explicitly, - * an array is used instead. + * - Passing string +command_line+ to the shell. + * - Invoking the executable at +exe_path+. * - * pid = spawn(command, :in=>["file"]) # read mode is assumed - * pid = spawn(command, :in=>["file", "r"]) - * pid = spawn(command, :out=>["log", "w"]) # 0644 assumed - * pid = spawn(command, :out=>["log", "w", 0600]) - * pid = spawn(command, :out=>["log", File::WRONLY|File::EXCL|File::CREAT, 0600]) + * This method has potential security vulnerabilities if called with untrusted input; + * see {Command Injection}[rdoc-ref:command_injection.rdoc]. * - * The array specifies a filename, flags and permission. - * The flags can be a string or an integer. - * If the flags is omitted or nil, File::RDONLY is assumed. - * The permission should be an integer. - * If the permission is omitted or nil, 0644 is assumed. + * Returns the process ID (pid) of the new process, + * without waiting for it to complete. * - * If an array of IOs and integers are specified as a hash key, - * all the elements are redirected. + * To avoid zombie processes, the parent process should call either: * - * # stdout and stderr is redirected to log file. - * # The file "log" is opened just once. - * pid = spawn(command, [:out, :err]=>["log", "w"]) + * - Process.wait, to collect the termination statuses of its children. + * - Process.detach, to register disinterest in their status. * - * Another way to merge multiple file descriptors is [:child, fd]. - * \[:child, fd] means the file descriptor in the child process. - * This is different from fd. - * For example, :err=>:out means redirecting child stderr to parent stdout. - * But :err=>[:child, :out] means redirecting child stderr to child stdout. - * They differ if stdout is redirected in the child process as follows. + * The new process is created using the + * {exec system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/execve.html]; + * it may inherit some of its environment from the calling program + * (possibly including open file descriptors). * - * # stdout and stderr is redirected to log file. - * # The file "log" is opened just once. - * pid = spawn(command, :out=>["log", "w"], :err=>[:child, :out]) + * Argument +env+, if given, is a hash that affects +ENV+ for the new process; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. * - * \[:child, :out] can be used to merge stderr into stdout in IO.popen. - * In this case, IO.popen redirects stdout to a pipe in the child process - * and [:child, :out] refers the redirected stdout. + * Argument +options+ is a hash of options for the new process; + * see {Execution Options}[rdoc-ref:Process@Execution+Options]. * - * io = IO.popen(["sh", "-c", "echo out; echo err >&2", :err=>[:child, :out]]) - * p io.read #=> "out\nerr\n" + * The first required argument is one of the following: * - * The :chdir key in +options+ specifies the current directory. + * - +command_line+ if it is a string, + * and if it begins with a shell reserved word or special built-in, + * or if it contains one or more metacharacters. + * - +exe_path+ otherwise. + * + * Argument +command_line+ + * + * \String argument +command_line+ is a command line to be passed to a shell; + * it must begin with a shell reserved word, begin with a special built-in, + * or contain meta characters: + * + * spawn('echo') # => 798847 + * Process.wait # => 798847 + * spawn('if true; then echo "Foo"; fi') # => 798848 + * Process.wait # => 798848 + * spawn('date > /tmp/date.tmp') # => 798879 + * Process.wait # => 798849 + * spawn('date > /nop/date.tmp') # => 798882 # Issues error message. + * Process.wait # => 798882 + * + * The command line may also contain arguments and options for the command: + * + * spawn('echo "Foo"') # => 799031 + * Process.wait # => 799031 + * + * Output: + * + * Foo + * + * On a Unix-like system, the shell is /bin/sh; + * otherwise the shell is determined by environment variable + * ENV['RUBYSHELL'], if defined, or ENV['COMSPEC'] otherwise. * - * pid = spawn(command, :chdir=>"/var/tmp") + * Except for the +COMSPEC+ case, + * the entire string +command_line+ is passed as an argument + * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * spawn closes all non-standard unspecified descriptors by default. - * The "standard" descriptors are 0, 1 and 2. - * This behavior is specified by :close_others option. - * :close_others doesn't affect the standard descriptors which are - * closed only if :close is specified explicitly. + * The shell performs normal shell expansion on the command line: * - * pid = spawn(command, :close_others=>true) # close 3,4,5,... (default) - * pid = spawn(command, :close_others=>false) # don't close 3,4,5,... + * spawn('echo C*') # => 799139 + * Process.wait # => 799139 + * + * Output: + * + * CONTRIBUTING.md COPYING COPYING.ja + * + * Raises an exception if the new process could not execute. + * + * Argument +exe_path+ * - * :close_others is false by default for spawn and IO.popen. + * Argument +exe_path+ is one of the following: * - * Note that fds which close-on-exec flag is already set are closed - * regardless of :close_others option. + * - The string path to an executable to be called. + * - A 2-element array containing the path to an executable + * and the string to be used as the name of the executing process. * - * So IO.pipe and spawn can be used as IO.popen. + * Example: * - * # similar to r = IO.popen(command) - * r, w = IO.pipe - * pid = spawn(command, :out=>w) # r, w is closed in the child process. - * w.close + * spawn('/usr/bin/date') # => 799198 # Path to date on Unix-style system. + * Process.wait # => 799198 * - * :close is specified as a hash value to close a fd individually. + * Output: * - * f = open(foo) - * system(command, f=>:close) # don't inherit f. + * Thu Aug 31 10:06:48 AM CDT 2023 * - * If a file descriptor need to be inherited, - * io=>io can be used. + * Ruby invokes the executable directly, with no shell and no shell expansion. * - * # valgrind has --log-fd option for log destination. - * # log_w=>log_w indicates log_w.fileno inherits to child process. - * log_r, log_w = IO.pipe - * pid = spawn("valgrind", "--log-fd=#{log_w.fileno}", "echo", "a", log_w=>log_w) - * log_w.close - * p log_r.read + * If one or more +args+ is given, each is an argument or option + * to be passed to the executable: * - * It is also possible to exchange file descriptors. + * spawn('echo', 'C*') # => 799392 + * Process.wait # => 799392 + * spawn('echo', 'hello', 'world') # => 799393 + * Process.wait # => 799393 * - * pid = spawn(command, :out=>:err, :err=>:out) + * Output: * - * The hash keys specify file descriptors in the child process. - * The hash values specifies file descriptors in the parent process. - * So the above specifies exchanging stdout and stderr. - * Internally, +spawn+ uses an extra file descriptor to resolve such cyclic - * file descriptor mapping. + * C* + * hello world * - * See Kernel.exec for the standard shell. + * Raises an exception if the new process could not execute. */ static VALUE @@ -8671,10 +8635,12 @@ static VALUE rb_mProcID_Syscall; * * On CRuby, +Process.warmup+: * - * * Perform a major GC. + * * Performs a major GC. * * Compacts the heap. * * Promotes all surviving objects to the old generation. - * * Precompute the coderange of all strings. + * * Precomputes the coderange of all strings. + * * Frees all empty heap pages and increments the allocatable pages counter + * by the number of pages freed. */ static VALUE diff --git a/ruby.c b/ruby.c index 2b4397c0011617..be2ea2d0cde267 100644 --- a/ruby.c +++ b/ruby.c @@ -1738,6 +1738,13 @@ ruby_opt_init(ruby_cmdline_options_t *opt) rb_warning_category_update(opt->warn.mask, opt->warn.set); + /* [Feature #19785] Warning for removed GC environment variable. + * Remove this in Ruby 3.4. */ + if (getenv("RUBY_GC_HEAP_INIT_SLOTS")) { + rb_warn_deprecated("The environment variable RUBY_GC_HEAP_INIT_SLOTS", + "environment variables RUBY_GC_HEAP_%d_INIT_SLOTS"); + } + #if USE_RJIT // rb_call_builtin_inits depends on RubyVM::RJIT.enabled? if (opt->rjit.on) diff --git a/spec/bundler/bundler/fetcher_spec.rb b/spec/bundler/bundler/fetcher_spec.rb index 27a63c476d321f..92790dc7d6f744 100644 --- a/spec/bundler/bundler/fetcher_spec.rb +++ b/spec/bundler/bundler/fetcher_spec.rb @@ -189,4 +189,70 @@ end end end + + describe "#specs_with_retry" do + let(:downloader) { double(:downloader) } + let(:remote) { double(:remote, :cache_slug => "slug", :uri => uri, :original_uri => nil, :anonymized_uri => uri) } + let(:compact_index) { double(Bundler::Fetcher::CompactIndex, :available? => true, :api_fetcher? => true) } + let(:dependency) { double(Bundler::Fetcher::Dependency, :available? => true, :api_fetcher? => true) } + let(:index) { double(Bundler::Fetcher::Index, :available? => true, :api_fetcher? => false) } + + before do + allow(Bundler::Fetcher::CompactIndex).to receive(:new).and_return(compact_index) + allow(Bundler::Fetcher::Dependency).to receive(:new).and_return(dependency) + allow(Bundler::Fetcher::Index).to receive(:new).and_return(index) + end + + it "picks the first fetcher that works" do + expect(compact_index).to receive(:specs).with("name").and_return([["name", "1.2.3", "ruby"]]) + expect(dependency).not_to receive(:specs) + expect(index).not_to receive(:specs) + fetcher.specs_with_retry("name", double(Bundler::Source::Rubygems)) + end + + context "when APIs are not available" do + before do + allow(compact_index).to receive(:available?).and_return(false) + allow(dependency).to receive(:available?).and_return(false) + end + + it "uses the index" do + expect(compact_index).not_to receive(:specs) + expect(dependency).not_to receive(:specs) + expect(index).to receive(:specs).with("name").and_return([["name", "1.2.3", "ruby"]]) + + fetcher.specs_with_retry("name", double(Bundler::Source::Rubygems)) + end + end + end + + describe "#api_fetcher?" do + let(:downloader) { double(:downloader) } + let(:remote) { double(:remote, :cache_slug => "slug", :uri => uri, :original_uri => nil, :anonymized_uri => uri) } + let(:compact_index) { double(Bundler::Fetcher::CompactIndex, :available? => false, :api_fetcher? => true) } + let(:dependency) { double(Bundler::Fetcher::Dependency, :available? => false, :api_fetcher? => true) } + let(:index) { double(Bundler::Fetcher::Index, :available? => true, :api_fetcher? => false) } + + before do + allow(Bundler::Fetcher::CompactIndex).to receive(:new).and_return(compact_index) + allow(Bundler::Fetcher::Dependency).to receive(:new).and_return(dependency) + allow(Bundler::Fetcher::Index).to receive(:new).and_return(index) + end + + context "when an api fetcher is available" do + before do + allow(compact_index).to receive(:available?).and_return(true) + end + + it "is truthy" do + expect(fetcher).to be_api_fetcher + end + end + + context "when only the index fetcher is available" do + it "is falsey" do + expect(fetcher).not_to be_api_fetcher + end + end + end end diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb index 1f89f9f0f9b933..8cb04768212e9b 100644 --- a/spec/bundler/install/gemfile/sources_spec.rb +++ b/spec/bundler/install/gemfile/sources_spec.rb @@ -78,6 +78,33 @@ end end + context "without source affinity, and a stdlib gem present in one of the sources", :ruby_repo do + let(:default_json_version) { ruby "gem 'json'; require 'json'; puts JSON::VERSION" } + + before do + build_repo2 do + build_gem "json", default_json_version + end + + build_repo4 do + build_gem "foo" do |s| + s.add_dependency "json", default_json_version + end + end + + gemfile <<-G + source "https://gem.repo2" + source "https://gem.repo4" + + gem "foo" + G + end + + it "works in standalone mode", :bundler => "< 3" do + bundle "install --standalone", :artifice => "compact_index" + end + end + context "with source affinity" do context "with sources given by a block" do before do diff --git a/spec/bundler/resolver/basic_spec.rb b/spec/bundler/resolver/basic_spec.rb index f739f8c02bb2fd..151d10c61cd1e5 100644 --- a/spec/bundler/resolver/basic_spec.rb +++ b/spec/bundler/resolver/basic_spec.rb @@ -347,4 +347,27 @@ should_resolve_as %w[rack-3.0.0 standalone_migrations-1.0.13] end + + it "does not ignore versions that incorrectly depend on themselves when dependency_api is not available" do + @index = build_index do + gem "rack", "3.0.0" + + gem "standalone_migrations", "7.1.0" do + dep "rack", "~> 2.0" + end + + gem "standalone_migrations", "2.0.4" do + dep "standalone_migrations", ">= 2.0.5" + end + + gem "standalone_migrations", "1.0.13" do + dep "rack", ">= 0" + end + end + + dep "rack", "~> 3.0" + dep "standalone_migrations" + + should_resolve_without_dependency_api %w[rack-3.0.0 standalone_migrations-2.0.4] + end end diff --git a/spec/bundler/runtime/self_management_spec.rb b/spec/bundler/runtime/self_management_spec.rb index 976437c332c553..3e49db4f32c3f7 100644 --- a/spec/bundler/runtime/self_management_spec.rb +++ b/spec/bundler/runtime/self_management_spec.rb @@ -129,6 +129,13 @@ expect(out).to eq(Bundler::VERSION[0] == "2" ? "Bundler version #{Bundler::VERSION}" : Bundler::VERSION) end + it "ignores malformed lockfile version" do + lockfile_bundled_with("2.3.") + + bundle "install --verbose" + expect(out).to include("Using bundler #{Bundler::VERSION}") + end + private def lockfile_bundled_with(version) diff --git a/spec/bundler/runtime/setup_spec.rb b/spec/bundler/runtime/setup_spec.rb index 09f179c51739dd..f02f81f6974104 100644 --- a/spec/bundler/runtime/setup_spec.rb +++ b/spec/bundler/runtime/setup_spec.rb @@ -1598,7 +1598,7 @@ module Gem::BUNDLED_GEMS require 'csv' R - expect(err).to be_empty + expect(err).not_to include("Add csv to your Gemfile") end it "don't warn with bundled gems when it's declared in Gemfile" do diff --git a/spec/bundler/support/indexes.rb b/spec/bundler/support/indexes.rb index 78372302f1b7d8..14f515f870b084 100644 --- a/spec/bundler/support/indexes.rb +++ b/spec/bundler/support/indexes.rb @@ -14,9 +14,9 @@ def platform(*args) alias_method :platforms, :platform - def resolve(args = []) + def resolve(args = [], dependency_api_available: true) @platforms ||= ["ruby"] - default_source = instance_double("Bundler::Source::Rubygems", :specs => @index, :to_s => "locally install gems") + default_source = instance_double("Bundler::Source::Rubygems", :specs => @index, :to_s => "locally install gems", :dependency_api_available? => dependency_api_available) source_requirements = { :default => default_source } base = args[0] || Bundler::SpecSet.new([]) base.each {|ls| ls.source = default_source } @@ -41,6 +41,12 @@ def should_resolve_as(specs) expect(got).to eq(specs.sort) end + def should_resolve_without_dependency_api(specs) + got = resolve(:dependency_api_available => false) + got = got.map(&:full_name).sort + expect(got).to eq(specs.sort) + end + def should_resolve_and_include(specs, args = []) got = resolve(args) got = got.map(&:full_name).sort diff --git a/spec/bundler/update/redownload_spec.rb b/spec/bundler/update/redownload_spec.rb index 147be823f57406..b55593d077642f 100644 --- a/spec/bundler/update/redownload_spec.rb +++ b/spec/bundler/update/redownload_spec.rb @@ -30,5 +30,15 @@ bundle "update rack --no-color --redownload" expect(err).not_to include "[DEPRECATED] The `--force` option has been renamed to `--redownload`" end + + it "re-installs installed gems" do + rack_lib = default_bundle_path("gems/rack-1.0.0/lib/rack.rb") + rack_lib.open("w") {|f| f.write("blah blah blah") } + bundle :update, :redownload => true + + expect(out).to include "Installing rack 1.0.0" + expect(rack_lib.open(&:read)).to eq("RACK = '1.0.0'\n") + expect(the_bundle).to include_gems "rack 1.0.0" + end end end diff --git a/spec/ruby/core/string/start_with_spec.rb b/spec/ruby/core/string/start_with_spec.rb index 3833289f96d488..81eed47f96e800 100644 --- a/spec/ruby/core/string/start_with_spec.rb +++ b/spec/ruby/core/string/start_with_spec.rb @@ -7,12 +7,14 @@ it_behaves_like :start_with, :to_s # Here and not in the shared examples because this is invalid as a Symbol - it "does not check that we are not starting to match at the head of a character" do + it "matches part of a character with the same part" do "\xA9".should.start_with?("\xA9") # A9 is not a character head for UTF-8 end - it "does not check we are matching only part of a character" do - "\xe3\x81\x82".size.should == 1 - "\xe3\x81\x82".should.start_with?("\xe3") + ruby_bug "#19784", ""..."3.3" do + it "checks we are matching only part of a character" do + "\xe3\x81\x82".size.should == 1 + "\xe3\x81\x82".should_not.start_with?("\xe3") + end end end diff --git a/spec/ruby/library/socket/basicsocket/recv_spec.rb b/spec/ruby/library/socket/basicsocket/recv_spec.rb index b6ccda5d00878d..a56114f4ab57d6 100644 --- a/spec/ruby/library/socket/basicsocket/recv_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_spec.rb @@ -32,6 +32,25 @@ ScratchPad.recorded.should == 'hello' end + ruby_version_is "3.3" do + it "returns nil on a closed stream socket" do + t = Thread.new do + client = @server.accept + packet = client.recv(10) + client.close + packet + end + + Thread.pass while t.status and t.status != "sleep" + t.status.should_not be_nil + + socket = TCPSocket.new('127.0.0.1', @port) + socket.close + + t.value.should be_nil + end + end + platform_is_not :solaris do it "accepts flags to specify unusual receiving behaviour" do t = Thread.new do diff --git a/spec/ruby/library/socket/basicsocket/send_spec.rb b/spec/ruby/library/socket/basicsocket/send_spec.rb index 868801df304a1c..041ce03d72c7ee 100644 --- a/spec/ruby/library/socket/basicsocket/send_spec.rb +++ b/spec/ruby/library/socket/basicsocket/send_spec.rb @@ -22,7 +22,7 @@ client = @server.accept loop do got = client.recv(5) - break if got.empty? + break if got.nil? || got.empty? data << got end client.close @@ -67,7 +67,7 @@ client = @server.accept loop do got = client.recv(5) - break if got.empty? + break if got.nil? || got.empty? data << got end client.close diff --git a/spec/ruby/library/socket/basicsocket/shutdown_spec.rb b/spec/ruby/library/socket/basicsocket/shutdown_spec.rb index 41d9581bdeb3b1..c78b32de38c3cd 100644 --- a/spec/ruby/library/socket/basicsocket/shutdown_spec.rb +++ b/spec/ruby/library/socket/basicsocket/shutdown_spec.rb @@ -23,7 +23,7 @@ it 'shuts down a socket for reading' do @client.shutdown(Socket::SHUT_RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing' do @@ -35,7 +35,7 @@ it 'shuts down a socket for reading and writing' do @client.shutdown(Socket::SHUT_RDWR) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end @@ -49,13 +49,13 @@ it 'shuts down a socket for reading using :RD' do @client.shutdown(:RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using :SHUT_RD' do @client.shutdown(:SHUT_RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing using :WR' do @@ -73,7 +73,7 @@ it 'shuts down a socket for reading and writing' do @client.shutdown(:RDWR) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end @@ -87,13 +87,13 @@ it 'shuts down a socket for reading using "RD"' do @client.shutdown('RD') - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using "SHUT_RD"' do @client.shutdown('SHUT_RD') - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing using "WR"' do @@ -123,7 +123,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using "SHUT_RD"' do @@ -131,7 +131,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading and writing' do @@ -139,7 +139,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end diff --git a/spec/ruby/library/socket/fixtures/classes.rb b/spec/ruby/library/socket/fixtures/classes.rb index 406bd7c7101e5e..786629d2eff048 100644 --- a/spec/ruby/library/socket/fixtures/classes.rb +++ b/spec/ruby/library/socket/fixtures/classes.rb @@ -113,7 +113,7 @@ def service(socket) begin data = socket.recv(1024) - return if data.empty? + return if data.nil? || data.empty? log "SpecTCPServer received: #{data.inspect}" return if data == "QUIT" diff --git a/spec/ruby/shared/string/start_with.rb b/spec/ruby/shared/string/start_with.rb index 6932a017b6d621..91fc50c4cdd3e3 100644 --- a/spec/ruby/shared/string/start_with.rb +++ b/spec/ruby/shared/string/start_with.rb @@ -70,7 +70,9 @@ $1.should be_nil end - it "does not check that we are not matching part of a character" do - "\xC3\xA9".send(@method).should.start_with?("\xC3") + ruby_bug "#19784", ""..."3.3" do + it "checks that we are not matching part of a character" do + "\xC3\xA9".send(@method).should_not.start_with?("\xC3") + end end end diff --git a/string.c b/string.c index 09d896bb5585dd..e962b32448c17a 100644 --- a/string.c +++ b/string.c @@ -3937,8 +3937,7 @@ str_ensure_byte_pos(VALUE str, long pos) const char *s = RSTRING_PTR(str); const char *e = RSTRING_END(str); const char *p = s + pos; - const char *pp = rb_enc_left_char_head(s, p, e, rb_enc_get(str)); - if (p != pp) { + if (!at_char_boundary(s, p, e, rb_enc_get(str))) { rb_raise(rb_eIndexError, "offset %ld does not land on character boundary", pos); } @@ -9528,7 +9527,7 @@ chompped_length(VALUE str, VALUE rs) if (p[len-1] == newline && (rslen <= 1 || memcmp(rsptr, pp, rslen) == 0)) { - if (rb_enc_left_char_head(p, pp, e, enc) == pp) + if (at_char_boundary(p, pp, e, enc)) return len - rslen; RB_GC_GUARD(rs); } @@ -10469,10 +10468,20 @@ rb_str_start_with(int argc, VALUE *argv, VALUE str) return Qtrue; } else { + const char *p, *s, *e; + long slen, tlen; + rb_encoding *enc; + StringValue(tmp); - rb_enc_check(str, tmp); - if (RSTRING_LEN(str) < RSTRING_LEN(tmp)) continue; - if (memcmp(RSTRING_PTR(str), RSTRING_PTR(tmp), RSTRING_LEN(tmp)) == 0) + enc = rb_enc_check(str, tmp); + if ((tlen = RSTRING_LEN(tmp)) == 0) return Qtrue; + if ((slen = RSTRING_LEN(str)) < tlen) continue; + p = RSTRING_PTR(str); + e = p + slen; + s = p + tlen; + if (!at_char_boundary(p, s, e, enc)) + continue; + if (memcmp(p, RSTRING_PTR(tmp), tlen) == 0) return Qtrue; } } @@ -10491,12 +10500,13 @@ static VALUE rb_str_end_with(int argc, VALUE *argv, VALUE str) { int i; - char *p, *s, *e; - rb_encoding *enc; for (i=0; i nil}) stdin = "#{no_core}register_sample_bug_reporter(12345); Process.kill :SEGV, $$" diff --git a/test/fiber/test_process.rb b/test/fiber/test_process.rb index a5990be2044a80..cc1694576eb9cd 100644 --- a/test/fiber/test_process.rb +++ b/test/fiber/test_process.rb @@ -34,6 +34,27 @@ def test_system end.join end + def test_system_faulty_process_wait + Thread.new do + scheduler = Scheduler.new + + def scheduler.process_wait(pid, flags) + Fiber.blocking{Process.wait(pid, flags)} + + # Don't return `Process::Status` instance. + return false + end + + Fiber.set_scheduler scheduler + + Fiber.schedule do + assert_raise TypeError do + system("true") + end + end + end.join + end + def test_fork omit 'fork not supported' unless Process.respond_to?(:fork) Thread.new do diff --git a/test/irb/test_cmd.rb b/test/irb/test_cmd.rb index 670078679f1152..e8c959ec39956f 100644 --- a/test/irb/test_cmd.rb +++ b/test/irb/test_cmd.rb @@ -211,8 +211,7 @@ def test_measure DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -241,8 +240,7 @@ def test_measure_keeps_previous_value DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -269,8 +267,7 @@ def test_measure_enabled_by_rc DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -300,8 +297,7 @@ def test_measure_enabled_by_rc_with_custom DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -331,8 +327,7 @@ def test_measure_with_custom DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -358,8 +353,7 @@ def test_measure_with_proc DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, diff --git a/test/irb/test_context.rb b/test/irb/test_context.rb index 5847df172e9e07..f59a23a99df9c4 100644 --- a/test/irb/test_context.rb +++ b/test/irb/test_context.rb @@ -90,6 +90,18 @@ def test_eval_input_raise2x ], out) end + def test_prompt_n_deprecation + irb = IRB::Irb.new(IRB::WorkSpace.new(Object.new)) + + _, err = capture_output do + irb.context.prompt_n = "foo" + irb.context.prompt_n + end + + assert_include err, "IRB::Context#prompt_n is deprecated" + assert_include err, "IRB::Context#prompt_n= is deprecated" + end + def test_output_to_pipe require 'stringio' input = TestInputMethod.new(["n=1"]) @@ -454,7 +466,6 @@ def main.inspect def test_default_return_format IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " # without :RETURN diff --git a/test/irb/test_debug_cmd.rb b/test/irb/test_debug_cmd.rb index a99f7a943f62cd..d669c174e6ef6a 100644 --- a/test/irb/test_debug_cmd.rb +++ b/test/irb/test_debug_cmd.rb @@ -331,6 +331,39 @@ def bar assert_include(output, "InputMethod: RelineInputMethod") end + def test_help_command_is_delegated_to_the_debugger + write_ruby <<~'ruby' + binding.irb + ruby + + output = run_ruby_file do + type "debug" + type "help" + type "continue" + end + + assert_include(output, "### Frame control") + end + + def test_show_cmds_display_different_content_when_debugger_is_enabled + write_ruby <<~'ruby' + # disable pager + STDIN.singleton_class.define_method(:tty?) { false } + binding.irb + ruby + + output = run_ruby_file do + type "debug" + type "show_cmds" + type "continue" + end + + # IRB's commands should still be listed + assert_match(/show_cmds\s+List all available commands and their description\./, output) + # debug gem's commands should be appended at the end + assert_match(/Debugging \(from debug\.gem\)\s+### Control flow/, output) + end + def test_input_is_evaluated_in_the_context_of_the_current_thread write_ruby <<~'ruby' current_thread = Thread.current diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index c685912093dffb..08fe41f5e7854e 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -94,8 +94,8 @@ def dynamic_prompt(&block) end def setup - @irb = build_irb save_encodings + @irb = build_irb end def teardown diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index 336f4fd3660ddf..09344e1b0efed9 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -203,11 +203,10 @@ def test_assignment_expression_with_local_variable end def test_initialising_the_old_top_level_ruby_lex - _, err = capture_output do + assert_in_out_err(["--disable-gems", "-W:deprecated"], <<~RUBY, [], /warning: constant ::RubyLex is deprecated/) + require "irb" ::RubyLex.new(nil) - end - - assert_match(/warning: constant ::RubyLex is deprecated/, err) + RUBY end private diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index 3da3935c195c07..279eff6515a144 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -47,6 +47,35 @@ def test_launch EOC end + def test_nomultiline + write_irbrc <<~'LINES' + puts 'start IRB' + LINES + start_terminal(25, 80, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb --nomultiline}, startup_message: 'start IRB') + write(<<~EOC) + if true + if false + a = "hello + world" + puts a + end + end + EOC + close + assert_screen(<<~EOC) + start IRB + irb(main):001> if true + irb(main):002* if false + irb(main):003* a = "hello + irb(main):004" world" + irb(main):005* puts a + irb(main):006* end + irb(main):007* end + => nil + irb(main):008> + EOC + end + def test_multiline_paste write_irbrc <<~'LINES' puts 'start IRB' @@ -179,7 +208,6 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_right write_irbrc <<~'LINES' IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " } @@ -214,7 +242,6 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_left write_irbrc <<~'LINES' IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " } diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index d5dab78fed2bda..2877bef7d8e16d 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'json/add/core' require 'json/add/complex' require 'json/add/rational' diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 9148b78c8ba13d..39d35fa7729346 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'stringio' require 'tempfile' diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index cc7b71553a3e02..7e7e5341b02cb6 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -1,6 +1,6 @@ # encoding: utf-8 #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONEncodingTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index c5a030ea8a7bad..b5b18fb20b7086 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONExtParserTest < Test::Unit::TestCase if defined?(JSON::Ext::Parser) diff --git a/test/json/json_fixtures_test.rb b/test/json/json_fixtures_test.rb index 845abb48673ca7..acc87499654802 100644 --- a/test/json/json_fixtures_test.rb +++ b/test/json/json_fixtures_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONFixturesTest < Test::Unit::TestCase def setup diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f71999b12c0560..3c4aad6528b80a 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -2,7 +2,7 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONGeneratorTest < Test::Unit::TestCase include JSON @@ -233,7 +233,7 @@ def test_buffer_initial_length def test_gc if respond_to?(:assert_in_out_err) && !(RUBY_PLATFORM =~ /java/) - assert_in_out_err(%w[-rjson --disable-gems], <<-EOS, [], []) + assert_in_out_err(%w[-rjson], <<-EOS, [], []) bignum_too_long_to_embed_as_string = 1234567890123456789012345 expect = bignum_too_long_to_embed_as_string.to_s GC.stress = true diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb index 82742dcd638967..c4d391208cc965 100644 --- a/test/json/json_generic_object_test.rb +++ b/test/json/json_generic_object_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONGenericObjectTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 802408b2cb78ff..220826a426b1f2 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1,6 +1,6 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'stringio' require 'tempfile' require 'ostruct' diff --git a/test/json/json_string_matching_test.rb b/test/json/json_string_matching_test.rb index 5d55dc31b0c4e0..b9cf904aaaf7ab 100644 --- a/test/json/json_string_matching_test.rb +++ b/test/json/json_string_matching_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'time' class JSONStringMatchingTest < Test::Unit::TestCase diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index 71105e55ecd22d..cca21b20f0b798 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -1,7 +1,7 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONInRactorTest < Test::Unit::TestCase def test_generate diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index da3ae5d67d2ebf..5fe37e2d6426d1 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -82,8 +82,6 @@ def test_hmac_sign_verify end def test_ed25519 - pend_on_openssl_issue_21493 - # Test vector from RFC 8032 Section 7.1 TEST 2 priv_pem = <<~EOF -----BEGIN PRIVATE KEY----- @@ -148,8 +146,6 @@ def test_ed25519 end def test_x25519 - pend_on_openssl_issue_21493 - # Test vector from RFC 7748 Section 6.1 alice_pem = <<~EOF -----BEGIN PRIVATE KEY----- @@ -202,8 +198,6 @@ def raw_initialize end def test_compare? - pend_on_openssl_issue_21493 - key1 = Fixtures.pkey("rsa1024") key2 = Fixtures.pkey("rsa1024") key3 = Fixtures.pkey("rsa2048") diff --git a/test/openssl/utils.rb b/test/openssl/utils.rb index b5ffbe1c999e95..3d4d05fe02ebc7 100644 --- a/test/openssl/utils.rb +++ b/test/openssl/utils.rb @@ -144,22 +144,6 @@ def libressl?(major = nil, minor = nil, fix = nil) return false unless version !major || (version.map(&:to_i) <=> [major, minor, fix]) >= 0 end - - # OpenSSL 3: x25519 a decode from and then encode to a pem file corrupts the - # key if fips+base provider is used - # This issue happens in OpenSSL between 3.0,0 and 3.0.10 or between 3.1.0 and - # 3.1.2. - # https://github.com/openssl/openssl/issues/21493 - # https://github.com/openssl/openssl/pull/21519 - def pend_on_openssl_issue_21493 - if OpenSSL.fips_mode && - ( - (openssl?(3, 0, 0, 0) && !openssl?(3, 0, 0, 11)) || - (openssl?(3, 1, 0, 0) && !openssl?(3, 1, 0, 3)) - ) - pend('See ') - end - end end class OpenSSL::TestCase < Test::Unit::TestCase diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb index f58f8a27789eba..6c0db0832b93f1 100644 --- a/test/ruby/test_array.rb +++ b/test/ruby/test_array.rb @@ -3336,6 +3336,8 @@ def test_bsearch_in_find_any_mode assert_equal(nil, a.bsearch {|x| 1 * (2**100) }) assert_equal(nil, a.bsearch {|x| (-1) * (2**100) }) + assert_equal(4, a.bsearch {|x| (4 - x).to_r }) + assert_include([4, 7], a.bsearch {|x| (2**100).coerce((1 - x / 4) * (2**100)).first }) end @@ -3371,6 +3373,8 @@ def test_bsearch_index_in_find_any_mode assert_equal(nil, a.bsearch_index {|x| 1 * (2**100) }) assert_equal(nil, a.bsearch_index {|x| (-1) * (2**100) }) + assert_equal(1, a.bsearch_index {|x| (4 - x).to_r }) + assert_include([1, 2], a.bsearch_index {|x| (2**100).coerce((1 - x / 4) * (2**100)).first }) end diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index bfcfbbb72f6d3c..0b4062e99f946c 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -228,12 +228,12 @@ def test_stat_heap_constraints def test_latest_gc_info omit 'stress' if GC.stress - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom' - GC.start - count = GC.stat(:heap_free_slots) + GC.stat(:heap_allocatable_pages) * GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] - count.times{ "a" + "b" } - assert_equal :newobj, GC.latest_gc_info[:gc_by] - eom + assert_separately([], __FILE__, __LINE__, <<-'RUBY') + GC.start + count = GC.stat(:heap_free_slots) + GC.stat(:heap_allocatable_pages) * GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] + count.times{ "a" + "b" } + assert_equal :newobj, GC.latest_gc_info[:gc_by] + RUBY GC.latest_gc_info(h = {}) # allocate hash and rehearsal GC.start @@ -335,7 +335,7 @@ def test_latest_gc_info_weak_references_count end def test_stress_compile_send - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "") + assert_in_out_err([], <<-EOS, [], [], "") GC.stress = true begin eval("A::B.c(1, 1, d: 234)") @@ -345,7 +345,7 @@ def test_stress_compile_send end def test_singleton_method - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "[ruby-dev:42832]") + assert_in_out_err([], <<-EOS, [], [], "[ruby-dev:42832]") GC.stress = true 10.times do obj = Object.new @@ -357,7 +357,7 @@ def obj.bar() raise "obj.foo is called, but this is obj.bar" end end def test_singleton_method_added - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "[ruby-dev:44436]") + assert_in_out_err([], <<-EOS, [], [], "[ruby-dev:44436]") class BasicObject undef singleton_method_added def singleton_method_added(mid) @@ -375,40 +375,22 @@ def test_gc_parameter env = { "RUBY_GC_HEAP_INIT_SLOTS" => "100" } - assert_in_out_err([env, "-W0", "-e", "exit"], "", [], [], "[Bug #19284]") - - env = { - "RUBY_GC_MALLOC_LIMIT" => "60000000", - "RUBY_GC_HEAP_INIT_SLOTS" => "100000" - } - assert_normal_exit("exit", "[ruby-core:39777]", :child_env => env) + assert_in_out_err([env, "-W0", "-e", "exit"], "", [], []) + assert_in_out_err([env, "-W:deprecated", "-e", "exit"], "", [], + /The environment variable RUBY_GC_HEAP_INIT_SLOTS is deprecated; use environment variables RUBY_GC_HEAP_%d_INIT_SLOTS instead/) env = {} - GC.stat_heap.each do |_, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = "200000" + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = "200000" end assert_normal_exit("exit", "", :child_env => env) - env["RUBY_GC_HEAP_INIT_SLOTS"] = "100000" - assert_normal_exit("exit", "", :child_env => env) - env = {} - GC.stat_heap.each do |_, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = "0" + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = "0" end assert_normal_exit("exit", "", :child_env => env) - env = { - "RUBYOPT" => "", - "RUBY_GC_HEAP_INIT_SLOTS" => "100000" - } - assert_in_out_err([env, "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-W0", "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-W1", "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_INIT_SLOTS=100000/, "[ruby-core:39795]") - # Value of GC_HEAP_INIT_SLOTS is 10000 - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /\(default value: 10000\)/) - env = { "RUBY_GC_HEAP_GROWTH_FACTOR" => "2.0", "RUBY_GC_HEAP_GROWTH_MAX_SLOTS" => "10000" @@ -417,17 +399,12 @@ def test_gc_parameter assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_GROWTH_FACTOR=2.0/, "") assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_GROWTH_MAX_SLOTS=10000/, "[ruby-core:57928]") - env = { - "RUBY_GC_HEAP_INIT_SLOTS" => "100000", - "RUBY_GC_HEAP_FREE_SLOTS" => "10000", - "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR" => "0.4", - } - assert_normal_exit("exit", "", :child_env => env) - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0\.4/, "") - if use_rgengc? + env = { + "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR" => "0.4", + } # always full GC when RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR < 1.0 - assert_in_out_err([env, "--disable-gems", "-e", "GC.start; 1000_000.times{Object.new}; p(GC.stat[:minor_gc_count] < GC.stat[:major_gc_count])"], "", ['true'], //, "") + assert_in_out_err([env, "-e", "GC.start; 1000_000.times{Object.new}; p(GC.stat[:minor_gc_count] < GC.stat[:major_gc_count])"], "", ['true'], //, "") end env = { @@ -467,28 +444,34 @@ def test_gc_parameter_init_slots # Constant from gc.c. GC_HEAP_INIT_SLOTS = 10_000 GC.stat_heap.each do |_, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - # Give a 0.9x delta because integer division in minimum_pages_for_size_pool can sometimes cause number to be - # less than GC_HEAP_INIT_SLOTS. - assert_operator(total_slots, :>=, GC_HEAP_INIT_SLOTS * 0.9, s) + assert_operator(total_slots, :>=, GC_HEAP_INIT_SLOTS, s) end RUBY env = {} # Make the heap big enough to ensure the heap never needs to grow. sizes = GC.stat_heap.keys.reverse.map { |i| (i + 1) * 100_000 } - GC.stat_heap.each do |i, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = sizes[i].to_s + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = sizes[heap].to_s end assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY) SIZES = #{sizes} GC.stat_heap.each do |i, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) end RUBY @@ -509,10 +492,61 @@ def test_gc_parameter_init_slots # Check that we still have the same number of slots as initially configured. GC.stat_heap.each do |i, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) + end + RUBY + + # Check that we don't grow the heap in minor GC if we have alloctable pages. + env["RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO"] = "0.3" + env["RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO"] = "0.99" + env["RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO"] = "1.0" + env["RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR"] = "100" # Large value to disable major GC + assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY) + SIZES = #{sizes} + + # Run a major GC to clear out dead objects. + GC.start + + # Disable GC so we can control when GC is ran. + GC.disable + + # Run minor GC enough times so that we don't grow the heap because we + # haven't yet ran RVALUE_OLD_AGE minor GC cycles. + GC::INTERNAL_CONSTANTS[:RVALUE_OLD_AGE].times { GC.start(full_mark: false) } + + # Fill size pool 0 to over 50% full so that the number of allocatable + # pages that will be created will be over the number in heap_allocatable_pages + # (calculated using RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO). + # 70% was chosen here to guarantee that. + ary = [] + while GC.stat_heap(0, :heap_allocatable_pages) > + (GC.stat_heap(0, :heap_allocatable_pages) + GC.stat_heap(0, :heap_eden_pages)) * 0.3 + ary << Object.new + end + + GC.start(full_mark: false) + + # Check that we still have the same number of slots as initially configured. + GC.stat_heap.each do |i, s| + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) end RUBY end @@ -528,19 +562,19 @@ def test_profiler_enabled def test_profiler_clear omit "for now" - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom', timeout: 30 - GC::Profiler.enable - - GC.start - assert_equal(1, GC::Profiler.raw_data.size) - GC::Profiler.clear - assert_equal(0, GC::Profiler.raw_data.size) + assert_separately([], __FILE__, __LINE__, <<-'RUBY', timeout: 30) + GC::Profiler.enable - 200.times{ GC.start } - assert_equal(200, GC::Profiler.raw_data.size) - GC::Profiler.clear - assert_equal(0, GC::Profiler.raw_data.size) - eom + GC.start + assert_equal(1, GC::Profiler.raw_data.size) + GC::Profiler.clear + assert_equal(0, GC::Profiler.raw_data.size) + + 200.times{ GC.start } + assert_equal(200, GC::Profiler.raw_data.size) + GC::Profiler.clear + assert_equal(0, GC::Profiler.raw_data.size) + RUBY end def test_profiler_total_time @@ -554,34 +588,34 @@ def test_profiler_total_time end def test_finalizing_main_thread - assert_in_out_err(%w[--disable-gems], <<-EOS, ["\"finalize\""], [], "[ruby-dev:46647]") + assert_in_out_err([], <<-EOS, ["\"finalize\""], [], "[ruby-dev:46647]") ObjectSpace.define_finalizer(Thread.main) { p 'finalize' } EOS end def test_expand_heap - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom' - GC.start - base_length = GC.stat[:heap_eden_pages] - (base_length * 500).times{ 'a' } - GC.start - base_length = GC.stat[:heap_eden_pages] - (base_length * 500).times{ 'a' } - GC.start - assert_in_epsilon base_length, (v = GC.stat[:heap_eden_pages]), 1/8r, - "invalid heap expanding (base_length: #{base_length}, GC.stat[:heap_eden_pages]: #{v})" + assert_separately([], __FILE__, __LINE__, <<~'RUBY') + GC.start + base_length = GC.stat[:heap_eden_pages] + (base_length * 500).times{ 'a' } + GC.start + base_length = GC.stat[:heap_eden_pages] + (base_length * 500).times{ 'a' } + GC.start + assert_in_epsilon base_length, (v = GC.stat[:heap_eden_pages]), 1/8r, + "invalid heap expanding (base_length: #{base_length}, GC.stat[:heap_eden_pages]: #{v})" - a = [] - (base_length * 500).times{ a << 'a'; nil } - GC.start - assert_operator base_length, :<, GC.stat[:heap_eden_pages] + 1 - eom + a = [] + (base_length * 500).times{ a << 'a'; nil } + GC.start + assert_operator base_length, :<, GC.stat[:heap_eden_pages] + 1 + RUBY end def test_thrashing_for_young_objects # This test prevents bugs like [Bug #18929] - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'RUBY' + assert_separately([], __FILE__, __LINE__, <<-'RUBY') # Grow the heap @ary = 100_000.times.map { Object.new } @@ -671,11 +705,11 @@ def test_interrupt_in_finalizer end def test_finalizer_passed_object_id - assert_in_out_err(%w[--disable-gems], <<-EOS, ["true"], []) + assert_in_out_err([], <<~RUBY, ["true"], []) o = Object.new obj_id = o.object_id ObjectSpace.define_finalizer(o, ->(id){ p id == obj_id }) - EOS + RUBY end def test_verify_internal_consistency diff --git a/test/ruby/test_integer.rb b/test/ruby/test_integer.rb index a3e64ddb388840..3e8d68702c0271 100644 --- a/test/ruby/test_integer.rb +++ b/test/ruby/test_integer.rb @@ -138,20 +138,6 @@ def test_Integer assert_equal(1234, Integer(1234)) assert_equal(1, Integer(1.234)) - # base argument - assert_equal(1234, Integer("1234", 10)) - assert_equal(668, Integer("1234", 8)) - assert_equal(4660, Integer("1234", 16)) - assert_equal(49360, Integer("1234", 36)) - # decimal, not octal - assert_equal(1234, Integer("01234", 10)) - assert_raise(ArgumentError) { Integer("0x123", 10) } - assert_raise(ArgumentError) { Integer(1234, 10) } - assert_raise(ArgumentError) { Integer(12.34, 10) } - assert_raise(ArgumentError) { Integer(Object.new, 1) } - - assert_raise(ArgumentError) { Integer(1, 1, 1) } - assert_equal(2 ** 50, Integer(2.0 ** 50)) assert_raise(TypeError) { Integer(nil) } @@ -252,6 +238,32 @@ def (obj = Object.new).to_str assert_equal(16, Integer(obj)) end + def test_Integer_with_base + assert_equal(1234, Integer("1234", 10)) + assert_equal(668, Integer("1234", 8)) + assert_equal(4660, Integer("1234", 16)) + assert_equal(49360, Integer("1234", 36)) + # decimal, not octal + assert_equal(1234, Integer("01234", 10)) + assert_raise(ArgumentError) { Integer("0x123", 10) } + assert_raise(ArgumentError) { Integer(1234, 10) } + assert_raise(ArgumentError) { Integer(12.34, 10) } + assert_raise(ArgumentError) { Integer(Object.new, 1) } + + assert_raise(ArgumentError) { Integer(1, 1, 1) } + + def (base = Object.new).to_int + 8 + end + assert_equal(8, Integer("10", base)) + + assert_raise(TypeError) { Integer("10", "8") } + def (base = Object.new).to_int + "8" + end + assert_raise(TypeError) { Integer("10", base) } + end + def test_int_p assert_not_predicate(1.0, :integer?) assert_predicate(1, :integer?) diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 38dcb8054fc1ca..095ab27f5d2c66 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -2725,4 +2725,26 @@ def test_warmup_precompute_string_coderange assert_include(ObjectSpace.dump(obj), '"coderange":"7bit"') end; end + + def test_warmup_frees_pages + assert_separately([{"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO" => "1.0"}, "-W0"], "#{<<~"begin;"}\n#{<<~'end;'}") + begin; + TIMES = 10_000 + ary = Array.new(TIMES) + TIMES.times do |i| + ary[i] = Object.new + end + ary.clear + ary = nil + + total_pages_before = GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages) + + Process.warmup + + # Number of pages freed should cause equal increase in number of allocatable pages. + assert_equal(total_pages_before, GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages)) + assert_equal(0, GC.stat(:heap_tomb_pages)) + assert_operator(GC.stat(:total_freed_pages), :>, 0) + end; + end end diff --git a/test/ruby/test_random_formatter.rb b/test/ruby/test_random_formatter.rb index a5072099e1bfb8..2c01d99b3dbe78 100644 --- a/test/ruby/test_random_formatter.rb +++ b/test/ruby/test_random_formatter.rb @@ -83,6 +83,20 @@ def test_alphanumeric end end + def test_alphanumeric_chars + [ + [[*"0".."9"], /\A\d*\z/], + [[*"a".."t"], /\A[a-t]*\z/], + ["一二三四五六七八九十".chars, /\A[一二三四五六七八九十]*\z/], + ].each do |chars, pattern| + 10.times do |n| + an = @it.alphanumeric(n, chars: chars) + assert_match(pattern, an) + assert_equal(n, an.length) + end + end + end + def assert_in_range(range, result, mesg = nil) assert(range.cover?(result), build_message(mesg, "Expected #{result} to be in #{range}")) end diff --git a/test/ruby/test_require.rb b/test/ruby/test_require.rb index 49c771c33cf6d2..cadab4f851ab3a 100644 --- a/test/ruby/test_require.rb +++ b/test/ruby/test_require.rb @@ -193,7 +193,7 @@ def test_require_twice File.write(req, "p :ok\n") assert_file.exist?(req) req[/.rb$/i] = "" - assert_in_out_err(['--disable-gems'], <<-INPUT, %w(:ok), []) + assert_in_out_err([], <<-INPUT, %w(:ok), []) require "#{req}" require "#{req}" INPUT @@ -681,7 +681,7 @@ def test_require_to_path_redefined_in_load_path Dir.mktmpdir {|tmp| Dir.chdir(tmp) { open("foo.rb", "w") {} - assert_in_out_err([{"RUBYOPT"=>nil}, '--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) + assert_in_out_err([{"RUBYOPT"=>nil}], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) begin; $:.replace([IO::NULL]) a = Object.new @@ -709,7 +709,7 @@ def test_require_to_str_redefined_in_load_path Dir.mktmpdir {|tmp| Dir.chdir(tmp) { open("foo.rb", "w") {} - assert_in_out_err([{"RUBYOPT"=>nil}, '--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) + assert_in_out_err([{"RUBYOPT"=>nil}], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) begin; $:.replace([IO::NULL]) a = Object.new @@ -739,7 +739,7 @@ def assert_require_with_shared_array_modified(add, del) open("foo.rb", "w") {} Dir.mkdir("a") open(File.join("a", "bar.rb"), "w") {} - assert_in_out_err(['--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7383) + assert_in_out_err([], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7383) begin; $:.replace([IO::NULL]) $:.#{add} "#{tmp}" @@ -963,7 +963,7 @@ def test_resolve_feature_path_with_missing_feature def test_require_with_public_method_missing # [Bug #19793] - assert_separately(["-W0", "--disable-gems", "-rtempfile"], __FILE__, __LINE__, <<~RUBY) + assert_separately(["-W0", "-rtempfile"], __FILE__, __LINE__, <<~RUBY) GC.stress = true class Object diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 3d98cd6f4e2526..20fa15604db9d1 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -135,12 +135,12 @@ def test_warning end def test_debug - assert_in_out_err(["--disable-gems", "-de", "p $DEBUG"], "", %w(true), []) + assert_in_out_err(["-de", "p $DEBUG"], "", %w(true), []) - assert_in_out_err(["--disable-gems", "--debug", "-e", "p $DEBUG"], + assert_in_out_err(["--debug", "-e", "p $DEBUG"], "", %w(true), []) - assert_in_out_err(["--disable-gems", "--debug-", "-e", "p $DEBUG"], "", %w(), /invalid option --debug-/) + assert_in_out_err(["--debug-", "-e", "p $DEBUG"], "", %w(), /invalid option --debug-/) end q = Regexp.method(:quote) @@ -211,9 +211,9 @@ def test_disable assert_in_out_err(%w(--disable foobarbazqux -e) + [""], "", [], /unknown argument for --disable: `foobarbazqux'/) assert_in_out_err(%w(--disable), "", [], /missing argument for --disable/) - assert_in_out_err(%w(--disable-gems -e) + ['p defined? Gem'], "", ["nil"], []) + assert_in_out_err(%w(-e) + ['p defined? Gem'], "", ["nil"], []) assert_in_out_err(%w(--disable-did_you_mean -e) + ['p defined? DidYouMean'], "", ["nil"], []) - assert_in_out_err(%w(--disable-gems -e) + ['p defined? DidYouMean'], "", ["nil"], []) + assert_in_out_err(%w(-e) + ['p defined? DidYouMean'], "", ["nil"], []) end def test_kanji diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 6cc958332f1679..1d8902baf1c738 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1938,10 +1938,14 @@ def test_start_with? assert_send([S("hello"), :start_with?, S("hel")]) assert_not_send([S("hello"), :start_with?, S("el")]) assert_send([S("hello"), :start_with?, S("el"), S("he")]) + assert_send([S("\xFF\xFE"), :start_with?, S("\xFF")]) + assert_not_send([S("\u{c4}"), :start_with?, S("\xC3")]) bug5536 = '[ruby-core:40623]' assert_raise(TypeError, bug5536) {S("str").start_with? :not_convertible_to_string} + end + def test_start_with_regexp assert_equal(true, S("hello").start_with?(/hel/)) assert_equal("hel", $&) assert_equal(false, S("hello").start_with?(/el/)) @@ -2891,11 +2895,13 @@ def test_lstrip_bang end - def test_delete_prefix + def test_delete_prefix_type_error assert_raise(TypeError) { S('hello').delete_prefix(nil) } assert_raise(TypeError) { S('hello').delete_prefix(1) } assert_raise(TypeError) { S('hello').delete_prefix(/hel/) } + end + def test_delete_prefix s = S("hello") assert_equal("lo", s.delete_prefix('hel')) assert_equal("hello", s) @@ -2915,8 +2921,9 @@ def test_delete_prefix s = S("hello") assert_equal("hello", s.delete_prefix("\u{3053 3093}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_prefix_broken_encoding s = S("\xe3\x81\x82") assert_equal("\xe3\x81\x82", s.delete_prefix("\xe3")) assert_equal("\xe3\x81\x82", s) @@ -2925,23 +2932,29 @@ def test_delete_prefix assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s.delete_prefix("\x95")) assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s) - # clear coderange + assert_equal("\xFE", S("\xFF\xFE").delete_prefix("\xFF")) + end + + def test_delete_prefix_clear_coderange s = S("\u{3053 3093}hello") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_prefix("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_prefix_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("bba", s.delete_prefix(klass.new)) assert_equal("abba", s) end - def test_delete_prefix_bang + def test_delete_prefix_bang_type_error assert_raise(TypeError) { S('hello').delete_prefix!(nil) } assert_raise(TypeError) { S('hello').delete_prefix!(1) } assert_raise(TypeError) { S('hello').delete_prefix!(/hel/) } + end + def test_delete_prefix_bang s = S("hello") assert_equal("lo", s.delete_prefix!('hel')) assert_equal("lo", s) @@ -2961,23 +2974,32 @@ def test_delete_prefix_bang s = S("hello") assert_equal(nil, s.delete_prefix!("\u{3053 3093}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_prefix_bang_broken_encoding s = S("\xe3\x81\x82") assert_equal(nil, s.delete_prefix!("\xe3")) assert_equal("\xe3\x81\x82", s) - # clear coderange + s = S("\xFF\xFE") + assert_equal("\xFE", s.delete_prefix!("\xFF")) + assert_equal("\xFE", s) + end + + def test_delete_prefix_bang_clear_coderange s = S("\u{3053 3093}hello") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_prefix!("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_prefix_bang_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("bba", s.delete_prefix!(klass.new)) assert_equal("bba", s) + end + def test_delete_prefix_bang_frozen_error s = S("ax").freeze assert_raise_with_message(FrozenError, /frozen/) {s.delete_prefix!("a")} @@ -2990,11 +3012,13 @@ def o.to_str assert_raise_with_message(FrozenError, /frozen/) {s.delete_prefix!(o)} end - def test_delete_suffix + def test_delete_suffix_type_error assert_raise(TypeError) { S('hello').delete_suffix(nil) } assert_raise(TypeError) { S('hello').delete_suffix(1) } assert_raise(TypeError) { S('hello').delete_suffix(/hel/) } + end + def test_delete_suffix s = S("hello") assert_equal("hel", s.delete_suffix('lo')) assert_equal("hello", s) @@ -3014,23 +3038,28 @@ def test_delete_suffix s = S("hello") assert_equal("hello", s.delete_suffix("\u{3061 306f}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_suffix_broken_encoding s = S("\xe3\x81\x82") assert_equal("\xe3\x81\x82", s.delete_suffix("\x82")) assert_equal("\xe3\x81\x82", s) + end - # clear coderange + def test_delete_suffix_clear_coderange s = S("hello\u{3053 3093}") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_suffix("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_suffix_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("abb", s.delete_suffix(klass.new)) assert_equal("abba", s) + end + def test_delete_suffix_newline # chomp removes any of "\n", "\r\n", "\r" when "\n" is specified, # but delete_suffix does not s = "foo\n" @@ -3041,11 +3070,13 @@ def test_delete_suffix assert_equal("foo\r", s.delete_suffix("\n")) end - def test_delete_suffix_bang + def test_delete_suffix_bang_type_error assert_raise(TypeError) { S('hello').delete_suffix!(nil) } assert_raise(TypeError) { S('hello').delete_suffix!(1) } assert_raise(TypeError) { S('hello').delete_suffix!(/hel/) } + end + def test_delete_suffix_bang_frozen_error s = S("hello").freeze assert_raise_with_message(FrozenError, /frozen/) {s.delete_suffix!('lo')} @@ -3056,7 +3087,9 @@ def o.to_str "x" end assert_raise_with_message(FrozenError, /frozen/) {s.delete_suffix!(o)} + end + def test_delete_suffix_bang s = S("hello") assert_equal("hel", s.delete_suffix!('lo')) assert_equal("hel", s) @@ -3076,8 +3109,9 @@ def o.to_str s = S("hello") assert_equal(nil, s.delete_suffix!("\u{3061 306f}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_suffix_bang_broken_encoding s = S("\xe3\x81\x82") assert_equal(nil, s.delete_suffix!("\x82")) assert_equal("\xe3\x81\x82", s) @@ -3085,18 +3119,22 @@ def o.to_str s = S("\x95\x5c").force_encoding("Shift_JIS") assert_equal(nil, s.delete_suffix!("\x5c")) assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s) + end - # clear coderange + def test_delete_suffix_bang_clear_coderange s = S("hello\u{3053 3093}") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_suffix!("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_suffix_bang_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("abb", s.delete_suffix!(klass.new)) assert_equal("abb", s) + end + def test_delete_suffix_bang_newline # chomp removes any of "\n", "\r\n", "\r" when "\n" is specified, # but delete_suffix does not s = "foo\n" diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index bcc37e7bbbda46..cda84c6368e383 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1640,6 +1640,29 @@ def test_classdef_in_cond def test_command_with_cmd_brace_block assert_valid_syntax('obj.foo (1) {}') assert_valid_syntax('obj::foo (1) {}') + assert_valid_syntax('bar {}') + assert_valid_syntax('Bar {}') + assert_valid_syntax('bar() {}') + assert_valid_syntax('Bar() {}') + assert_valid_syntax('Foo::bar {}') + assert_valid_syntax('Foo::Bar {}') + assert_valid_syntax('Foo::bar() {}') + assert_valid_syntax('Foo::Bar() {}') + end + + def test_command_newline_in_tlparen_args + assert_valid_syntax("p (1\n2\n),(3),(4)") + assert_valid_syntax("p (\n),(),()") + assert_valid_syntax("a.b (1\n2\n),(3),(4)") + assert_valid_syntax("a.b (\n),(),()") + end + + def test_command_semicolon_in_tlparen_at_the_first_arg + bug19281 = '[ruby-core:111499] [Bug #19281]' + assert_valid_syntax('p (1;2),(3),(4)', bug19281) + assert_valid_syntax('p (;),(),()', bug19281) + assert_valid_syntax('a.b (1;2),(3),(4)', bug19281) + assert_valid_syntax('a.b (;),(),()', bug19281) end def test_numbered_parameter diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index e6f347c4869fe3..703373b11ea75a 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -395,7 +395,7 @@ def test_abort_on_exception end INPUT - assert_in_out_err(%w(--disable-gems -d), <<-INPUT, %w(false 2), %r".+") + assert_in_out_err(%w(-d), <<-INPUT, %w(false 2), %r".+") p Thread.abort_on_exception begin t = Thread.new { raise } diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index 5347028550dfca..cc9507aff490e4 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -547,8 +547,7 @@ def foo &blk end def test_getblockparamproxy - # Currently two side exits as OPTIMIZED_METHOD_TYPE_CALL is unimplemented - assert_compiles(<<~'RUBY', insns: [:getblockparamproxy]) + assert_compiles(<<~'RUBY', insns: [:getblockparamproxy], exits: {}) def foo &blk p blk.call p blk.call @@ -559,6 +558,24 @@ def foo &blk RUBY end + def test_ifunc_getblockparamproxy + assert_compiles(<<~'RUBY', insns: [:getblockparamproxy], exits: {}) + class Foo + include Enumerable + + def each(&block) + block.call 1 + block.call 2 + block.call 3 + end + end + + foo = Foo.new + foo.map { _1 * 2 } + foo.map { _1 * 2 } + RUBY + end + def test_send_blockarg assert_compiles(<<~'RUBY', insns: [:getblockparamproxy, :send], exits: {}) def bar diff --git a/test/socket/test_unix.rb b/test/socket/test_unix.rb index e206339db09596..b2da1e439dd019 100644 --- a/test/socket/test_unix.rb +++ b/test/socket/test_unix.rb @@ -420,9 +420,10 @@ def test_dgram_pair s1.recv_nonblock(10) fail rescue => e - assert(IO::EAGAINWaitReadable === e) - assert(IO::WaitReadable === e) + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) end + s2.send("", 0) s2.send("haha", 0) s2.send("", 0) @@ -446,6 +447,67 @@ def test_dgram_pair s2.close if s2 end + def test_stream_pair + s1, s2 = UNIXSocket.pair(Socket::SOCK_STREAM) + begin + s1.recv_nonblock(10) + fail + rescue => e + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) + end + + s2.send("", 0) + s2.send("haha", 0) + assert_equal("haha", s1.recv(10)) + assert_raise(IO::EWOULDBLOCKWaitReadable) { s1.recv_nonblock(10) } + + buf = "".dup + s2.send("BBBBBB", 0) + IO.select([s1]) + rv = s1.recv(100, 0, buf) + assert_equal buf.object_id, rv.object_id + assert_equal "BBBBBB", rv + + s2.close + assert_nil(s1.recv(10)) + rescue Errno::EPROTOTYPE => error + omit error.message + ensure + s1.close if s1 + s2.close if s2 + end + + def test_seqpacket_pair + s1, s2 = UNIXSocket.pair(Socket::SOCK_SEQPACKET) + begin + s1.recv_nonblock(10) + fail + rescue => e + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) + end + + s2.send("haha", 0) + assert_equal("haha", s1.recv(10)) + assert_raise(IO::EWOULDBLOCKWaitReadable) { s1.recv_nonblock(10) } + + buf = "".dup + s2.send("BBBBBB", 0) + IO.select([s1]) + rv = s1.recv(100, 0, buf) + assert_equal buf.object_id, rv.object_id + assert_equal "BBBBBB", rv + + s2.close + assert_nil(s1.recv(10)) + rescue Errno::EPROTOTYPE, Errno::EPROTONOSUPPORT => error + omit error.message + ensure + s1.close if s1 + s2.close if s2 + end + def test_dgram_pair_sendrecvmsg_errno_set if /mswin|mingw/ =~ RUBY_PLATFORM omit("AF_UNIX + SOCK_DGRAM is not supported on windows") diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 246f107d053899..d2d96c5719a43c 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -14,6 +14,10 @@ def open_file(content) include TestEOF::Seek + def test_version + assert_kind_of(String, StringIO::VERSION) + end + def test_initialize assert_kind_of StringIO, StringIO.new assert_kind_of StringIO, StringIO.new('str') diff --git a/test/test_tempfile.rb b/test/test_tempfile.rb index 6b087f920742b4..87c1241df85f0f 100644 --- a/test/test_tempfile.rb +++ b/test/test_tempfile.rb @@ -209,8 +209,7 @@ def test_tempfile_is_unlinked_when_ruby_exits def test_tempfile_finalizer_does_not_run_if_unlinked bug8768 = '[ruby-core:56521] [Bug #8768]' - args = %w(--disable-gems -rtempfile) - assert_in_out_err(args, <<-'EOS') do |(filename), (error)| + assert_in_out_err(%w(-rtempfile), <<-'EOS') do |(filename), (error)| tmp = Tempfile.new('foo') puts tmp.path tmp.close diff --git a/test/yarp/bom_test.rb b/test/yarp/bom_test.rb index 7dc7eabe92409d..3a4e04a900afd2 100644 --- a/test/yarp/bom_test.rb +++ b/test/yarp/bom_test.rb @@ -4,54 +4,56 @@ # test. return if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby" -require "yarp_test_helper" - -class BOMTest < Test::Unit::TestCase - def test_ident - assert_bom("foo") - end - - def test_back_reference - assert_bom("$+") - end - - def test_instance_variable - assert_bom("@foo") - end - - def test_class_variable - assert_bom("@@foo") - end - - def test_global_variable - assert_bom("$foo") - end - - def test_numbered_reference - assert_bom("$1") - end - - def test_percents - assert_bom("%i[]") - assert_bom("%r[]") - assert_bom("%s[]") - assert_bom("%q{}") - assert_bom("%w[]") - assert_bom("%x[]") - assert_bom("%I[]") - assert_bom("%W[]") - assert_bom("%Q{}") - end - - def test_string - assert_bom("\"\"") - assert_bom("''") - end - - private - - def assert_bom(source) - bommed = "\xEF\xBB\xBF#{source}" - assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value +require_relative "test_helper" + +module YARP + class BOMTest < TestCase + def test_ident + assert_bom("foo") + end + + def test_back_reference + assert_bom("$+") + end + + def test_instance_variable + assert_bom("@foo") + end + + def test_class_variable + assert_bom("@@foo") + end + + def test_global_variable + assert_bom("$foo") + end + + def test_numbered_reference + assert_bom("$1") + end + + def test_percents + assert_bom("%i[]") + assert_bom("%r[]") + assert_bom("%s[]") + assert_bom("%q{}") + assert_bom("%w[]") + assert_bom("%x[]") + assert_bom("%I[]") + assert_bom("%W[]") + assert_bom("%Q{}") + end + + def test_string + assert_bom("\"\"") + assert_bom("''") + end + + private + + def assert_bom(source) + bommed = "\xEF\xBB\xBF#{source}" + assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value + end end end diff --git a/test/yarp/comments_test.rb b/test/yarp/comments_test.rb index fdb70045ca9462..ac91eab4ac5188 100644 --- a/test/yarp/comments_test.rb +++ b/test/yarp/comments_test.rb @@ -1,68 +1,68 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" -class CommentsTest < Test::Unit::TestCase - include ::YARP::DSL +module YARP + class CommentsTest < TestCase + def test_comment_inline + source = "# comment" - def test_comment_inline - source = "# comment" + assert_comment source, :inline, 0..9 + assert_equal [0], Debug.newlines(source) + end - assert_comment source, :inline, 0..9 - assert_equal [0], YARP.const_get(:Debug).newlines(source) - end + def test_comment_inline_def + source = <<~RUBY + def foo + # a comment + end + RUBY - def test_comment_inline_def - source = <<~RUBY - def foo - # a comment + assert_comment source, :inline, 10..22 end - RUBY - - assert_comment source, :inline, 10..22 - end - def test_comment___END__ - source = <<~RUBY - __END__ - comment - RUBY + def test_comment___END__ + source = <<~RUBY + __END__ + comment + RUBY - assert_comment source, :__END__, 0..16 - end + assert_comment source, :__END__, 0..16 + end - def test_comment___END__crlf - source = "__END__\r\ncomment\r\n" + def test_comment___END__crlf + source = "__END__\r\ncomment\r\n" - assert_comment source, :__END__, 0..18 - end + assert_comment source, :__END__, 0..18 + end - def test_comment_embedded_document - source = <<~RUBY - =begin - comment - =end - RUBY + def test_comment_embedded_document + source = <<~RUBY + =begin + comment + =end + RUBY - assert_comment source, :embdoc, 0..20 - end + assert_comment source, :embdoc, 0..20 + end - def test_comment_embedded_document_with_content_on_same_line - source = <<~RUBY - =begin other stuff - =end - RUBY + def test_comment_embedded_document_with_content_on_same_line + source = <<~RUBY + =begin other stuff + =end + RUBY - assert_comment source, :embdoc, 0..24 - end + assert_comment source, :embdoc, 0..24 + end - private + private - def assert_comment(source, type, location) - result = YARP.parse(source) - assert result.errors.empty?, result.errors.map(&:message).join("\n") - assert_equal result.comments.first.type, type - assert_equal result.comments.first.location.start_offset, location.begin - assert_equal result.comments.first.location.end_offset, location.end + def assert_comment(source, type, location) + result = YARP.parse(source) + assert result.errors.empty?, result.errors.map(&:message).join("\n") + assert_equal result.comments.first.type, type + assert_equal result.comments.first.location.start_offset, location.begin + assert_equal result.comments.first.location.end_offset, location.end + end end end diff --git a/test/yarp/compiler_test.rb b/test/yarp/compiler_test.rb new file mode 100644 index 00000000000000..668908d4230606 --- /dev/null +++ b/test/yarp/compiler_test.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +module YARP + class CompilerTest < Test::Unit::TestCase + ############################################################################ + # Literals # + ############################################################################ + + def test_FalseNode + assert_equal false, compile("false") + end + + def test_FloatNode + assert_equal 1.0, compile("1.0") + assert_equal 1.0e0, compile("1.0e0") + assert_equal +1.0e+0, compile("+1.0e+0") + assert_equal -1.0e-0, compile("-1.0e-0") + end + + def test_ImaginaryNode + # assert_equal 1i, compile("1i") + # assert_equal +1.0i, compile("+1.0i") + # assert_equal 1ri, compile("1ri") + end + + def test_IntegerNode + assert_equal 1, compile("1") + assert_equal +1, compile("+1") + assert_equal -1, compile("-1") + # assert_equal 0x10, compile("0x10") + # assert_equal 0b10, compile("0b10") + # assert_equal 0o10, compile("0o10") + # assert_equal 010, compile("010") + end + + def test_NilNode + assert_nil compile("nil") + end + + def test_SelfNode + assert_equal TOPLEVEL_BINDING.eval("self"), compile("self") + end + + def test_TrueNode + assert_equal true, compile("true") + end + + ############################################################################ + # Reads # + ############################################################################ + + def test_ClassVariableReadNode + assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; @@yct; end") + end + + def test_ConstantPathNode + assert_equal YARP::CompilerTest, compile("YARP::CompilerTest") + end + + def test_ConstantReadNode + assert_equal YARP, compile("YARP") + end + + def test_GlobalVariableReadNode + assert_equal 1, compile("$yct = 1; $yct") + end + + def test_InstanceVariableReadNode + assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; @yct; end") + end + + def test_LocalVariableReadNode + assert_equal 1, compile("yct = 1; yct") + end + + ############################################################################ + # Writes # + ############################################################################ + + def test_ClassVariableWriteNode + assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end") + end + + def test_ConstantWriteNode + assert_equal 1, compile("YCT = 1") + end + + def test_ConstantPathWriteNode + assert_equal 1, compile("YARP::YCT = 1") + end + + def test_GlobalVariableWriteNode + assert_equal 1, compile("$yct = 1") + end + + def test_InstanceVariableWriteNode + assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; end") + end + + def test_LocalVariableWriteNode + assert_equal 1, compile("yct = 1") + end + + ############################################################################ + # String-likes # + ############################################################################ + + def test_EmbeddedVariableNode + # assert_equal "1", compile('class YARP::CompilerTest; @yct = 1; "#@yct"; end') + # assert_equal "1", compile('class YARP::CompilerTest; @@yct = 1; "#@@yct"; end') + assert_equal "1", compile('$yct = 1; "#$yct"') + end + + def test_InterpolatedStringNode + assert_equal "1 1 1", compile('$yct = 1; "1 #$yct 1"') + assert_equal "1 3 1", compile('"1 #{1 + 2} 1"') + end + + def test_InterpolatedSymbolNode + assert_equal :"1 1 1", compile('$yct = 1; :"1 #$yct 1"') + assert_equal :"1 3 1", compile(':"1 #{1 + 2} 1"') + end + + def test_StringConcatNode + # assert_equal "YARP::CompilerTest", compile('"YARP" "::" "CompilerTest"') + end + + def test_StringNode + assert_equal "yct", compile('"yct"') + end + + def test_SymbolNode + assert_equal :yct, compile(":yct") + end + + def test_XStringNode + # assert_equal "yctyct", compile(<<~RUBY) + # class YARP::CompilerTest + # def self.`(command) = command * 2 + # `yct` + # end + # RUBY + end + + ############################################################################ + # Jumps # + ############################################################################ + + def test_AndNode + assert_equal 1, compile("true && 1") + assert_equal false, compile("false && 1") + end + + def test_OrNode + assert_equal true, compile("true || 1") + assert_equal 1, compile("false || 1") + end + + ############################################################################ + # Scopes/statements # + ############################################################################ + + def test_ParenthesesNode + assert_equal (), compile("()") + assert_equal (1), compile("(1)") + end + + private + + def compile(source) + RubyVM::InstructionSequence.compile_yarp(source).eval + end + end +end diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb new file mode 100644 index 00000000000000..3966d7bfcb6078 --- /dev/null +++ b/test/yarp/desugar_visitor_test.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module YARP + class DesugarVisitorTest < TestCase + def test_and_write + assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") + assert_not_desugared("Foo::Bar &&= baz", "Desugaring would execute Foo twice or need temporary variables") + assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") + assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") + assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") + end + + def test_or_write + assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") + assert_not_desugared("Foo::Bar ||= baz", "Desugaring would execute Foo twice or need temporary variables") + assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") + assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") + assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") + end + + def test_operator_write + assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") + assert_not_desugared("Foo::Bar += baz", "Desugaring would execute Foo twice or need temporary variables") + assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") + assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") + assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") + end + + private + + def ast_inspect(node) + parts = [node.class.name.split("::").last] + + node.deconstruct_keys(nil).each do |_, value| + case value + when Node + parts << ast_inspect(value) + when Array + parts.concat(value.map { |element| ast_inspect(element) }) + end + end + + "(#{parts.join(" ")})" + end + + # Ensure every node is only present once in the AST. + # If the same node is present twice it would most likely indicate it is executed twice, which is invalid semantically. + # This also acts as a sanity check that Node#child_nodes returns only nodes or nil (which caught a couple bugs). + class EnsureEveryNodeOnceInAST < Visitor + def initialize + @all_nodes = {}.compare_by_identity + end + + def visit(node) + if node + if @all_nodes.include?(node) + raise "#{node.inspect} is present multiple times in the desugared AST and likely executed multiple times" + else + @all_nodes[node] = true + end + end + super(node) + end + end + + def assert_desugars(expected, source) + ast = YARP.parse(source).value.accept(DesugarVisitor.new) + assert_equal expected, ast_inspect(ast.statements.body.last) + + ast.accept(EnsureEveryNodeOnceInAST.new) + end + + def assert_not_desugared(source, reason) + ast = YARP.parse(source).value + assert_equal_nodes(ast, ast.accept(DesugarVisitor.new)) + end + end +end diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index c96a08e60e6c23..828b45be7302c3 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -1,93 +1,100 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" -class EncodingTest < Test::Unit::TestCase - %w[ - ascii - ascii-8bit - big5 - binary - euc-jp - gbk - iso-8859-1 - iso-8859-2 - iso-8859-3 - iso-8859-4 - iso-8859-5 - iso-8859-6 - iso-8859-7 - iso-8859-8 - iso-8859-9 - iso-8859-10 - iso-8859-11 - iso-8859-13 - iso-8859-14 - iso-8859-15 - iso-8859-16 - koi8-r - shift_jis - sjis - us-ascii - utf-8 - utf8-mac - windows-31j - windows-1251 - windows-1252 - CP1251 - CP1252 - ].each do |encoding| - define_method "test_encoding_#{encoding}" do - result = YARP.parse("# encoding: #{encoding}\nident") +module YARP + class EncodingTest < TestCase + %w[ + ascii + ascii-8bit + big5 + binary + euc-jp + gbk + iso-8859-1 + iso-8859-2 + iso-8859-3 + iso-8859-4 + iso-8859-5 + iso-8859-6 + iso-8859-7 + iso-8859-8 + iso-8859-9 + iso-8859-10 + iso-8859-11 + iso-8859-13 + iso-8859-14 + iso-8859-15 + iso-8859-16 + koi8-r + shift_jis + sjis + us-ascii + utf-8 + utf8-mac + windows-31j + windows-1251 + windows-1252 + CP1251 + CP1252 + ].each do |encoding| + define_method "test_encoding_#{encoding}" do + result = YARP.parse("# encoding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find(encoding), actual + end + end + + def test_coding + result = YARP.parse("# coding: utf-8\nident") actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find(encoding), actual + assert_equal Encoding.find("utf-8"), actual end - end - def test_coding - result = YARP.parse("# coding: utf-8\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + def test_coding_with_whitespace + result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("ascii-8bit"), actual + end - def test_coding_with_whitespace - result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("ascii-8bit"), actual - end + def test_emacs_style + result = YARP.parse("# -*- coding: utf-8 -*-\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end - def test_emacs_style - result = YARP.parse("# -*- coding: utf-8 -*-\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + # This test may be a little confusing. Basically when we use our strpbrk, it + # takes into account the encoding of the file. + def test_strpbrk_multibyte + result = YARP.parse(<<~RUBY) + # encoding: Shift_JIS + %w[\x81\x5c] + RUBY - # This test may be a little confusing. Basically when we use our strpbrk, it - # takes into account the encoding of the file. - def test_strpbrk_multibyte - result = YARP.parse(<<~RUBY) - # encoding: Shift_JIS - %w[\x81\x5c] - RUBY + assert(result.errors.empty?) + assert_equal( + (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), + result.value.statements.body.first.elements.first.unescaped + ) + end - assert(result.errors.empty?) - assert_equal( - (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), - result.value.statements.body.first.elements.first.unescaped - ) - end + def test_utf_8_variations + %w[ + utf-8-unix + utf-8-dos + utf-8-mac + utf-8-* + ].each do |encoding| + result = YARP.parse("# coding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end + end - def test_utf_8_variations - %w[ - utf-8-unix - utf-8-dos - utf-8-mac - utf-8-* - ].each do |encoding| - result = YARP.parse("# coding: #{encoding}\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual + def test_first_lexed_token + encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding + assert_equal Encoding.find("ascii-8bit"), encoding end end end diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index d58fd27448a1ad..2af3c605e4b4cf 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -1,1127 +1,1147 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" -class ErrorsTest < Test::Unit::TestCase - include ::YARP::DSL +module YARP + class ErrorsTest < TestCase + include DSL - def test_constant_path_with_invalid_token_after - assert_error_messages "A::$b", [ - "Expected identifier or constant after '::'", - "Expected a newline or semicolon after statement." - ] - end - - def test_module_name_recoverable - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode( - [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] - ), - Location(), - "Parent" - ) - - assert_errors expected, "module Parent module end", [ - ["Expected to find a module name after `module`.", 20..20] - ] - end - - def test_for_loops_index_missing - expected = ForNode( - MissingNode(), - expression("1..10"), - StatementsNode([expression("i")]), - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for in 1..10\ni\nend", [ - ["Expected index after for.", 0..0] - ] - end - - def test_for_loops_only_end - expected = ForNode( - MissingNode(), - MissingNode(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for end", [ - ["Expected index after for.", 0..0], - ["Expected keyword in.", 3..3], - ["Expected collection.", 3..3] - ] - end - - def test_pre_execution_missing_brace - expected = PreExecutionNode( - StatementsNode([expression("1")]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN 1 }", [ - ["Expected '{' after 'BEGIN'.", 5..5] - ] - end + def test_constant_path_with_invalid_token_after + assert_error_messages "A::$b", [ + "Expected identifier or constant after '::'", + "Expected a newline or semicolon after statement." + ] + end - def test_pre_execution_context - expected = PreExecutionNode( - StatementsNode([ - CallNode( - expression("1"), - nil, - Location(), - nil, - ArgumentsNode([MissingNode()]), - nil, - nil, - 0, - "+" - ) - ]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN { 1 + }", [ - ["Expected a value after the operator.", 11..11] - ] - end + def test_module_name_recoverable + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode( + [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] + ), + Location(), + "Parent" + ) - def test_unterminated_embdoc - assert_errors expression("1"), "1\n=begin\n", [ - ["Unterminated embdoc", 2..9] - ] - end + assert_errors expected, "module Parent module end", [ + ["Expected to find a module name after `module`.", 20..20] + ] + end - def test_unterminated_i_list - assert_errors expression("%i["), "%i[", [ - ["Expected a closing delimiter for a `%i` list.", 3..3] - ] - end + def test_for_loops_index_missing + expected = ForNode( + MissingNode(), + expression("1..10"), + StatementsNode([expression("i")]), + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_w_list - assert_errors expression("%w["), "%w[", [ - ["Expected a closing delimiter for a `%w` list.", 3..3] - ] - end + assert_errors expected, "for in 1..10\ni\nend", [ + ["Expected index after for.", 0..0] + ] + end - def test_unterminated_W_list - assert_errors expression("%W["), "%W[", [ - ["Expected a closing delimiter for a `%W` list.", 3..3] - ] - end + def test_for_loops_only_end + expected = ForNode( + MissingNode(), + MissingNode(), + nil, + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_regular_expression - assert_errors expression("/hello"), "/hello", [ - ["Expected a closing delimiter for a regular expression.", 1..1] - ] - end + assert_errors expected, "for end", [ + ["Expected index after for.", 0..0], + ["Expected keyword in.", 3..3], + ["Expected collection.", 3..3] + ] + end - def test_unterminated_regular_expression_with_heredoc - source = "<<-END + /b\nEND\n" + def test_pre_execution_missing_brace + expected = PreExecutionNode( + StatementsNode([expression("1")]), + Location(), + Location(), + Location() + ) - assert_errors expression(source), source, [ - ["Expected a closing delimiter for a regular expression.", 10..10] - ] - end + assert_errors expected, "BEGIN 1 }", [ + ["Expected '{' after 'BEGIN'.", 5..5] + ] + end - def test_unterminated_xstring - assert_errors expression("`hello"), "`hello", [ - ["Expected a closing delimiter for an xstring.", 1..1] - ] - end + def test_pre_execution_context + expected = PreExecutionNode( + StatementsNode([ + CallNode( + expression("1"), + nil, + Location(), + nil, + ArgumentsNode([MissingNode()]), + nil, + nil, + 0, + "+" + ) + ]), + Location(), + Location(), + Location() + ) - def test_unterminated_string - assert_errors expression('"hello'), '"hello', [ - ["Expected a closing delimiter for an interpolated string.", 1..1] - ] - end + assert_errors expected, "BEGIN { 1 + }", [ + ["Expected a value after the operator.", 11..11] + ] + end - def test_unterminated_s_symbol - assert_errors expression("%s[abc"), "%s[abc", [ - ["Expected a closing delimiter for a dynamic symbol.", 3..3] - ] - end + def test_unterminated_embdoc + assert_errors expression("1"), "1\n=begin\n", [ + ["Unterminated embdoc", 2..9] + ] + end - def test_unterminated_parenthesized_expression - assert_errors expression('(1 + 2'), '(1 + 2', [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6] - ] - end + def test_unterminated_i_list + assert_errors expression("%i["), "%i[", [ + ["Expected a closing delimiter for a `%i` list.", 3..3] + ] + end - def test_unterminated_argument_expression - assert_errors expression('a %'), 'a %', [ - ["Unexpected end of input", 2..3], - ["Expected a value after the operator.", 3..3], - ] - end + def test_unterminated_w_list + assert_errors expression("%w["), "%w[", [ + ["Expected a closing delimiter for a `%w` list.", 3..3] + ] + end - def test_cr_without_lf_in_percent_expression - assert_errors expression("%\r"), "%\r", [ - ["Invalid %% token", 0..2], - ] - end + def test_unterminated_W_list + assert_errors expression("%W["), "%W[", [ + ["Expected a closing delimiter for a `%W` list.", 3..3] + ] + end - def test_1_2_3 - assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 2..2], - ["Expected a closing parenthesis.", 2..2], - ["Expected a newline or semicolon after statement.", 2..2], - ["Expected to be able to parse an expression.", 2..2], - ["Expected a newline or semicolon after statement.", 5..5], - ["Expected to be able to parse an expression.", 5..5], - ["Expected a newline or semicolon after statement.", 8..8], - ["Expected to be able to parse an expression.", 8..8], - ] - end + def test_unterminated_regular_expression + assert_errors expression("/hello"), "/hello", [ + ["Expected a closing delimiter for a regular expression.", 1..1] + ] + end - def test_return_1_2_3 - assert_error_messages "return(1, 2, 3)", [ - "Expected to be able to parse an expression.", - "Expected a closing parenthesis.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_unterminated_regular_expression_with_heredoc + source = "<<-END + /b\nEND\n" - def test_return_1 - assert_errors expression("return 1,;"), "return 1,;", [ - ["Expected to be able to parse an argument.", 9..9] - ] - end + assert_errors expression(source), source, [ + ["Expected a closing delimiter for a regular expression.", 10..10] + ] + end - def test_next_1_2_3 - assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6], - ["Expected a newline or semicolon after statement.", 12..12], - ["Expected to be able to parse an expression.", 12..12] - ] - end + def test_unterminated_xstring + assert_errors expression("`hello"), "`hello", [ + ["Expected a closing delimiter for an xstring.", 1..1] + ] + end - def test_next_1 - assert_errors expression("next 1,;"), "next 1,;", [ - ["Expected to be able to parse an argument.", 7..7] - ] - end + def test_unterminated_string + assert_errors expression('"hello'), '"hello', [ + ["Expected a closing delimiter for an interpolated string.", 1..1] + ] + end - def test_break_1_2_3 - assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 7..7], - ["Expected a closing parenthesis.", 7..7], - ["Expected a newline or semicolon after statement.", 13..13], - ["Expected to be able to parse an expression.", 13..13], - ] - end + def test_unterminated_s_symbol + assert_errors expression("%s[abc"), "%s[abc", [ + ["Expected a closing delimiter for a dynamic symbol.", 3..3] + ] + end - def test_break_1 - assert_errors expression("break 1,;"), "break 1,;", [ - ["Expected to be able to parse an argument.", 8..8] - ] - end + def test_unterminated_parenthesized_expression + assert_errors expression('(1 + 2'), '(1 + 2', [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6] + ] + end - def test_argument_forwarding_when_parent_is_not_forwarding - assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 18..21] - ] - end + def test_unterminated_argument_expression + assert_errors expression('a %'), 'a %', [ + ["Unexpected end of input", 2..3], + ["Expected a value after the operator.", 3..3], + ] + end - def test_argument_forwarding_only_effects_its_own_internals - assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), - 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 43..46] + def test_cr_without_lf_in_percent_expression + assert_errors expression("%\r"), "%\r", [ + ["Invalid %% token", 0..2], ] - end + end - def test_top_level_constant_with_downcased_identifier - assert_error_messages "::foo", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_1_2_3 + assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 2..2], + ["Expected a closing parenthesis.", 2..2], + ["Expected a newline or semicolon after statement.", 2..2], + ["Expected to be able to parse an expression.", 2..2], + ["Expected a newline or semicolon after statement.", 5..5], + ["Expected to be able to parse an expression.", 5..5], + ["Expected a newline or semicolon after statement.", 8..8], + ["Expected to be able to parse an expression.", 8..8], + ] + end - def test_top_level_constant_starting_with_downcased_identifier - assert_error_messages "::foo::A", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_return_1_2_3 + assert_error_messages "return(1, 2, 3)", [ + "Expected to be able to parse an expression.", + "Expected a closing parenthesis.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_aliasing_global_variable_with_non_global_variable - assert_errors expression("alias $a b"), "alias $a b", [ - ["Expected a global variable.", 9..10] - ] - end + def test_return_1 + assert_errors expression("return 1,;"), "return 1,;", [ + ["Expected to be able to parse an argument.", 9..9] + ] + end - def test_aliasing_non_global_variable_with_global_variable - assert_errors expression("alias a $b"), "alias a $b", [ - ["Expected a bare word or symbol argument.", 8..10] - ] - end + def test_next_1_2_3 + assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6], + ["Expected a newline or semicolon after statement.", 12..12], + ["Expected to be able to parse an expression.", 12..12] + ] + end - def test_aliasing_global_variable_with_global_number_variable - assert_errors expression("alias $a $1"), "alias $a $1", [ - ["Can't make alias for number variables.", 9..11] - ] - end + def test_next_1 + assert_errors expression("next 1,;"), "next 1,;", [ + ["Expected to be able to parse an argument.", 7..7] + ] + end - def test_def_with_expression_receiver_and_no_identifier - assert_errors expression("def (a); end"), "def (a); end", [ - ["Expected '.' or '::' after receiver", 7..7] - ] - end + def test_break_1_2_3 + assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 7..7], + ["Expected a closing parenthesis.", 7..7], + ["Expected a newline or semicolon after statement.", 13..13], + ["Expected to be able to parse an expression.", 13..13], + ] + end - def test_def_with_multiple_statements_receiver - assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ - ["Expected closing ')' for receiver.", 7..7], - ["Expected '.' or '::' after receiver", 7..7], - ["Expected to be able to parse an expression.", 10..10], - ["Expected to be able to parse an expression.", 11..11] - ] - end + def test_break_1 + assert_errors expression("break 1,;"), "break 1,;", [ + ["Expected to be able to parse an argument.", 8..8] + ] + end - def test_def_with_empty_expression_receiver - assert_errors expression("def ().a; end"), "def ().a; end", [ - ["Expected to be able to parse receiver.", 5..5] - ] - end + def test_argument_forwarding_when_parent_is_not_forwarding + assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 18..21] + ] + end - def test_block_beginning_with_brace_and_ending_with_end - assert_error_messages "x.each { x end", [ - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression.", - "Expected to be able to parse an expression.", - "Expected block beginning with '{' to end with '}'." - ] - end + def test_argument_forwarding_only_effects_its_own_internals + assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), + 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 43..46] + ] + end - def test_double_splat_followed_by_splat_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(**kwargs, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_top_level_constant_with_downcased_identifier + assert_error_messages "::foo", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_after_block - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - BlockArgumentNode(expression("block"), Location()), - expression("foo") - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(&block, foo)", [ - ["Unexpected argument after block argument.", 10..13] - ] - end + def test_top_level_constant_starting_with_downcased_identifier + assert_error_messages "::foo::A", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_binding_power_for_and - assert_error_messages "foo(*bar and baz)", [ - "Expected a ')' to close the argument list.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_aliasing_global_variable_with_non_global_variable + assert_errors expression("alias $a b"), "alias $a b", [ + ["Expected a global variable.", 9..10] + ] + end - def test_splat_argument_after_keyword_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode( - [AssocNode( - SymbolNode(nil, Location(), Location(), "foo"), - expression("bar"), - nil - )] - ), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(foo: bar, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_aliasing_non_global_variable_with_global_variable + assert_errors expression("alias a $b"), "alias a $b", [ + ["Expected a bare word or symbol argument.", 8..10] + ] + end - def test_module_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;module A;end;end", [ - ["Module definition in method body", 8..14] - ] - end + def test_aliasing_global_variable_with_global_number_variable + assert_errors expression("alias $a $1"), "alias $a $1", [ + ["Can't make alias for number variables.", 9..11] + ] + end - def test_module_definition_in_method_body_within_block - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [], - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), - Location(), - Location() - ), - 0, - "bar" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, " - def foo - bar do - module Foo;end - end - end - ", [ - ["Module definition in method body", 40..46] - ] - end + def test_def_with_expression_receiver_and_no_identifier + assert_errors expression("def (a); end"), "def (a); end", [ + ["Expected '.' or '::' after receiver", 7..7] + ] + end - def test_class_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - nil, - Location(), - "A" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;class A;end;end", [ - ["Class definition in method body", 8..13] - ] - end + def test_def_with_multiple_statements_receiver + assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ + ["Expected closing ')' for receiver.", 7..7], + ["Expected '.' or '::' after receiver", 7..7], + ["Expected to be able to parse an expression.", 10..10], + ["Expected to be able to parse an expression.", 11..11] + ] + end - def test_bad_arguments - expected = DefNode( - Location(), - nil, - ParametersNode([ - RequiredParameterNode(:A), - RequiredParameterNode(:@a), - RequiredParameterNode(:$A), - RequiredParameterNode(:@@a), - ], [], [], nil, [], nil, nil), - nil, - [:A, :@a, :$A, :@@a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(A, @a, $A, @@a);end", [ - ["Formal argument cannot be a constant", 8..9], - ["Formal argument cannot be an instance variable", 11..13], - ["Formal argument cannot be a global variable", 15..17], - ["Formal argument cannot be a class variable", 19..22], - ] - end + def test_def_with_empty_expression_receiver + assert_errors expression("def ().a; end"), "def ().a; end", [ + ["Expected to be able to parse receiver.", 5..5] + ] + end - def test_cannot_assign_to_a_reserved_numbered_parameter - expected = BeginNode( - Location(), - StatementsNode([ - LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) - ]), - nil, - nil, - nil, - Location() - ) - source = <<~RUBY - begin - _1=:a;_2=:a;_3=:a;_4=:a;_5=:a - _6=:a;_7=:a;_8=:a;_9=:a;_10=:a - end - RUBY - assert_errors expected, source, [ - ["reserved for numbered parameter", 8..10], - ["reserved for numbered parameter", 14..16], - ["reserved for numbered parameter", 20..22], - ["reserved for numbered parameter", 26..28], - ["reserved for numbered parameter", 32..34], - ["reserved for numbered parameter", 40..42], - ["reserved for numbered parameter", 46..48], - ["reserved for numbered parameter", 52..54], - ["reserved for numbered parameter", 58..60], - ] - end + def test_block_beginning_with_brace_and_ending_with_end + assert_error_messages "x.each { x end", [ + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression.", + "Expected to be able to parse an expression.", + "Expected block beginning with '{' to end with '}'." + ] + end - def test_do_not_allow_trailing_commas_in_method_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], - [], - [], + def test_double_splat_followed_by_splat_argument + expected = CallNode( nil, - [], nil, - nil - ), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,c,);end", [ - ["Unexpected ','.", 13..14] - ] - end - - def test_do_not_allow_trailing_commas_in_lambda_parameters - expected = LambdaNode( - [:a, :b], - Location(), - BlockParametersNode( - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), - [], Location(), - Location() - ), - nil - ) - assert_errors expected, "-> (a, b, ) {}", [ - ["Unexpected ','.", 8..9] - ] - end - - def test_do_not_allow_multiple_codepoints_in_a_single_character_literal - expected = StringNode(Location(), Location(), nil, "\u0001\u0002") - - assert_errors expected, '?\u{0001 0002}', [ - ["Multiple codepoints at single character literal", 9..12] - ] - end - - def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0001") - - assert_errors expected, '"\u{0000001}"', [ - ["invalid Unicode escape.", 4..11], - ["invalid Unicode escape.", 4..11] - ] - end - - def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0000z}") + Location(), + ArgumentsNode([ + KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), + SplatNode(Location(), expression("args")) + ]), + Location(), + nil, + 0, + "a" + ) - assert_errors expected, '"\u{000z}"', [ - ["unterminated Unicode escape", 7..7], - ["unterminated Unicode escape", 7..7] - ] - end + assert_errors expected, "a(**kwargs, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_method_parameters_after_block - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_arguments_after_block + expected = CallNode( nil, - [], nil, - BlockParameterNode(Location(), Location()) - ), - nil, - [:block, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(&block, a)\nend", [ - ["Unexpected parameter order", 16..17] - ] - end + Location(), + Location(), + ArgumentsNode([ + BlockArgumentNode(expression("block"), Location()), + expression("foo") + ]), + Location(), + nil, + 0, + "a" + ) - def test_method_with_arguments_after_anonymous_block - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), - nil, - [:&, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(&, a)\nend", [ - ["Unexpected parameter order", 11..12] - ] - end + assert_errors expected, "a(&block, foo)", [ + ["Unexpected argument after block argument.", 10..13] + ] + end - def test_method_parameters_after_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], - nil, - [], - ForwardingParameterNode(), - nil - ), - nil, - [:"...", :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(..., a)\nend", [ - ["Unexpected parameter order", 13..14] - ] - end + def test_arguments_binding_power_for_and + assert_error_messages "foo(*bar and baz)", [ + "Expected a ')' to close the argument list.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_splat_argument_after_keyword_argument + expected = CallNode( nil, - [KeywordParameterNode(Location(), nil)], nil, - nil - ), - nil, - [:b, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(b:, a)\nend", [ - ["Unexpected parameter order", 12..13] - ] - end - - def test_rest_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [], + Location(), + Location(), + ArgumentsNode([ + KeywordHashNode( + [AssocNode( + SymbolNode(nil, Location(), Location(), "foo"), + expression("bar"), + nil + )] + ), + SplatNode(Location(), expression("args")) + ]), + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:rest, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(**rest, b:)\nend", [ - ["Unexpected parameter order", 16..18] - ] - end + 0, + "a" + ) - def test_double_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), - nil, - [:"..."], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(..., ...)\nend", [ - ["Unexpected parameter order", 13..16] - ] - end + assert_errors expected, "a(foo: bar, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_multiple_error_in_parameters_order - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_module_definition_in_method_body + expected = DefNode( + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_optional_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_named_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), [], - [RequiredParameterNode(:a)], + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_returning_to_optional_parameters_multiple_times - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a)], - [ - OptionalParameterNode(:b, Location(), Location(), IntegerNode()), - OptionalParameterNode(:d, Location(), Location(), IntegerNode()) - ], - [RequiredParameterNode(:c), RequiredParameterNode(:e)], nil, - [], nil, - nil - ), - nil, - [:a, :b, :c, :d, :e], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ - ["Unexpected parameter order", 23..24] - ] - end - - def test_case_without_when_clauses_errors_on_else_clause - expected = CaseNode( - SymbolNode(Location(), Location(), nil, "a"), - [], - ElseNode(Location(), nil, Location()), - Location(), - Location() - ) - - assert_errors expected, "case :a\nelse\nend", [ - ["Unexpected else without no when clauses in case statement.", 8..12] - ] - end - - def test_setter_method_cannot_be_defined_in_an_endless_method_definition - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([IntegerNode()]), - [], - Location(), - nil, - Location(), - Location(), - Location(), - nil - ) - - assert_errors expected, "def a=() = 42", [ - ["Setter method cannot be defined in an endless method definition", 4..6] - ] - end - - def test_do_not_allow_forward_arguments_in_lambda_literals - expected = LambdaNode( - [:"..."], - Location(), - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), - nil - ) - - assert_errors expected, "->(...) {}", [ - ["Unexpected ...", 3..6] - ] - end - - def test_do_not_allow_forward_arguments_in_blocks - expected = CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [:"..."], - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), nil, - Location(), Location() - ), - 0, - "a" - ) - - assert_errors expected, "a {|...|}", [ - ["Unexpected ...", 4..7] - ] - end - - def test_dont_allow_return_inside_class_body - expected = ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "class A; return; end", [ - ["Invalid return in class/module body", 15..16] - ] - end - - def test_dont_allow_return_inside_module_body - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "module A; return; end", [ - ["Invalid return in class/module body", 16..17] - ] - end + ) - def test_dont_allow_setting_to_back_and_nth_reference - expected = BeginNode( - Location(), - StatementsNode([ - GlobalVariableWriteNode(Location(), Location(), NilNode()), - GlobalVariableWriteNode(Location(), Location(), NilNode()) - ]), - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ - ["Can't set variable", 6..8], - ["Can't set variable", 15..20] - ] - end + assert_errors expected, "def foo;module A;end;end", [ + ["Module definition in method body", 8..14] + ] + end - def test_duplicated_parameter_names - # For some reason, Ripper reports no error for Ruby 3.0 when you have - # duplicated parameter names for positional parameters. - unless RUBY_VERSION < "3.1.0" + def test_module_definition_in_method_body_within_block expected = DefNode( Location(), nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), nil, - [:a, :b], + StatementsNode( + [CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [], + nil, + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), + Location(), + Location() + ), + 0, + "bar" + )] + ), + [], Location(), nil, + nil, + nil, + nil, + Location() + ) + + assert_errors expected, <<~RUBY, [["Module definition in method body", 21..27]] + def foo + bar do + module Foo;end + end + end + RUBY + end + + def test_class_definition_in_method_body + expected = DefNode( Location(), + nil, + nil, + StatementsNode( + [ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + nil, + Location(), + "A" + )] + ), + [], Location(), nil, + nil, + nil, + nil, Location() ) - assert_errors expected, "def foo(a,b,a);end", [ - ["Duplicated parameter name.", 12..13] - ] - end - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,*a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,**a);end", [ - ["Duplicated parameter name.", 14..15] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,&a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] - end + assert_errors expected, "def foo;class A;end;end", [ + ["Class definition in method body", 8..13] + ] + end - private + def test_bad_arguments + expected = DefNode( + Location(), + nil, + ParametersNode([ + RequiredParameterNode(:A), + RequiredParameterNode(:@a), + RequiredParameterNode(:$A), + RequiredParameterNode(:@@a), + ], [], [], nil, [], nil, nil), + nil, + [:A, :@a, :$A, :@@a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - def assert_errors(expected, source, errors) - # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby - assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + assert_errors expected, "def foo(A, @a, $A, @@a);end", [ + ["Formal argument cannot be a constant", 8..9], + ["Formal argument cannot be an instance variable", 11..13], + ["Formal argument cannot be a global variable", 15..17], + ["Formal argument cannot be a class variable", 19..22], + ] + end - result = YARP.parse(source) - node = result.value.statements.body.last + def test_cannot_assign_to_a_reserved_numbered_parameter + expected = BeginNode( + Location(), + StatementsNode([ + LocalVariableWriteNode(:_1, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_2, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_3, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_4, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_5, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_6, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_7, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_8, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_9, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_10, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()) + ]), + nil, + nil, + nil, + Location() + ) + source = <<~RUBY + begin + _1=:a;_2=:a;_3=:a;_4=:a;_5=:a + _6=:a;_7=:a;_8=:a;_9=:a;_10=:a + end + RUBY + assert_errors expected, source, [ + ["reserved for numbered parameter", 8..10], + ["reserved for numbered parameter", 14..16], + ["reserved for numbered parameter", 20..22], + ["reserved for numbered parameter", 26..28], + ["reserved for numbered parameter", 32..34], + ["reserved for numbered parameter", 40..42], + ["reserved for numbered parameter", 46..48], + ["reserved for numbered parameter", 52..54], + ["reserved for numbered parameter", 58..60], + ] + end - assert_equal_nodes(expected, node, compare_location: false) - assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) - end + def test_do_not_allow_trailing_commas_in_method_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], + [], + [], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - def assert_error_messages(source, errors) - assert_nil Ripper.sexp_raw(source) - result = YARP.parse(source) - assert_equal(errors, result.errors.map(&:message)) - end + assert_errors expected, "def foo(a,b,c,);end", [ + ["Unexpected ','.", 13..14] + ] + end + + def test_do_not_allow_trailing_commas_in_lambda_parameters + expected = LambdaNode( + [:a, :b], + Location(), + Location(), + Location(), + BlockParametersNode( + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), + [], + Location(), + Location() + ), + nil + ) + assert_errors expected, "-> (a, b, ) {}", [ + ["Unexpected ','.", 8..9] + ] + end + + def test_do_not_allow_multiple_codepoints_in_a_single_character_literal + expected = StringNode(Location(), Location(), nil, "\u0001\u0002") + + assert_errors expected, '?\u{0001 0002}', [ + ["Multiple codepoints at single character literal", 9..12] + ] + end + + def test_invalid_hex_escape + assert_errors expression('"\\xx"'), '"\\xx"', [ + ["Invalid hex escape.", 1..3], + ] + end + + def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0001") + + assert_errors expected, '"\u{0000001}"', [ + ["invalid Unicode escape.", 4..11], + ] + end + + def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0000z}") + + assert_errors expected, '"\u{000z}"', [ + ["unterminated Unicode escape", 7..7], + ] + end + + def test_unterminated_unicode_brackets_should_be_a_syntax_error + assert_errors expression('?\\u{3'), '?\\u{3', [ + ["invalid Unicode escape.", 1..5], + ] + end + + def test_method_parameters_after_block + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + nil, + BlockParameterNode(Location(), Location()) + ), + nil, + [:block, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(&block, a)\nend", [ + ["Unexpected parameter order", 16..17] + ] + end + + def test_method_with_arguments_after_anonymous_block + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), + nil, + [:&, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(&, a)\nend", [ + ["Unexpected parameter order", 11..12] + ] + end + + def test_method_parameters_after_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + ForwardingParameterNode(), + nil + ), + nil, + [:"...", :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(..., a)\nend", [ + ["Unexpected parameter order", 13..14] + ] + end + + def test_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + nil, + nil + ), + nil, + [:b, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(b:, a)\nend", [ + ["Unexpected parameter order", 12..13] + ] + end + + def test_rest_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:rest, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(**rest, b:)\nend", [ + ["Unexpected parameter order", 16..18] + ] + end + + def test_double_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), + nil, + [:"..."], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(..., ...)\nend", [ + ["Unexpected parameter order", 13..16] + ] + end + + def test_multiple_error_in_parameters_order + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - def expression(source) - YARP.parse(source).value.statements.body.last + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_optional_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_named_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_returning_to_optional_parameters_multiple_times + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a)], + [ + OptionalParameterNode(:b, Location(), Location(), IntegerNode()), + OptionalParameterNode(:d, Location(), Location(), IntegerNode()) + ], + [RequiredParameterNode(:c), RequiredParameterNode(:e)], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c, :d, :e], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ + ["Unexpected parameter order", 23..24] + ] + end + + def test_case_without_when_clauses_errors_on_else_clause + expected = CaseNode( + SymbolNode(Location(), Location(), nil, "a"), + [], + ElseNode(Location(), nil, Location()), + Location(), + Location() + ) + + assert_errors expected, "case :a\nelse\nend", [ + ["Unexpected else without no when clauses in case statement.", 8..12] + ] + end + + def test_setter_method_cannot_be_defined_in_an_endless_method_definition + expected = DefNode( + Location(), + nil, + nil, + StatementsNode([IntegerNode()]), + [], + Location(), + nil, + Location(), + Location(), + Location(), + nil + ) + + assert_errors expected, "def a=() = 42", [ + ["Setter method cannot be defined in an endless method definition", 4..6] + ] + end + + def test_do_not_allow_forward_arguments_in_lambda_literals + expected = LambdaNode( + [:"..."], + Location(), + Location(), + Location(), + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil + ) + + assert_errors expected, "->(...) {}", [ + ["Unexpected ...", 3..6] + ] + end + + def test_do_not_allow_forward_arguments_in_blocks + expected = CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [:"..."], + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil, + Location(), + Location() + ), + 0, + "a" + ) + + assert_errors expected, "a {|...|}", [ + ["Unexpected ...", 4..7] + ] + end + + def test_dont_allow_return_inside_class_body + expected = ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "class A; return; end", [ + ["Invalid return in class/module body", 15..16] + ] + end + + def test_dont_allow_return_inside_module_body + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "module A; return; end", [ + ["Invalid return in class/module body", 16..17] + ] + end + + def test_dont_allow_setting_to_back_and_nth_reference + expected = BeginNode( + Location(), + StatementsNode([ + GlobalVariableWriteNode(Location(), NilNode(), Location()), + GlobalVariableWriteNode(Location(), NilNode(), Location()) + ]), + nil, + nil, + nil, + Location() + ) + + assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ + ["Can't set variable", 6..8], + ["Can't set variable", 15..20] + ] + end + + def test_duplicated_parameter_names + # For some reason, Ripper reports no error for Ruby 3.0 when you have + # duplicated parameter names for positional parameters. + unless RUBY_VERSION < "3.1.0" + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,a);end", [ + ["Duplicated parameter name.", 12..13] + ] + end + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,*a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,**a);end", [ + ["Duplicated parameter name.", 14..15] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,&a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] + end + + def test_unterminated_global_variable + assert_errors expression("$"), "$", [ + ["Invalid global variable.", 0..1] + ] + end + + private + + def assert_errors(expected, source, errors) + # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby + assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + + result = YARP.parse(source) + node = result.value.statements.body.last + + assert_equal_nodes(expected, node, compare_location: false) + assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) + end + + def assert_error_messages(source, errors) + assert_nil Ripper.sexp_raw(source) + result = YARP.parse(source) + assert_equal(errors, result.errors.map(&:message)) + end + + def expression(source) + YARP.parse(source).value.statements.body.last + end end end diff --git a/test/yarp/fixtures/begin_ensure.txt b/test/yarp/fixtures/begin_ensure.txt index 1c6f9dac359290..6cfb6274539100 100644 --- a/test/yarp/fixtures/begin_ensure.txt +++ b/test/yarp/fixtures/begin_ensure.txt @@ -12,3 +12,10 @@ begin a begin a; ensure b; end +begin begin:s.l begin ensure Module.new do + begin + break + ensure Module.new do + end + end +end end end end diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt new file mode 100644 index 00000000000000..a52a4c3c27b37a --- /dev/null +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -0,0 +1,51 @@ +# test regex, string, and lists that span a heredoc thanks to an escaped newline + +# ripper incorrectly creates a "b\nb" token instead of two separate string tokens +pp <<-A.gsub(/b\ +a +A +b/, "") + +# ripper incorrectly creates a "d\nd" token instead of two separate string tokens +pp <<-A, "d\ +c +A +d" + +# ripper gets this right +pp <<-A, %q[f\ +e +A +f] + +# ripper incorrectly creates a "h\nh" token instead of two separate string tokens +pp <<-A, %Q[h\ +g +A +h] + +# ripper can't parse this successfully, though ruby runs it correctly +pp <<-A, %w[j\ +i +A +j] + +# ripper can't parse this successfully, though ruby runs it correctly +# TODO: yarp does not include the "\n" in "l\nl" in the AST like ruby does +pp <<-A, %W[l\ +k +A +l] + +# ripper can't parse this successfully, though ruby runs it correctly +pp <<-A, %i[n\ +m +A +n] + +# ripper gets this one wrong in the same way that YARP does ... +# TODO: yarp does not include the "\n" in "p\np" in the AST like ruby does +pp <<-A, %I[p\ +o +A +p] diff --git a/test/yarp/fixtures/wrapping_heredoc.txt b/test/yarp/fixtures/wrapping_heredoc.txt deleted file mode 100644 index d5fc7101780c21..00000000000000 --- a/test/yarp/fixtures/wrapping_heredoc.txt +++ /dev/null @@ -1,13 +0,0 @@ -# test regex, string, and lists that wrap a heredoc thanks to an escaped newline - -# ripper incorrectly creates a "b\nc" string instead of two separate string tokens -pp <<-A.gsub(/b\ -a -A -c/, "") - -# ripper incorrectly creates a "e\nf" string instead of two separate string tokens -pp <<-A + "e\ -d -A -f" diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb new file mode 100644 index 00000000000000..f4abcd4ac8000f --- /dev/null +++ b/test/yarp/fuzzer_test.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module YARP + # These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. + class FuzzerTest < TestCase + def self.snippet(name, source) + define_method(:"test_fuzzer_#{name}") { YARP.dump(source) } + end + + snippet "incomplete global variable", "$" + snippet "incomplete symbol", ":" + snippet "incomplete escaped string", '"\\' + snippet "trailing comment", "1\n#\n" + snippet "comment followed by whitespace at end of file", "1\n#\n " + snippet "trailing asterisk", "a *" + snippet "incomplete decimal number", "0d" + snippet "incomplete binary number", "0b" + snippet "incomplete octal number", "0o" + snippet "incomplete hex number", "0x" + snippet "incomplete escaped list", "%w[\\" + snippet "incomplete escaped regex", "/a\\" + snippet "unterminated heredoc with unterminated escape at end of file", "< do foo end") end + def test_LocalVariableAndWriteNode + assert_location(LocalVariableAndWriteNode, "foo &&= bar") + assert_location(LocalVariableAndWriteNode, "foo = 1; foo &&= bar", 9...20) + end + + def test_LocalVariableOperatorWriteNode + assert_location(LocalVariableOperatorWriteNode, "foo += bar") + assert_location(LocalVariableOperatorWriteNode, "foo = 1; foo += bar", 9...19) + end + + def test_LocalVariableOrWriteNode + assert_location(LocalVariableOrWriteNode, "foo ||= bar") + assert_location(LocalVariableOrWriteNode, "foo = 1; foo ||= bar", 9...20) + end + def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) end + def test_LocalVariableTargetNode + assert_location(LocalVariableTargetNode, "foo, bar = baz", 0...3) do |node| + node.targets.first + end + end + def test_LocalVariableWriteNode assert_location(LocalVariableWriteNode, "foo = bar") end @@ -445,16 +554,6 @@ def test_NumberedReferenceReadNode assert_location(NumberedReferenceReadNode, "$1") end - def test_OperatorWriteNode - assert_location(OperatorWriteNode, "@@foo += bar") - assert_location(OperatorWriteNode, "Parent::Child += bar") - assert_location(OperatorWriteNode, "Foo += bar") - assert_location(OperatorWriteNode, "$foo += bar") - assert_location(OperatorWriteNode, "@foo += bar") - assert_location(OperatorWriteNode, "foo += bar") - assert_location(OperatorWriteNode, "foo = 1; foo += bar", 9...19) - end - def test_OptionalParameterNode assert_location(OptionalParameterNode, "def foo(bar = nil); end", 8...17) do |node| node.parameters.optionals.first @@ -466,16 +565,6 @@ def test_OrNode assert_location(OrNode, "foo or bar") end - def test_OrWriteNode - assert_location(OrWriteNode, "@@foo ||= bar") - assert_location(OrWriteNode, "Parent::Child ||= bar") - assert_location(OrWriteNode, "Foo ||= bar") - assert_location(OrWriteNode, "$foo ||= bar") - assert_location(OrWriteNode, "@foo ||= bar") - assert_location(OrWriteNode, "foo ||= bar") - assert_location(OrWriteNode, "foo = 1; foo ||= bar", 9...20) - end - def test_ParametersNode assert_location(ParametersNode, "def foo(bar, baz); end", 8...16, &:parameters) end @@ -706,6 +795,12 @@ def test_YieldNode assert_location(YieldNode, "yield(foo)") end + def test_all_tested + expected = YARP.constants.grep(/.Node$/).sort - %i[MissingNode ProgramNode] + actual = LocationTest.instance_methods(false).grep(/.Node$/).map { |name| name[5..].to_sym }.sort + assert_equal expected, actual + end + private def assert_location(kind, source, expected = 0...source.length) diff --git a/test/yarp/memsize_test.rb b/test/yarp/memsize_test.rb index 30de1085cf275e..07c85ce329504a 100644 --- a/test/yarp/memsize_test.rb +++ b/test/yarp/memsize_test.rb @@ -1,15 +1,17 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI -class MemsizeTest < Test::Unit::TestCase - def test_memsize - result = YARP.const_get(:Debug).memsize("2 + 3") +module YARP + class MemsizeTest < TestCase + def test_memsize + result = Debug.memsize("2 + 3") - assert_equal 5, result[:length] - assert_kind_of Integer, result[:memsize] - assert_equal 6, result[:node_count] + assert_equal 5, result[:length] + assert_kind_of Integer, result[:memsize] + assert_equal 6, result[:node_count] + end end end diff --git a/test/yarp/newline_test.rb b/test/yarp/newline_test.rb index 80f6329d94d6c2..5a85f856f39f5f 100644 --- a/test/yarp/newline_test.rb +++ b/test/yarp/newline_test.rb @@ -1,100 +1,96 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return unless defined?(RubyVM::InstructionSequence) -# It is useful to have a diff even if the strings to compare are big -# However, ruby/ruby does not have a version of Test::Unit with access to -# max_diff_target_string_size -if defined?(Test::Unit::Assertions::AssertionMessage) - Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 -end +module YARP + class NewlineTest < TestCase + class NewlineVisitor < Visitor + attr_reader :source, :newlines -class NewlineTest < Test::Unit::TestCase - class NewlineVisitor < YARP::Visitor - attr_reader :source, :newlines + def initialize(source) + @source = source + @newlines = [] + end - def initialize(source) - @source = source - @newlines = [] + def visit(node) + newlines << source.line(node.location.start_offset) if node&.newline? + super(node) + end end - def visit(node) - newlines << source.line(node.location.start_offset) if node&.newline? - super(node) - end - end - - base = File.dirname(__dir__) - Dir["{lib,test}/**/*.rb", base: base].each do |relative| - define_method("test_newline_flags_#{relative}") do - assert_newlines(base, relative) + base = File.dirname(__dir__) + Dir["{lib,test}/**/*.rb", base: base].each do |relative| + define_method("test_newline_flags_#{relative}") do + assert_newlines(base, relative) + end end - end - private + private - def assert_newlines(base, relative) - filepath = File.join(base, relative) - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - expected = rubyvm_lines(source) + def assert_newlines(base, relative) + filepath = File.join(base, relative) + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + expected = rubyvm_lines(source) - result = YARP.parse_file(filepath) - assert_empty result.errors + result = YARP.parse_file(filepath) + assert_empty result.errors - result.mark_newlines - visitor = NewlineVisitor.new(result.source) + result.mark_newlines + visitor = NewlineVisitor.new(result.source) - result.value.accept(visitor) - actual = visitor.newlines + result.value.accept(visitor) + actual = visitor.newlines - source.each_line.with_index(1) do |line, line_number| - # Lines like `while (foo = bar)` result in two line flags in the bytecode - # but only one newline flag in the AST. We need to remove the extra line - # flag from the bytecode to make the test pass. - if line.match?(/while \(/) - index = expected.index(line_number) - expected.delete_at(index) if index - end + source.each_line.with_index(1) do |line, line_number| + # Lines like `while (foo = bar)` result in two line flags in the + # bytecode but only one newline flag in the AST. We need to remove the + # extra line flag from the bytecode to make the test pass. + if line.match?(/while \(/) + index = expected.index(line_number) + expected.delete_at(index) if index + end - # Lines like `foo =` where the value is on the next line result in another - # line flag in the bytecode but only one newline flag in the AST. - if line.match?(/^\s+\w+ =$/) - if source.lines[line_number].match?(/^\s+case/) - actual[actual.index(line_number)] += 1 - else - actual.delete_at(actual.index(line_number)) + # Lines like `foo =` where the value is on the next line result in + # another line flag in the bytecode but only one newline flag in the + # AST. + if line.match?(/^\s+\w+ =$/) + if source.lines[line_number].match?(/^\s+case/) + actual[actual.index(line_number)] += 1 + else + actual.delete_at(actual.index(line_number)) + end end - end - if line.match?(/^\s+\w+ = \[$/) - if !expected.include?(line_number) && !expected.include?(line_number + 2) - actual[actual.index(line_number)] += 1 + if line.match?(/^\s+\w+ = \[$/) + if !expected.include?(line_number) && !expected.include?(line_number + 2) + actual[actual.index(line_number)] += 1 + end end end + + assert_equal expected, actual end - assert_equal expected, actual - end + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity - end + def rubyvm_lines(source) + queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] + lines = [] - def rubyvm_lines(source) - queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] - lines = [] + while iseq = queue.shift + lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) + iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + end - while iseq = queue.shift - lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) - iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + lines.sort end - - lines.sort end end diff --git a/test/yarp/parse_serialize_test.rb b/test/yarp/parse_serialize_test.rb index 6789ba9c438499..82a1c29d48a10f 100644 --- a/test/yarp/parse_serialize_test.rb +++ b/test/yarp/parse_serialize_test.rb @@ -1,26 +1,28 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI -class ParseSerializeTest < Test::Unit::TestCase - def test_parse_serialize - dumped = YARP.const_get(:Debug).parse_serialize_file(__FILE__) - result = YARP.load(File.read(__FILE__), dumped) +module YARP + class ParseSerializeTest < TestCase + def test_parse_serialize + dumped = Debug.parse_serialize_file(__FILE__) + result = YARP.load(File.read(__FILE__), dumped) - assert_kind_of YARP::ParseResult, result, "Expected the return value to be a ParseResult" - assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" - end + assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult" + assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" + end - private + private - def find_file_node(result) - queue = [result.value] + def find_file_node(result) + queue = [result.value] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end end end diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index f8c1fe12d13443..5299cfd7b11ed4 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -1,202 +1,222 @@ # frozen_string_literal: true -require "yarp_test_helper" - -class ParseTest < Test::Unit::TestCase - # When we pretty-print the trees to compare against the snapshots, we want to - # be certain that we print with the same external encoding. This is because - # methods like Symbol#inspect take into account external encoding and it could - # change how the snapshot is generated. On machines with certain settings - # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're - # going to force it to be UTF-8 to keep the snapshots consistent. - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end +require_relative "test_helper" + +module YARP + class ParseTest < TestCase + # When we pretty-print the trees to compare against the snapshots, we want to + # be certain that we print with the same external encoding. This is because + # methods like Symbol#inspect take into account external encoding and it could + # change how the snapshot is generated. On machines with certain settings + # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're + # going to force it to be UTF-8 to keep the snapshots consistent. + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end - def test_empty_string - result = YARP.parse("") - assert_equal [], result.value.statements.body - end + def test_empty_string + result = YARP.parse("") + assert_equal [], result.value.statements.body + end - def test_parse_takes_file_path - filepath = "filepath.rb" - result = YARP.parse("def foo; __FILE__; end", filepath) + def test_parse_takes_file_path + filepath = "filepath.rb" + result = YARP.parse("def foo; __FILE__; end", filepath) - assert_equal filepath, find_source_file_node(result.value).filepath - end + assert_equal filepath, find_source_file_node(result.value).filepath + end - # To accurately compare against Ripper, we need to make sure that we're - # running on Ruby 3.2+. - check_ripper = RUBY_VERSION >= "3.2.0" + def test_parse_lex + node, tokens = YARP.parse_lex("def foo; end").value - # The FOCUS environment variable allows you to specify one particular fixture - # to test, instead of all of them. - base = File.join(__dir__, "fixtures") - relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] + assert_kind_of ProgramNode, node + assert_equal 5, tokens.length + end - relatives.each do |relative| - # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" - next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) + def test_parse_lex_file + node, tokens = YARP.parse_lex_file(__FILE__).value - filepath = File.join(base, relative) - snapshot = File.expand_path(File.join("snapshots", relative), __dir__) + assert_kind_of ProgramNode, node + refute_empty tokens + end - directory = File.dirname(snapshot) - FileUtils.mkdir_p(directory) unless File.directory?(directory) + # To accurately compare against Ripper, we need to make sure that we're + # running on Ruby 3.2+. + ripper_enabled = RUBY_VERSION >= "3.2.0" - define_method "test_filepath_#{relative}" do - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # The FOCUS environment variable allows you to specify one particular fixture + # to test, instead of all of them. + base = File.join(__dir__, "fixtures") + relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] - # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture - # that is invalid Ruby. - refute_nil Ripper.sexp_raw(source) if check_ripper + relatives.each do |relative| + # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" + next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) - # Next, assert that there were no errors during parsing. - result = YARP.parse(source, relative) - assert_empty result.errors + filepath = File.join(base, relative) + snapshot = File.expand_path(File.join("snapshots", relative), __dir__) - # Next, pretty print the source. - printed = PP.pp(result.value, +"", 79) + directory = File.dirname(snapshot) + FileUtils.mkdir_p(directory) unless File.directory?(directory) - if File.exist?(snapshot) - saved = File.read(snapshot) + ripper_should_parse = ripper_should_match = ripper_enabled - # If the snapshot file exists, but the printed value does not match the - # snapshot, then update the snapshot file. - if printed != saved - File.write(snapshot, printed) - warn("Updated snapshot at #{snapshot}.") - end + # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if + # we're on an earlier version. + ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" - # If the snapshot file exists, then assert that the printed value - # matches the snapshot. - assert_equal(saved, printed) - else - # If the snapshot file does not yet exist, then write it out now. - File.write(snapshot, printed) - warn("Created snapshot at #{snapshot}.") - end + # It seems like there are some oddities with nested heredocs and ripper. + # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. + ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" - # Next, assert that the value can be serialized and deserialized without - # changing the shape of the tree. - assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) + # Ripper seems to have a bug that the regex portions before and after the heredoc are combined + # into a single token. See https://bugs.ruby-lang.org/issues/19838. + # + # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. + ripper_should_parse = false if relative == "spanning_heredoc.txt" + + define_method "test_filepath_#{relative}" do + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + + # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture + # that is invalid Ruby. + refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse + + # Next, assert that there were no errors during parsing. + result = YARP.parse(source, relative) + assert_empty result.errors - # Next, check that the location ranges of each node in the tree are a - # superset of their respective child nodes. - assert_non_overlapping_locations(result.value) + # Next, pretty print the source. + printed = PP.pp(result.value, +"", 79) - # Next, assert that the newlines are in the expected places. - expected_newlines = [0] - source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } + if File.exist?(snapshot) + saved = File.read(snapshot) - # If there's a __END__, then we should trip out those newlines because we - # don't actually scan them during parsing (because we don't need to). - if found = result.comments.find { |comment| comment.type == :__END__ } - expected_newlines = expected_newlines[...found.location.start_line] - end + # If the snapshot file exists, but the printed value does not match the + # snapshot, then update the snapshot file. + if printed != saved + File.write(snapshot, printed) + warn("Updated snapshot at #{snapshot}.") + end - assert_equal expected_newlines, YARP.const_get(:Debug).newlines(source) + # If the snapshot file exists, then assert that the printed value + # matches the snapshot. + assert_equal(saved, printed) + else + # If the snapshot file does not yet exist, then write it out now. + File.write(snapshot, printed) + warn("Created snapshot at #{snapshot}.") + end - # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if - # we're on an earlier version. - return if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" + # Next, assert that the value can be serialized and deserialized without + # changing the shape of the tree. + assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - return if relative == "seattlerb/heredoc_nested.txt" + # Next, check that the location ranges of each node in the tree are a + # superset of their respective child nodes. + assert_non_overlapping_locations(result.value) - # Ripper seems to have a bug that the regex portions before and after the heredoc are combined - # into a single token. - return if relative == "wrapping_heredoc.txt" - - # Finally, assert that we can lex the source and get the same tokens as - # Ripper. - lex_result = YARP.lex_compat(source) - assert_equal [], lex_result.errors - tokens = lex_result.value - - if check_ripper - begin - YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| - assert_equal ripper, yarp + # Next, assert that the newlines are in the expected places. + expected_newlines = [0] + source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } + + # If there's a __END__, then we should trip out those newlines because we + # don't actually scan them during parsing (because we don't need to). + if found = result.comments.find { |comment| comment.type == :__END__ } + expected_newlines = expected_newlines[...found.location.start_line] + end + + assert_equal expected_newlines, Debug.newlines(source) + + if ripper_should_parse && ripper_should_match + # Finally, assert that we can lex the source and get the same tokens as + # Ripper. + lex_result = YARP.lex_compat(source) + assert_equal [], lex_result.errors + tokens = lex_result.value + + begin + YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| + assert_equal ripper, yarp + end + rescue SyntaxError + raise ArgumentError, "Test file has invalid syntax #{filepath}" end - rescue SyntaxError - raise ArgumentError, "Test file has invalid syntax #{filepath}" end end end - end - Dir["*.txt", base: base].each do |relative| - next if relative == "newline_terminated.txt" + Dir["*.txt", base: base].each do |relative| + next if relative == "newline_terminated.txt" - # We test every snippet (separated by \n\n) in isolation - # to ensure the parser does not try to read bytes further than the end of each snippet - define_method "test_individual_snippets_#{relative}" do - filepath = File.join(base, relative) + # We test every snippet (separated by \n\n) in isolation + # to ensure the parser does not try to read bytes further than the end of each snippet + define_method "test_individual_snippets_#{relative}" do + filepath = File.join(base, relative) - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| - snippet = snippet.rstrip - result = YARP.parse(snippet, relative) - assert_empty result.errors + file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| + snippet = snippet.rstrip + result = YARP.parse(snippet, relative) + assert_empty result.errors - assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + end end end - end - private + private - # Check that the location ranges of each node in the tree are a superset of - # their respective child nodes. - def assert_non_overlapping_locations(node) - queue = [node] + # Check that the location ranges of each node in the tree are a superset of + # their respective child nodes. + def assert_non_overlapping_locations(node) + queue = [node] - while (current = queue.shift) - # We only want to compare parent/child location overlap in the case that - # we are not looking at a heredoc. That's because heredoc locations are - # special in that they only use the declaration of the heredoc. - compare = !(current.is_a?(YARP::InterpolatedStringNode) || current.is_a?(YARP::InterpolatedXStringNode)) || !current.opening&.start_with?("<<") + while (current = queue.shift) + # We only want to compare parent/child location overlap in the case that + # we are not looking at a heredoc. That's because heredoc locations are + # special in that they only use the declaration of the heredoc. + compare = !(current.is_a?(InterpolatedStringNode) || current.is_a?(InterpolatedXStringNode)) || !current.opening&.start_with?("<<") - current.child_nodes.each do |child| - # child_nodes can return nil values, so we need to skip those. - next unless child + current.child_nodes.each do |child| + # child_nodes can return nil values, so we need to skip those. + next unless child - # Now that we know we have a child node, add that to the queue. - queue << child + # Now that we know we have a child node, add that to the queue. + queue << child - if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset + if compare + assert_operator current.location.start_offset, :<=, child.location.start_offset + assert_operator current.location.end_offset, :>=, child.location.end_offset + end end end end - end - def find_source_file_node(program) - queue = [program] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + def find_source_file_node(program) + queue = [program] + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end - end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end end end diff --git a/test/yarp/regexp_test.rb b/test/yarp/regexp_test.rb index bb236e2f1f9daf..9863a54758aa1f 100644 --- a/test/yarp/regexp_test.rb +++ b/test/yarp/regexp_test.rb @@ -1,199 +1,201 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI -class RegexpTest < Test::Unit::TestCase - ############################################################################## - # These tests test the actual use case of extracting named capture groups - ############################################################################## +module YARP + class RegexpTest < TestCase + ############################################################################## + # These tests test the actual use case of extracting named capture groups + ############################################################################## - def test_named_captures_with_arrows - assert_equal(["foo"], named_captures("(?bar)")) - end + def test_named_captures_with_arrows + assert_equal(["foo"], named_captures("(?bar)")) + end - def test_named_captures_with_single_quotes - assert_equal(["foo"], named_captures("(?'foo'bar)")) - end + def test_named_captures_with_single_quotes + assert_equal(["foo"], named_captures("(?'foo'bar)")) + end - def test_nested_named_captures_with_arrows - assert_equal(["foo", "bar"], named_captures("(?(?baz))")) - end + def test_nested_named_captures_with_arrows + assert_equal(["foo", "bar"], named_captures("(?(?baz))")) + end - def test_nested_named_captures_with_single_quotes - assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) - end + def test_nested_named_captures_with_single_quotes + assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) + end - def test_allows_duplicate_named_captures - assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) - end + def test_allows_duplicate_named_captures + assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) + end - def test_named_capture_inside_fake_range_quantifier - assert_equal(["foo"], named_captures("foo{1, (?2)}")) - end + def test_named_capture_inside_fake_range_quantifier + assert_equal(["foo"], named_captures("foo{1, (?2)}")) + end - ############################################################################## - # These tests test the rest of the AST. They are not exhaustive, but they - # should cover the most common cases. We test these to make sure we don't - # accidentally regress and stop being able to extract named captures. - ############################################################################## + ############################################################################## + # These tests test the rest of the AST. They are not exhaustive, but they + # should cover the most common cases. We test these to make sure we don't + # accidentally regress and stop being able to extract named captures. + ############################################################################## - def test_alternation - refute_nil(named_captures("foo|bar")) - end + def test_alternation + refute_nil(named_captures("foo|bar")) + end - def test_anchors - refute_nil(named_captures("^foo$")) - end + def test_anchors + refute_nil(named_captures("^foo$")) + end - def test_any - refute_nil(named_captures(".")) - end + def test_any + refute_nil(named_captures(".")) + end - def test_posix_character_classes - refute_nil(named_captures("[[:digit:]]")) - end + def test_posix_character_classes + refute_nil(named_captures("[[:digit:]]")) + end - def test_negated_posix_character_classes - refute_nil(named_captures("[[:^digit:]]")) - end + def test_negated_posix_character_classes + refute_nil(named_captures("[[:^digit:]]")) + end - def test_invalid_posix_character_classes_should_fall_back_to_regular_classes - refute_nil(named_captures("[[:foo]]")) - end + def test_invalid_posix_character_classes_should_fall_back_to_regular_classes + refute_nil(named_captures("[[:foo]]")) + end - def test_character_sets - refute_nil(named_captures("[abc]")) - end + def test_character_sets + refute_nil(named_captures("[abc]")) + end - def test_nested_character_sets - refute_nil(named_captures("[[abc]]")) - end + def test_nested_character_sets + refute_nil(named_captures("[[abc]]")) + end - def test_nested_character_sets_with_operators - refute_nil(named_captures("[[abc] && [def]]")) - end + def test_nested_character_sets_with_operators + refute_nil(named_captures("[[abc] && [def]]")) + end - def test_named_capture_inside_nested_character_set - assert_equal([], named_captures("[foo (?bar)]")) - end + def test_named_capture_inside_nested_character_set + assert_equal([], named_captures("[foo (?bar)]")) + end - def test_negated_character_sets - refute_nil(named_captures("[^abc]")) - end + def test_negated_character_sets + refute_nil(named_captures("[^abc]")) + end - def test_character_ranges - refute_nil(named_captures("[a-z]")) - end + def test_character_ranges + refute_nil(named_captures("[a-z]")) + end - def test_negated_character_ranges - refute_nil(named_captures("[^a-z]")) - end + def test_negated_character_ranges + refute_nil(named_captures("[^a-z]")) + end - def test_fake_named_captures_inside_character_sets - assert_equal([], named_captures("[a-z(?)]")) - end + def test_fake_named_captures_inside_character_sets + assert_equal([], named_captures("[a-z(?)]")) + end - def test_fake_named_capture_inside_character_set_with_escaped_ending - assert_equal([], named_captures("[a-z\\](?)]")) - end + def test_fake_named_capture_inside_character_set_with_escaped_ending + assert_equal([], named_captures("[a-z\\](?)]")) + end - def test_comments - refute_nil(named_captures("(?#foo)")) - end + def test_comments + refute_nil(named_captures("(?#foo)")) + end - def test_comments_with_escaped_parentheses - refute_nil(named_captures("(?#foo\\)\\))")) - end + def test_comments_with_escaped_parentheses + refute_nil(named_captures("(?#foo\\)\\))")) + end - def test_non_capturing_groups - refute_nil(named_captures("(?:foo)")) - end + def test_non_capturing_groups + refute_nil(named_captures("(?:foo)")) + end - def test_positive_lookaheads - refute_nil(named_captures("(?=foo)")) - end + def test_positive_lookaheads + refute_nil(named_captures("(?=foo)")) + end - def test_negative_lookaheads - refute_nil(named_captures("(?!foo)")) - end + def test_negative_lookaheads + refute_nil(named_captures("(?!foo)")) + end - def test_positive_lookbehinds - refute_nil(named_captures("(?<=foo)")) - end + def test_positive_lookbehinds + refute_nil(named_captures("(?<=foo)")) + end - def test_negative_lookbehinds - refute_nil(named_captures("(?foo)")) - end + def test_atomic_groups + refute_nil(named_captures("(?>foo)")) + end - def test_absence_operator - refute_nil(named_captures("(?~foo)")) - end + def test_absence_operator + refute_nil(named_captures("(?~foo)")) + end - def test_conditional_expression_with_index - refute_nil(named_captures("(?(1)foo)")) - end + def test_conditional_expression_with_index + refute_nil(named_captures("(?(1)foo)")) + end - def test_conditional_expression_with_name - refute_nil(named_captures("(?(foo)bar)")) - end + def test_conditional_expression_with_name + refute_nil(named_captures("(?(foo)bar)")) + end - def test_conditional_expression_with_group - refute_nil(named_captures("(?()bar)")) - end + def test_conditional_expression_with_group + refute_nil(named_captures("(?()bar)")) + end - def test_options_on_groups - refute_nil(named_captures("(?imxdau:foo)")) - end + def test_options_on_groups + refute_nil(named_captures("(?imxdau:foo)")) + end - def test_options_on_groups_with_invalid_options - assert_nil(named_captures("(?z:bar)")) - end + def test_options_on_groups_with_invalid_options + assert_nil(named_captures("(?z:bar)")) + end - def test_options_on_groups_getting_turned_off - refute_nil(named_captures("(?-imx:foo)")) - end + def test_options_on_groups_getting_turned_off + refute_nil(named_captures("(?-imx:foo)")) + end - def test_options_on_groups_some_getting_turned_on_some_getting_turned_off - refute_nil(named_captures("(?im-x:foo)")) - end + def test_options_on_groups_some_getting_turned_on_some_getting_turned_off + refute_nil(named_captures("(?im-x:foo)")) + end - def test_star_quantifier - refute_nil(named_captures("foo*")) - end + def test_star_quantifier + refute_nil(named_captures("foo*")) + end - def test_plus_quantifier - refute_nil(named_captures("foo+")) - end + def test_plus_quantifier + refute_nil(named_captures("foo+")) + end - def test_question_mark_quantifier - refute_nil(named_captures("foo?")) - end + def test_question_mark_quantifier + refute_nil(named_captures("foo?")) + end - def test_endless_range_quantifier - refute_nil(named_captures("foo{1,}")) - end + def test_endless_range_quantifier + refute_nil(named_captures("foo{1,}")) + end - def test_beginless_range_quantifier - refute_nil(named_captures("foo{,1}")) - end + def test_beginless_range_quantifier + refute_nil(named_captures("foo{,1}")) + end - def test_range_quantifier - refute_nil(named_captures("foo{1,2}")) - end + def test_range_quantifier + refute_nil(named_captures("foo{1,2}")) + end - def test_fake_range_quantifier_because_of_spaces - refute_nil(named_captures("foo{1, 2}")) - end + def test_fake_range_quantifier_because_of_spaces + refute_nil(named_captures("foo{1, 2}")) + end - private + private - def named_captures(source) - YARP.const_get(:Debug).named_captures(source) + def named_captures(source) + Debug.named_captures(source) + end end end diff --git a/test/yarp/ripper_compat_test.rb b/test/yarp/ripper_compat_test.rb index 4350ba7f81f7f3..9fcdfe63c685ca 100644 --- a/test/yarp/ripper_compat_test.rb +++ b/test/yarp/ripper_compat_test.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" module YARP - class RipperCompatTest < Test::Unit::TestCase + class RipperCompatTest < TestCase def test_1_plus_2 assert_equivalent("1 + 2") end diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index a26b971b824732..dc12012f44abf2 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -1,33 +1,64 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" -class YARPRubyAPITest < Test::Unit::TestCase - def test_ruby_api - filepath = __FILE__ - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) +module YARP + class RubyAPITest < TestCase + def test_ruby_api + filepath = __FILE__ + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value + assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value + assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) - assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) + serialized = YARP.dump(source, filepath) + ast1 = YARP.load(source, serialized).value + ast2 = YARP.parse(source, filepath).value + ast3 = YARP.parse_file(filepath).value - serialized = YARP.dump(source, filepath) - ast1 = YARP.load(source, serialized).value - ast2 = YARP.parse(source, filepath).value - ast3 = YARP.parse_file(filepath).value + assert_equal_nodes ast1, ast2 + assert_equal_nodes ast2, ast3 + end - assert_equal_nodes ast1, ast2 - assert_equal_nodes ast2, ast3 - end + def test_literal_value_method + assert_equal 123, parse_expression("123").value + assert_equal 3.14, parse_expression("3.14").value + assert_equal 42i, parse_expression("42i").value + assert_equal 42.1ri, parse_expression("42.1ri").value + assert_equal 3.14i, parse_expression("3.14i").value + assert_equal 42r, parse_expression("42r").value + assert_equal 0.5r, parse_expression("0.5r").value + assert_equal 42ri, parse_expression("42ri").value + assert_equal 0.5ri, parse_expression("0.5ri").value + end + + def test_location_join + recv, args_node, _ = parse_expression("1234 + 567").child_nodes + arg = args_node.arguments[0] + + joined = recv.location.join(arg.location) + assert_equal 0, joined.start_offset + assert_equal 10, joined.length + + assert_raise RuntimeError, "Incompatible locations" do + arg.location.join(recv.location) + end + + other_arg = parse_expression("1234 + 567").arguments.arguments[0] + + assert_raise RuntimeError, "Incompatible sources" do + other_arg.location.join(recv.location) + end + + assert_raise RuntimeError, "Incompatible sources" do + recv.location.join(other_arg.location) + end + end + + private - def test_literal_value_method - assert_equal 123, YARP.parse("123").value.statements.body.first.value - assert_equal 3.14, YARP.parse("3.14").value.statements.body.first.value - assert_equal 42i, YARP.parse("42i").value.statements.body.first.value - assert_equal 3.14i, YARP.parse("3.14i").value.statements.body.first.value - assert_equal 42r, YARP.parse("42r").value.statements.body.first.value - assert_equal 0.5r, YARP.parse("0.5r").value.statements.body.first.value - assert_equal 42ri, YARP.parse("42ri").value.statements.body.first.value - assert_equal 0.5ri, YARP.parse("0.5ri").value.statements.body.first.value + def parse_expression(source) + YARP.parse(source).value.statements.body.first + end end end diff --git a/test/yarp/snapshots/begin_ensure.txt b/test/yarp/snapshots/begin_ensure.txt index e04a1236df3ee0..8e9873122bb589 100644 --- a/test/yarp/snapshots/begin_ensure.txt +++ b/test/yarp/snapshots/begin_ensure.txt @@ -1,6 +1,6 @@ -ProgramNode(0...94)( +ProgramNode(0...211)( [], - StatementsNode(0...94)( + StatementsNode(0...211)( [BeginNode(0...20)( (0...5), StatementsNode(6...7)( @@ -64,6 +64,99 @@ ProgramNode(0...94)( (91...94) ), (91...94) + ), + BeginNode(96...211)( + (96...101), + StatementsNode(102...207)( + [BeginNode(102...207)( + (102...107), + StatementsNode(107...203)( + [CallNode(107...203)( + SymbolNode(107...109)((107...108), (108...109), nil, "s"), + (109...110), + (110...111), + nil, + ArgumentsNode(112...203)( + [BeginNode(112...203)( + (112...117), + nil, + nil, + nil, + EnsureNode(118...203)( + (118...124), + StatementsNode(125...199)( + [CallNode(125...199)( + ConstantReadNode(125...131)(), + (131...132), + (132...135), + nil, + nil, + nil, + BlockNode(136...199)( + [], + nil, + StatementsNode(141...195)( + [BeginNode(141...195)( + (141...146), + StatementsNode(151...156)( + [BreakNode(151...156)(nil, (151...156))] + ), + nil, + nil, + EnsureNode(161...195)( + (161...167), + StatementsNode(168...189)( + [CallNode(168...189)( + ConstantReadNode(168...174)(), + (174...175), + (175...178), + nil, + nil, + nil, + BlockNode(179...189)( + [], + nil, + nil, + (179...181), + (186...189) + ), + 0, + "new" + )] + ), + (192...195) + ), + (192...195) + )] + ), + (136...138), + (196...199) + ), + 0, + "new" + )] + ), + (200...203) + ), + (200...203) + )] + ), + nil, + nil, + 0, + "l" + )] + ), + nil, + nil, + nil, + (204...207) + )] + ), + nil, + nil, + nil, + (208...211) )] ) ) diff --git a/test/yarp/snapshots/begin_rescue.txt b/test/yarp/snapshots/begin_rescue.txt index f7195f80a08d54..4c902e53509898 100644 --- a/test/yarp/snapshots/begin_rescue.txt +++ b/test/yarp/snapshots/begin_rescue.txt @@ -223,7 +223,7 @@ ProgramNode(0...578)( (187...193), [ConstantReadNode(194...203)()], (204...206), - LocalVariableWriteNode(207...209)(:ex, 0, nil, (207...209), nil), + LocalVariableTargetNode(207...209)(:ex, 0), StatementsNode(212...213)( [CallNode(212...213)( nil, @@ -241,7 +241,7 @@ ProgramNode(0...578)( (214...220), [ConstantReadNode(221...237)(), ConstantReadNode(239...255)()], (256...258), - LocalVariableWriteNode(259...261)(:ex, 0, nil, (259...261), nil), + LocalVariableTargetNode(259...261)(:ex, 0), StatementsNode(264...265)( [CallNode(264...265)( nil, @@ -281,7 +281,7 @@ ProgramNode(0...578)( (281...287), [ConstantReadNode(288...297)()], (298...300), - LocalVariableWriteNode(301...303)(:ex, 0, nil, (301...303), nil), + LocalVariableTargetNode(301...303)(:ex, 0), StatementsNode(306...307)( [CallNode(306...307)( nil, @@ -532,7 +532,7 @@ ProgramNode(0...578)( (489...495), [ConstantReadNode(496...505)(), ConstantReadNode(507...522)()], (523...525), - LocalVariableWriteNode(526...528)(:ex, 0, nil, (526...528), nil), + LocalVariableTargetNode(526...528)(:ex, 0), StatementsNode(531...532)( [CallNode(531...532)( nil, @@ -571,7 +571,7 @@ ProgramNode(0...578)( (548...554), [ConstantReadNode(555...564)()], (565...567), - LocalVariableWriteNode(568...570)(:ex, 0, nil, (568...570), nil), + LocalVariableTargetNode(568...570)(:ex, 0), StatementsNode(573...574)( [CallNode(573...574)( nil, diff --git a/test/yarp/snapshots/blocks.txt b/test/yarp/snapshots/blocks.txt index 089b63b89ca539..4a2c3f7585eeff 100644 --- a/test/yarp/snapshots/blocks.txt +++ b/test/yarp/snapshots/blocks.txt @@ -88,11 +88,13 @@ ProgramNode(0...402)( (61...62) ), StatementsNode(63...72)( - [OperatorWriteNode(63...72)( - LocalVariableWriteNode(63...67)(:memo, 0, nil, (63...67), nil), + [LocalVariableOperatorWriteNode(63...72)( + (63...67), (68...70), + LocalVariableReadNode(71...72)(:x, 0), + :memo, :+, - LocalVariableReadNode(71...72)(:x, 0) + 0 )] ), (51...52), @@ -451,8 +453,8 @@ ProgramNode(0...402)( LocalVariableWriteNode(292...300)( :fork, 0, - IntegerNode(299...300)(), (292...296), + IntegerNode(299...300)(), (297...298) ), CallNode(301...316)( diff --git a/test/yarp/snapshots/boolean_operators.txt b/test/yarp/snapshots/boolean_operators.txt index c6cfc311e560a8..f1fc5cd4b29472 100644 --- a/test/yarp/snapshots/boolean_operators.txt +++ b/test/yarp/snapshots/boolean_operators.txt @@ -1,21 +1,27 @@ ProgramNode(0...24)( [:a], StatementsNode(0...24)( - [AndWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableAndWriteNode(0...7)( + (0...1), + (2...5), CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), - (2...5) + :a, + 0 ), - OperatorWriteNode(9...15)( - LocalVariableWriteNode(9...10)(:a, 0, nil, (9...10), nil), + LocalVariableOperatorWriteNode(9...15)( + (9...10), (11...13), + CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b"), + :a, :+, - CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b") + 0 ), - OrWriteNode(17...24)( - LocalVariableWriteNode(17...18)(:a, 0, nil, (17...18), nil), + LocalVariableOrWriteNode(17...24)( + (17...18), + (19...22), CallNode(23...24)(nil, nil, (23...24), nil, nil, nil, nil, 2, "b"), - (19...22) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/classes.txt b/test/yarp/snapshots/classes.txt index 33b3d042148f9b..1d8813f40e342f 100644 --- a/test/yarp/snapshots/classes.txt +++ b/test/yarp/snapshots/classes.txt @@ -11,8 +11,8 @@ ProgramNode(0...370)( [LocalVariableWriteNode(8...13)( :a, 0, - IntegerNode(12...13)(), (8...9), + IntegerNode(12...13)(), (10...11) )] ), @@ -63,8 +63,8 @@ ProgramNode(0...370)( [LocalVariableWriteNode(89...94)( :a, 0, - IntegerNode(93...94)(), (89...90), + IntegerNode(93...94)(), (91...92) )] ), diff --git a/test/yarp/snapshots/defined.txt b/test/yarp/snapshots/defined.txt index 6c2cad656e0feb..1d4b5a5502e876 100644 --- a/test/yarp/snapshots/defined.txt +++ b/test/yarp/snapshots/defined.txt @@ -8,11 +8,13 @@ ProgramNode(0...78)( ), DefinedNode(27...43)( (35...36), - OperatorWriteNode(36...42)( - LocalVariableWriteNode(36...37)(:x, 0, nil, (36...37), nil), + LocalVariableOperatorWriteNode(36...42)( + (36...37), (38...40), + IntegerNode(41...42)(), + :x, :%, - IntegerNode(41...42)() + 0 ), (42...43), (27...35) diff --git a/test/yarp/snapshots/dos_endings.txt b/test/yarp/snapshots/dos_endings.txt index 240ae043a42477..7048028b2a193b 100644 --- a/test/yarp/snapshots/dos_endings.txt +++ b/test/yarp/snapshots/dos_endings.txt @@ -35,13 +35,14 @@ ProgramNode(0...108)( LocalVariableWriteNode(75...84)( :x, 0, - StringNode(79...84)((79...82), (82...82), (82...84), ""), (75...76), + StringNode(79...84)((79...82), (82...82), (82...84), ""), (77...78) ), LocalVariableWriteNode(88...108)( :a, 0, + (88...89), CallNode(92...108)( nil, nil, @@ -74,7 +75,6 @@ ProgramNode(0...108)( 0, "foo" ), - (88...89), (90...91) )] ) diff --git a/test/yarp/snapshots/for.txt b/test/yarp/snapshots/for.txt index 0bd8319aa1af36..53eb1caf581484 100644 --- a/test/yarp/snapshots/for.txt +++ b/test/yarp/snapshots/for.txt @@ -3,7 +3,7 @@ ProgramNode(0...143)( StatementsNode(0...143)( [ForNode(0...20)( MultiWriteNode(4...5)( - [LocalVariableWriteNode(4...5)(:i, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(4...5)(:i, 0)], nil, nil, nil, @@ -23,7 +23,7 @@ ProgramNode(0...143)( ), ForNode(22...44)( MultiWriteNode(26...27)( - [LocalVariableWriteNode(26...27)(:i, 0, nil, (26...27), nil)], + [LocalVariableTargetNode(26...27)(:i, 0)], nil, nil, nil, @@ -43,8 +43,8 @@ ProgramNode(0...143)( ), ForNode(46...68)( MultiWriteNode(50...53)( - [LocalVariableWriteNode(50...51)(:i, 0, nil, (50...51), nil), - LocalVariableWriteNode(52...53)(:j, 0, nil, (52...53), nil)], + [LocalVariableTargetNode(50...51)(:i, 0), + LocalVariableTargetNode(52...53)(:j, 0)], nil, nil, nil, @@ -64,9 +64,9 @@ ProgramNode(0...143)( ), ForNode(70...94)( MultiWriteNode(74...79)( - [LocalVariableWriteNode(74...75)(:i, 0, nil, (74...75), nil), - LocalVariableWriteNode(76...77)(:j, 0, nil, (76...77), nil), - LocalVariableWriteNode(78...79)(:k, 0, nil, (78...79), nil)], + [LocalVariableTargetNode(74...75)(:i, 0), + LocalVariableTargetNode(76...77)(:j, 0), + LocalVariableTargetNode(78...79)(:k, 0)], nil, nil, nil, @@ -86,7 +86,7 @@ ProgramNode(0...143)( ), ForNode(96...119)( MultiWriteNode(100...101)( - [LocalVariableWriteNode(100...101)(:i, 0, nil, (100...101), nil)], + [LocalVariableTargetNode(100...101)(:i, 0)], nil, nil, nil, @@ -106,7 +106,7 @@ ProgramNode(0...143)( ), ForNode(121...143)( MultiWriteNode(125...126)( - [LocalVariableWriteNode(125...126)(:i, 0, nil, (125...126), nil)], + [LocalVariableTargetNode(125...126)(:i, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/lambda.txt b/test/yarp/snapshots/lambda.txt index 23c4ebc072e31b..6640b585c32140 100644 --- a/test/yarp/snapshots/lambda.txt +++ b/test/yarp/snapshots/lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...92)( [LambdaNode(0...14)( [:foo], (0...2), + (12...13), + (13...14), BlockParametersNode(2...11)( ParametersNode(6...9)( [RequiredParameterNode(6...9)(:foo)], @@ -23,6 +25,8 @@ ProgramNode(0...92)( LambdaNode(16...34)( [:x], (16...18), + (31...32), + (33...34), BlockParametersNode(18...30)( ParametersNode(19...29)( [], @@ -66,6 +70,8 @@ ProgramNode(0...92)( LambdaNode(36...51)( [:a], (36...38), + (49...50), + (50...51), BlockParametersNode(38...48)( ParametersNode(39...47)( [], @@ -108,6 +114,8 @@ ProgramNode(0...92)( LambdaNode(53...72)( [:foo], (53...55), + (66...68), + (69...72), BlockParametersNode(56...65)( ParametersNode(56...65)( [], @@ -142,6 +150,8 @@ ProgramNode(0...92)( LambdaNode(74...92)( [:foo], (74...76), + (86...88), + (89...92), BlockParametersNode(77...85)( ParametersNode(77...85)( [], diff --git a/test/yarp/snapshots/method_calls.txt b/test/yarp/snapshots/method_calls.txt index b2c84d0d3573c2..cfbb0784711b44 100644 --- a/test/yarp/snapshots/method_calls.txt +++ b/test/yarp/snapshots/method_calls.txt @@ -64,7 +64,7 @@ ProgramNode(0...1237)( CallNode(58...62)( CallNode(58...59)(nil, nil, (58...59), nil, nil, nil, nil, 2, "a"), (59...60), - (0...0), + nil, (60...61), nil, (61...62), @@ -75,7 +75,7 @@ ProgramNode(0...1237)( CallNode(64...75)( CallNode(64...65)(nil, nil, (64...65), nil, nil, nil, nil, 2, "a"), (65...66), - (0...0), + nil, (66...67), ArgumentsNode(67...74)( [IntegerNode(67...68)(), @@ -413,7 +413,7 @@ ProgramNode(0...1237)( CallNode(212...217)( CallNode(212...213)(nil, nil, (212...213), nil, nil, nil, nil, 2, "a"), (213...215), - (0...0), + nil, (215...216), nil, (216...217), @@ -1369,6 +1369,7 @@ ProgramNode(0...1237)( [SymbolNode(1067...1069)((1067...1068), (1068...1069), nil, "a"), WhileNode(1073...1117)( (1073...1078), + (1114...1117), CallNode(1079...1080)( nil, nil, @@ -1418,6 +1419,7 @@ ProgramNode(0...1237)( ), UntilNode(1121...1153)( (1121...1126), + (1150...1153), CallNode(1127...1128)( nil, nil, diff --git a/test/yarp/snapshots/methods.txt b/test/yarp/snapshots/methods.txt index a2cfa8e06d0626..79d11f10381466 100644 --- a/test/yarp/snapshots/methods.txt +++ b/test/yarp/snapshots/methods.txt @@ -199,7 +199,7 @@ ProgramNode(0...1194)( ), DefNode(190...204)( (199...200), - InstanceVariableReadNode(194...198)(), + InstanceVariableReadNode(194...198)(:@var), nil, nil, [], @@ -298,8 +298,8 @@ ProgramNode(0...1194)( LocalVariableWriteNode(275...280)( :a, 0, - IntegerNode(279...280)(), (275...276), + IntegerNode(279...280)(), (277...278) ), DefNode(282...291)( @@ -589,8 +589,8 @@ ProgramNode(0...1194)( [LocalVariableWriteNode(537...542)( :b, 0, - IntegerNode(541...542)(), (537...538), + IntegerNode(541...542)(), (539...540) )] ), @@ -839,6 +839,7 @@ ProgramNode(0...1194)( LocalVariableWriteNode(769...774)( :c, 0, + (769...770), CallNode(773...774)( nil, nil, @@ -850,7 +851,6 @@ ProgramNode(0...1194)( 2, "b" ), - (769...770), (771...772) ), (768...769), @@ -910,7 +910,7 @@ ProgramNode(0...1194)( ), DefNode(811...826)( (821...822), - ClassVariableReadNode(815...820)(), + ClassVariableReadNode(815...820)(:@@var), nil, nil, [], @@ -927,6 +927,7 @@ ProgramNode(0...1194)( LocalVariableWriteNode(833...838)( :a, 0, + (833...834), CallNode(837...838)( nil, nil, @@ -938,7 +939,6 @@ ProgramNode(0...1194)( 2, "b" ), - (833...834), (835...836) ), (832...833), diff --git a/test/yarp/snapshots/modules.txt b/test/yarp/snapshots/modules.txt index a3cf30b9ff8786..aa8c1a743d749b 100644 --- a/test/yarp/snapshots/modules.txt +++ b/test/yarp/snapshots/modules.txt @@ -9,8 +9,8 @@ ProgramNode(0...140)( [LocalVariableWriteNode(9...14)( :a, 0, - IntegerNode(13...14)(), (9...10), + IntegerNode(13...14)(), (11...12) )] ), @@ -62,8 +62,8 @@ ProgramNode(0...140)( [LocalVariableWriteNode(67...72)( :x, 0, - IntegerNode(71...72)(), (67...68), + IntegerNode(71...72)(), (69...70) )] ), diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index d578a43f066cb2..01292cbf3caabd 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -3,7 +3,7 @@ ProgramNode(0...3743)( StatementsNode(0...3743)( [MatchRequiredNode(0...10)( CallNode(0...3)(nil, nil, (0...3), nil, nil, nil, nil, 2, "foo"), - LocalVariableWriteNode(7...10)(:bar, 0, nil, (7...10), nil), + LocalVariableTargetNode(7...10)(:bar, 0), (4...6) ), MatchRequiredNode(11...19)( @@ -38,7 +38,7 @@ ProgramNode(0...3743)( ), MatchRequiredNode(78...91)( CallNode(78...81)(nil, nil, (78...81), nil, nil, nil, nil, 2, "foo"), - SymbolNode(85...91)(nil, (87...90), nil, "foo"), + SymbolNode(85...91)((85...87), (87...90), (90...91), "foo"), (82...84) ), MatchRequiredNode(92...104)( @@ -323,6 +323,8 @@ ProgramNode(0...3743)( LambdaNode(343...353)( [], (343...345), + (346...347), + (352...353), nil, StatementsNode(348...351)([LocalVariableReadNode(348...351)(:bar, 1)]) ), @@ -461,8 +463,8 @@ ProgramNode(0...3743)( "foo" ), RangeNode(472...488)( - SymbolNode(472...478)(nil, (474...477), nil, "foo"), - SymbolNode(482...488)(nil, (484...487), nil, "foo"), + SymbolNode(472...478)((472...474), (474...477), (477...478), "foo"), + SymbolNode(482...488)((482...484), (484...487), (487...488), "foo"), (479...481), 0 ), @@ -868,6 +870,8 @@ ProgramNode(0...3743)( LambdaNode(916...926)( [], (916...918), + (919...920), + (925...926), nil, StatementsNode(921...924)( [LocalVariableReadNode(921...924)(:bar, 1)] @@ -876,6 +880,8 @@ ProgramNode(0...3743)( LambdaNode(930...940)( [], (930...932), + (933...934), + (939...940), nil, StatementsNode(935...938)( [LocalVariableReadNode(935...938)(:bar, 1)] @@ -917,7 +923,7 @@ ProgramNode(0...3743)( "foo" ), PinnedVariableNode(961...966)( - InstanceVariableReadNode(962...966)(), + InstanceVariableReadNode(962...966)(:@bar), (961...962) ), (958...960) @@ -935,7 +941,7 @@ ProgramNode(0...3743)( "foo" ), PinnedVariableNode(974...980)( - ClassVariableReadNode(975...980)(), + ClassVariableReadNode(975...980)(:@@bar), (974...975) ), (971...973) @@ -1206,13 +1212,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1169...1177)( ConstantReadNode(1169...1172)(), - [LocalVariableWriteNode(1173...1176)( - :bar, - 0, - nil, - (1173...1176), - nil - )], + [LocalVariableTargetNode(1173...1176)(:bar, 0)], nil, [], (1172...1173), @@ -1237,21 +1237,9 @@ ProgramNode(0...3743)( [], SplatNode(1189...1193)( (1189...1190), - LocalVariableWriteNode(1190...1193)( - :bar, - 0, - nil, - (1190...1193), - nil - ) + LocalVariableTargetNode(1190...1193)(:bar, 0) ), - [LocalVariableWriteNode(1195...1198)( - :baz, - 0, - nil, - (1195...1198), - nil - )], + [LocalVariableTargetNode(1195...1198)(:baz, 0)], (1188...1189), (1198...1199) ), @@ -1271,22 +1259,10 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1207...1221)( ConstantReadNode(1207...1210)(), - [LocalVariableWriteNode(1211...1214)( - :bar, - 0, - nil, - (1211...1214), - nil - )], + [LocalVariableTargetNode(1211...1214)(:bar, 0)], SplatNode(1216...1220)( (1216...1217), - LocalVariableWriteNode(1217...1220)( - :baz, - 0, - nil, - (1217...1220), - nil - ) + LocalVariableTargetNode(1217...1220)(:baz, 0) ), [], (1210...1211), @@ -1310,30 +1286,12 @@ ProgramNode(0...3743)( ConstantReadNode(1229...1232)(), SplatNode(1233...1237)( (1233...1234), - LocalVariableWriteNode(1234...1237)( - :bar, - 0, - nil, - (1234...1237), - nil - ) + LocalVariableTargetNode(1234...1237)(:bar, 0) ), - [LocalVariableWriteNode(1239...1242)( - :baz, - 0, - nil, - (1239...1242), - nil - )], + [LocalVariableTargetNode(1239...1242)(:baz, 0)], SplatNode(1244...1248)( (1244...1245), - LocalVariableWriteNode(1245...1248)( - :qux, - 0, - nil, - (1245...1248), - nil - ) + LocalVariableTargetNode(1245...1248)(:qux, 0) ), (1232...1233), (1248...1249) @@ -1451,13 +1409,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1323...1331)( ConstantReadNode(1323...1326)(), - [LocalVariableWriteNode(1327...1330)( - :bar, - 0, - nil, - (1327...1330), - nil - )], + [LocalVariableTargetNode(1327...1330)(:bar, 0)], nil, [], (1326...1327), @@ -1482,21 +1434,9 @@ ProgramNode(0...3743)( [], SplatNode(1343...1347)( (1343...1344), - LocalVariableWriteNode(1344...1347)( - :bar, - 0, - nil, - (1344...1347), - nil - ) + LocalVariableTargetNode(1344...1347)(:bar, 0) ), - [LocalVariableWriteNode(1349...1352)( - :baz, - 0, - nil, - (1349...1352), - nil - )], + [LocalVariableTargetNode(1349...1352)(:baz, 0)], (1342...1343), (1352...1353) ), @@ -1516,22 +1456,10 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1361...1375)( ConstantReadNode(1361...1364)(), - [LocalVariableWriteNode(1365...1368)( - :bar, - 0, - nil, - (1365...1368), - nil - )], + [LocalVariableTargetNode(1365...1368)(:bar, 0)], SplatNode(1370...1374)( (1370...1371), - LocalVariableWriteNode(1371...1374)( - :baz, - 0, - nil, - (1371...1374), - nil - ) + LocalVariableTargetNode(1371...1374)(:baz, 0) ), [], (1364...1365), @@ -1555,30 +1483,12 @@ ProgramNode(0...3743)( ConstantReadNode(1383...1386)(), SplatNode(1387...1391)( (1387...1388), - LocalVariableWriteNode(1388...1391)( - :bar, - 0, - nil, - (1388...1391), - nil - ) + LocalVariableTargetNode(1388...1391)(:bar, 0) ), - [LocalVariableWriteNode(1393...1396)( - :baz, - 0, - nil, - (1393...1396), - nil - )], + [LocalVariableTargetNode(1393...1396)(:baz, 0)], SplatNode(1398...1402)( (1398...1399), - LocalVariableWriteNode(1399...1402)( - :qux, - 0, - nil, - (1399...1402), - nil - ) + LocalVariableTargetNode(1399...1402)(:qux, 0) ), (1386...1387), (1402...1403) @@ -1602,13 +1512,7 @@ ProgramNode(0...3743)( [], SplatNode(1412...1416)( (1412...1413), - LocalVariableWriteNode(1413...1416)( - :bar, - 0, - nil, - (1413...1416), - nil - ) + LocalVariableTargetNode(1413...1416)(:bar, 0) ), [], nil, @@ -1633,28 +1537,10 @@ ProgramNode(0...3743)( [], SplatNode(1424...1428)( (1424...1425), - LocalVariableWriteNode(1425...1428)( - :bar, - 0, - nil, - (1425...1428), - nil - ) + LocalVariableTargetNode(1425...1428)(:bar, 0) ), - [LocalVariableWriteNode(1430...1433)( - :baz, - 0, - nil, - (1430...1433), - nil - ), - LocalVariableWriteNode(1435...1438)( - :qux, - 0, - nil, - (1435...1438), - nil - )], + [LocalVariableTargetNode(1430...1433)(:baz, 0), + LocalVariableTargetNode(1435...1438)(:qux, 0)], nil, nil ), @@ -1674,30 +1560,12 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1446...1460)( nil, - [LocalVariableWriteNode(1446...1449)( - :bar, - 0, - nil, - (1446...1449), - nil - )], + [LocalVariableTargetNode(1446...1449)(:bar, 0)], SplatNode(1451...1455)( (1451...1452), - LocalVariableWriteNode(1452...1455)( - :baz, - 0, - nil, - (1452...1455), - nil - ) + LocalVariableTargetNode(1452...1455)(:baz, 0) ), - [LocalVariableWriteNode(1457...1460)( - :qux, - 0, - nil, - (1457...1460), - nil - )], + [LocalVariableTargetNode(1457...1460)(:qux, 0)], nil, nil ), @@ -1717,29 +1585,11 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1468...1482)( nil, - [LocalVariableWriteNode(1468...1471)( - :bar, - 0, - nil, - (1468...1471), - nil - ), - LocalVariableWriteNode(1473...1476)( - :baz, - 0, - nil, - (1473...1476), - nil - )], + [LocalVariableTargetNode(1468...1471)(:bar, 0), + LocalVariableTargetNode(1473...1476)(:baz, 0)], SplatNode(1478...1482)( (1478...1479), - LocalVariableWriteNode(1479...1482)( - :qux, - 0, - nil, - (1479...1482), - nil - ) + LocalVariableTargetNode(1479...1482)(:qux, 0) ), [], nil, @@ -1763,30 +1613,12 @@ ProgramNode(0...3743)( nil, SplatNode(1490...1494)( (1490...1491), - LocalVariableWriteNode(1491...1494)( - :bar, - 0, - nil, - (1491...1494), - nil - ) + LocalVariableTargetNode(1491...1494)(:bar, 0) ), - [LocalVariableWriteNode(1496...1499)( - :baz, - 0, - nil, - (1496...1499), - nil - )], + [LocalVariableTargetNode(1496...1499)(:baz, 0)], SplatNode(1501...1505)( (1501...1502), - LocalVariableWriteNode(1502...1505)( - :qux, - 0, - nil, - (1502...1505), - nil - ) + LocalVariableTargetNode(1502...1505)(:qux, 0) ), nil, nil @@ -1882,13 +1714,7 @@ ProgramNode(0...3743)( [], SplatNode(1544...1548)( (1544...1545), - LocalVariableWriteNode(1545...1548)( - :bar, - 0, - nil, - (1545...1548), - nil - ) + LocalVariableTargetNode(1545...1548)(:bar, 0) ), [], (1543...1544), @@ -1913,28 +1739,10 @@ ProgramNode(0...3743)( [], SplatNode(1558...1562)( (1558...1559), - LocalVariableWriteNode(1559...1562)( - :bar, - 0, - nil, - (1559...1562), - nil - ) + LocalVariableTargetNode(1559...1562)(:bar, 0) ), - [LocalVariableWriteNode(1564...1567)( - :baz, - 0, - nil, - (1564...1567), - nil - ), - LocalVariableWriteNode(1569...1572)( - :qux, - 0, - nil, - (1569...1572), - nil - )], + [LocalVariableTargetNode(1564...1567)(:baz, 0), + LocalVariableTargetNode(1569...1572)(:qux, 0)], (1557...1558), (1572...1573) ), @@ -1954,30 +1762,12 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1581...1597)( nil, - [LocalVariableWriteNode(1582...1585)( - :bar, - 0, - nil, - (1582...1585), - nil - )], + [LocalVariableTargetNode(1582...1585)(:bar, 0)], SplatNode(1587...1591)( (1587...1588), - LocalVariableWriteNode(1588...1591)( - :baz, - 0, - nil, - (1588...1591), - nil - ) + LocalVariableTargetNode(1588...1591)(:baz, 0) ), - [LocalVariableWriteNode(1593...1596)( - :qux, - 0, - nil, - (1593...1596), - nil - )], + [LocalVariableTargetNode(1593...1596)(:qux, 0)], (1581...1582), (1596...1597) ), @@ -1997,29 +1787,11 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1605...1621)( nil, - [LocalVariableWriteNode(1606...1609)( - :bar, - 0, - nil, - (1606...1609), - nil - ), - LocalVariableWriteNode(1611...1614)( - :baz, - 0, - nil, - (1611...1614), - nil - )], + [LocalVariableTargetNode(1606...1609)(:bar, 0), + LocalVariableTargetNode(1611...1614)(:baz, 0)], SplatNode(1616...1620)( (1616...1617), - LocalVariableWriteNode(1617...1620)( - :qux, - 0, - nil, - (1617...1620), - nil - ) + LocalVariableTargetNode(1617...1620)(:qux, 0) ), [], (1605...1606), @@ -2043,30 +1815,12 @@ ProgramNode(0...3743)( nil, SplatNode(1630...1634)( (1630...1631), - LocalVariableWriteNode(1631...1634)( - :bar, - 0, - nil, - (1631...1634), - nil - ) + LocalVariableTargetNode(1631...1634)(:bar, 0) ), - [LocalVariableWriteNode(1636...1639)( - :baz, - 0, - nil, - (1636...1639), - nil - )], + [LocalVariableTargetNode(1636...1639)(:baz, 0)], SplatNode(1641...1645)( (1641...1642), - LocalVariableWriteNode(1642...1645)( - :qux, - 0, - nil, - (1642...1645), - nil - ) + LocalVariableTargetNode(1642...1645)(:qux, 0) ), (1629...1630), (1645...1646) @@ -2085,7 +1839,7 @@ ProgramNode(0...3743)( 2, "foo" ), - LocalVariableWriteNode(1655...1658)(:bar, 0, nil, (1655...1658), nil), + LocalVariableTargetNode(1655...1658)(:bar, 0), (1652...1654) ), MatchPredicateNode(1659...1667)( @@ -2195,7 +1949,12 @@ ProgramNode(0...3743)( 2, "foo" ), - SymbolNode(1733...1739)(nil, (1735...1738), nil, "foo"), + SymbolNode(1733...1739)( + (1733...1735), + (1735...1738), + (1738...1739), + "foo" + ), (1730...1732) ), MatchPredicateNode(1740...1752)( @@ -2515,6 +2274,8 @@ ProgramNode(0...3743)( LambdaNode(1991...2001)( [], (1991...1993), + (1994...1995), + (2000...2001), nil, StatementsNode(1996...1999)( [LocalVariableReadNode(1996...1999)(:bar, 1)] @@ -2535,13 +2296,7 @@ ProgramNode(0...3743)( "foo" ), [InNode(2013...2024)( - LocalVariableWriteNode(2016...2019)( - :bar, - 0, - nil, - (2016...2019), - nil - ), + LocalVariableTargetNode(2016...2019)(:bar, 0), nil, (2013...2015), (2020...2024) @@ -2700,7 +2455,12 @@ ProgramNode(0...3743)( "foo" ), [InNode(2196...2210)( - SymbolNode(2199...2205)(nil, (2201...2204), nil, "foo"), + SymbolNode(2199...2205)( + (2199...2201), + (2201...2204), + (2204...2205), + "foo" + ), nil, (2196...2198), (2206...2210) @@ -3146,6 +2906,8 @@ ProgramNode(0...3743)( LambdaNode(2727...2737)( [], (2727...2729), + (2730...2731), + (2736...2737), nil, StatementsNode(2732...2735)( [LocalVariableReadNode(2732...2735)(:bar, 1)] @@ -3176,13 +2938,7 @@ ProgramNode(0...3743)( (2765...2767), LocalVariableReadNode(2768...2771)(:baz, 0), StatementsNode(2761...2764)( - [LocalVariableWriteNode(2761...2764)( - :bar, - 0, - nil, - (2761...2764), - nil - )] + [LocalVariableTargetNode(2761...2764)(:bar, 0)] ), nil, nil @@ -3398,7 +3154,12 @@ ProgramNode(0...3743)( (3000...3002), LocalVariableReadNode(3003...3006)(:baz, 0), StatementsNode(2993...2999)( - [SymbolNode(2993...2999)(nil, (2995...2998), nil, "foo")] + [SymbolNode(2993...2999)( + (2993...2995), + (2995...2998), + (2998...2999), + "foo" + )] ), nil, nil @@ -3976,6 +3737,8 @@ ProgramNode(0...3743)( [LambdaNode(3647...3657)( [], (3647...3649), + (3650...3651), + (3656...3657), nil, StatementsNode(3652...3655)( [LocalVariableReadNode(3652...3655)(:bar, 1)] @@ -4035,7 +3798,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(3696...3703)( nil, - [LocalVariableWriteNode(3700...3701)(:b, 0, nil, (3700...3701), nil)], + [LocalVariableTargetNode(3700...3701)(:b, 0)], nil, [], (3696...3697), @@ -4068,13 +3831,7 @@ ProgramNode(0...3743)( (3734...3735), "value" ), - LocalVariableWriteNode(3736...3737)( - :a, - 0, - nil, - (3736...3737), - nil - ), + LocalVariableTargetNode(3736...3737)(:a, 0), nil )], nil, diff --git a/test/yarp/snapshots/procs.txt b/test/yarp/snapshots/procs.txt index 3db8ec5b1378d9..adfb0637079532 100644 --- a/test/yarp/snapshots/procs.txt +++ b/test/yarp/snapshots/procs.txt @@ -4,6 +4,8 @@ ProgramNode(0...266)( [LambdaNode(0...21)( [:a, :b, :c, :d], (0...2), + (16...17), + (20...21), BlockParametersNode(3...15)( ParametersNode(4...5)( [RequiredParameterNode(4...5)(:a)], @@ -23,6 +25,8 @@ ProgramNode(0...266)( LambdaNode(23...39)( [], (23...25), + (26...28), + (36...39), nil, BeginNode(29...39)( nil, @@ -36,6 +40,8 @@ ProgramNode(0...266)( LambdaNode(41...69)( [], (41...43), + (44...46), + (66...69), nil, BeginNode(47...69)( nil, @@ -49,6 +55,8 @@ ProgramNode(0...266)( LambdaNode(71...81)( [], (71...73), + (74...75), + (80...81), nil, StatementsNode(76...79)( [CallNode(76...79)(nil, nil, (76...79), nil, nil, nil, nil, 2, "foo")] @@ -57,6 +65,8 @@ ProgramNode(0...266)( LambdaNode(83...98)( [], (83...85), + (86...88), + (95...98), nil, StatementsNode(90...93)( [CallNode(90...93)(nil, nil, (90...93), nil, nil, nil, nil, 2, "foo")] @@ -65,6 +75,8 @@ ProgramNode(0...266)( LambdaNode(100...129)( [:a, :b, :c, :d, :e], (100...102), + (124...125), + (128...129), BlockParametersNode(103...123)( ParametersNode(103...123)( [RequiredParameterNode(103...104)(:a)], @@ -90,6 +102,8 @@ ProgramNode(0...266)( LambdaNode(131...171)( [:a, :b, :c, :d, :e, :f, :g], (131...133), + (166...167), + (170...171), BlockParametersNode(134...165)( ParametersNode(135...164)( [RequiredParameterNode(135...136)(:a)], @@ -115,6 +129,8 @@ ProgramNode(0...266)( LambdaNode(173...218)( [:a, :b, :c, :d, :e, :f, :g], (173...175), + (208...210), + (215...218), BlockParametersNode(176...207)( ParametersNode(177...206)( [RequiredParameterNode(177...178)(:a)], @@ -140,6 +156,8 @@ ProgramNode(0...266)( LambdaNode(220...245)( [:a], (220...222), + (227...228), + (244...245), BlockParametersNode(223...226)( ParametersNode(224...225)( [RequiredParameterNode(224...225)(:a)], @@ -158,6 +176,8 @@ ProgramNode(0...266)( [LambdaNode(229...243)( [:b], (229...231), + (234...235), + (242...243), BlockParametersNode(232...233)( ParametersNode(232...233)( [RequiredParameterNode(232...233)(:b)], @@ -193,6 +213,8 @@ ProgramNode(0...266)( LambdaNode(247...266)( [:a, :b, :c], (247...249), + (263...264), + (265...266), BlockParametersNode(250...262)( ParametersNode(251...261)( [RequiredDestructuredParameterNode(251...257)( diff --git a/test/yarp/snapshots/rescue.txt b/test/yarp/snapshots/rescue.txt index b8944afd19bcf2..c52a261fceee6f 100644 --- a/test/yarp/snapshots/rescue.txt +++ b/test/yarp/snapshots/rescue.txt @@ -192,6 +192,7 @@ ProgramNode(0...316)( LocalVariableWriteNode(217...235)( :a, 0, + (217...218), RescueModifierNode(221...235)( CallNode(221...224)( nil, @@ -207,7 +208,6 @@ ProgramNode(0...316)( (225...231), NilNode(232...235)() ), - (217...218), (219...220) ), StatementsNode(238...241)( diff --git a/test/yarp/snapshots/seattlerb/bug202.txt b/test/yarp/snapshots/seattlerb/bug202.txt index 7bb548f79830b6..cbed9a6712f87e 100644 --- a/test/yarp/snapshots/seattlerb/bug202.txt +++ b/test/yarp/snapshots/seattlerb/bug202.txt @@ -1,12 +1,12 @@ ProgramNode(0...22)( [:测试], StatementsNode(0...22)( - [GlobalVariableWriteNode(0...11)((0...7), (8...9), IntegerNode(10...11)()), + [GlobalVariableWriteNode(0...11)((0...7), IntegerNode(10...11)(), (8...9)), LocalVariableWriteNode(12...22)( :测试, 0, - IntegerNode(21...22)(), (12...18), + IntegerNode(21...22)(), (19...20) )] ) diff --git a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt index c672ef6b79bcae..2bd4f430e3549f 100644 --- a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt +++ b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt @@ -1,14 +1,16 @@ ProgramNode(0...18)( [:a], StatementsNode(0...18)( - [OrWriteNode(0...18)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...18)( + (0...1), + (2...5), RescueModifierNode(6...18)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (8...14), NilNode(15...18)() ), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt b/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt index 1d8c2f1472c4f1..c83adb235f72be 100644 --- a/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt +++ b/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt @@ -11,6 +11,8 @@ ProgramNode(0...18)( [LambdaNode(3...10)( [], (3...5), + (8...9), + (9...10), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), nil )], diff --git a/test/yarp/snapshots/seattlerb/call_colon_parens.txt b/test/yarp/snapshots/seattlerb/call_colon_parens.txt index 2e707afcfb6717..14c7247bcd47cb 100644 --- a/test/yarp/snapshots/seattlerb/call_colon_parens.txt +++ b/test/yarp/snapshots/seattlerb/call_colon_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...5)( [CallNode(0...5)( IntegerNode(0...1)(), (1...3), - (0...0), + nil, (3...4), nil, (4...5), diff --git a/test/yarp/snapshots/seattlerb/call_dot_parens.txt b/test/yarp/snapshots/seattlerb/call_dot_parens.txt index 5a32c3b9cf23a4..073336ed48525b 100644 --- a/test/yarp/snapshots/seattlerb/call_dot_parens.txt +++ b/test/yarp/snapshots/seattlerb/call_dot_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...4)( [CallNode(0...4)( IntegerNode(0...1)(), (1...2), - (0...0), + nil, (2...3), nil, (3...4), diff --git a/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt b/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt index e48578da2aad53..f541c529851ce2 100644 --- a/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt +++ b/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt @@ -10,6 +10,8 @@ ProgramNode(0...22)( [LambdaNode(2...13)( [], (2...4), + (5...7), + (10...13), nil, StatementsNode(8...9)([IntegerNode(8...9)()]) )] diff --git a/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt b/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt index 36fbd7fb1a8778..83c62571fd7c44 100644 --- a/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt +++ b/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt @@ -10,6 +10,8 @@ ProgramNode(0...19)( [LambdaNode(2...10)( [], (2...4), + (5...6), + (9...10), nil, StatementsNode(7...8)([IntegerNode(7...8)()]) )] diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index 8277b654c117ea..aaee410ef325bd 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -195,7 +195,7 @@ ProgramNode(0...747)( [SymbolNode(292...294)((292...293), (293...294), nil, "b")], SplatNode(296...298)( (296...297), - LocalVariableWriteNode(297...298)(:_, 0, nil, (297...298), nil) + LocalVariableTargetNode(297...298)(:_, 0) ), [SymbolNode(300...302)((300...301), (301...302), nil, "c")], nil, @@ -262,12 +262,12 @@ ProgramNode(0...747)( ConstantReadNode(369...375)(), SplatNode(376...380)( (376...377), - LocalVariableWriteNode(377...380)(:lhs, 0, nil, (377...380), nil) + LocalVariableTargetNode(377...380)(:lhs, 0) ), - [LocalVariableWriteNode(382...383)(:x, 0, nil, (382...383), nil)], + [LocalVariableTargetNode(382...383)(:x, 0)], SplatNode(385...389)( (385...386), - LocalVariableWriteNode(386...389)(:rhs, 0, nil, (386...389), nil) + LocalVariableTargetNode(386...389)(:rhs, 0) ), (375...376), (389...390) @@ -287,12 +287,12 @@ ProgramNode(0...747)( ConstantReadNode(407...413)(), SplatNode(414...418)( (414...415), - LocalVariableWriteNode(415...418)(:lhs, 0, nil, (415...418), nil) + LocalVariableTargetNode(415...418)(:lhs, 0) ), - [LocalVariableWriteNode(420...421)(:x, 0, nil, (420...421), nil)], + [LocalVariableTargetNode(420...421)(:x, 0)], SplatNode(423...427)( (423...424), - LocalVariableWriteNode(424...427)(:rhs, 0, nil, (424...427), nil) + LocalVariableTargetNode(424...427)(:rhs, 0) ), (413...414), (427...428) @@ -313,6 +313,8 @@ ProgramNode(0...747)( [LambdaNode(446...460)( [:b], (446...448), + (452...453), + (459...460), BlockParametersNode(448...451)( ParametersNode(449...450)( [RequiredParameterNode(449...450)(:b)], @@ -329,7 +331,7 @@ ProgramNode(0...747)( ), StatementsNode(454...458)([TrueNode(454...458)()]) ), - LocalVariableWriteNode(462...463)(:c, 0, nil, (462...463), nil)], + LocalVariableTargetNode(462...463)(:c, 0)], nil, [], (445...446), @@ -349,20 +351,14 @@ ProgramNode(0...747)( ArrayPatternNode(481...506)( nil, [SymbolNode(482...484)((482...483), (483...484), nil, "a"), - LocalVariableWriteNode(486...487)(:b, 0, nil, (486...487), nil), - LocalVariableWriteNode(489...490)(:c, 0, nil, (489...490), nil), + LocalVariableTargetNode(486...487)(:b, 0), + LocalVariableTargetNode(489...490)(:c, 0), ArrayPatternNode(492...505)( nil, [SymbolNode(493...495)((493...494), (494...495), nil, "d")], SplatNode(497...499)( (497...498), - LocalVariableWriteNode(498...499)( - :e, - 0, - nil, - (498...499), - nil - ) + LocalVariableTargetNode(498...499)(:e, 0) ), [NilNode(501...504)()], (492...493), @@ -408,13 +404,7 @@ ProgramNode(0...747)( [ArrayPatternNode(550...557)( nil, [SymbolNode(551...553)((551...552), (552...553), nil, "b"), - LocalVariableWriteNode(555...556)( - :c, - 0, - nil, - (555...556), - nil - )], + LocalVariableTargetNode(555...556)(:c, 0)], nil, [], (550...551), @@ -504,7 +494,7 @@ ProgramNode(0...747)( ArrayPatternNode(627...643)( nil, [PinnedVariableNode(628...631)( - InstanceVariableReadNode(629...631)(), + InstanceVariableReadNode(629...631)(:@a), (628...629) ), PinnedVariableNode(633...636)( @@ -512,7 +502,7 @@ ProgramNode(0...747)( (633...634) ), PinnedVariableNode(638...642)( - ClassVariableReadNode(639...642)(), + ClassVariableReadNode(639...642)(:@@c), (638...639) )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_31.txt b/test/yarp/snapshots/seattlerb/case_in_31.txt index 26ff5c913fa23c..c34b271b848e76 100644 --- a/test/yarp/snapshots/seattlerb/case_in_31.txt +++ b/test/yarp/snapshots/seattlerb/case_in_31.txt @@ -9,7 +9,7 @@ ProgramNode(0...28)( [SymbolNode(12...14)((12...13), (13...14), nil, "b")], SplatNode(16...18)( (16...17), - LocalVariableWriteNode(17...18)(:c, 0, nil, (17...18), nil) + LocalVariableTargetNode(17...18)(:c, 0) ), [], (11...12), diff --git a/test/yarp/snapshots/seattlerb/case_in_42.txt b/test/yarp/snapshots/seattlerb/case_in_42.txt index 7cd17bb0e1ca50..b98fd9f4b78f60 100644 --- a/test/yarp/snapshots/seattlerb/case_in_42.txt +++ b/test/yarp/snapshots/seattlerb/case_in_42.txt @@ -9,7 +9,7 @@ ProgramNode(0...30)( [SymbolNode(11...13)((11...12), (12...13), nil, "b")], SplatNode(15...17)( (15...16), - LocalVariableWriteNode(16...17)(:_, 0, nil, (16...17), nil) + LocalVariableTargetNode(16...17)(:_, 0) ), [], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_42_2.txt b/test/yarp/snapshots/seattlerb/case_in_42_2.txt index 7c0c87e98c03a4..945be4ddcf574a 100644 --- a/test/yarp/snapshots/seattlerb/case_in_42_2.txt +++ b/test/yarp/snapshots/seattlerb/case_in_42_2.txt @@ -9,7 +9,7 @@ ProgramNode(0...32)( [], SplatNode(13...18)( (13...14), - LocalVariableWriteNode(14...18)(:list, 0, nil, (14...18), nil) + LocalVariableTargetNode(14...18)(:list, 0) ), [], (12...13), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt index b8eee482039b25..d503434f7e5ec7 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt @@ -6,7 +6,7 @@ ProgramNode(0...24)( [InNode(8...20)( ArrayPatternNode(11...15)( ConstantReadNode(11...12)(), - [LocalVariableWriteNode(13...14)(:c, 0, nil, (13...14), nil)], + [LocalVariableTargetNode(13...14)(:c, 0)], nil, [], (12...13), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt index 2e29ef9d16c9ec..828d83dd52dc92 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt @@ -10,7 +10,7 @@ ProgramNode(0...27)( ConstantReadNode(14...15)(), (12...14) ), - [LocalVariableWriteNode(16...17)(:d, 0, nil, (16...17), nil)], + [LocalVariableTargetNode(16...17)(:d, 0)], nil, [], (15...16), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt index 85b069fb001cfb..c4e1380f7b1748 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt @@ -8,7 +8,7 @@ ProgramNode(0...29)( ConstantReadNode(11...12)(), [CapturePatternNode(13...19)( ConstantReadNode(13...14)(), - LocalVariableWriteNode(18...19)(:d, 0, nil, (18...19), nil), + LocalVariableTargetNode(18...19)(:d, 0), (15...17) )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_find.txt b/test/yarp/snapshots/seattlerb/case_in_find.txt index 2d366423694d2a..3c8e6442448236 100644 --- a/test/yarp/snapshots/seattlerb/case_in_find.txt +++ b/test/yarp/snapshots/seattlerb/case_in_find.txt @@ -8,12 +8,12 @@ ProgramNode(0...27)( nil, SplatNode(13...15)( (13...14), - LocalVariableWriteNode(14...15)(:a, 0, nil, (14...15), nil) + LocalVariableTargetNode(14...15)(:a, 0) ), [SymbolNode(17...19)((17...18), (18...19), nil, "+")], SplatNode(21...23)( (21...22), - LocalVariableWriteNode(22...23)(:b, 0, nil, (22...23), nil) + LocalVariableTargetNode(22...23)(:b, 0) ), nil, nil diff --git a/test/yarp/snapshots/seattlerb/case_in_find_array.txt b/test/yarp/snapshots/seattlerb/case_in_find_array.txt index 410fda576aa5ca..616c8f864bc634 100644 --- a/test/yarp/snapshots/seattlerb/case_in_find_array.txt +++ b/test/yarp/snapshots/seattlerb/case_in_find_array.txt @@ -8,7 +8,7 @@ ProgramNode(0...28)( nil, SplatNode(12...13)((12...13), nil), [SymbolNode(15...17)((15...16), (16...17), nil, "b"), - LocalVariableWriteNode(19...20)(:c, 0, nil, (19...20), nil)], + LocalVariableTargetNode(19...20)(:c, 0)], SplatNode(22...23)((22...23), nil), (11...12), (23...24) diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt index 92933ca0718055..6c6c9e8b675c2b 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt @@ -10,7 +10,7 @@ ProgramNode(0...56)( SymbolNode(13...15)(nil, (13...14), (14...15), "b"), CapturePatternNode(16...28)( ConstantReadNode(16...23)(), - LocalVariableWriteNode(27...28)(:x, 0, nil, (27...28), nil), + LocalVariableTargetNode(27...28)(:x, 0), (24...26) ), nil diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt index ed7b16685280e2..9998de3cfe6b15 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt @@ -8,11 +8,11 @@ ProgramNode(0...35)( nil, [AssocNode(11...15)( SymbolNode(11...13)(nil, (11...12), (12...13), "b"), - LocalVariableWriteNode(14...15)(:c, 0, nil, (14...15), nil), + LocalVariableTargetNode(14...15)(:c, 0), nil ), AssocSplatNode(17...23)( - LocalVariableWriteNode(19...23)(:rest, 0, nil, (19...23), nil), + LocalVariableTargetNode(19...23)(:rest, 0), (17...19) )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt index 7bdc02e6fb8af4..7d6634c7f8d3d6 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt @@ -7,7 +7,7 @@ ProgramNode(0...29)( HashPatternNode(11...17)( nil, [AssocSplatNode(11...17)( - LocalVariableWriteNode(13...17)(:rest, 0, nil, (13...17), nil), + LocalVariableTargetNode(13...17)(:rest, 0), (11...13) )], nil, diff --git a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt index 7405ad8e5d9d22..48a4d61f5247b7 100644 --- a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt +++ b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt @@ -1,18 +1,14 @@ ProgramNode(0...12)( [], StatementsNode(0...12)( - [OrWriteNode(0...12)( - ConstantPathWriteNode(0...6)( - ConstantPathNode(0...6)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - ConstantReadNode(5...6)(), - (3...5) - ), - nil, - nil + [ConstantPathOrWriteNode(0...12)( + ConstantPathNode(0...6)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + ConstantReadNode(5...6)(), + (3...5) ), - IntegerNode(11...12)(), - (7...10) + (7...10), + IntegerNode(11...12)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt index 838287c59fa9f7..327e0a96431f1a 100644 --- a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt @@ -1,14 +1,10 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [OrWriteNode(0...9)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - IntegerNode(8...9)(), - (4...7) + [ConstantPathOrWriteNode(0...9)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + (4...7), + IntegerNode(8...9)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt index 2fdcd97205fd82..1e7dce022689a5 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt @@ -1,15 +1,11 @@ ProgramNode(0...8)( [], StatementsNode(0...8)( - [OperatorWriteNode(0...8)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), + [ConstantPathOperatorWriteNode(0...8)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), (4...6), - :&, - IntegerNode(7...8)() + IntegerNode(7...8)(), + :& )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt index 275aa8238d4735..8c9d3ac353a61a 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt @@ -1,14 +1,10 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [AndWriteNode(0...9)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - IntegerNode(8...9)(), - (4...7) + [ConstantPathAndWriteNode(0...9)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + (4...7), + IntegerNode(8...9)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt index c14175c4e3a8a2..d812fe5a55e675 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt @@ -1,18 +1,14 @@ ProgramNode(0...10)( [], StatementsNode(0...10)( - [OrWriteNode(0...10)( - ConstantPathWriteNode(0...4)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) - ), - nil, - nil + [ConstantPathOrWriteNode(0...10)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) ), - IntegerNode(9...10)(), - (5...8) + (5...8), + IntegerNode(9...10)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt index 26d5ca491c6b6e..8fbbf8ec0ab0e3 100644 --- a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt +++ b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt @@ -15,8 +15,8 @@ ProgramNode(0...76)( [LocalVariableWriteNode(7...14)( :v, 0, - NilNode(11...14)(), (7...8), + NilNode(11...14)(), (9...10) ), BeginNode(17...72)( @@ -28,7 +28,7 @@ ProgramNode(0...76)( (35...41), [ConstantReadNode(42...51)()], (52...54), - LocalVariableWriteNode(55...56)(:v, 0, nil, (55...56), nil), + LocalVariableTargetNode(55...56)(:v, 0), StatementsNode(61...66)([BreakNode(61...66)(nil, (61...66))]), nil ), diff --git a/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt b/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt index 86433437ce1ea6..d19d6d817b6e7d 100644 --- a/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt +++ b/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt @@ -21,8 +21,8 @@ ProgramNode(0...48)( LocalVariableWriteNode(24...29)( :b, 0, - IntegerNode(28...29)(), (24...25), + IntegerNode(28...29)(), (26...27) ), CallNode(32...35)( @@ -39,8 +39,8 @@ ProgramNode(0...48)( LocalVariableWriteNode(38...42)( :c, 0, - IntegerNode(41...42)(), (38...39), + IntegerNode(41...42)(), (40...41) )] ), diff --git a/test/yarp/snapshots/seattlerb/difficult3_4.txt b/test/yarp/snapshots/seattlerb/difficult3_4.txt index afca77c5fa310a..40ebfca6612174 100644 --- a/test/yarp/snapshots/seattlerb/difficult3_4.txt +++ b/test/yarp/snapshots/seattlerb/difficult3_4.txt @@ -4,6 +4,7 @@ ProgramNode(0...17)( [LocalVariableWriteNode(0...17)( :a, 0, + (0...1), IfNode(2...17)( nil, CallNode(2...3)(nil, nil, (2...3), nil, nil, nil, nil, 2, "b"), @@ -15,7 +16,6 @@ ProgramNode(0...17)( ), nil ), - (0...1), (1...2) )] ) diff --git a/test/yarp/snapshots/seattlerb/difficult3_5.txt b/test/yarp/snapshots/seattlerb/difficult3_5.txt index e7aaafc87a1261..746786b8a5fb10 100644 --- a/test/yarp/snapshots/seattlerb/difficult3_5.txt +++ b/test/yarp/snapshots/seattlerb/difficult3_5.txt @@ -10,6 +10,8 @@ ProgramNode(0...19)( [LambdaNode(2...19)( [], (2...4), + (7...8), + (18...19), BlockParametersNode(4...6)(nil, [], (4...5), (5...6)), StatementsNode(9...17)( [CallNode(9...17)( diff --git a/test/yarp/snapshots/seattlerb/difficult6_.txt b/test/yarp/snapshots/seattlerb/difficult6_.txt index 20e419bbc45455..2d1677e108f7e9 100644 --- a/test/yarp/snapshots/seattlerb/difficult6_.txt +++ b/test/yarp/snapshots/seattlerb/difficult6_.txt @@ -4,6 +4,8 @@ ProgramNode(0...25)( [LambdaNode(0...25)( [:a, :b], (0...2), + (13...14), + (24...25), BlockParametersNode(2...12)( ParametersNode(3...11)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/seattlerb/do_lambda.txt b/test/yarp/snapshots/seattlerb/do_lambda.txt index 86b3548eb4b278..a7eb211623e8fe 100644 --- a/test/yarp/snapshots/seattlerb/do_lambda.txt +++ b/test/yarp/snapshots/seattlerb/do_lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [], (0...2), + (5...7), + (8...11), BlockParametersNode(2...4)(nil, [], (2...3), (3...4)), nil )] diff --git a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt index 16cd806d018f85..4f6697237b1a70 100644 --- a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt @@ -1,4 +1,6 @@ ProgramNode(0...17)( [], - StatementsNode(0...17)([SymbolNode(0...17)(nil, (2...16), nil, "Varietà")]) + StatementsNode(0...17)( + [SymbolNode(0...17)((0...2), (2...16), (16...17), "Varietà")] + ) ) diff --git a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt index f2e3452dc12e9b..d383b67184c1dc 100644 --- a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt @@ -2,8 +2,8 @@ ProgramNode(0...32)( [], StatementsNode(0...32)( [AliasNode(0...17)( - SymbolNode(6...11)(nil, (8...10), nil, "<<"), - SymbolNode(12...17)(nil, (14...16), nil, ">>"), + SymbolNode(6...11)((6...8), (8...10), (10...11), "<<"), + SymbolNode(12...17)((12...14), (14...16), (16...17), ">>"), (0...5) ), AliasNode(19...32)( diff --git a/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt b/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt index d8a5d9354f388a..befdff81ee76ce 100644 --- a/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt +++ b/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt @@ -4,12 +4,12 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :str, 0, + (0...3), InterpolatedStringNode(6...12)( (6...12), [StringNode(14...30)(nil, (14...30), nil, "beforeafter\r\n")], (30...35) ), - (0...3), (4...5) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt b/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt index 4bc1846ef128c1..842da16d1d52f6 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt @@ -4,12 +4,12 @@ ProgramNode(0...9)( [LocalVariableWriteNode(0...9)( :s, 0, + (0...1), InterpolatedStringNode(4...9)( (4...9), [StringNode(10...17)(nil, (10...17), nil, "a\xE9b\n")], (17...21) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt b/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt index ebfb3b4da35c24..14c02bdfb33c37 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :s, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...23)(nil, (11...23), nil, "a\xA7b\n" + "cöd\n")], (23...27) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_lineno.txt b/test/yarp/snapshots/seattlerb/heredoc_lineno.txt index 27708e25381e78..9bbc817c20f30d 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_lineno.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_lineno.txt @@ -4,6 +4,7 @@ ProgramNode(0...41)( [LocalVariableWriteNode(0...11)( :c, 0, + (0...1), InterpolatedStringNode(4...11)( (4...11), [StringNode(12...30)( @@ -14,14 +15,13 @@ ProgramNode(0...41)( )], (30...34) ), - (0...1), (2...3) ), LocalVariableWriteNode(35...41)( :d, 0, - IntegerNode(39...41)(), (35...36), + IntegerNode(39...41)(), (37...38) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt index a1013d9e84e231..6491c86c04c5aa 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt @@ -4,12 +4,12 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...25)(nil, (13...25), nil, "x\n" + "y\n" + "z\n")], (25...31) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt index e78764e926461f..f2c3c1b1a902b9 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt @@ -4,6 +4,7 @@ ProgramNode(0...20)( [LocalVariableWriteNode(0...20)( :a, 0, + (0...1), CallNode(4...20)( nil, nil, @@ -49,7 +50,6 @@ ProgramNode(0...20)( 0, "foo" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt index 3bb7807c5c6d6a..4b59c352461881 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...20)(nil, (11...20), nil, "x\n" + "\n" + "z\n")], (20...24) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt index f895e997e7ca18..e03304a7d147b0 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt @@ -4,6 +4,7 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...22)(nil, (11...22), nil, " w\n" + "x"), @@ -15,7 +16,6 @@ ProgramNode(0...10)( StringNode(27...36)(nil, (27...36), nil, " y\n" + " z\n")], (36...42) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt index 13d1844e3c7425..1592d1310476e9 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...43)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( )], (43...49) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt index da631943fceab6..63ff5887c9e25e 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...37)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( )], (37...43) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt index 680ffbe0a68dd0..866c5fac084cd5 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...21)(nil, (11...21), nil, "x\n" + "\n" + "z\n")], (21...25) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt index cfb1f9644370c7..452e5b605b8ef5 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(6...11)(nil, (6...11), nil, "foo\r"), EmbeddedVariableNode(11...16)( (11...12), - InstanceVariableReadNode(12...16)() + InstanceVariableReadNode(12...16)(:@bar) ), StringNode(16...17)(nil, (16...17), nil, "\n")], (17...21) diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt index ee013f51378747..99b42f6122a97e 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(7...12)(nil, (7...12), nil, "foo\r"), EmbeddedVariableNode(12...17)( (12...13), - InstanceVariableReadNode(13...17)() + InstanceVariableReadNode(13...17)(:@bar) ), StringNode(17...19)(nil, (17...19), nil, "\r\n")], (19...24) diff --git a/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt b/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt index 1c9e2efc33940f..c7f83a9b512023 100644 --- a/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt +++ b/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt @@ -6,7 +6,9 @@ ProgramNode(0...46)( nil, (0...1), nil, - ArgumentsNode(2...11)([LambdaNode(2...11)([], (2...4), nil, nil)]), + ArgumentsNode(2...11)( + [LambdaNode(2...11)([], (2...4), (5...7), (8...11), nil, nil)] + ), nil, nil, 0, @@ -17,7 +19,9 @@ ProgramNode(0...46)( nil, (13...14), nil, - ArgumentsNode(15...20)([LambdaNode(15...20)([], (15...17), nil, nil)]), + ArgumentsNode(15...20)( + [LambdaNode(15...20)([], (15...17), (18...19), (19...20), nil, nil)] + ), nil, nil, 0, @@ -32,6 +36,8 @@ ProgramNode(0...46)( [LambdaNode(24...35)( [], (24...26), + (29...31), + (32...35), BlockParametersNode(26...28)(nil, [], (26...27), (27...28)), nil )] @@ -50,6 +56,8 @@ ProgramNode(0...46)( [LambdaNode(39...46)( [], (39...41), + (44...45), + (45...46), BlockParametersNode(41...43)(nil, [], (41...42), (42...43)), nil )] diff --git a/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt index 03ec05d3a348c0..ad36a5edb4eab8 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt @@ -4,12 +4,12 @@ ProgramNode(0...14)( [LocalVariableWriteNode(0...14)( :a, 0, + (0...1), RescueModifierNode(4...14)( IntegerNode(4...5)(), (6...12), IntegerNode(13...14)() ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt index c28b8c2178ac29..86469f1c9129e5 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt @@ -4,6 +4,7 @@ ProgramNode(0...17)( [LocalVariableWriteNode(0...17)( :a, 0, + (0...1), RescueModifierNode(4...17)( CallNode(4...8)( nil, @@ -19,7 +20,6 @@ ProgramNode(0...17)( (9...15), IntegerNode(16...17)() ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt index bfb7c9b678854d..72e54e010160ce 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt @@ -4,6 +4,7 @@ ProgramNode(0...16)( [LocalVariableWriteNode(0...16)( :a, 0, + (0...1), CallNode(4...16)( nil, nil, @@ -21,7 +22,6 @@ ProgramNode(0...16)( 0, "b" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_command.txt b/test/yarp/snapshots/seattlerb/lasgn_command.txt index 696be7871609cc..b1912a7c6d22cf 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_command.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_command.txt @@ -4,6 +4,7 @@ ProgramNode(0...9)( [LocalVariableWriteNode(0...9)( :a, 0, + (0...1), CallNode(4...9)( CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "b"), (5...6), @@ -15,7 +16,6 @@ ProgramNode(0...9)( 0, "c" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_env.txt b/test/yarp/snapshots/seattlerb/lasgn_env.txt index 33f2f4701c53f5..cfd276b1b28557 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_env.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_env.txt @@ -4,8 +4,8 @@ ProgramNode(0...6)( [LocalVariableWriteNode(0...6)( :a, 0, - IntegerNode(4...6)(), (0...1), + IntegerNode(4...6)(), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt index b71eeaf2a7916d..cf47f6dc7e5ded 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt @@ -1,6 +1,11 @@ ProgramNode(0...7)( [], StatementsNode(0...7)( - [InstanceVariableWriteNode(0...7)((0...2), IntegerNode(5...7)(), (3...4))] + [InstanceVariableWriteNode(0...7)( + :@a, + (0...2), + IntegerNode(5...7)(), + (3...4) + )] ) ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt b/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt index 4c272c16808a7d..49f22eeb4311b1 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt @@ -4,9 +4,11 @@ ProgramNode(0...11)( [LocalVariableWriteNode(0...11)( :a, 0, + (0...1), LocalVariableWriteNode(4...11)( :b, 0, + (4...5), CallNode(8...11)( nil, nil, @@ -18,10 +20,8 @@ ProgramNode(0...11)( 0, "c" ), - (4...5), (6...7) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt b/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt index 623f6dc1257f95..acb6c844673f99 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), ArrayNode(4...12)( [CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "b"), SplatNode(7...9)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( nil, nil ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt index 08c8d9ea3fca24..b8b1ec343d09c9 100644 --- a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt +++ b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt @@ -16,8 +16,8 @@ ProgramNode(18...90)( [LocalVariableWriteNode(67...81)( :così, 0, - SymbolNode(75...81)((75...76), (76...81), nil, "però"), (67...72), + SymbolNode(75...81)((75...76), (76...81), nil, "però"), (73...74) )] ), diff --git a/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt index 738ee25165893a..c57ea8778fa388 100644 --- a/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt @@ -9,7 +9,7 @@ ProgramNode(0...8)( nil, nil ), - LocalVariableWriteNode(3...4)(:a, 0, nil, (3...4), nil)], + LocalVariableTargetNode(3...4)(:a, 0)], (5...6), CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "b"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt b/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt index 1b1c0f43badc08..1a8275332e534b 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt @@ -2,7 +2,7 @@ ProgramNode(0...11)( [:a], StatementsNode(0...11)( [MultiWriteNode(0...11)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), CallNode(3...7)( CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), (4...6), diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt b/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt index 53ecd8fd183fc6..da590680c034a3 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt @@ -2,7 +2,7 @@ ProgramNode(0...10)( [:a], StatementsNode(0...10)( [MultiWriteNode(0...10)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), CallNode(3...6)( CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), (4...5), diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt index 285c22f660ba87..c22781a5d01f29 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt @@ -2,12 +2,9 @@ ProgramNode(0...12)( [:a, :b, :c], StatementsNode(0...12)( [MultiWriteNode(0...12)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - SplatNode(3...5)( - (3...4), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil) - ), - LocalVariableWriteNode(7...8)(:c, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + SplatNode(3...5)((3...4), LocalVariableTargetNode(4...5)(:b, 0)), + LocalVariableTargetNode(7...8)(:c, 0)], (9...10), CallNode(11...12)(nil, nil, (11...12), nil, nil, nil, nil, 2, "d"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_colon2.txt b/test/yarp/snapshots/seattlerb/masgn_colon2.txt index 490d1657b03762..99288bf229f285 100644 --- a/test/yarp/snapshots/seattlerb/masgn_colon2.txt +++ b/test/yarp/snapshots/seattlerb/masgn_colon2.txt @@ -2,15 +2,11 @@ ProgramNode(0...14)( [:a], StatementsNode(0...14)( [MultiWriteNode(0...14)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - ConstantPathWriteNode(3...7)( - ConstantPathNode(3...7)( - CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), - ConstantReadNode(6...7)(), - (4...6) - ), - nil, - nil + [LocalVariableTargetNode(0...1)(:a, 0), + ConstantPathTargetNode(3...7)( + CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), + ConstantReadNode(6...7)(), + (4...6) )], (8...9), ArrayNode(10...14)( diff --git a/test/yarp/snapshots/seattlerb/masgn_colon3.txt b/test/yarp/snapshots/seattlerb/masgn_colon3.txt index f02dcc1f50756e..abb7125505e734 100644 --- a/test/yarp/snapshots/seattlerb/masgn_colon3.txt +++ b/test/yarp/snapshots/seattlerb/masgn_colon3.txt @@ -2,15 +2,11 @@ ProgramNode(0...15)( [], StatementsNode(0...15)( [MultiWriteNode(0...15)( - [ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + [ConstantPathTargetNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + ConstantPathTargetNode(5...8)( nil, - nil - ), - ConstantPathWriteNode(5...8)( - ConstantPathNode(5...8)(nil, ConstantReadNode(7...8)(), (5...7)), - nil, - nil + ConstantReadNode(7...8)(), + (5...7) )], (9...10), ArrayNode(11...15)( diff --git a/test/yarp/snapshots/seattlerb/masgn_command_call.txt b/test/yarp/snapshots/seattlerb/masgn_command_call.txt index ef2eab9d34ed0a..edb11e8dc07c49 100644 --- a/test/yarp/snapshots/seattlerb/masgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/masgn_command_call.txt @@ -2,8 +2,7 @@ ProgramNode(0...10)( [:a], StatementsNode(0...10)( [MultiWriteNode(0...10)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - SplatNode(1...2)((1...2), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), SplatNode(1...2)((1...2), nil)], (3...4), CallNode(5...10)( CallNode(5...6)(nil, nil, (5...6), nil, nil, nil, nil, 2, "b"), diff --git a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt index 4f19128a219d20..3d0c2cf7ab6fbd 100644 --- a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt @@ -1,10 +1,10 @@ -ProgramNode(2...9)( +ProgramNode(0...9)( [:a, :b], - StatementsNode(2...9)( - [MultiWriteNode(2...9)( - [MultiWriteNode(2...5)( - [LocalVariableWriteNode(2...3)(:a, 0, nil, (2...3), nil), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + StatementsNode(0...9)( + [MultiWriteNode(0...9)( + [MultiWriteNode(1...6)( + [LocalVariableTargetNode(2...3)(:a, 0), + LocalVariableTargetNode(4...5)(:b, 0)], nil, nil, (1...2), diff --git a/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt b/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt index e8606e7152857b..0fe5b400fae43b 100644 --- a/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt +++ b/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt @@ -2,10 +2,7 @@ ProgramNode(0...12)( [:a], StatementsNode(0...12)( [MultiWriteNode(0...12)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], (3...4), ArrayNode(5...12)( [IntegerNode(5...6)(), IntegerNode(8...9)(), IntegerNode(11...12)()], diff --git a/test/yarp/snapshots/seattlerb/masgn_paren.txt b/test/yarp/snapshots/seattlerb/masgn_paren.txt index b47b00c417f004..91dd386d8c4a35 100644 --- a/test/yarp/snapshots/seattlerb/masgn_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_paren.txt @@ -1,9 +1,9 @@ -ProgramNode(1...12)( +ProgramNode(0...12)( [:a, :b], - StatementsNode(1...12)( - [MultiWriteNode(1...12)( - [LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + StatementsNode(0...12)( + [MultiWriteNode(0...12)( + [LocalVariableTargetNode(1...2)(:a, 0), + LocalVariableTargetNode(4...5)(:b, 0)], (7...8), CallNode(9...12)( CallNode(9...10)(nil, nil, (9...10), nil, nil, nil, nil, 2, "c"), diff --git a/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt index 21844a7c542b26..2832f367da1d7f 100644 --- a/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt @@ -3,16 +3,13 @@ ProgramNode(0...9)( StatementsNode(0...9)( [MultiWriteNode(0...9)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + LocalVariableTargetNode(4...5)(:b, 0)], (6...7), CallNode(8...9)(nil, nil, (8...9), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt b/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt index af7f0b1b35721f..9046d08d2bb091 100644 --- a/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt @@ -3,17 +3,14 @@ ProgramNode(0...12)( StatementsNode(0...12)( [MultiWriteNode(0...12)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:c, 0, nil, (7...8), nil)], + LocalVariableTargetNode(4...5)(:b, 0), + LocalVariableTargetNode(7...8)(:c, 0)], (9...10), CallNode(11...12)(nil, nil, (11...12), nil, nil, nil, nil, 2, "d"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt b/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt index b2ce2af1738528..64d21d637316be 100644 --- a/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt +++ b/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt @@ -2,9 +2,9 @@ ProgramNode(0...11)( [:a, :b], StatementsNode(0...11)( [MultiWriteNode(0...11)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), SplatNode(3...4)((3...4), nil), - LocalVariableWriteNode(6...7)(:b, 0, nil, (6...7), nil)], + LocalVariableTargetNode(6...7)(:b, 0)], (8...9), CallNode(10...11)(nil, nil, (10...11), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt index 8c678e890d92d8..71a28870bd2240 100644 --- a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt +++ b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt @@ -9,18 +9,13 @@ ProgramNode(0...15)( ArgumentsNode(2...15)( [ParenthesesNode(2...15)( StatementsNode(3...14)( - [OperatorWriteNode(3...14)( - ConstantPathWriteNode(3...7)( - ConstantPathNode(3...7)( - ConstantReadNode(3...4)(), - ConstantReadNode(6...7)(), - (4...6) - ), - nil, - nil + [ConstantPathOperatorWriteNode(3...14)( + ConstantPathNode(3...7)( + ConstantReadNode(3...4)(), + ConstantReadNode(6...7)(), + (4...6) ), (8...10), - :*, CallNode(11...14)( nil, nil, @@ -43,7 +38,8 @@ ProgramNode(0...15)( nil, 0, "d" - ) + ), + :* )] ), (2...3), diff --git a/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt index c046218e1ee69e..ba484f89b0374a 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt @@ -2,9 +2,9 @@ ProgramNode(0...14)( [:a, :b, :c], StatementsNode(0...14)( [MultiWriteNode(0...14)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), SplatNode(9...10)((9...10), nil)], (11...12), CallNode(13...14)(nil, nil, (13...14), nil, nil, nil, nil, 2, "f"), diff --git a/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt index 1ba49176842da2..9578258257e340 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt @@ -2,13 +2,10 @@ ProgramNode(0...15)( [:a, :b, :c, :s], StatementsNode(0...15)( [MultiWriteNode(0...15)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), - SplatNode(9...11)( - (9...10), - LocalVariableWriteNode(10...11)(:s, 0, nil, (10...11), nil) - )], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), + SplatNode(9...11)((9...10), LocalVariableTargetNode(10...11)(:s, 0))], (12...13), CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt index b035fbb8b7d883..8ed6e38ebdac89 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt @@ -9,9 +9,9 @@ ProgramNode(0...14)( nil, nil ), - LocalVariableWriteNode(3...4)(:x, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:y, 0, nil, (6...7), nil), - LocalVariableWriteNode(9...10)(:z, 0, nil, (9...10), nil)], + LocalVariableTargetNode(3...4)(:x, 0), + LocalVariableTargetNode(6...7)(:y, 0), + LocalVariableTargetNode(9...10)(:z, 0)], (11...12), CallNode(13...14)(nil, nil, (13...14), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt index 653b0e167ff5e8..07866df9edc139 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt @@ -3,18 +3,15 @@ ProgramNode(0...15)( StatementsNode(0...15)( [MultiWriteNode(0...15)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:s, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:s, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:x, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:y, 0, nil, (7...8), nil), - LocalVariableWriteNode(10...11)(:z, 0, nil, (10...11), nil)], + LocalVariableTargetNode(4...5)(:x, 0), + LocalVariableTargetNode(7...8)(:y, 0), + LocalVariableTargetNode(10...11)(:z, 0)], (12...13), CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt index 313a4590f7ff5d..ce114d05287340 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt @@ -2,13 +2,13 @@ ProgramNode(0...23)( [:a, :b, :c, :x, :y, :z], StatementsNode(0...23)( [MultiWriteNode(0...23)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), SplatNode(9...10)((9...10), nil), - LocalVariableWriteNode(12...13)(:x, 0, nil, (12...13), nil), - LocalVariableWriteNode(15...16)(:y, 0, nil, (15...16), nil), - LocalVariableWriteNode(18...19)(:z, 0, nil, (18...19), nil)], + LocalVariableTargetNode(12...13)(:x, 0), + LocalVariableTargetNode(15...16)(:y, 0), + LocalVariableTargetNode(18...19)(:z, 0)], (20...21), CallNode(22...23)(nil, nil, (22...23), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt index 5e307e81944223..672de03089e4be 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt @@ -2,16 +2,13 @@ ProgramNode(0...24)( [:a, :b, :c, :s, :x, :y, :z], StatementsNode(0...24)( [MultiWriteNode(0...24)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), - SplatNode(9...11)( - (9...10), - LocalVariableWriteNode(10...11)(:s, 0, nil, (10...11), nil) - ), - LocalVariableWriteNode(13...14)(:x, 0, nil, (13...14), nil), - LocalVariableWriteNode(16...17)(:y, 0, nil, (16...17), nil), - LocalVariableWriteNode(19...20)(:z, 0, nil, (19...20), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), + SplatNode(9...11)((9...10), LocalVariableTargetNode(10...11)(:s, 0)), + LocalVariableTargetNode(13...14)(:x, 0), + LocalVariableTargetNode(16...17)(:y, 0), + LocalVariableTargetNode(19...20)(:z, 0)], (21...22), CallNode(23...24)(nil, nil, (23...24), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_rescue.txt b/test/yarp/snapshots/seattlerb/mlhs_rescue.txt index 13d418461bccb0..3e1cd57291d87b 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_rescue.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_rescue.txt @@ -2,8 +2,8 @@ ProgramNode(0...18)( [:a, :b], StatementsNode(0...18)( [MultiWriteNode(0...18)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0)], (5...6), RescueModifierNode(7...18)( CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "f"), diff --git a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt index deb7e17066b389..cf092dd5de2cfc 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt @@ -1,8 +1,9 @@ ProgramNode(0...11)( [:a], StatementsNode(0...11)( - [OrWriteNode(0...11)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...11)( + (0...1), + (2...5), CallNode(6...11)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (7...8), @@ -14,7 +15,8 @@ ProgramNode(0...11)( 0, "c" ), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt index d071a45442fe3c..0cd3775202dff6 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt @@ -1,18 +1,13 @@ ProgramNode(0...11)( [], StatementsNode(0...11)( - [OperatorWriteNode(0...11)( - ConstantPathWriteNode(0...4)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) - ), - nil, - nil + [ConstantPathOperatorWriteNode(0...11)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) ), (5...7), - :*, CallNode(8...11)( nil, nil, @@ -25,7 +20,8 @@ ProgramNode(0...11)( nil, 0, "c" - ) + ), + :* )] ) ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_block.txt b/test/yarp/snapshots/seattlerb/parse_line_block.txt index 51282de9773dc5..7ca169a86b5cec 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_block.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_block.txt @@ -4,8 +4,8 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...6)( :a, 0, - IntegerNode(4...6)(), (0...1), + IntegerNode(4...6)(), (2...3) ), CallNode(7...10)( diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt index b7cca1732b5225..84ee2d450bd7aa 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt @@ -6,7 +6,7 @@ ProgramNode(0...4)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), nil, nil, 0, diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt index ae67416e67a765..1f9d067c76bf7d 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt @@ -6,7 +6,7 @@ ProgramNode(0...6)( nil, (0...1), (1...2), - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), (5...6), nil, 0, diff --git a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt index 5d4820bef3cf7f..170b9bc81e321e 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt @@ -25,11 +25,13 @@ ProgramNode(0...40)( 0, "p" ), - OperatorWriteNode(18...24)( - LocalVariableWriteNode(18...19)(:y, 0, nil, (18...19), nil), + LocalVariableOperatorWriteNode(18...24)( + (18...19), (20...22), + IntegerNode(23...24)(), + :y, :*, - IntegerNode(23...24)() + 0 ), ReturnNode(27...35)( (27...33), diff --git a/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt b/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt index 8e41b1648bc6f3..955d94e98fb289 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt @@ -4,6 +4,7 @@ ProgramNode(6...88)( [LocalVariableWriteNode(6...31)( :string, 0, + (6...12), CallNode(15...31)( InterpolatedStringNode(15...25)( (15...25), @@ -24,7 +25,6 @@ ProgramNode(6...88)( 0, "strip" ), - (6...12), (13...14) ), CallNode(77...88)( diff --git a/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt b/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt index b3ed88d90eb66d..3d52fcbff0cec7 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt @@ -4,6 +4,7 @@ ProgramNode(6...74)( [LocalVariableWriteNode(6...22)( :string, 0, + (6...12), InterpolatedStringNode(15...22)( (15...22), [StringNode(23...48)( @@ -14,7 +15,6 @@ ProgramNode(6...74)( )], (48...57) ), - (6...12), (13...14) ), CallNode(63...74)( diff --git a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt index d8bd9891848f16..d3aa84c9869457 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt @@ -1,11 +1,13 @@ ProgramNode(6...34)( [:foo], StatementsNode(6...34)( - [OperatorWriteNode(6...24)( - LocalVariableWriteNode(6...9)(:foo, 0, nil, (6...9), nil), + [LocalVariableOperatorWriteNode(6...24)( + (6...9), (10...12), + CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar"), + :foo, :+, - CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar") + 0 ), CallNode(31...34)(nil, nil, (31...34), nil, nil, nil, nil, 2, "baz")] ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt b/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt index ed1ab308658bce..7f0a75913c6395 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt @@ -2,8 +2,8 @@ ProgramNode(0...10)( [:a, :b], StatementsNode(0...10)( [MultiWriteNode(0...8)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0)], (5...6), CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/seattlerb/parse_pattern_058.txt b/test/yarp/snapshots/seattlerb/parse_pattern_058.txt index 23b0ba5d102d0e..d9416d4a61c613 100644 --- a/test/yarp/snapshots/seattlerb/parse_pattern_058.txt +++ b/test/yarp/snapshots/seattlerb/parse_pattern_058.txt @@ -20,7 +20,7 @@ ProgramNode(0...43)( nil ), AssocSplatNode(20...26)( - LocalVariableWriteNode(22...26)(:rest, 0, nil, (22...26), nil), + LocalVariableTargetNode(22...26)(:rest, 0), (20...22) )], nil, diff --git a/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt b/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt index acac8dd857298b..e84c4bf8a92476 100644 --- a/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [UntilNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt b/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt index acac8dd857298b..e84c4bf8a92476 100644 --- a/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [UntilNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt b/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt index b113e0ad537a68..8d152ba76ec6f6 100644 --- a/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [WhileNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt b/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt index b113e0ad537a68..8d152ba76ec6f6 100644 --- a/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [WhileNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/pct_nl.txt b/test/yarp/snapshots/seattlerb/pct_nl.txt index 8435c33e6abfb4..4cdba9d6d621ee 100644 --- a/test/yarp/snapshots/seattlerb/pct_nl.txt +++ b/test/yarp/snapshots/seattlerb/pct_nl.txt @@ -4,8 +4,8 @@ ProgramNode(0...7)( [LocalVariableWriteNode(0...7)( :x, 0, - StringNode(4...7)((4...6), (6...6), (6...7), ""), (0...1), + StringNode(4...7)((4...6), (6...6), (6...7), ""), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/rhs_asgn.txt b/test/yarp/snapshots/seattlerb/rhs_asgn.txt index 7ba161b170a660..604a14667b1e6d 100644 --- a/test/yarp/snapshots/seattlerb/rhs_asgn.txt +++ b/test/yarp/snapshots/seattlerb/rhs_asgn.txt @@ -3,7 +3,7 @@ ProgramNode(0...7)( StatementsNode(0...7)( [MatchRequiredNode(0...7)( IntegerNode(0...2)(), - LocalVariableWriteNode(6...7)(:n, 0, nil, (6...7), nil), + LocalVariableTargetNode(6...7)(:n, 0), (3...5) )] ) diff --git a/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt b/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt index 63a6f3822afae9..5800090fbbb9d5 100644 --- a/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt +++ b/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...5)( [CallNode(0...5)( CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), (1...3), - (0...0), + nil, (3...4), nil, (4...5), diff --git a/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt b/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt index 4000ecf59b70a1..7785c7921ece14 100644 --- a/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt +++ b/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt @@ -4,6 +4,7 @@ ProgramNode(0...8)( [LocalVariableWriteNode(0...8)( :c, 0, + (0...1), CallNode(4...8)( CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "a"), (5...7), @@ -15,7 +16,6 @@ ProgramNode(0...8)( 1, "b" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt b/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt index bbbd0a3fcf5bda..d999e4ea4729ba 100644 --- a/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt +++ b/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt @@ -4,6 +4,8 @@ ProgramNode(0...5)( [LambdaNode(0...5)( [:a], (0...2), + (3...4), + (4...5), BlockParametersNode(2...3)( ParametersNode(2...3)( [RequiredParameterNode(2...3)(:a)], diff --git a/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt b/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt index 9475ba9c3f960e..0f64173a920c99 100644 --- a/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt +++ b/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt @@ -4,6 +4,8 @@ ProgramNode(0...23)( [LambdaNode(0...23)( [:b, :c, :d, :e, :f], (0...2), + (21...22), + (22...23), BlockParametersNode(2...21)( ParametersNode(3...20)( [RequiredParameterNode(3...4)(:b)], diff --git a/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt b/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt index f84e8227f954e0..719c895b99b89c 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt @@ -10,6 +10,8 @@ ProgramNode(0...25)( [LambdaNode(2...25)( [], (2...4), + (8...10), + (22...25), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), StatementsNode(11...21)( [CallNode(11...21)( diff --git a/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt b/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt index c24b3c63c75e1e..14b7a3eb982ed5 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt @@ -10,6 +10,8 @@ ProgramNode(0...26)( [LambdaNode(2...26)( [], (2...4), + (8...10), + (23...26), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), StatementsNode(11...22)( [CallNode(11...22)( diff --git a/test/yarp/snapshots/seattlerb/stabby_block_kw.txt b/test/yarp/snapshots/seattlerb/stabby_block_kw.txt index 9554c9d37a6983..8df95715c9f04e 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_kw.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_kw.txt @@ -4,6 +4,8 @@ ProgramNode(0...13)( [LambdaNode(0...13)( [:k], (0...2), + (10...11), + (12...13), BlockParametersNode(3...9)( ParametersNode(4...8)( [], diff --git a/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt b/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt index e2bdaf8124cd6b..8d3e73af8b7766 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [:k], (0...2), + (8...9), + (10...11), BlockParametersNode(3...7)( ParametersNode(4...6)( [], diff --git a/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt b/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt index d8aef65941e5b0..c4594997ebd59e 100644 --- a/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt +++ b/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [:a, :b], (0...2), + (9...10), + (10...11), BlockParametersNode(2...8)( ParametersNode(3...4)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/seattlerb/thingy.txt b/test/yarp/snapshots/seattlerb/thingy.txt index f12a16ed4fc808..929dd9fd96899e 100644 --- a/test/yarp/snapshots/seattlerb/thingy.txt +++ b/test/yarp/snapshots/seattlerb/thingy.txt @@ -4,7 +4,7 @@ ProgramNode(0...15)( [CallNode(0...6)( CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "f"), (1...2), - (0...0), + nil, (2...3), ArgumentsNode(3...5)([IntegerNode(3...5)()]), (5...6), @@ -15,7 +15,7 @@ ProgramNode(0...15)( CallNode(8...15)( CallNode(8...9)(nil, nil, (8...9), nil, nil, nil, nil, 2, "f"), (9...11), - (0...0), + nil, (11...12), ArgumentsNode(12...14)([IntegerNode(12...14)()]), (14...15), diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt new file mode 100644 index 00000000000000..301c70adf2a67c --- /dev/null +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -0,0 +1,215 @@ +ProgramNode(164...964)( + [], + StatementsNode(164...964)( + [CallNode(164...192)( + nil, + nil, + (164...166), + nil, + ArgumentsNode(167...192)( + [CallNode(167...192)( + InterpolatedStringNode(167...171)( + (167...171), + [StringNode(181...183)(nil, (181...183), nil, "a\n")], + (183...185) + ), + (171...172), + (172...176), + (176...177), + ArgumentsNode(177...191)( + [InterpolatedRegularExpressionNode(177...187)( + (177...178), + [StringNode(178...181)(nil, (178...181), nil, "b"), + StringNode(185...186)(nil, (185...186), nil, "b")], + (186...187), + 0 + ), + StringNode(189...191)( + (189...190), + (190...190), + (190...191), + "" + )] + ), + (191...192), + nil, + 0, + "gsub" + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(276...295)( + nil, + nil, + (276...278), + nil, + ArgumentsNode(279...295)( + [InterpolatedStringNode(279...283)( + (279...283), + [StringNode(289...291)(nil, (289...291), nil, "c\n")], + (291...293) + ), + InterpolatedStringNode(285...295)( + (285...286), + [StringNode(286...289)(nil, (286...289), nil, "d"), + StringNode(293...294)(nil, (293...294), nil, "d")], + (294...295) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(322...343)( + nil, + nil, + (322...324), + nil, + ArgumentsNode(325...343)( + [InterpolatedStringNode(325...329)( + (325...329), + [StringNode(337...339)(nil, (337...339), nil, "e\n")], + (339...341) + ), + InterpolatedStringNode(331...343)( + (331...334), + [StringNode(334...337)(nil, (334...337), nil, "f\\\n"), + StringNode(341...342)(nil, (341...342), nil, "f")], + (342...343) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(427...448)( + nil, + nil, + (427...429), + nil, + ArgumentsNode(430...448)( + [InterpolatedStringNode(430...434)( + (430...434), + [StringNode(442...444)(nil, (442...444), nil, "g\n")], + (444...446) + ), + InterpolatedStringNode(436...448)( + (436...439), + [StringNode(439...442)(nil, (439...442), nil, "h"), + StringNode(446...447)(nil, (446...447), nil, "h")], + (447...448) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(520...541)( + nil, + nil, + (520...522), + nil, + ArgumentsNode(523...541)( + [InterpolatedStringNode(523...527)( + (523...527), + [StringNode(535...537)(nil, (535...537), nil, "i\n")], + (537...539) + ), + ArrayNode(529...541)( + [StringNode(532...535)(nil, (532...535), nil, "j\\\n"), + StringNode(539...540)(nil, (539...540), nil, "j")], + (529...532), + (540...541) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(688...709)( + nil, + nil, + (688...690), + nil, + ArgumentsNode(691...709)( + [InterpolatedStringNode(691...695)( + (691...695), + [StringNode(703...705)(nil, (703...705), nil, "k\n")], + (705...707) + ), + ArrayNode(697...709)( + [InterpolatedStringNode(700...708)( + nil, + [StringNode(700...703)(nil, (700...703), nil, "l"), + StringNode(707...708)(nil, (707...708), nil, "l")], + nil + )], + (697...700), + (708...709) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(781...802)( + nil, + nil, + (781...783), + nil, + ArgumentsNode(784...802)( + [InterpolatedStringNode(784...788)( + (784...788), + [StringNode(796...798)(nil, (796...798), nil, "m\n")], + (798...800) + ), + ArrayNode(790...802)( + [SymbolNode(793...796)(nil, (793...796), nil, "n\\\n"), + SymbolNode(800...801)(nil, (800...801), nil, "n")], + (790...793), + (801...802) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(943...964)( + nil, + nil, + (943...945), + nil, + ArgumentsNode(946...964)( + [InterpolatedStringNode(946...950)( + (946...950), + [StringNode(958...960)(nil, (958...960), nil, "o\n")], + (960...962) + ), + ArrayNode(952...964)( + [InterpolatedSymbolNode(955...963)( + nil, + [SymbolNode(955...958)(nil, (955...958), nil, "p"), + StringNode(962...963)(nil, (962...963), nil, "p")], + nil + )], + (952...955), + (963...964) + )] + ), + nil, + nil, + 0, + "pp" + )] + ) +) diff --git a/test/yarp/snapshots/strings.txt b/test/yarp/snapshots/strings.txt index 38033453af3da3..86a732d85d5fbe 100644 --- a/test/yarp/snapshots/strings.txt +++ b/test/yarp/snapshots/strings.txt @@ -20,7 +20,7 @@ ProgramNode(0...498)( (122...123), [EmbeddedVariableNode(123...129)( (123...124), - ClassVariableReadNode(124...129)() + ClassVariableReadNode(124...129)(:@@foo) )], (129...130) ), @@ -185,7 +185,7 @@ ProgramNode(0...498)( (414...415), [EmbeddedVariableNode(415...420)( (415...416), - InstanceVariableReadNode(416...420)() + InstanceVariableReadNode(416...420)(:@foo) )], (420...421) ), diff --git a/test/yarp/snapshots/ternary_operator.txt b/test/yarp/snapshots/ternary_operator.txt index 6055e2434ffb92..9751d53a317c21 100644 --- a/test/yarp/snapshots/ternary_operator.txt +++ b/test/yarp/snapshots/ternary_operator.txt @@ -147,8 +147,8 @@ ProgramNode(0...131)( [LocalVariableWriteNode(124...129)( :_a, 0, - IntegerNode(128...129)(), (124...126), + IntegerNode(128...129)(), (127...128) )] ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index ffaedf4f1834b5..7680b90f618165 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -1,10 +1,10 @@ ProgramNode(0...704)( [:a, :b, :foo, :c, :x], StatementsNode(0...704)( - [GlobalVariableWriteNode(0...6)((0...2), (3...4), IntegerNode(5...6)()), - MultiWriteNode(8...24)( - [GlobalVariableWriteNode(8...10)((8...10), nil, nil), - GlobalVariableWriteNode(12...14)((12...14), nil, nil)], + [GlobalVariableWriteNode(0...6)((0...2), IntegerNode(5...6)(), (3...4)), + MultiWriteNode(7...24)( + [GlobalVariableTargetNode(8...10)(), + GlobalVariableTargetNode(12...14)()], (16...17), ArrayNode(18...24)( [IntegerNode(19...20)(), IntegerNode(22...23)()], @@ -14,35 +14,35 @@ ProgramNode(0...704)( (7...8), (14...15) ), - MultiWriteNode(27...38)( - [MultiWriteNode(27...29)( - [LocalVariableWriteNode(27...28)(:a, 0, nil, (27...28), nil), + MultiWriteNode(25...38)( + [MultiWriteNode(26...30)( + [LocalVariableTargetNode(27...28)(:a, 0), SplatNode(28...29)((28...29), nil)], nil, nil, (26...27), (29...30) ), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + LocalVariableTargetNode(32...33)(:b, 0)], (35...36), IntegerNode(37...38)(), (25...26), (33...34) ), - MultiWriteNode(40...48)( + MultiWriteNode(39...48)( [SplatNode(40...42)( (40...41), - LocalVariableWriteNode(41...42)(:a, 0, nil, (41...42), nil) + LocalVariableTargetNode(41...42)(:a, 0) )], (44...45), ArrayNode(46...48)([], (46...47), (47...48)), (39...40), (42...43) ), - MultiWriteNode(50...64)( + MultiWriteNode(49...64)( [SplatNode(50...54)( (50...51), - LocalVariableWriteNode(51...54)(:foo, 0, nil, (51...54), nil) + LocalVariableTargetNode(51...54)(:foo, 0) )], (56...57), ArrayNode(58...64)( @@ -53,9 +53,9 @@ ProgramNode(0...704)( (49...50), (54...55) ), - MultiWriteNode(66...84)( - [ClassVariableWriteNode(66...69)((66...69), nil, nil), - ClassVariableWriteNode(71...74)((71...74), nil, nil)], + MultiWriteNode(65...84)( + [ClassVariableTargetNode(66...69)(:@@a), + ClassVariableTargetNode(71...74)(:@@b)], (76...77), ArrayNode(78...84)( [IntegerNode(79...80)(), IntegerNode(82...83)()], @@ -65,9 +65,9 @@ ProgramNode(0...704)( (65...66), (74...75) ), - MultiWriteNode(86...102)( - [InstanceVariableWriteNode(86...88)((86...88), nil, nil), - InstanceVariableWriteNode(90...92)((90...92), nil, nil)], + MultiWriteNode(85...102)( + [InstanceVariableTargetNode(86...88)(:@a), + InstanceVariableTargetNode(90...92)(:@b)], (94...95), ArrayNode(96...102)( [IntegerNode(97...98)(), IntegerNode(100...101)()], @@ -77,11 +77,11 @@ ProgramNode(0...704)( (85...86), (92...93) ), - MultiWriteNode(104...128)( - [LocalVariableWriteNode(104...105)(:a, 0, nil, (104...105), nil), - MultiWriteNode(108...113)( - [LocalVariableWriteNode(108...109)(:b, 0, nil, (108...109), nil), - LocalVariableWriteNode(111...112)(:c, 0, nil, (111...112), nil)], + MultiWriteNode(103...128)( + [LocalVariableTargetNode(104...105)(:a, 0), + MultiWriteNode(107...113)( + [LocalVariableTargetNode(108...109)(:b, 0), + LocalVariableTargetNode(111...112)(:c, 0)], nil, nil, (107...108), @@ -101,8 +101,8 @@ ProgramNode(0...704)( (103...104), (113...114) ), - MultiWriteNode(130...144)( - [LocalVariableWriteNode(130...131)(:a, 0, nil, (130...131), nil), + MultiWriteNode(129...144)( + [LocalVariableTargetNode(130...131)(:a, 0), SplatNode(133...134)((133...134), nil)], (136...137), ArrayNode(138...144)( @@ -113,11 +113,11 @@ ProgramNode(0...704)( (129...130), (134...135) ), - MultiWriteNode(146...163)( - [LocalVariableWriteNode(146...147)(:a, 0, nil, (146...147), nil), + MultiWriteNode(145...163)( + [LocalVariableTargetNode(146...147)(:a, 0), SplatNode(149...153)( (149...150), - LocalVariableWriteNode(150...153)(:foo, 0, nil, (150...153), nil) + LocalVariableTargetNode(150...153)(:foo, 0) )], (155...156), ArrayNode(157...163)( @@ -128,9 +128,9 @@ ProgramNode(0...704)( (145...146), (153...154) ), - MultiWriteNode(165...179)( - [LocalVariableWriteNode(165...166)(:a, 0, nil, (165...166), nil), - LocalVariableWriteNode(168...169)(:b, 0, nil, (168...169), nil)], + MultiWriteNode(164...179)( + [LocalVariableTargetNode(165...166)(:a, 0), + LocalVariableTargetNode(168...169)(:b, 0)], (171...172), ArrayNode(173...179)( [IntegerNode(174...175)(), IntegerNode(177...178)()], @@ -140,23 +140,23 @@ ProgramNode(0...704)( (164...165), (169...170) ), - MultiWriteNode(181...192)( - [LocalVariableWriteNode(181...182)(:a, 0, nil, (181...182), nil), - LocalVariableWriteNode(184...185)(:b, 0, nil, (184...185), nil)], + MultiWriteNode(180...192)( + [LocalVariableTargetNode(181...182)(:a, 0), + LocalVariableTargetNode(184...185)(:b, 0)], (187...188), LocalVariableReadNode(189...192)(:foo, 0), (180...181), (185...186) ), - MultiWriteNode(194...203)( - [LocalVariableWriteNode(194...195)(:a, 0, nil, (194...195), nil), + MultiWriteNode(193...203)( + [LocalVariableTargetNode(194...195)(:a, 0), SplatNode(195...196)((195...196), nil)], (198...199), LocalVariableReadNode(200...203)(:foo, 0), (193...194), (196...197) ), - MultiWriteNode(205...227)( + MultiWriteNode(204...227)( [CallNode(205...210)( LocalVariableReadNode(205...206)(:a, 0), (206...207), @@ -188,7 +188,7 @@ ProgramNode(0...704)( (204...205), (217...218) ), - MultiWriteNode(229...252)( + MultiWriteNode(228...252)( [CallNode(229...236)( LocalVariableReadNode(229...230)(:a, 0), nil, @@ -225,7 +225,7 @@ ProgramNode(0...704)( (228...229), (242...243) ), - MultiWriteNode(254...274)( + MultiWriteNode(253...274)( [CallNode(254...258)( LocalVariableReadNode(254...255)(:a, 0), nil, @@ -257,7 +257,7 @@ ProgramNode(0...704)( (253...254), (264...265) ), - MultiWriteNode(276...287)( + MultiWriteNode(275...287)( [SplatNode(276...282)( (276...277), CallNode(277...282)( @@ -291,11 +291,13 @@ ProgramNode(0...704)( ) ), ClassVariableWriteNode(302...309)( + :@@a, (302...305), IntegerNode(308...309)(), (306...307) ), InstanceVariableWriteNode(310...316)( + :@a, (310...312), IntegerNode(315...316)(), (313...314) @@ -321,17 +323,12 @@ ProgramNode(0...704)( LocalVariableWriteNode(351...367)( :a, 0, + (351...352), ParenthesesNode(355...367)( - StatementsNode(357...366)( - [MultiWriteNode(357...366)( - [LocalVariableWriteNode(357...358)(:b, 0, nil, (357...358), nil), - LocalVariableWriteNode(360...361)( - :c, - 0, - nil, - (360...361), - nil - )], + StatementsNode(356...366)( + [MultiWriteNode(356...366)( + [LocalVariableTargetNode(357...358)(:b, 0), + LocalVariableTargetNode(360...361)(:c, 0)], (363...364), IntegerNode(365...366)(), (356...357), @@ -341,19 +338,19 @@ ProgramNode(0...704)( (355...356), (366...367) ), - (351...352), (353...354) ), LocalVariableWriteNode(368...373)( :a, 0, - IntegerNode(372...373)(), (368...369), + IntegerNode(372...373)(), (370...371) ), LocalVariableWriteNode(374...385)( :foo, 0, + (374...377), CallNode(380...385)( nil, nil, @@ -365,7 +362,6 @@ ProgramNode(0...704)( 0, "foo" ), - (374...377), (378...379) ), CallNode(386...395)( @@ -542,8 +538,8 @@ ProgramNode(0...704)( LocalVariableWriteNode(507...514)( :x, 0, - StringNode(511...514)((511...513), (513...513), (513...514), ""), (507...508), + StringNode(511...514)((511...513), (513...513), (513...514), ""), (509...510) ), CallNode(515...522)( @@ -610,14 +606,16 @@ ProgramNode(0...704)( ), (543...546) ), - OrWriteNode(551...561)( - InstanceVariableWriteNode(551...553)((551...553), nil, nil), - StringNode(558...561)((558...560), (560...560), (560...561), ""), - (554...557) + InstanceVariableOrWriteNode(551...561)( + :@a, + (551...553), + (554...557), + StringNode(558...561)((558...560), (560...560), (560...561), "") ), LocalVariableWriteNode(562...576)( :x, 0, + (562...563), InterpolatedStringNode(566...576)( (566...576), [StringNode(577...579)(nil, (577...579), nil, " "), @@ -625,7 +623,6 @@ ProgramNode(0...704)( StringNode(582...583)(nil, (582...583), nil, "\n")], (583...591) ), - (562...563), (564...565) ), CallNode(591...605)( @@ -703,16 +700,17 @@ ProgramNode(0...704)( ), (665...668) ), - OrWriteNode(687...704)( - InstanceVariableWriteNode(687...689)((687...689), nil, nil), + InstanceVariableOrWriteNode(687...704)( + :@a, + (687...689), + (690...693), InterpolatedStringNode(694...704)( (694...704), [StringNode(705...707)(nil, (705...707), nil, " "), EmbeddedStatementsNode(707...710)((707...709), nil, (709...710)), StringNode(710...711)(nil, (710...711), nil, "\n")], (711...719) - ), - (690...693) + ) )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/block.txt b/test/yarp/snapshots/unparser/corpus/literal/block.txt index 787c8cb5929594..a115e84135b5f9 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/block.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/block.txt @@ -743,7 +743,7 @@ ProgramNode(0...737)( (365...371), [ConstantReadNode(372...381)()], (382...384), - LocalVariableWriteNode(385...386)(:e, 0, nil, (385...386), nil), + LocalVariableTargetNode(385...386)(:e, 0), nil, nil ), @@ -786,7 +786,7 @@ ProgramNode(0...737)( (402...408), [ConstantReadNode(409...418)()], (419...421), - LocalVariableWriteNode(422...425)(:bar, 0, nil, (422...425), nil), + LocalVariableTargetNode(422...425)(:bar, 0), StatementsNode(428...431)( [LocalVariableReadNode(428...431)(:bar, 0)] ), @@ -914,13 +914,7 @@ ProgramNode(0...737)( ) )], (514...516), - LocalVariableWriteNode(517...526)( - :exception, - 0, - nil, - (517...526), - nil - ), + LocalVariableTargetNode(517...526)(:exception, 0), StatementsNode(529...532)( [CallNode(529...532)( nil, @@ -1151,13 +1145,7 @@ ProgramNode(0...737)( ) )], (658...660), - LocalVariableWriteNode(661...670)( - :exception, - 0, - nil, - (661...670), - nil - ), + LocalVariableTargetNode(661...670)(:exception, 0), StatementsNode(673...676)( [CallNode(673...676)( nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/defined.txt b/test/yarp/snapshots/unparser/corpus/literal/defined.txt index 1ea9cb9d87a7b0..9229badf44efbe 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/defined.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...56)( StatementsNode(0...56)( [DefinedNode(0...14)( (8...9), - InstanceVariableReadNode(9...13)(), + InstanceVariableReadNode(9...13)(:@foo), (13...14), (0...8) ), @@ -16,10 +16,10 @@ ProgramNode(0...56)( DefinedNode(29...56)( (37...38), ParenthesesNode(38...55)( - StatementsNode(40...54)( - [MultiWriteNode(40...54)( - [LocalVariableWriteNode(40...41)(:a, 0, nil, (40...41), nil), - LocalVariableWriteNode(43...44)(:b, 0, nil, (43...44), nil)], + StatementsNode(39...54)( + [MultiWriteNode(39...54)( + [LocalVariableTargetNode(40...41)(:a, 0), + LocalVariableTargetNode(43...44)(:b, 0)], (46...47), ArrayNode(48...54)( [IntegerNode(49...50)(), IntegerNode(52...53)()], diff --git a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt index 846614936967e2..396ebcd3daf896 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt @@ -57,7 +57,7 @@ ProgramNode(0...299)( [StringNode(146...147)(nil, (146...147), nil, "a"), EmbeddedVariableNode(147...150)( (147...148), - NumberedReferenceReadNode(148...150)() + NumberedReferenceReadNode(148...150)(1) )], (150...151) ), @@ -75,7 +75,7 @@ ProgramNode(0...299)( [StringNode(160...161)(nil, (160...161), nil, "a"), EmbeddedVariableNode(161...164)( (161...162), - InstanceVariableReadNode(162...164)() + InstanceVariableReadNode(162...164)(:@a) )], (164...165) ), @@ -84,7 +84,7 @@ ProgramNode(0...299)( [StringNode(167...168)(nil, (167...168), nil, "a"), EmbeddedVariableNode(168...172)( (168...169), - ClassVariableReadNode(169...172)() + ClassVariableReadNode(169...172)(:@@a) )], (172...173) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/for.txt b/test/yarp/snapshots/unparser/corpus/literal/for.txt index 1bd6587bd7ad26..d7a916867674f3 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/for.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/for.txt @@ -9,7 +9,7 @@ ProgramNode(0...119)( ArgumentsNode(4...29)( [ForNode(4...29)( MultiWriteNode(8...9)( - [LocalVariableWriteNode(8...9)(:a, 0, nil, (8...9), nil)], + [LocalVariableTargetNode(8...9)(:a, 0)], nil, nil, nil, @@ -52,7 +52,7 @@ ProgramNode(0...119)( ), ForNode(31...56)( MultiWriteNode(35...36)( - [LocalVariableWriteNode(35...36)(:a, 0, nil, (35...36), nil)], + [LocalVariableTargetNode(35...36)(:a, 0)], nil, nil, nil, @@ -69,10 +69,10 @@ ProgramNode(0...119)( ), ForNode(57...88)( MultiWriteNode(61...68)( - [LocalVariableWriteNode(62...63)(:a, 0, nil, (62...63), nil), + [LocalVariableTargetNode(62...63)(:a, 0), SplatNode(65...67)( (65...66), - LocalVariableWriteNode(66...67)(:b, 0, nil, (66...67), nil) + LocalVariableTargetNode(66...67)(:b, 0) )], nil, nil, @@ -90,8 +90,8 @@ ProgramNode(0...119)( ), ForNode(89...119)( MultiWriteNode(93...99)( - [LocalVariableWriteNode(94...95)(:a, 0, nil, (94...95), nil), - LocalVariableWriteNode(97...98)(:b, 0, nil, (97...98), nil)], + [LocalVariableTargetNode(94...95)(:a, 0), + LocalVariableTargetNode(97...98)(:b, 0)], nil, nil, (93...94), diff --git a/test/yarp/snapshots/unparser/corpus/literal/if.txt b/test/yarp/snapshots/unparser/corpus/literal/if.txt index 1bfbe7e26ff5c5..77fb5c8c636042 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/if.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/if.txt @@ -61,6 +61,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(113...122)( :foo, 0, + (113...116), CallNode(119...122)( nil, nil, @@ -72,7 +73,6 @@ ProgramNode(0...246)( 2, "bar" ), - (113...116), (117...118) )] ), @@ -95,6 +95,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(146...155)( :foo, 0, + (146...149), CallNode(152...155)( nil, nil, @@ -106,7 +107,6 @@ ProgramNode(0...246)( 2, "bar" ), - (146...149), (150...151) )] ), @@ -134,6 +134,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(184...193)( :foo, 0, + (184...187), CallNode(190...193)( nil, nil, @@ -145,7 +146,6 @@ ProgramNode(0...246)( 2, "bar" ), - (184...187), (188...189) )] ), @@ -190,8 +190,8 @@ ProgramNode(0...246)( [LocalVariableWriteNode(225...236)( :pair, 0, - SymbolNode(232...236)((232...233), (233...236), nil, "foo"), (225...229), + SymbolNode(232...236)((232...233), (233...236), nil, "foo"), (230...231) ), LocalVariableReadNode(239...242)(:foo, 0)] diff --git a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt index 0553be37eab3f6..1f28ad87b416b0 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt @@ -98,7 +98,7 @@ ProgramNode(0...530)( (133...139), [ConstantReadNode(140...141)()], (142...144), - LocalVariableWriteNode(145...148)(:foo, 0, nil, (145...148), nil), + LocalVariableTargetNode(145...148)(:foo, 0), nil, nil ), @@ -259,6 +259,7 @@ ProgramNode(0...530)( LocalVariableWriteNode(307...316)( :foo, 0, + (307...310), CallNode(313...316)( nil, nil, @@ -270,7 +271,6 @@ ProgramNode(0...530)( 2, "bar" ), - (307...310), (311...312) ) )] @@ -294,7 +294,7 @@ ProgramNode(0...530)( (351...357), [], (358...360), - LocalVariableWriteNode(361...364)(:bar, 0, nil, (361...364), nil), + LocalVariableTargetNode(361...364)(:bar, 0), StatementsNode(367...370)( [LocalVariableReadNode(367...370)(:bar, 0)] ), @@ -311,7 +311,7 @@ ProgramNode(0...530)( (388...394), [ConstantReadNode(395...404)(), ConstantReadNode(406...411)()], (412...414), - LocalVariableWriteNode(415...418)(:bar, 0, nil, (415...418), nil), + LocalVariableTargetNode(415...418)(:bar, 0), StatementsNode(421...424)( [LocalVariableReadNode(421...424)(:bar, 0)] ), @@ -332,13 +332,7 @@ ProgramNode(0...530)( LocalVariableReadNode(461...464)(:bar, 0) )], (465...467), - LocalVariableWriteNode(468...477)( - :exception, - 0, - nil, - (468...477), - nil - ), + LocalVariableTargetNode(468...477)(:exception, 0), StatementsNode(480...483)( [CallNode(480...483)( nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/lambda.txt b/test/yarp/snapshots/unparser/corpus/literal/lambda.txt index 66300fd726cda4..13161f9c881262 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/lambda.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/lambda.txt @@ -46,12 +46,16 @@ ProgramNode(0...80)( LambdaNode(33...41)( [], (33...35), + (38...39), + (40...41), BlockParametersNode(35...37)(nil, [], (35...36), (36...37)), nil ), LambdaNode(42...51)( [:a], (42...44), + (48...49), + (50...51), BlockParametersNode(44...47)( ParametersNode(45...46)( [RequiredParameterNode(45...46)(:a)], @@ -71,6 +75,8 @@ ProgramNode(0...80)( LambdaNode(52...64)( [:a, :b], (52...54), + (61...62), + (63...64), BlockParametersNode(54...60)( ParametersNode(55...59)( [RequiredParameterNode(55...56)(:a), @@ -91,6 +97,8 @@ ProgramNode(0...80)( LambdaNode(65...80)( [:a, :b, :c], (65...67), + (77...78), + (79...80), BlockParametersNode(67...76)( ParametersNode(68...72)( [RequiredParameterNode(68...69)(:a), diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index 9b0fad50720ceb..3b85d3e95fb34f 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -153,12 +153,12 @@ ProgramNode(0...916)( (206...207), [EmbeddedVariableNode(207...210)( (207...208), - InstanceVariableReadNode(208...210)() + InstanceVariableReadNode(208...210)(:@a) ), StringNode(210...211)(nil, (210...211), nil, " "), EmbeddedVariableNode(211...215)( (211...212), - ClassVariableReadNode(212...215)() + ClassVariableReadNode(212...215)(:@@a) ), StringNode(215...216)(nil, (215...216), nil, " "), EmbeddedVariableNode(216...219)( @@ -267,7 +267,9 @@ ProgramNode(0...916)( [StringNode(419...422)(nil, (419...422), nil, "foo"), EmbeddedStatementsNode(422...429)( (422...424), - StatementsNode(424...428)([InstanceVariableReadNode(424...428)()]), + StatementsNode(424...428)( + [InstanceVariableReadNode(424...428)(:@bar)] + ), (428...429) )], (429...430) @@ -276,11 +278,11 @@ ProgramNode(0...916)( XStringNode(435...439)((435...436), (436...438), (438...439), "`"), XStringNode(440...443)((440...441), (441...442), (442...443), "\""), SymbolNode(444...448)((444...445), (445...448), nil, "foo"), - SymbolNode(449...455)(nil, (451...454), nil, "A B"), + SymbolNode(449...455)((449...451), (451...454), (454...455), "A B"), SymbolNode(456...460)((456...457), (457...460), nil, "foo"), - SymbolNode(461...467)(nil, (463...466), nil, "A B"), - SymbolNode(468...475)(nil, (470...474), nil, "A\"B"), - InterpolatedSymbolNode(476...479)((476...478), [], (478...479)), + SymbolNode(461...467)((461...463), (463...466), (466...467), "A B"), + SymbolNode(468...475)((468...470), (470...474), (474...475), "A\"B"), + SymbolNode(476...479)((476...478), (0...0), (478...479), ""), RegularExpressionNode(480...485)( (480...481), (481...484), @@ -300,7 +302,9 @@ ProgramNode(0...916)( [StringNode(516...519)(nil, (516...519), nil, "foo"), EmbeddedStatementsNode(519...526)( (519...521), - StatementsNode(521...525)([InstanceVariableReadNode(521...525)()]), + StatementsNode(521...525)( + [InstanceVariableReadNode(521...525)(:@bar)] + ), (525...526) )], (526...527), @@ -311,7 +315,9 @@ ProgramNode(0...916)( [StringNode(529...532)(nil, (529...532), nil, "foo"), EmbeddedStatementsNode(532...539)( (532...534), - StatementsNode(534...538)([InstanceVariableReadNode(534...538)()]), + StatementsNode(534...538)( + [InstanceVariableReadNode(534...538)(:@bar)] + ), (538...539) )], (539...543), @@ -501,7 +507,7 @@ ProgramNode(0...916)( [IntegerNode(693...694)(), SplatNode(696...701)( (696...697), - InstanceVariableReadNode(697...701)() + InstanceVariableReadNode(697...701)(:@foo) )], (692...693), (701...702) @@ -509,7 +515,7 @@ ProgramNode(0...916)( ArrayNode(703...713)( [SplatNode(704...709)( (704...705), - InstanceVariableReadNode(705...709)() + InstanceVariableReadNode(705...709)(:@foo) ), IntegerNode(711...712)()], (703...704), @@ -518,11 +524,11 @@ ProgramNode(0...916)( ArrayNode(714...728)( [SplatNode(715...720)( (715...716), - InstanceVariableReadNode(716...720)() + InstanceVariableReadNode(716...720)(:@foo) ), SplatNode(722...727)( (722...723), - InstanceVariableReadNode(723...727)() + InstanceVariableReadNode(723...727)(:@baz) )], (714...715), (727...728) @@ -620,7 +626,7 @@ ProgramNode(0...916)( HashNode(828...843)( (828...829), [AssocNode(830...841)( - SymbolNode(830...836)(nil, (832...835), nil, "a b"), + SymbolNode(830...836)((830...832), (832...835), (835...836), "a b"), IntegerNode(840...841)(), (837...839) )], @@ -677,7 +683,12 @@ ProgramNode(0...916)( 0, "foo" ), - SymbolNode(893...901)(nil, (895...900), nil, "a\\\n" + "b"), + SymbolNode(893...901)( + (893...895), + (895...900), + (900...901), + "a\\\n" + "b" + ), InterpolatedXStringNode(902...916)( (902...903), [StringNode(903...907)(nil, (903...907), nil, " x\n"), diff --git a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt index 980459766311ca..15d621cbf6de63 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt @@ -1,53 +1,69 @@ ProgramNode(0...233)( [:a, :h], StatementsNode(0...233)( - [OperatorWriteNode(0...6)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOperatorWriteNode(0...6)( + (0...1), (2...4), + IntegerNode(5...6)(), + :a, :+, - IntegerNode(5...6)() + 0 ), - OperatorWriteNode(7...13)( - LocalVariableWriteNode(7...8)(:a, 0, nil, (7...8), nil), + LocalVariableOperatorWriteNode(7...13)( + (7...8), (9...11), + IntegerNode(12...13)(), + :a, :-, - IntegerNode(12...13)() + 0 ), - OperatorWriteNode(14...21)( - LocalVariableWriteNode(14...15)(:a, 0, nil, (14...15), nil), + LocalVariableOperatorWriteNode(14...21)( + (14...15), (16...19), + IntegerNode(20...21)(), + :a, :**, - IntegerNode(20...21)() + 0 ), - OperatorWriteNode(22...28)( - LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), + LocalVariableOperatorWriteNode(22...28)( + (22...23), (24...26), + IntegerNode(27...28)(), + :a, :*, - IntegerNode(27...28)() + 0 ), - OperatorWriteNode(29...35)( - LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), + LocalVariableOperatorWriteNode(29...35)( + (29...30), (31...33), + IntegerNode(34...35)(), + :a, :/, - IntegerNode(34...35)() + 0 ), - AndWriteNode(36...43)( - LocalVariableWriteNode(36...37)(:a, 0, nil, (36...37), nil), + LocalVariableAndWriteNode(36...43)( + (36...37), + (38...41), CallNode(42...43)(nil, nil, (42...43), nil, nil, nil, nil, 2, "b"), - (38...41) + :a, + 0 ), - OrWriteNode(44...51)( - LocalVariableWriteNode(44...45)(:a, 0, nil, (44...45), nil), + LocalVariableOrWriteNode(44...51)( + (44...45), + (46...49), IntegerNode(50...51)(), - (46...49) + :a, + 0 ), CallNode(52...65)( ParenthesesNode(52...61)( StatementsNode(53...60)( - [OrWriteNode(53...60)( - LocalVariableWriteNode(53...54)(:a, 0, nil, (53...54), nil), + [LocalVariableOrWriteNode(53...60)( + (53...54), + (55...58), IntegerNode(59...60)(), - (55...58) + :a, + 0 )] ), (52...53), @@ -65,10 +81,12 @@ ProgramNode(0...233)( CallNode(66...83)( ParenthesesNode(66...76)( StatementsNode(67...75)( - [OrWriteNode(67...75)( - LocalVariableWriteNode(67...68)(:h, 0, nil, (67...68), nil), + [LocalVariableOrWriteNode(67...75)( + (67...68), + (69...72), HashNode(73...75)((73...74), [], (74...75)), - (69...72) + :h, + 0 )] ), (66...67), diff --git a/test/yarp/snapshots/unparser/corpus/literal/pattern.txt b/test/yarp/snapshots/unparser/corpus/literal/pattern.txt index 9488cc78502614..227938293bdc78 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/pattern.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/pattern.txt @@ -9,7 +9,7 @@ ProgramNode(0...408)( [IntegerNode(14...15)(), IntegerNode(17...18)()], SplatNode(20...22)( (20...21), - LocalVariableWriteNode(21...22)(:a, 0, nil, (21...22), nil) + LocalVariableTargetNode(21...22)(:a, 0) ), [IntegerNode(24...25)()], (13...14), @@ -65,7 +65,7 @@ ProgramNode(0...408)( nil, [AssocNode(85...88)( AssocSplatNode(85...88)( - LocalVariableWriteNode(87...88)(:a, 0, nil, (87...88), nil), + LocalVariableTargetNode(87...88)(:a, 0), (85...87) ), nil, @@ -102,8 +102,8 @@ ProgramNode(0...408)( InNode(128...152)( ArrayPatternNode(131...140)( nil, - [LocalVariableWriteNode(132...133)(:x, 0, nil, (132...133), nil), - LocalVariableWriteNode(135...136)(:y, 0, nil, (135...136), nil)], + [LocalVariableTargetNode(132...133)(:x, 0), + LocalVariableTargetNode(135...136)(:y, 0)], SplatNode(138...139)((138...139), nil), [], (131...132), @@ -190,7 +190,7 @@ ProgramNode(0...408)( InNode(268...289)( CapturePatternNode(271...277)( IntegerNode(271...272)(), - LocalVariableWriteNode(276...277)(:a, 0, nil, (276...277), nil), + LocalVariableTargetNode(276...277)(:a, 0), (273...275) ), StatementsNode(285...289)([TrueNode(285...289)()]), @@ -239,7 +239,7 @@ ProgramNode(0...408)( [IntegerNode(360...361)(), IntegerNode(363...364)()], SplatNode(366...368)( (366...367), - LocalVariableWriteNode(367...368)(:a, 0, nil, (367...368), nil) + LocalVariableTargetNode(367...368)(:a, 0) ), [IntegerNode(370...371)()], (359...360), @@ -279,7 +279,7 @@ ProgramNode(0...408)( IntegerNode(400...401)(), ArrayPatternNode(405...408)( nil, - [LocalVariableWriteNode(406...407)(:a, 0, nil, (406...407), nil)], + [LocalVariableTargetNode(406...407)(:a, 0)], nil, [], (405...406), diff --git a/test/yarp/snapshots/unparser/corpus/literal/rescue.txt b/test/yarp/snapshots/unparser/corpus/literal/rescue.txt index ed653b5d1362e9..8898a6e1a0db24 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/rescue.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/rescue.txt @@ -29,6 +29,7 @@ ProgramNode(0...64)( LocalVariableWriteNode(37...64)( :x, 0, + (37...38), ParenthesesNode(41...64)( StatementsNode(42...63)( [RescueModifierNode(42...63)( @@ -65,7 +66,6 @@ ProgramNode(0...64)( (41...42), (63...64) ), - (37...38), (39...40) )] ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index 083a0140d9b974..7ec3a338ce9ffc 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -6,25 +6,14 @@ ProgramNode(0...999)( (0...6), ConstantReadNode(7...8)(), StatementsNode(11...31)( - [OrWriteNode(11...31)( - LocalVariableWriteNode(11...14)(:foo, 0, nil, (11...14), nil), + [LocalVariableOrWriteNode(11...31)( + (11...14), + (15...18), ParenthesesNode(19...31)( - StatementsNode(21...30)( - [MultiWriteNode(21...30)( - [LocalVariableWriteNode(21...22)( - :a, - 0, - nil, - (21...22), - nil - ), - LocalVariableWriteNode(24...25)( - :_, - 0, - nil, - (24...25), - nil - )], + StatementsNode(20...30)( + [MultiWriteNode(20...30)( + [LocalVariableTargetNode(21...22)(:a, 0), + LocalVariableTargetNode(24...25)(:_, 0)], (27...28), CallNode(29...30)( nil, @@ -44,7 +33,8 @@ ProgramNode(0...999)( (19...20), (30...31) ), - (15...18) + :foo, + 0 )] ), (32...35), @@ -58,8 +48,8 @@ ProgramNode(0...999)( [LocalVariableWriteNode(48...57)( :local, 0, - IntegerNode(56...57)(), (48...53), + IntegerNode(56...57)(), (54...55) ), CallNode(60...69)( @@ -291,6 +281,7 @@ ProgramNode(0...999)( CallNode(255...272)( UntilNode(255...268)( (255...260), + (265...268), CallNode(261...264)( nil, nil, @@ -317,6 +308,7 @@ ProgramNode(0...999)( CallNode(273...290)( WhileNode(273...286)( (273...278), + (283...286), CallNode(279...282)( nil, nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/since/27.txt b/test/yarp/snapshots/unparser/corpus/literal/since/27.txt index cccfbc64b47061..f8709ae21af282 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/since/27.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/since/27.txt @@ -4,6 +4,8 @@ ProgramNode(0...22)( [LambdaNode(0...16)( [], (0...2), + (3...4), + (15...16), nil, StatementsNode(7...14)( [CallNode(7...14)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/since/30.txt b/test/yarp/snapshots/unparser/corpus/literal/since/30.txt index 15d5136d8308f9..4aba7677950132 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/since/30.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/since/30.txt @@ -5,7 +5,7 @@ ProgramNode(0...51)( IntegerNode(0...1)(), ArrayPatternNode(5...8)( nil, - [LocalVariableWriteNode(6...7)(:a, 0, nil, (6...7), nil)], + [LocalVariableTargetNode(6...7)(:a, 0)], nil, [], (5...6), @@ -42,10 +42,10 @@ ProgramNode(0...51)( FindPatternNode(39...51)( nil, SplatNode(40...41)((40...41), nil), - [LocalVariableWriteNode(43...44)(:a, 0, nil, (43...44), nil)], + [LocalVariableTargetNode(43...44)(:a, 0)], SplatNode(46...50)( (46...47), - LocalVariableWriteNode(47...50)(:foo, 0, nil, (47...50), nil) + LocalVariableTargetNode(47...50)(:foo, 0) ), (39...40), (50...51) diff --git a/test/yarp/snapshots/unparser/corpus/literal/variables.txt b/test/yarp/snapshots/unparser/corpus/literal/variables.txt index b7ab14e36c35f7..fe98a8cd57f251 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/variables.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/variables.txt @@ -2,10 +2,10 @@ ProgramNode(0...66)( [], StatementsNode(0...66)( [CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), - InstanceVariableReadNode(2...4)(), - ClassVariableReadNode(5...8)(), + InstanceVariableReadNode(2...4)(:@a), + ClassVariableReadNode(5...8)(:@@a), GlobalVariableReadNode(9...11)(), - NumberedReferenceReadNode(12...14)(), + NumberedReferenceReadNode(12...14)(1), BackReferenceReadNode(15...17)(), ConstantReadNode(18...23)(), ConstantPathNode(24...37)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/while.txt b/test/yarp/snapshots/unparser/corpus/literal/while.txt index dae7e0f0f31215..7fcd66a01e61d1 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/while.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/while.txt @@ -32,6 +32,7 @@ ProgramNode(0...620)( StatementsNode(27...60)( [WhileNode(27...60)( (27...32), + (57...60), CallNode(33...36)( nil, nil, @@ -47,8 +48,8 @@ ProgramNode(0...620)( [LocalVariableWriteNode(43...52)( :foo, 0, - LocalVariableReadNode(49...52)(:bar, 0), (43...46), + LocalVariableReadNode(49...52)(:bar, 0), (47...48) )] ), @@ -72,6 +73,7 @@ ProgramNode(0...620)( StatementsNode(80...106)( [WhileNode(80...106)( (90...95), + nil, CallNode(96...106)( LocalVariableReadNode(96...99)(:foo, 0), nil, @@ -99,6 +101,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(80...89)( :foo, 0, + (80...83), CallNode(86...89)( nil, nil, @@ -110,7 +113,6 @@ ProgramNode(0...620)( 2, "bar" ), - (80...83), (84...85) )] ), @@ -132,11 +134,13 @@ ProgramNode(0...620)( StatementsNode(123...142)( [WhileNode(123...142)( (133...138), + nil, LocalVariableReadNode(139...142)(:foo, 0), StatementsNode(123...132)( [LocalVariableWriteNode(123...132)( :foo, 0, + (123...126), CallNode(129...132)( nil, nil, @@ -148,7 +152,6 @@ ProgramNode(0...620)( 2, "bar" ), - (123...126), (127...128) )] ), @@ -165,11 +168,13 @@ ProgramNode(0...620)( StatementsNode(159...178)( [UntilNode(159...178)( (169...174), + nil, LocalVariableReadNode(175...178)(:foo, 0), StatementsNode(159...168)( [LocalVariableWriteNode(159...168)( :foo, 0, + (159...162), CallNode(165...168)( nil, nil, @@ -181,7 +186,6 @@ ProgramNode(0...620)( 2, "bar" ), - (159...162), (163...164) )] ), @@ -198,6 +202,7 @@ ProgramNode(0...620)( StatementsNode(195...224)( [WhileNode(195...224)( (195...200), + (221...224), CallNode(201...204)( nil, nil, @@ -213,6 +218,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(209...218)( :foo, 0, + (209...212), CallNode(215...218)( nil, nil, @@ -224,7 +230,6 @@ ProgramNode(0...620)( 2, "bar" ), - (209...212), (213...214) )] ), @@ -265,6 +270,7 @@ ProgramNode(0...620)( StatementsNode(258...291)( [WhileNode(258...291)( (258...263), + (288...291), CallNode(264...267)( nil, nil, @@ -280,6 +286,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(274...283)( :foo, 0, + (274...277), CallNode(280...283)( nil, nil, @@ -291,7 +298,6 @@ ProgramNode(0...620)( 2, "bar" ), - (274...277), (278...279) )] ), @@ -339,11 +345,13 @@ ProgramNode(0...620)( StatementsNode(329...362)( [WhileNode(329...362)( (329...334), + (359...362), LocalVariableReadNode(335...338)(:foo, 0), StatementsNode(345...354)( [LocalVariableWriteNode(345...354)( :foo, 0, + (345...348), CallNode(351...354)( nil, nil, @@ -355,7 +363,6 @@ ProgramNode(0...620)( 2, "bar" ), - (345...348), (349...350) )] ), @@ -375,10 +382,12 @@ ProgramNode(0...620)( LocalVariableWriteNode(371...402)( :x, 0, + (371...372), ParenthesesNode(375...402)( StatementsNode(376...401)( [WhileNode(376...401)( (392...397), + nil, CallNode(398...401)( nil, nil, @@ -418,11 +427,11 @@ ProgramNode(0...620)( (375...376), (401...402) ), - (371...372), (373...374) ), WhileNode(403...428)( (419...424), + nil, CallNode(425...428)( nil, nil, @@ -460,6 +469,7 @@ ProgramNode(0...620)( ), UntilNode(429...460)( (451...456), + nil, CallNode(457...460)( nil, nil, @@ -508,6 +518,7 @@ ProgramNode(0...620)( ), WhileNode(461...492)( (483...488), + nil, CallNode(489...492)( nil, nil, @@ -554,15 +565,23 @@ ProgramNode(0...620)( ), 1 ), - WhileNode(493...508)((493...498), FalseNode(499...504)(), nil, 0), + WhileNode(493...508)( + (493...498), + (505...508), + FalseNode(499...504)(), + nil, + 0 + ), WhileNode(509...528)( (509...514), + (525...528), FalseNode(515...520)(), StatementsNode(523...524)([IntegerNode(523...524)()]), 0 ), WhileNode(529...556)( (529...534), + (553...556), ParenthesesNode(535...544)( StatementsNode(536...543)( [CallNode(536...543)( @@ -585,15 +604,23 @@ ProgramNode(0...620)( ), 0 ), - UntilNode(557...572)((557...562), FalseNode(563...568)(), nil, 0), + UntilNode(557...572)( + (557...562), + (569...572), + FalseNode(563...568)(), + nil, + 0 + ), UntilNode(573...592)( (573...578), + (589...592), FalseNode(579...584)(), StatementsNode(587...588)([IntegerNode(587...588)()]), 0 ), UntilNode(593...620)( (593...598), + (617...620), ParenthesesNode(599...608)( StatementsNode(600...607)( [CallNode(600...607)( diff --git a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt index 14843a83a65114..8822eb53337c33 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt @@ -207,7 +207,9 @@ ProgramNode(0...608)( [StringNode(563...564)(nil, (563...564), nil, "a"), EmbeddedStatementsNode(564...569)( (564...566), - StatementsNode(566...568)([InstanceVariableReadNode(566...568)()]), + StatementsNode(566...568)( + [InstanceVariableReadNode(566...568)(:@a)] + ), (568...569) )], (569...570) @@ -220,7 +222,7 @@ ProgramNode(0...608)( [StringNode(576...577)(nil, (576...577), nil, "a"), EmbeddedVariableNode(577...580)( (577...578), - InstanceVariableReadNode(578...580)() + InstanceVariableReadNode(578...580)(:@a) )], (580...581) ), @@ -244,7 +246,7 @@ ProgramNode(0...608)( [StringNode(598...599)(nil, (598...599), nil, "a"), EmbeddedVariableNode(599...603)( (599...600), - ClassVariableReadNode(600...603)() + ClassVariableReadNode(600...603)(:@@a) )], (603...604) ), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt index d8578058df0b35..5e18c54da412a6 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt @@ -15,7 +15,7 @@ ProgramNode(0...131)( (71...74), [EmbeddedStatementsNode(74...81)( (74...76), - StatementsNode(76...80)([InstanceVariableReadNode(76...80)()]), + StatementsNode(76...80)([InstanceVariableReadNode(76...80)(:@bar)]), (80...81) ), StringNode(81...84)(nil, (81...84), nil, "baz")], diff --git a/test/yarp/snapshots/unparser/corpus/semantic/while.txt b/test/yarp/snapshots/unparser/corpus/semantic/while.txt index 148db33da6523f..d6d4198447aabe 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/while.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/while.txt @@ -3,6 +3,7 @@ ProgramNode(0...188)( StatementsNode(0...188)( [UntilNode(0...13)( (2...7), + nil, CallNode(8...13)( nil, nil, @@ -21,6 +22,7 @@ ProgramNode(0...188)( ), UntilNode(15...34)( (15...20), + (31...34), CallNode(21...26)( nil, nil, @@ -39,11 +41,13 @@ ProgramNode(0...188)( ), WhileNode(36...55)( (46...51), + nil, LocalVariableReadNode(52...55)(:foo, 0), StatementsNode(36...45)( [LocalVariableWriteNode(36...45)( :foo, 0, + (36...39), CallNode(42...45)( nil, nil, @@ -55,7 +59,6 @@ ProgramNode(0...188)( 2, "bar" ), - (36...39), (40...41) )] ), @@ -63,6 +66,7 @@ ProgramNode(0...188)( ), UntilNode(57...75)( (59...64), + nil, AndNode(65...75)( CallNode(65...66)(nil, nil, (65...66), nil, nil, nil, nil, 2, "b"), CallNode(70...75)( @@ -85,11 +89,12 @@ ProgramNode(0...188)( ), WhileNode(77...96)( (77...82), + (93...96), LocalVariableWriteNode(83...88)( :a, 0, - CallNode(87...88)(nil, nil, (87...88), nil, nil, nil, nil, 2, "b"), (83...84), + CallNode(87...88)(nil, nil, (87...88), nil, nil, nil, nil, 2, "b"), (85...86) ), StatementsNode(91...92)([LocalVariableReadNode(91...92)(:a, 0)]), @@ -97,6 +102,7 @@ ProgramNode(0...188)( ), UntilNode(98...130)( (100...105), + nil, CallNode(106...130)( nil, nil, @@ -139,6 +145,7 @@ ProgramNode(0...188)( [LocalVariableWriteNode(143...152)( :foo, 0, + (143...146), CallNode(149...152)( nil, nil, @@ -150,16 +157,17 @@ ProgramNode(0...188)( 2, "exp" ), - (143...146), (147...148) ), WhileNode(155...184)( (155...160), + (181...184), LocalVariableReadNode(161...164)(:foo, 0), StatementsNode(169...178)( [LocalVariableWriteNode(169...178)( :foo, 0, + (169...172), CallNode(175...178)( nil, nil, @@ -171,7 +179,6 @@ ProgramNode(0...188)( 2, "bar" ), - (169...172), (173...174) )] ), diff --git a/test/yarp/snapshots/until.txt b/test/yarp/snapshots/until.txt index 3218ce2d423b1b..6a93708e5b6da2 100644 --- a/test/yarp/snapshots/until.txt +++ b/test/yarp/snapshots/until.txt @@ -3,36 +3,42 @@ ProgramNode(0...109)( StatementsNode(0...109)( [UntilNode(0...18)( (0...5), + (15...18), TrueNode(6...10)(), StatementsNode(12...13)([IntegerNode(12...13)()]), 0 ), UntilNode(20...32)( (22...27), + nil, TrueNode(28...32)(), StatementsNode(20...21)([IntegerNode(20...21)()]), 0 ), UntilNode(34...50)( (40...45), + nil, TrueNode(46...50)(), StatementsNode(34...39)([BreakNode(34...39)(nil, (34...39))]), 0 ), UntilNode(52...67)( (57...62), + nil, TrueNode(63...67)(), StatementsNode(52...56)([NextNode(52...56)(nil, (52...56))]), 0 ), UntilNode(69...86)( (76...81), + nil, TrueNode(82...86)(), StatementsNode(69...75)([ReturnNode(69...75)((69...75), nil)]), 0 ), UntilNode(88...109)( (99...104), + nil, CallNode(105...109)( nil, nil, diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index a4373717f511d9..edd27690a1a2ec 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -1,21 +1,23 @@ ProgramNode(0...293)( [:abc, :foo, :bar, :baz], StatementsNode(0...293)( - [ClassVariableReadNode(0...5)(), + [ClassVariableReadNode(0...5)(:@@abc), ClassVariableWriteNode(7...16)( + :@@abc, (7...12), IntegerNode(15...16)(), (13...14) ), MultiWriteNode(18...34)( - [ClassVariableWriteNode(18...23)((18...23), nil, nil), - ClassVariableWriteNode(25...30)((25...30), nil, nil)], + [ClassVariableTargetNode(18...23)(:@@foo), + ClassVariableTargetNode(25...30)(:@@bar)], (31...32), IntegerNode(33...34)(), nil, nil ), ClassVariableWriteNode(36...48)( + :@@foo, (36...41), ArrayNode(44...48)( [IntegerNode(44...45)(), IntegerNode(47...48)()], @@ -26,12 +28,13 @@ ProgramNode(0...293)( ), GlobalVariableWriteNode(50...58)( (50...54), - (55...56), - IntegerNode(57...58)() + IntegerNode(57...58)(), + (55...56) ), GlobalVariableReadNode(60...64)(), - InstanceVariableReadNode(66...70)(), + InstanceVariableReadNode(66...70)(:@abc), InstanceVariableWriteNode(72...80)( + :@abc, (72...76), IntegerNode(79...80)(), (77...78) @@ -40,13 +43,13 @@ ProgramNode(0...293)( LocalVariableWriteNode(85...92)( :abc, 0, - IntegerNode(91...92)(), (85...88), + IntegerNode(91...92)(), (89...90) ), MultiWriteNode(94...108)( - [GlobalVariableWriteNode(94...98)((94...98), nil, nil), - GlobalVariableWriteNode(100...104)((100...104), nil, nil)], + [GlobalVariableTargetNode(94...98)(), + GlobalVariableTargetNode(100...104)()], (105...106), IntegerNode(107...108)(), nil, @@ -54,22 +57,23 @@ ProgramNode(0...293)( ), GlobalVariableWriteNode(110...121)( (110...114), - (115...116), ArrayNode(117...121)( [IntegerNode(117...118)(), IntegerNode(120...121)()], nil, nil - ) + ), + (115...116) ), MultiWriteNode(123...137)( - [InstanceVariableWriteNode(123...127)((123...127), nil, nil), - InstanceVariableWriteNode(129...133)((129...133), nil, nil)], + [InstanceVariableTargetNode(123...127)(:@foo), + InstanceVariableTargetNode(129...133)(:@bar)], (134...135), IntegerNode(136...137)(), nil, nil ), InstanceVariableWriteNode(139...150)( + :@foo, (139...143), ArrayNode(146...150)( [IntegerNode(146...147)(), IntegerNode(149...150)()], @@ -81,34 +85,34 @@ ProgramNode(0...293)( LocalVariableWriteNode(152...159)( :foo, 0, - IntegerNode(158...159)(), (152...155), + IntegerNode(158...159)(), (156...157) ), LocalVariableWriteNode(161...171)( :foo, 0, + (161...164), ArrayNode(167...171)( [IntegerNode(167...168)(), IntegerNode(170...171)()], nil, nil ), - (161...164), (165...166) ), LocalVariableWriteNode(173...183)( :foo, 0, + (173...176), ArrayNode(179...183)( [IntegerNode(179...180)(), IntegerNode(182...183)()], nil, nil ), - (173...176), (177...178) ), MultiWriteNode(185...198)( - [LocalVariableWriteNode(185...188)(:foo, 0, nil, (185...188), nil), + [LocalVariableTargetNode(185...188)(:foo, 0), SplatNode(190...191)((190...191), nil)], (192...193), ArrayNode(194...198)( @@ -120,7 +124,7 @@ ProgramNode(0...293)( nil ), MultiWriteNode(200...211)( - [LocalVariableWriteNode(200...203)(:foo, 0, nil, (200...203), nil), + [LocalVariableTargetNode(200...203)(:foo, 0), SplatNode(203...204)((203...204), nil)], (205...206), ArrayNode(207...211)( @@ -132,10 +136,10 @@ ProgramNode(0...293)( nil ), MultiWriteNode(213...229)( - [LocalVariableWriteNode(213...216)(:foo, 0, nil, (213...216), nil), + [LocalVariableTargetNode(213...216)(:foo, 0), SplatNode(218...222)( (218...219), - LocalVariableWriteNode(219...222)(:bar, 0, nil, (219...222), nil) + LocalVariableTargetNode(219...222)(:bar, 0) )], (223...224), ArrayNode(225...229)( @@ -147,10 +151,10 @@ ProgramNode(0...293)( nil ), MultiWriteNode(231...258)( - [LocalVariableWriteNode(231...234)(:foo, 0, nil, (231...234), nil), - MultiWriteNode(237...246)( - [LocalVariableWriteNode(237...240)(:bar, 0, nil, (237...240), nil), - LocalVariableWriteNode(242...245)(:baz, 0, nil, (242...245), nil)], + [LocalVariableTargetNode(231...234)(:foo, 0), + MultiWriteNode(236...246)( + [LocalVariableTargetNode(237...240)(:bar, 0), + LocalVariableTargetNode(242...245)(:baz, 0)], nil, nil, (236...237), @@ -173,6 +177,7 @@ ProgramNode(0...293)( LocalVariableWriteNode(260...270)( :foo, 0, + (260...263), ArrayNode(266...270)( [SplatNode(266...270)( (266...267), @@ -181,7 +186,6 @@ ProgramNode(0...293)( nil, nil ), - (260...263), (264...265) ), ConstantWriteNode(272...282)( diff --git a/test/yarp/snapshots/while.txt b/test/yarp/snapshots/while.txt index 484eb68b2c9c02..4aa5fe85f1428d 100644 --- a/test/yarp/snapshots/while.txt +++ b/test/yarp/snapshots/while.txt @@ -3,36 +3,42 @@ ProgramNode(0...314)( StatementsNode(0...314)( [WhileNode(0...18)( (0...5), + (15...18), TrueNode(6...10)(), StatementsNode(12...13)([IntegerNode(12...13)()]), 0 ), WhileNode(20...32)( (22...27), + nil, TrueNode(28...32)(), StatementsNode(20...21)([IntegerNode(20...21)()]), 0 ), WhileNode(34...50)( (40...45), + nil, TrueNode(46...50)(), StatementsNode(34...39)([BreakNode(34...39)(nil, (34...39))]), 0 ), WhileNode(52...67)( (57...62), + nil, TrueNode(63...67)(), StatementsNode(52...56)([NextNode(52...56)(nil, (52...56))]), 0 ), WhileNode(69...86)( (76...81), + nil, TrueNode(82...86)(), StatementsNode(69...75)([ReturnNode(69...75)((69...75), nil)]), 0 ), WhileNode(88...109)( (99...104), + nil, CallNode(105...109)( nil, nil, @@ -64,6 +70,7 @@ ProgramNode(0...314)( ), WhileNode(111...161)( (111...116), + (158...161), DefNode(117...149)( (126...129), SelfNode(121...125)(), @@ -105,6 +112,7 @@ ProgramNode(0...314)( ), WhileNode(163...210)( (163...168), + (207...210), ClassNode(169...198)( [:a], (169...174), @@ -115,6 +123,7 @@ ProgramNode(0...314)( [LocalVariableWriteNode(179...193)( :a, 0, + (179...180), CallNode(183...193)( nil, nil, @@ -126,7 +135,6 @@ ProgramNode(0...314)( 0, "tap" ), - (179...180), (181...182) )] ), @@ -138,6 +146,7 @@ ProgramNode(0...314)( ), WhileNode(212...260)( (212...217), + (257...260), SingletonClassNode(218...248)( [], (218...223), @@ -163,6 +172,7 @@ ProgramNode(0...314)( ), WhileNode(262...314)( (262...267), + (311...314), SingletonClassNode(268...302)( [:a], (268...273), @@ -172,6 +182,7 @@ ProgramNode(0...314)( [LocalVariableWriteNode(283...297)( :a, 0, + (283...284), CallNode(287...297)( nil, nil, @@ -183,7 +194,6 @@ ProgramNode(0...314)( 0, "tap" ), - (283...284), (285...286) )] ), diff --git a/test/yarp/snapshots/whitequark/and_or_masgn.txt b/test/yarp/snapshots/whitequark/and_or_masgn.txt index eb768ac90a4731..bc64788ca43665 100644 --- a/test/yarp/snapshots/whitequark/and_or_masgn.txt +++ b/test/yarp/snapshots/whitequark/and_or_masgn.txt @@ -6,8 +6,8 @@ ProgramNode(0...40)( ParenthesesNode(7...19)( StatementsNode(8...18)( [MultiWriteNode(8...18)( - [LocalVariableWriteNode(8...9)(:a, 0, nil, (8...9), nil), - LocalVariableWriteNode(11...12)(:b, 0, nil, (11...12), nil)], + [LocalVariableTargetNode(8...9)(:a, 0), + LocalVariableTargetNode(11...12)(:b, 0)], (13...14), CallNode(15...18)( nil, @@ -34,8 +34,8 @@ ProgramNode(0...40)( ParenthesesNode(28...40)( StatementsNode(29...39)( [MultiWriteNode(29...39)( - [LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + [LocalVariableTargetNode(29...30)(:a, 0), + LocalVariableTargetNode(32...33)(:b, 0)], (34...35), CallNode(36...39)( nil, diff --git a/test/yarp/snapshots/whitequark/array_words_interp.txt b/test/yarp/snapshots/whitequark/array_words_interp.txt index 3cc21323f7f910..d4bc9292e6ea64 100644 --- a/test/yarp/snapshots/whitequark/array_words_interp.txt +++ b/test/yarp/snapshots/whitequark/array_words_interp.txt @@ -51,7 +51,7 @@ ProgramNode(0...38)( StringNode(29...32)(nil, (29...32), nil, "foo"), EmbeddedVariableNode(32...37)( (32...33), - InstanceVariableReadNode(33...37)() + InstanceVariableReadNode(33...37)(:@baz) )], nil )], diff --git a/test/yarp/snapshots/whitequark/asgn_cmd.txt b/test/yarp/snapshots/whitequark/asgn_cmd.txt index 218c28dc0f24ff..ef24cc0477f9e8 100644 --- a/test/yarp/snapshots/whitequark/asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/asgn_cmd.txt @@ -4,9 +4,11 @@ ProgramNode(0...30)( [LocalVariableWriteNode(0...17)( :foo, 0, + (0...3), LocalVariableWriteNode(6...17)( :bar, 0, + (6...9), CallNode(12...17)( nil, nil, @@ -18,15 +20,14 @@ ProgramNode(0...30)( 0, "m" ), - (6...9), (10...11) ), - (0...3), (4...5) ), LocalVariableWriteNode(19...30)( :foo, 0, + (19...22), CallNode(25...30)( nil, nil, @@ -38,7 +39,6 @@ ProgramNode(0...30)( 0, "m" ), - (19...22), (23...24) )] ) diff --git a/test/yarp/snapshots/whitequark/asgn_mrhs.txt b/test/yarp/snapshots/whitequark/asgn_mrhs.txt index fddbc439e5ae05..aaf2bedb6f340c 100644 --- a/test/yarp/snapshots/whitequark/asgn_mrhs.txt +++ b/test/yarp/snapshots/whitequark/asgn_mrhs.txt @@ -4,6 +4,7 @@ ProgramNode(0...41)( [LocalVariableWriteNode(0...10)( :foo, 0, + (0...3), ArrayNode(6...10)( [SplatNode(6...10)( (6...7), @@ -12,24 +13,24 @@ ProgramNode(0...41)( nil, nil ), - (0...3), (4...5) ), LocalVariableWriteNode(12...24)( :foo, 0, + (12...15), ArrayNode(18...24)( [CallNode(18...21)(nil, nil, (18...21), nil, nil, nil, nil, 2, "bar"), IntegerNode(23...24)()], nil, nil ), - (12...15), (16...17) ), LocalVariableWriteNode(26...41)( :foo, 0, + (26...29), ArrayNode(32...41)( [CallNode(32...35)(nil, nil, (32...35), nil, nil, nil, nil, 2, "baz"), SplatNode(37...41)( @@ -49,7 +50,6 @@ ProgramNode(0...41)( nil, nil ), - (26...29), (30...31) )] ) diff --git a/test/yarp/snapshots/whitequark/bug_435.txt b/test/yarp/snapshots/whitequark/bug_435.txt index ed5a985e3a65ac..7d74c8c8a2af64 100644 --- a/test/yarp/snapshots/whitequark/bug_435.txt +++ b/test/yarp/snapshots/whitequark/bug_435.txt @@ -9,6 +9,8 @@ ProgramNode(0...14)( [LambdaNode(3...12)( [:foo], (3...5), + (10...11), + (11...12), BlockParametersNode(6...9)( ParametersNode(6...9)( [RequiredParameterNode(6...9)(:foo)], diff --git a/test/yarp/snapshots/whitequark/bug_cmdarg.txt b/test/yarp/snapshots/whitequark/bug_cmdarg.txt index f953e561456ef8..b3f79aafd460fc 100644 --- a/test/yarp/snapshots/whitequark/bug_cmdarg.txt +++ b/test/yarp/snapshots/whitequark/bug_cmdarg.txt @@ -55,6 +55,8 @@ ProgramNode(0...56)( LambdaNode(35...56)( [], (35...37), + (38...40), + (53...56), nil, StatementsNode(41...52)( [CallNode(41...52)( diff --git a/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt b/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt index c59109d0a95c16..7792fa495af6ce 100644 --- a/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt +++ b/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt @@ -4,6 +4,8 @@ ProgramNode(0...19)( [LambdaNode(0...12)( [:scope], (0...2), + (10...11), + (11...12), BlockParametersNode(2...9)( ParametersNode(3...8)( [RequiredParameterNode(3...8)(:scope)], diff --git a/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt b/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt index 0f517f935c115f..f5e086059a86c3 100644 --- a/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt +++ b/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt @@ -3,6 +3,7 @@ ProgramNode(0...23)( StatementsNode(0...23)( [WhileNode(0...23)( (0...5), + (20...23), CallNode(6...16)( ParenthesesNode(10...16)( StatementsNode(11...15)([TrueNode(11...15)()]), diff --git a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt index 0af9f40e8a7ec7..1e954972a8df0b 100644 --- a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt @@ -3,6 +3,7 @@ ProgramNode(0...197)( StatementsNode(0...197)( [WhileNode(0...52)( (0...5), + (49...52), SingletonClassNode(6...40)( [:a], (6...11), @@ -12,6 +13,7 @@ ProgramNode(0...197)( [LocalVariableWriteNode(21...35)( :a, 0, + (21...22), CallNode(25...35)( nil, nil, @@ -23,7 +25,6 @@ ProgramNode(0...197)( 0, "tap" ), - (21...22), (23...24) )] ), @@ -34,6 +35,7 @@ ProgramNode(0...197)( ), WhileNode(54...102)( (54...59), + (99...102), SingletonClassNode(60...90)( [], (60...65), @@ -59,6 +61,7 @@ ProgramNode(0...197)( ), WhileNode(104...151)( (104...109), + (148...151), ClassNode(110...139)( [:a], (110...115), @@ -69,6 +72,7 @@ ProgramNode(0...197)( [LocalVariableWriteNode(120...134)( :a, 0, + (120...121), CallNode(124...134)( nil, nil, @@ -80,7 +84,6 @@ ProgramNode(0...197)( 0, "tap" ), - (120...121), (122...123) )] ), @@ -92,6 +95,7 @@ ProgramNode(0...197)( ), WhileNode(153...197)( (153...158), + (194...197), ClassNode(159...185)( [], (159...164), diff --git a/test/yarp/snapshots/whitequark/cond_begin_masgn.txt b/test/yarp/snapshots/whitequark/cond_begin_masgn.txt index 10e2f5d8dc72dc..8114bd1cb4491d 100644 --- a/test/yarp/snapshots/whitequark/cond_begin_masgn.txt +++ b/test/yarp/snapshots/whitequark/cond_begin_masgn.txt @@ -7,8 +7,8 @@ ProgramNode(0...25)( StatementsNode(4...19)( [CallNode(4...7)(nil, nil, (4...7), nil, nil, nil, nil, 2, "bar"), MultiWriteNode(9...19)( - [LocalVariableWriteNode(9...10)(:a, 0, nil, (9...10), nil), - LocalVariableWriteNode(12...13)(:b, 0, nil, (12...13), nil)], + [LocalVariableTargetNode(9...10)(:a, 0), + LocalVariableTargetNode(12...13)(:b, 0)], (14...15), CallNode(16...19)( nil, diff --git a/test/yarp/snapshots/whitequark/const_op_asgn.txt b/test/yarp/snapshots/whitequark/const_op_asgn.txt index e212170a1d799d..8a7bb4eae86a92 100644 --- a/test/yarp/snapshots/whitequark/const_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/const_op_asgn.txt @@ -1,53 +1,41 @@ ProgramNode(0...77)( [], StatementsNode(0...77)( - [OperatorWriteNode(0...8)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), + [ConstantPathOperatorWriteNode(0...8)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), (4...6), - :+, - IntegerNode(7...8)() + IntegerNode(7...8)(), + :+ ), - OperatorWriteNode(10...16)( - ConstantWriteNode(10...11)((10...11), nil, nil), + ConstantOperatorWriteNode(10...16)( + (10...11), (12...14), - :+, - IntegerNode(15...16)() + IntegerNode(15...16)(), + :+ ), - OperatorWriteNode(18...27)( - ConstantPathWriteNode(18...22)( - ConstantPathNode(18...22)( - ConstantReadNode(18...19)(), - ConstantReadNode(21...22)(), - (19...21) - ), - nil, - nil + ConstantPathOperatorWriteNode(18...27)( + ConstantPathNode(18...22)( + ConstantReadNode(18...19)(), + ConstantReadNode(21...22)(), + (19...21) ), (23...25), - :+, - IntegerNode(26...27)() + IntegerNode(26...27)(), + :+ ), DefNode(29...50)( (33...34), nil, nil, StatementsNode(36...45)( - [OrWriteNode(36...45)( - ConstantPathWriteNode(36...39)( - ConstantPathNode(36...39)( - nil, - ConstantReadNode(38...39)(), - (36...38) - ), + [ConstantPathOrWriteNode(36...45)( + ConstantPathNode(36...39)( nil, - nil + ConstantReadNode(38...39)(), + (36...38) ), - IntegerNode(44...45)(), - (40...43) + (40...43), + IntegerNode(44...45)() )] ), [], @@ -63,18 +51,14 @@ ProgramNode(0...77)( nil, nil, StatementsNode(59...72)( - [OrWriteNode(59...72)( - ConstantPathWriteNode(59...66)( - ConstantPathNode(59...66)( - SelfNode(59...63)(), - ConstantReadNode(65...66)(), - (63...65) - ), - nil, - nil + [ConstantPathOrWriteNode(59...72)( + ConstantPathNode(59...66)( + SelfNode(59...63)(), + ConstantReadNode(65...66)(), + (63...65) ), - IntegerNode(71...72)(), - (67...70) + (67...70), + IntegerNode(71...72)() )] ), [], diff --git a/test/yarp/snapshots/whitequark/cvar.txt b/test/yarp/snapshots/whitequark/cvar.txt index 069ff24b17ea3a..6a1f2e50f48566 100644 --- a/test/yarp/snapshots/whitequark/cvar.txt +++ b/test/yarp/snapshots/whitequark/cvar.txt @@ -1 +1,4 @@ -ProgramNode(0...5)([], StatementsNode(0...5)([ClassVariableReadNode(0...5)()])) +ProgramNode(0...5)( + [], + StatementsNode(0...5)([ClassVariableReadNode(0...5)(:@@foo)]) +) diff --git a/test/yarp/snapshots/whitequark/cvasgn.txt b/test/yarp/snapshots/whitequark/cvasgn.txt index b972caaab79f0e..7e07810eb6cc0f 100644 --- a/test/yarp/snapshots/whitequark/cvasgn.txt +++ b/test/yarp/snapshots/whitequark/cvasgn.txt @@ -1,6 +1,11 @@ ProgramNode(0...10)( [], StatementsNode(0...10)( - [ClassVariableWriteNode(0...10)((0...5), IntegerNode(8...10)(), (6...7))] + [ClassVariableWriteNode(0...10)( + :@@var, + (0...5), + IntegerNode(8...10)(), + (6...7) + )] ) ) diff --git a/test/yarp/snapshots/whitequark/defined.txt b/test/yarp/snapshots/whitequark/defined.txt index 12a30055939bca..3ddfdd84bbf5ea 100644 --- a/test/yarp/snapshots/whitequark/defined.txt +++ b/test/yarp/snapshots/whitequark/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [DefinedNode(0...13)( nil, - InstanceVariableReadNode(9...13)(), + InstanceVariableReadNode(9...13)(:@foo), nil, (0...8) ), diff --git a/test/yarp/snapshots/whitequark/for.txt b/test/yarp/snapshots/whitequark/for.txt index e62da36cd540fa..1250760a01f976 100644 --- a/test/yarp/snapshots/whitequark/for.txt +++ b/test/yarp/snapshots/whitequark/for.txt @@ -3,7 +3,7 @@ ProgramNode(0...48)( StatementsNode(0...48)( [ForNode(0...24)( MultiWriteNode(4...5)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(4...5)(:a, 0)], nil, nil, nil, @@ -30,7 +30,7 @@ ProgramNode(0...48)( ), ForNode(26...48)( MultiWriteNode(30...31)( - [LocalVariableWriteNode(30...31)(:a, 0, nil, (30...31), nil)], + [LocalVariableTargetNode(30...31)(:a, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/whitequark/for_mlhs.txt b/test/yarp/snapshots/whitequark/for_mlhs.txt index 07bdb0662a5b9f..fecaeb10caf949 100644 --- a/test/yarp/snapshots/whitequark/for_mlhs.txt +++ b/test/yarp/snapshots/whitequark/for_mlhs.txt @@ -3,8 +3,8 @@ ProgramNode(0...28)( StatementsNode(0...28)( [ForNode(0...28)( MultiWriteNode(4...8)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:b, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(4...5)(:a, 0), + LocalVariableTargetNode(7...8)(:b, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/whitequark/gvasgn.txt b/test/yarp/snapshots/whitequark/gvasgn.txt index d6a53275909939..c4030831e3f3f5 100644 --- a/test/yarp/snapshots/whitequark/gvasgn.txt +++ b/test/yarp/snapshots/whitequark/gvasgn.txt @@ -1,6 +1,6 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [GlobalVariableWriteNode(0...9)((0...4), (5...6), IntegerNode(7...9)())] + [GlobalVariableWriteNode(0...9)((0...4), IntegerNode(7...9)(), (5...6))] ) ) diff --git a/test/yarp/snapshots/whitequark/if_masgn__24.txt b/test/yarp/snapshots/whitequark/if_masgn__24.txt index 94e63ac4f7ff99..24c9ef784ddc58 100644 --- a/test/yarp/snapshots/whitequark/if_masgn__24.txt +++ b/test/yarp/snapshots/whitequark/if_masgn__24.txt @@ -6,8 +6,8 @@ ProgramNode(0...20)( ParenthesesNode(3...15)( StatementsNode(4...14)( [MultiWriteNode(4...14)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:b, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(4...5)(:a, 0), + LocalVariableTargetNode(7...8)(:b, 0)], (9...10), CallNode(11...14)( nil, diff --git a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt index 2096097f5c6ee9..1ebb67a457477a 100644 --- a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt +++ b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt @@ -27,6 +27,7 @@ ProgramNode(0...178)( ConstantPathNode(46...82)( WhileNode(46...74)( (46...51), + (71...74), TrueNode(52...56)(), StatementsNode(58...70)( [BreakNode(58...70)( @@ -69,6 +70,7 @@ ProgramNode(0...178)( ConstantPathNode(137...173)( WhileNode(137...165)( (137...142), + (162...165), TrueNode(143...147)(), StatementsNode(149...161)( [BreakNode(149...161)( diff --git a/test/yarp/snapshots/whitequark/interp_digit_var.txt b/test/yarp/snapshots/whitequark/interp_digit_var.txt index 4af55a97a8bb00..48d7ac96716907 100644 --- a/test/yarp/snapshots/whitequark/interp_digit_var.txt +++ b/test/yarp/snapshots/whitequark/interp_digit_var.txt @@ -83,8 +83,8 @@ ProgramNode(1...465)( "\#@@1", 0 ), - SymbolNode(294...300)(nil, (296...299), nil, "\#@1"), - SymbolNode(304...311)(nil, (306...310), nil, "\#@@1"), + SymbolNode(294...300)((294...296), (296...299), (299...300), "\#@1"), + SymbolNode(304...311)((304...306), (306...310), (310...311), "\#@@1"), SymbolNode(315...321)((315...317), (317...320), (320...321), "\#@1"), SymbolNode(325...332)((325...327), (327...331), (331...332), "\#@@1"), XStringNode(336...341)((336...337), (337...340), (340...341), "\#@1"), diff --git a/test/yarp/snapshots/whitequark/ivar.txt b/test/yarp/snapshots/whitequark/ivar.txt index 88bb2ace357f5f..ae2c8c34b48306 100644 --- a/test/yarp/snapshots/whitequark/ivar.txt +++ b/test/yarp/snapshots/whitequark/ivar.txt @@ -1,4 +1,4 @@ ProgramNode(0...4)( [], - StatementsNode(0...4)([InstanceVariableReadNode(0...4)()]) + StatementsNode(0...4)([InstanceVariableReadNode(0...4)(:@foo)]) ) diff --git a/test/yarp/snapshots/whitequark/ivasgn.txt b/test/yarp/snapshots/whitequark/ivasgn.txt index 4f6f90bbd71dde..8e952de2465dd1 100644 --- a/test/yarp/snapshots/whitequark/ivasgn.txt +++ b/test/yarp/snapshots/whitequark/ivasgn.txt @@ -1,6 +1,11 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [InstanceVariableWriteNode(0...9)((0...4), IntegerNode(7...9)(), (5...6))] + [InstanceVariableWriteNode(0...9)( + :@var, + (0...4), + IntegerNode(7...9)(), + (5...6) + )] ) ) diff --git a/test/yarp/snapshots/whitequark/kwnilarg.txt b/test/yarp/snapshots/whitequark/kwnilarg.txt index d7ca2692996c3c..0995a597c26dfa 100644 --- a/test/yarp/snapshots/whitequark/kwnilarg.txt +++ b/test/yarp/snapshots/whitequark/kwnilarg.txt @@ -4,6 +4,8 @@ ProgramNode(0...46)( [LambdaNode(0...12)( [], (0...2), + (10...11), + (11...12), BlockParametersNode(2...9)( ParametersNode(3...8)( [], diff --git a/test/yarp/snapshots/whitequark/lvasgn.txt b/test/yarp/snapshots/whitequark/lvasgn.txt index 403c7f8042c9b2..bb762001ba0d17 100644 --- a/test/yarp/snapshots/whitequark/lvasgn.txt +++ b/test/yarp/snapshots/whitequark/lvasgn.txt @@ -4,8 +4,8 @@ ProgramNode(0...13)( [LocalVariableWriteNode(0...8)( :var, 0, - IntegerNode(6...8)(), (0...3), + IntegerNode(6...8)(), (4...5) ), LocalVariableReadNode(10...13)(:var, 0)] diff --git a/test/yarp/snapshots/whitequark/masgn.txt b/test/yarp/snapshots/whitequark/masgn.txt index 9945a348c11745..69667e1a38b5a9 100644 --- a/test/yarp/snapshots/whitequark/masgn.txt +++ b/test/yarp/snapshots/whitequark/masgn.txt @@ -1,9 +1,9 @@ -ProgramNode(1...56)( +ProgramNode(0...56)( [:foo, :bar, :baz], - StatementsNode(1...56)( - [MultiWriteNode(1...17)( - [LocalVariableWriteNode(1...4)(:foo, 0, nil, (1...4), nil), - LocalVariableWriteNode(6...9)(:bar, 0, nil, (6...9), nil)], + StatementsNode(0...56)( + [MultiWriteNode(0...17)( + [LocalVariableTargetNode(1...4)(:foo, 0), + LocalVariableTargetNode(6...9)(:bar, 0)], (11...12), ArrayNode(13...17)( [IntegerNode(13...14)(), IntegerNode(16...17)()], @@ -14,8 +14,8 @@ ProgramNode(1...56)( (9...10) ), MultiWriteNode(19...34)( - [LocalVariableWriteNode(19...22)(:foo, 0, nil, (19...22), nil), - LocalVariableWriteNode(24...27)(:bar, 0, nil, (24...27), nil)], + [LocalVariableTargetNode(19...22)(:foo, 0), + LocalVariableTargetNode(24...27)(:bar, 0)], (28...29), ArrayNode(30...34)( [IntegerNode(30...31)(), IntegerNode(33...34)()], @@ -26,9 +26,9 @@ ProgramNode(1...56)( nil ), MultiWriteNode(36...56)( - [LocalVariableWriteNode(36...39)(:foo, 0, nil, (36...39), nil), - LocalVariableWriteNode(41...44)(:bar, 0, nil, (41...44), nil), - LocalVariableWriteNode(46...49)(:baz, 0, nil, (46...49), nil)], + [LocalVariableTargetNode(36...39)(:foo, 0), + LocalVariableTargetNode(41...44)(:bar, 0), + LocalVariableTargetNode(46...49)(:baz, 0)], (50...51), ArrayNode(52...56)( [IntegerNode(52...53)(), IntegerNode(55...56)()], diff --git a/test/yarp/snapshots/whitequark/masgn_attr.txt b/test/yarp/snapshots/whitequark/masgn_attr.txt index c5ab22eb0e980e..1c71499f78f448 100644 --- a/test/yarp/snapshots/whitequark/masgn_attr.txt +++ b/test/yarp/snapshots/whitequark/masgn_attr.txt @@ -13,7 +13,7 @@ ProgramNode(0...63)( 0, "A=" ), - LocalVariableWriteNode(8...11)(:foo, 0, nil, (8...11), nil)], + LocalVariableTargetNode(8...11)(:foo, 0)], (12...13), LocalVariableReadNode(14...17)(:foo, 0), nil, @@ -61,7 +61,7 @@ ProgramNode(0...63)( 0, "a=" ), - LocalVariableWriteNode(54...57)(:foo, 0, nil, (54...57), nil)], + LocalVariableTargetNode(54...57)(:foo, 0)], (58...59), LocalVariableReadNode(60...63)(:foo, 0), nil, diff --git a/test/yarp/snapshots/whitequark/masgn_cmd.txt b/test/yarp/snapshots/whitequark/masgn_cmd.txt index 1ca3b64a0a92e4..14d81d79bcf2b3 100644 --- a/test/yarp/snapshots/whitequark/masgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/masgn_cmd.txt @@ -2,8 +2,8 @@ ProgramNode(0...16)( [:foo, :bar], StatementsNode(0...16)( [MultiWriteNode(0...16)( - [LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), - LocalVariableWriteNode(5...8)(:bar, 0, nil, (5...8), nil)], + [LocalVariableTargetNode(0...3)(:foo, 0), + LocalVariableTargetNode(5...8)(:bar, 0)], (9...10), CallNode(11...16)( nil, diff --git a/test/yarp/snapshots/whitequark/masgn_const.txt b/test/yarp/snapshots/whitequark/masgn_const.txt index 4db22ceb7a3208..7aeb91557c4914 100644 --- a/test/yarp/snapshots/whitequark/masgn_const.txt +++ b/test/yarp/snapshots/whitequark/masgn_const.txt @@ -2,28 +2,20 @@ ProgramNode(0...34)( [:foo], StatementsNode(0...34)( [MultiWriteNode(0...14)( - [ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - LocalVariableWriteNode(5...8)(:foo, 0, nil, (5...8), nil)], + [ConstantPathTargetNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + LocalVariableTargetNode(5...8)(:foo, 0)], (9...10), LocalVariableReadNode(11...14)(:foo, 0), nil, nil ), MultiWriteNode(16...34)( - [ConstantPathWriteNode(16...23)( - ConstantPathNode(16...23)( - SelfNode(16...20)(), - ConstantReadNode(22...23)(), - (20...22) - ), - nil, - nil + [ConstantPathTargetNode(16...23)( + SelfNode(16...20)(), + ConstantReadNode(22...23)(), + (20...22) ), - LocalVariableWriteNode(25...28)(:foo, 0, nil, (25...28), nil)], + LocalVariableTargetNode(25...28)(:foo, 0)], (29...30), LocalVariableReadNode(31...34)(:foo, 0), nil, diff --git a/test/yarp/snapshots/whitequark/masgn_nested.txt b/test/yarp/snapshots/whitequark/masgn_nested.txt index 98f1c094d00e70..b1601b8aa7436a 100644 --- a/test/yarp/snapshots/whitequark/masgn_nested.txt +++ b/test/yarp/snapshots/whitequark/masgn_nested.txt @@ -1,9 +1,9 @@ -ProgramNode(2...30)( +ProgramNode(0...30)( [:b, :a, :c], - StatementsNode(2...30)( - [MultiWriteNode(2...13)( - [MultiWriteNode(2...4)( - [LocalVariableWriteNode(2...3)(:b, 0, nil, (2...3), nil), + StatementsNode(0...30)( + [MultiWriteNode(0...13)( + [MultiWriteNode(1...6)( + [LocalVariableTargetNode(2...3)(:b, 0), SplatNode(3...4)((3...4), nil)], nil, nil, @@ -16,10 +16,10 @@ ProgramNode(2...30)( (6...7) ), MultiWriteNode(15...30)( - [LocalVariableWriteNode(15...16)(:a, 0, nil, (15...16), nil), - MultiWriteNode(19...24)( - [LocalVariableWriteNode(19...20)(:b, 0, nil, (19...20), nil), - LocalVariableWriteNode(22...23)(:c, 0, nil, (22...23), nil)], + [LocalVariableTargetNode(15...16)(:a, 0), + MultiWriteNode(18...24)( + [LocalVariableTargetNode(19...20)(:b, 0), + LocalVariableTargetNode(22...23)(:c, 0)], nil, nil, (18...19), diff --git a/test/yarp/snapshots/whitequark/masgn_splat.txt b/test/yarp/snapshots/whitequark/masgn_splat.txt index 635e1ae6fdd25b..d21760a65b3921 100644 --- a/test/yarp/snapshots/whitequark/masgn_splat.txt +++ b/test/yarp/snapshots/whitequark/masgn_splat.txt @@ -16,8 +16,8 @@ ProgramNode(0...139)( nil, nil ), - LocalVariableWriteNode(12...13)(:c, 0, nil, (12...13), nil), - LocalVariableWriteNode(15...16)(:d, 0, nil, (15...16), nil)], + LocalVariableTargetNode(12...13)(:c, 0), + LocalVariableTargetNode(15...16)(:d, 0)], (17...18), CallNode(19...22)(nil, nil, (19...22), nil, nil, nil, nil, 2, "bar"), nil, @@ -26,7 +26,7 @@ ProgramNode(0...139)( MultiWriteNode(24...32)( [SplatNode(24...26)( (24...25), - LocalVariableWriteNode(25...26)(:b, 0, nil, (25...26), nil) + LocalVariableTargetNode(25...26)(:b, 0) )], (27...28), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "bar"), @@ -37,22 +37,22 @@ ProgramNode(0...139)( [MultiWriteNode(34...36)( [SplatNode(34...36)( (34...35), - LocalVariableWriteNode(35...36)(:b, 0, nil, (35...36), nil) + LocalVariableTargetNode(35...36)(:b, 0) )], nil, nil, nil, nil ), - LocalVariableWriteNode(38...39)(:c, 0, nil, (38...39), nil)], + LocalVariableTargetNode(38...39)(:c, 0)], (40...41), CallNode(42...45)(nil, nil, (42...45), nil, nil, nil, nil, 2, "bar"), nil, nil ), MultiWriteNode(47...65)( - [InstanceVariableWriteNode(47...51)((47...51), nil, nil), - ClassVariableWriteNode(53...58)((53...58), nil, nil)], + [InstanceVariableTargetNode(47...51)(:@foo), + ClassVariableTargetNode(53...58)(:@@bar)], (59...60), ArrayNode(61...65)( [SplatNode(61...65)( @@ -76,7 +76,7 @@ ProgramNode(0...139)( nil ), MultiWriteNode(67...77)( - [LocalVariableWriteNode(67...68)(:a, 0, nil, (67...68), nil), + [LocalVariableTargetNode(67...68)(:a, 0), SplatNode(70...71)((70...71), nil)], (72...73), CallNode(74...77)(nil, nil, (74...77), nil, nil, nil, nil, 2, "bar"), @@ -84,19 +84,19 @@ ProgramNode(0...139)( nil ), MultiWriteNode(79...92)( - [LocalVariableWriteNode(79...80)(:a, 0, nil, (79...80), nil), + [LocalVariableTargetNode(79...80)(:a, 0), SplatNode(82...83)((82...83), nil), - LocalVariableWriteNode(85...86)(:c, 0, nil, (85...86), nil)], + LocalVariableTargetNode(85...86)(:c, 0)], (87...88), CallNode(89...92)(nil, nil, (89...92), nil, nil, nil, nil, 2, "bar"), nil, nil ), MultiWriteNode(94...105)( - [LocalVariableWriteNode(94...95)(:a, 0, nil, (94...95), nil), + [LocalVariableTargetNode(94...95)(:a, 0), SplatNode(97...99)( (97...98), - LocalVariableWriteNode(98...99)(:b, 0, nil, (98...99), nil) + LocalVariableTargetNode(98...99)(:b, 0) )], (100...101), CallNode(102...105)( @@ -114,12 +114,12 @@ ProgramNode(0...139)( nil ), MultiWriteNode(107...121)( - [LocalVariableWriteNode(107...108)(:a, 0, nil, (107...108), nil), + [LocalVariableTargetNode(107...108)(:a, 0), SplatNode(110...112)( (110...111), - LocalVariableWriteNode(111...112)(:b, 0, nil, (111...112), nil) + LocalVariableTargetNode(111...112)(:b, 0) ), - LocalVariableWriteNode(114...115)(:c, 0, nil, (114...115), nil)], + LocalVariableTargetNode(114...115)(:c, 0)], (116...117), CallNode(118...121)( nil, @@ -136,8 +136,8 @@ ProgramNode(0...139)( nil ), MultiWriteNode(123...139)( - [LocalVariableWriteNode(123...124)(:a, 0, nil, (123...124), nil), - LocalVariableWriteNode(126...127)(:b, 0, nil, (126...127), nil)], + [LocalVariableTargetNode(123...124)(:a, 0), + LocalVariableTargetNode(126...127)(:b, 0)], (128...129), ArrayNode(130...139)( [SplatNode(130...134)( diff --git a/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt index 3b67815c7b8647..3179d39b959edc 100644 --- a/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt @@ -3,6 +3,7 @@ ProgramNode(0...190)( StatementsNode(0...190)( [WhileNode(0...45)( (0...5), + (42...45), DefNode(6...33)( (10...13), nil, @@ -44,6 +45,7 @@ ProgramNode(0...190)( ), WhileNode(47...89)( (47...52), + (86...89), DefNode(53...77)( (57...60), nil, @@ -74,6 +76,7 @@ ProgramNode(0...190)( ), WhileNode(91...141)( (91...96), + (138...141), DefNode(97...129)( (106...109), SelfNode(101...105)(), @@ -115,6 +118,7 @@ ProgramNode(0...190)( ), WhileNode(143...190)( (143...148), + (187...190), DefNode(149...178)( (158...161), SelfNode(153...157)(), diff --git a/test/yarp/snapshots/whitequark/not_masgn__24.txt b/test/yarp/snapshots/whitequark/not_masgn__24.txt index badac67e44fb05..47895d8c462466 100644 --- a/test/yarp/snapshots/whitequark/not_masgn__24.txt +++ b/test/yarp/snapshots/whitequark/not_masgn__24.txt @@ -5,8 +5,8 @@ ProgramNode(0...13)( ParenthesesNode(1...13)( StatementsNode(2...12)( [MultiWriteNode(2...12)( - [LocalVariableWriteNode(2...3)(:a, 0, nil, (2...3), nil), - LocalVariableWriteNode(5...6)(:b, 0, nil, (5...6), nil)], + [LocalVariableTargetNode(2...3)(:a, 0), + LocalVariableTargetNode(5...6)(:b, 0)], (7...8), CallNode(9...12)( nil, diff --git a/test/yarp/snapshots/whitequark/nth_ref.txt b/test/yarp/snapshots/whitequark/nth_ref.txt index 3bfe4c9a5c3413..ca131a208ce53f 100644 --- a/test/yarp/snapshots/whitequark/nth_ref.txt +++ b/test/yarp/snapshots/whitequark/nth_ref.txt @@ -1,4 +1,4 @@ ProgramNode(0...3)( [], - StatementsNode(0...3)([NumberedReferenceReadNode(0...3)()]) + StatementsNode(0...3)([NumberedReferenceReadNode(0...3)(10)]) ) diff --git a/test/yarp/snapshots/whitequark/numbered_args_after_27.txt b/test/yarp/snapshots/whitequark/numbered_args_after_27.txt index 45decfc3b708e5..c2a67b4cc32dc0 100644 --- a/test/yarp/snapshots/whitequark/numbered_args_after_27.txt +++ b/test/yarp/snapshots/whitequark/numbered_args_after_27.txt @@ -4,6 +4,8 @@ ProgramNode(0...65)( [LambdaNode(0...17)( [], (0...2), + (3...5), + (14...17), nil, StatementsNode(6...13)( [CallNode(6...13)( @@ -34,6 +36,8 @@ ProgramNode(0...65)( LambdaNode(19...32)( [], (19...21), + (22...23), + (31...32), nil, StatementsNode(24...31)( [CallNode(24...31)( diff --git a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt index ccc8730d85e7c9..409aa26ee25c84 100644 --- a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt @@ -77,28 +77,13 @@ ProgramNode(0...64)( ), :+ ), - OperatorWriteNode(32...47)( - ConstantPathWriteNode(32...38)( - ConstantPathNode(32...38)( - CallNode(32...35)( - nil, - nil, - (32...35), - nil, - nil, - nil, - nil, - 2, - "foo" - ), - ConstantReadNode(37...38)(), - (35...37) - ), - nil, - nil + ConstantPathOperatorWriteNode(32...47)( + ConstantPathNode(32...38)( + CallNode(32...35)(nil, nil, (32...35), nil, nil, nil, nil, 2, "foo"), + ConstantReadNode(37...38)(), + (35...37) ), (39...41), - :+, CallNode(42...47)( nil, nil, @@ -121,7 +106,8 @@ ProgramNode(0...64)( nil, 0, "m" - ) + ), + :+ ), CallOperatorWriteNode(49...64)( CallNode(49...55)( diff --git a/test/yarp/snapshots/whitequark/parser_bug_272.txt b/test/yarp/snapshots/whitequark/parser_bug_272.txt index eeb981d694fbb3..0cb194d1ae4126 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_272.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_272.txt @@ -6,7 +6,7 @@ ProgramNode(0...15)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), nil, BlockNode(5...15)( [:c], diff --git a/test/yarp/snapshots/whitequark/parser_bug_507.txt b/test/yarp/snapshots/whitequark/parser_bug_507.txt index e0b69e4c8b80df..f305b7daeab39a 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_507.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_507.txt @@ -4,9 +4,12 @@ ProgramNode(0...19)( [LocalVariableWriteNode(0...19)( :m, 0, + (0...1), LambdaNode(4...19)( [:args], (4...6), + (13...15), + (16...19), BlockParametersNode(7...12)( ParametersNode(7...12)( [], @@ -23,7 +26,6 @@ ProgramNode(0...19)( ), nil ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/whitequark/parser_bug_645.txt b/test/yarp/snapshots/whitequark/parser_bug_645.txt index 22ae40a011c68a..27680b00aecf85 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_645.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_645.txt @@ -4,6 +4,8 @@ ProgramNode(0...14)( [LambdaNode(0...14)( [:arg], (0...2), + (12...13), + (13...14), BlockParametersNode(3...11)( ParametersNode(4...10)( [], diff --git a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt index 16db486beb243e..3a3377f96d6aa2 100644 --- a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt +++ b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt @@ -41,7 +41,7 @@ ProgramNode(0...210)( "ab", 0 ), - SymbolNode(123...130)(nil, (125...129), nil, "ab"), + SymbolNode(123...130)((123...125), (125...129), (129...130), "ab"), SymbolNode(132...139)((132...134), (134...138), (138...139), "ab"), InterpolatedStringNode(141...150)( (141...150), diff --git a/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt b/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt index 7a0546c56951ec..c428d83cda5e3d 100644 --- a/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt +++ b/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt @@ -5,7 +5,7 @@ ProgramNode(0...24)( IntegerNode(0...1)(), ArrayPatternNode(5...8)( nil, - [LocalVariableWriteNode(6...7)(:a, 0, nil, (6...7), nil)], + [LocalVariableTargetNode(6...7)(:a, 0)], nil, [], (5...6), @@ -18,7 +18,7 @@ ProgramNode(0...24)( IntegerNode(13...14)(), ArrayPatternNode(18...21)( nil, - [LocalVariableWriteNode(19...20)(:a, 0, nil, (19...20), nil)], + [LocalVariableTargetNode(19...20)(:a, 0)], nil, [], (18...19), diff --git a/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt b/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt index c6d53befee0d44..1e009aa3db8a71 100644 --- a/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt +++ b/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt @@ -9,8 +9,8 @@ ProgramNode(0...142)( ), ArrayPatternNode(10...14)( nil, - [LocalVariableWriteNode(10...11)(:a, 0, nil, (10...11), nil), - LocalVariableWriteNode(13...14)(:b, 0, nil, (13...14), nil)], + [LocalVariableTargetNode(10...11)(:a, 0), + LocalVariableTargetNode(13...14)(:b, 0)], nil, [], nil, @@ -27,8 +27,8 @@ ProgramNode(0...142)( ), ArrayPatternNode(29...33)( nil, - [LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + [LocalVariableTargetNode(29...30)(:a, 0), + LocalVariableTargetNode(32...33)(:b, 0)], nil, [], nil, @@ -99,7 +99,7 @@ ProgramNode(0...142)( nil, [AssocNode(89...99)( SymbolNode(89...93)(nil, (89...92), (92...93), "key"), - LocalVariableWriteNode(94...99)(:value, 0, nil, (94...99), nil), + LocalVariableTargetNode(94...99)(:value, 0), nil )], nil, @@ -123,13 +123,7 @@ ProgramNode(0...142)( nil, [AssocNode(125...135)( SymbolNode(125...129)(nil, (125...128), (128...129), "key"), - LocalVariableWriteNode(130...135)( - :value, - 0, - nil, - (130...135), - nil - ), + LocalVariableTargetNode(130...135)(:value, 0), nil )], nil, diff --git a/test/yarp/snapshots/whitequark/resbody_list_var.txt b/test/yarp/snapshots/whitequark/resbody_list_var.txt index 7552914e005237..660abcc181dfff 100644 --- a/test/yarp/snapshots/whitequark/resbody_list_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_list_var.txt @@ -20,7 +20,7 @@ ProgramNode(0...39)( "foo" )], (24...26), - LocalVariableWriteNode(27...29)(:ex, 0, nil, (27...29), nil), + LocalVariableTargetNode(27...29)(:ex, 0), StatementsNode(31...34)( [CallNode(31...34)( nil, diff --git a/test/yarp/snapshots/whitequark/resbody_var.txt b/test/yarp/snapshots/whitequark/resbody_var.txt index 8e2d566c19642f..e88a7612dc1b2a 100644 --- a/test/yarp/snapshots/whitequark/resbody_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_var.txt @@ -10,7 +10,7 @@ ProgramNode(0...73)( (13...19), [], (20...22), - InstanceVariableWriteNode(23...26)((23...26), nil, nil), + InstanceVariableTargetNode(23...26)(:@ex), StatementsNode(28...31)( [CallNode(28...31)( nil, @@ -49,7 +49,7 @@ ProgramNode(0...73)( (51...57), [], (58...60), - LocalVariableWriteNode(61...63)(:ex, 0, nil, (61...63), nil), + LocalVariableTargetNode(61...63)(:ex, 0), StatementsNode(65...68)( [CallNode(65...68)( nil, diff --git a/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt b/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt index 475b72dbe80e4e..89825a7ac412ac 100644 --- a/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt +++ b/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt @@ -4,6 +4,8 @@ ProgramNode(0...17)( [LambdaNode(0...17)( [], (0...2), + (3...5), + (14...17), nil, BeginNode(6...17)( nil, diff --git a/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt b/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt index c0c1b20cef787c..b0ca3f6c951009 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt @@ -4,12 +4,12 @@ ProgramNode(0...21)( [LocalVariableWriteNode(0...21)( :foo, 0, + (0...3), RescueModifierNode(6...21)( CallNode(6...10)(nil, nil, (6...10), nil, nil, nil, nil, 2, "meth"), (11...17), CallNode(18...21)(nil, nil, (18...21), nil, nil, nil, nil, 2, "bar") ), - (0...3), (4...5) )] ) diff --git a/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt b/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt index 60464244f58dbc..13023afc175e47 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt @@ -2,8 +2,8 @@ ProgramNode(0...29)( [:foo, :bar], StatementsNode(0...29)( [MultiWriteNode(0...29)( - [LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), - LocalVariableWriteNode(5...8)(:bar, 0, nil, (5...8), nil)], + [LocalVariableTargetNode(0...3)(:foo, 0), + LocalVariableTargetNode(5...8)(:bar, 0)], (9...10), RescueModifierNode(11...29)( CallNode(11...15)(nil, nil, (11...15), nil, nil, nil, nil, 2, "meth"), diff --git a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt index 8ae81caa8f5b1b..f9bfc7bfeb3649 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt @@ -1,15 +1,17 @@ ProgramNode(0...22)( [:foo], StatementsNode(0...22)( - [OperatorWriteNode(0...22)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...22)( + (0...3), (4...6), - :+, RescueModifierNode(7...22)( CallNode(7...11)(nil, nil, (7...11), nil, nil, nil, nil, 2, "meth"), (12...18), CallNode(19...22)(nil, nil, (19...22), nil, nil, nil, nil, 2, "bar") - ) + ), + :foo, + :+, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_11107.txt b/test/yarp/snapshots/whitequark/ruby_bug_11107.txt index 0f5b129c5cde37..c61ee9ef80d316 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_11107.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_11107.txt @@ -10,6 +10,8 @@ ProgramNode(0...24)( [LambdaNode(2...24)( [], (2...4), + (7...9), + (21...24), BlockParametersNode(4...6)(nil, [], (4...5), (5...6)), StatementsNode(10...20)( [CallNode(10...20)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_11380.txt b/test/yarp/snapshots/whitequark/ruby_bug_11380.txt index 5962097a6a4075..9e89161e7817e9 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_11380.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_11380.txt @@ -10,6 +10,8 @@ ProgramNode(0...28)( [LambdaNode(2...15)( [], (2...4), + (5...6), + (14...15), nil, StatementsNode(7...13)( [SymbolNode(7...13)((7...8), (8...13), nil, "hello")] diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12073.txt b/test/yarp/snapshots/whitequark/ruby_bug_12073.txt index 7614f1b1201d99..8a3aee68a6998d 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12073.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12073.txt @@ -4,8 +4,8 @@ ProgramNode(0...49)( [LocalVariableWriteNode(0...5)( :a, 0, - IntegerNode(4...5)(), (0...1), + IntegerNode(4...5)(), (2...3) ), CallNode(7...13)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt index 4ca03bab0109ad..7bcef82926f8f0 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt @@ -1,10 +1,9 @@ ProgramNode(0...437)( [:foo], StatementsNode(0...437)( - [OperatorWriteNode(0...27)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...27)( + (0...3), (4...6), - :+, CallNode(7...27)( nil, nil, @@ -31,12 +30,14 @@ ProgramNode(0...437)( nil, 0, "raise" - ) + ), + :foo, + :+, + 0 ), - OperatorWriteNode(29...57)( - LocalVariableWriteNode(29...32)(:foo, 0, nil, (29...32), nil), + LocalVariableOperatorWriteNode(29...57)( + (29...32), (33...35), - :+, RescueModifierNode(36...57)( CallNode(36...46)( nil, @@ -63,11 +64,15 @@ ProgramNode(0...437)( ), (47...53), NilNode(54...57)() - ) + ), + :foo, + :+, + 0 ), LocalVariableWriteNode(59...85)( :foo, 0, + (59...62), CallNode(65...85)( nil, nil, @@ -95,12 +100,12 @@ ProgramNode(0...437)( 0, "raise" ), - (59...62), (63...64) ), LocalVariableWriteNode(87...114)( :foo, 0, + (87...90), RescueModifierNode(93...114)( CallNode(93...103)( nil, @@ -128,7 +133,6 @@ ProgramNode(0...437)( (104...110), NilNode(111...114)() ), - (87...90), (91...92) ), CallOperatorWriteNode(116...145)( @@ -299,16 +303,13 @@ ProgramNode(0...437)( ), :+ ), - OrWriteNode(242...273)( - ConstantPathWriteNode(242...248)( - ConstantPathNode(242...248)( - LocalVariableReadNode(242...245)(:foo, 0), - ConstantReadNode(247...248)(), - (245...247) - ), - nil, - nil + ConstantPathOrWriteNode(242...273)( + ConstantPathNode(242...248)( + LocalVariableReadNode(242...245)(:foo, 0), + ConstantReadNode(247...248)(), + (245...247) ), + (249...252), CallNode(253...273)( nil, nil, @@ -335,19 +336,15 @@ ProgramNode(0...437)( nil, 0, "raise" - ), - (249...252) + ) ), - OrWriteNode(275...307)( - ConstantPathWriteNode(275...281)( - ConstantPathNode(275...281)( - LocalVariableReadNode(275...278)(:foo, 0), - ConstantReadNode(280...281)(), - (278...280) - ), - nil, - nil + ConstantPathOrWriteNode(275...307)( + ConstantPathNode(275...281)( + LocalVariableReadNode(275...278)(:foo, 0), + ConstantReadNode(280...281)(), + (278...280) ), + (282...285), RescueModifierNode(286...307)( CallNode(286...296)( nil, @@ -374,8 +371,7 @@ ProgramNode(0...437)( ), (297...303), NilNode(304...307)() - ), - (282...285) + ) ), CallOperatorWriteNode(309...339)( CallNode(309...315)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt index 6eb9e9a0c81ba0..689a3d9a77a245 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt @@ -1,14 +1,12 @@ ProgramNode(0...74)( [:a, :b], StatementsNode(0...74)( - [OperatorWriteNode(0...18)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOperatorWriteNode(0...18)( + (0...1), (2...4), - :+, - OperatorWriteNode(5...18)( - LocalVariableWriteNode(5...6)(:b, 0, nil, (5...6), nil), + LocalVariableOperatorWriteNode(5...18)( + (5...6), (7...9), - :+, CallNode(10...18)( nil, nil, @@ -21,16 +19,22 @@ ProgramNode(0...74)( nil, 0, "raise" - ) - ) + ), + :b, + :+, + 0 + ), + :a, + :+, + 0 ), - OperatorWriteNode(20...37)( - LocalVariableWriteNode(20...21)(:a, 0, nil, (20...21), nil), + LocalVariableOperatorWriteNode(20...37)( + (20...21), (22...24), - :+, LocalVariableWriteNode(25...37)( :b, 0, + (25...26), CallNode(29...37)( nil, nil, @@ -44,17 +48,19 @@ ProgramNode(0...74)( 0, "raise" ), - (25...26), (27...28) - ) + ), + :a, + :+, + 0 ), LocalVariableWriteNode(39...56)( :a, 0, - OperatorWriteNode(43...56)( - LocalVariableWriteNode(43...44)(:b, 0, nil, (43...44), nil), + (39...40), + LocalVariableOperatorWriteNode(43...56)( + (43...44), (45...47), - :+, CallNode(48...56)( nil, nil, @@ -67,17 +73,21 @@ ProgramNode(0...74)( nil, 0, "raise" - ) + ), + :b, + :+, + 0 ), - (39...40), (41...42) ), LocalVariableWriteNode(58...74)( :a, 0, + (58...59), LocalVariableWriteNode(62...74)( :b, 0, + (62...63), CallNode(66...74)( nil, nil, @@ -91,10 +101,8 @@ ProgramNode(0...74)( 0, "raise" ), - (62...63), (64...65) ), - (58...59), (60...61) )] ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_15789.txt b/test/yarp/snapshots/whitequark/ruby_bug_15789.txt index 331c1e5cd858ad..0621e71c8cc3f5 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_15789.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_15789.txt @@ -10,6 +10,8 @@ ProgramNode(0...41)( [LambdaNode(2...20)( [:a], (2...4), + (17...18), + (19...20), BlockParametersNode(4...16)( ParametersNode(5...15)( [], @@ -20,6 +22,8 @@ ProgramNode(0...41)( LambdaNode(9...15)( [], (9...11), + (11...12), + (14...15), nil, StatementsNode(12...14)( [CallNode(12...14)( @@ -63,6 +67,8 @@ ProgramNode(0...41)( [LambdaNode(24...41)( [:a], (24...26), + (38...39), + (40...41), BlockParametersNode(26...37)( ParametersNode(27...36)( [], @@ -74,6 +80,8 @@ ProgramNode(0...41)( LambdaNode(30...36)( [], (30...32), + (32...33), + (35...36), nil, StatementsNode(33...35)( [CallNode(33...35)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_9669.txt b/test/yarp/snapshots/whitequark/ruby_bug_9669.txt index 3cdaf914070f44..6302c3d317b073 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_9669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_9669.txt @@ -25,6 +25,7 @@ ProgramNode(0...33)( LocalVariableWriteNode(21...33)( :o, 0, + (21...22), HashNode(25...33)( (25...26), [AssocNode(27...31)( @@ -34,7 +35,6 @@ ProgramNode(0...33)( )], (32...33) ), - (21...22), (23...24) )] ) diff --git a/test/yarp/snapshots/whitequark/send_call.txt b/test/yarp/snapshots/whitequark/send_call.txt index 4280323078034f..f96195145baf54 100644 --- a/test/yarp/snapshots/whitequark/send_call.txt +++ b/test/yarp/snapshots/whitequark/send_call.txt @@ -4,7 +4,7 @@ ProgramNode(0...17)( [CallNode(0...7)( CallNode(0...3)(nil, nil, (0...3), nil, nil, nil, nil, 2, "foo"), (3...4), - (0...0), + nil, (4...5), ArgumentsNode(5...6)([IntegerNode(5...6)()]), (6...7), @@ -15,7 +15,7 @@ ProgramNode(0...17)( CallNode(9...17)( CallNode(9...12)(nil, nil, (9...12), nil, nil, nil, nil, 2, "foo"), (12...14), - (0...0), + nil, (14...15), ArgumentsNode(15...16)([IntegerNode(15...16)()]), (16...17), diff --git a/test/yarp/snapshots/whitequark/send_lambda.txt b/test/yarp/snapshots/whitequark/send_lambda.txt index 3c9f4f56cb47c4..c30ec09249f4e6 100644 --- a/test/yarp/snapshots/whitequark/send_lambda.txt +++ b/test/yarp/snapshots/whitequark/send_lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...26)( [LambdaNode(0...8)( [:*], (0...2), + (5...6), + (7...8), BlockParametersNode(3...4)( ParametersNode(3...4)( [], @@ -20,7 +22,7 @@ ProgramNode(0...26)( ), nil ), - LambdaNode(10...19)([], (10...12), nil, nil), - LambdaNode(21...26)([], (21...23), nil, nil)] + LambdaNode(10...19)([], (10...12), (13...15), (16...19), nil, nil), + LambdaNode(21...26)([], (21...23), (23...24), (25...26), nil, nil)] ) ) diff --git a/test/yarp/snapshots/whitequark/send_lambda_args.txt b/test/yarp/snapshots/whitequark/send_lambda_args.txt index f904384781ba55..ae41cfa463fa6f 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args.txt @@ -4,6 +4,8 @@ ProgramNode(0...21)( [LambdaNode(0...10)( [:a], (0...2), + (7...8), + (9...10), BlockParametersNode(3...6)( ParametersNode(4...5)( [RequiredParameterNode(4...5)(:a)], @@ -23,6 +25,8 @@ ProgramNode(0...21)( LambdaNode(12...21)( [:a], (12...14), + (18...19), + (20...21), BlockParametersNode(14...17)( ParametersNode(15...16)( [RequiredParameterNode(15...16)(:a)], diff --git a/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt b/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt index 8767cf48fac00d..ebb6fb106518f6 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt @@ -4,6 +4,8 @@ ProgramNode(0...22)( [LambdaNode(0...11)( [:a], (0...2), + (8...9), + (10...11), BlockParametersNode(3...7)( ParametersNode(3...7)( [], @@ -23,6 +25,8 @@ ProgramNode(0...22)( LambdaNode(13...22)( [:a], (13...15), + (19...20), + (21...22), BlockParametersNode(16...18)( ParametersNode(16...18)( [], diff --git a/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt b/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt index a2e92a67dcc056..9a6c888a930c90 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt @@ -4,6 +4,8 @@ ProgramNode(0...19)( [LambdaNode(0...19)( [:a, :foo, :bar], (0...2), + (16...17), + (18...19), BlockParametersNode(2...15)( ParametersNode(3...4)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/whitequark/send_lambda_legacy.txt b/test/yarp/snapshots/whitequark/send_lambda_legacy.txt index fdc30c30dcb16e..57b827cb7edd27 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_legacy.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_legacy.txt @@ -1,4 +1,6 @@ ProgramNode(0...5)( [], - StatementsNode(0...5)([LambdaNode(0...5)([], (0...2), nil, nil)]) + StatementsNode(0...5)( + [LambdaNode(0...5)([], (0...2), (2...3), (4...5), nil, nil)] + ) ) diff --git a/test/yarp/snapshots/whitequark/string_concat.txt b/test/yarp/snapshots/whitequark/string_concat.txt index c20f4ac35e7b2b..a51362805e4997 100644 --- a/test/yarp/snapshots/whitequark/string_concat.txt +++ b/test/yarp/snapshots/whitequark/string_concat.txt @@ -7,7 +7,7 @@ ProgramNode(0...14)( [StringNode(1...4)(nil, (1...4), nil, "foo"), EmbeddedVariableNode(4...7)( (4...5), - InstanceVariableReadNode(5...7)() + InstanceVariableReadNode(5...7)(:@a) )], (7...8) ), diff --git a/test/yarp/snapshots/whitequark/string_dvar.txt b/test/yarp/snapshots/whitequark/string_dvar.txt index c89e3595653eb7..12f083660bb8d7 100644 --- a/test/yarp/snapshots/whitequark/string_dvar.txt +++ b/test/yarp/snapshots/whitequark/string_dvar.txt @@ -5,10 +5,13 @@ ProgramNode(0...14)( (0...1), [EmbeddedVariableNode(1...4)( (1...2), - InstanceVariableReadNode(2...4)() + InstanceVariableReadNode(2...4)(:@a) ), StringNode(4...5)(nil, (4...5), nil, " "), - EmbeddedVariableNode(5...9)((5...6), ClassVariableReadNode(6...9)()), + EmbeddedVariableNode(5...9)( + (5...6), + ClassVariableReadNode(6...9)(:@@a) + ), StringNode(9...10)(nil, (9...10), nil, " "), EmbeddedVariableNode(10...13)( (10...11), diff --git a/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt b/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt index 245f8a07fe6656..3f77bfd3e5c30d 100644 --- a/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt +++ b/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt @@ -4,8 +4,8 @@ ProgramNode(0...13)( [LocalVariableWriteNode(0...3)( :t, 0, - IntegerNode(2...3)(), (0...1), + IntegerNode(2...3)(), (1...2) ), IfNode(4...13)( diff --git a/test/yarp/snapshots/whitequark/until.txt b/test/yarp/snapshots/whitequark/until.txt index 2eae2d5307f9b7..5a40a615bd68ef 100644 --- a/test/yarp/snapshots/whitequark/until.txt +++ b/test/yarp/snapshots/whitequark/until.txt @@ -3,6 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [UntilNode(0...21)( (0...5), + (18...21), CallNode(6...9)(nil, nil, (6...9), nil, nil, nil, nil, 2, "foo"), StatementsNode(13...17)( [CallNode(13...17)( @@ -21,6 +22,7 @@ ProgramNode(0...42)( ), UntilNode(23...42)( (23...28), + (39...42), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "foo"), StatementsNode(34...38)( [CallNode(34...38)( diff --git a/test/yarp/snapshots/whitequark/until_mod.txt b/test/yarp/snapshots/whitequark/until_mod.txt index 62f9d4992e8a93..ab5acaf4b37eb4 100644 --- a/test/yarp/snapshots/whitequark/until_mod.txt +++ b/test/yarp/snapshots/whitequark/until_mod.txt @@ -3,6 +3,7 @@ ProgramNode(0...14)( StatementsNode(0...14)( [UntilNode(0...14)( (5...10), + nil, CallNode(11...14)(nil, nil, (11...14), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...4)( [CallNode(0...4)(nil, nil, (0...4), nil, nil, nil, nil, 2, "meth")] diff --git a/test/yarp/snapshots/whitequark/until_post.txt b/test/yarp/snapshots/whitequark/until_post.txt index 27e0a60e407e5f..87a0fa3a12e0f0 100644 --- a/test/yarp/snapshots/whitequark/until_post.txt +++ b/test/yarp/snapshots/whitequark/until_post.txt @@ -3,6 +3,7 @@ ProgramNode(0...24)( StatementsNode(0...24)( [UntilNode(0...24)( (15...20), + nil, CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...14)( [BeginNode(0...14)( diff --git a/test/yarp/snapshots/whitequark/var_and_asgn.txt b/test/yarp/snapshots/whitequark/var_and_asgn.txt index a082413a7d05a7..a613761d87e163 100644 --- a/test/yarp/snapshots/whitequark/var_and_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_and_asgn.txt @@ -1,10 +1,12 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [AndWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableAndWriteNode(0...7)( + (0...1), + (2...5), IntegerNode(6...7)(), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 08b724031b7743..0927e8b8f3aebc 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -1,34 +1,39 @@ ProgramNode(0...53)( [:a], StatementsNode(0...53)( - [OperatorWriteNode(0...11)( - ClassVariableWriteNode(0...5)((0...5), nil, nil), + [ClassVariableOperatorWriteNode(0...11)( + :@@var, + (0...5), (6...8), - :|, - IntegerNode(9...11)() + IntegerNode(9...11)(), + :| ), - OperatorWriteNode(13...20)( - InstanceVariableWriteNode(13...15)((13...15), nil, nil), + InstanceVariableOperatorWriteNode(13...20)( + :@a, + (13...15), (16...18), - :|, - IntegerNode(19...20)() + IntegerNode(19...20)(), + :| ), - OperatorWriteNode(22...28)( - LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), + LocalVariableOperatorWriteNode(22...28)( + (22...23), (24...26), + IntegerNode(27...28)(), + :a, :+, - IntegerNode(27...28)() + 0 ), DefNode(30...53)( (34...35), nil, nil, StatementsNode(37...48)( - [OperatorWriteNode(37...48)( - ClassVariableWriteNode(37...42)((37...42), nil, nil), + [ClassVariableOperatorWriteNode(37...48)( + :@@var, + (37...42), (43...45), - :|, - IntegerNode(46...48)() + IntegerNode(46...48)(), + :| )] ), [], diff --git a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt index f78329a1eba157..21c1fb6e1056a6 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt @@ -1,10 +1,9 @@ ProgramNode(0...12)( [:foo], StatementsNode(0...12)( - [OperatorWriteNode(0...12)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...12)( + (0...3), (4...6), - :+, CallNode(7...12)( nil, nil, @@ -15,7 +14,10 @@ ProgramNode(0...12)( nil, 0, "m" - ) + ), + :foo, + :+, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_or_asgn.txt b/test/yarp/snapshots/whitequark/var_or_asgn.txt index bb2f8c6b39cfaa..3fb56b7f4a88d8 100644 --- a/test/yarp/snapshots/whitequark/var_or_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_or_asgn.txt @@ -1,10 +1,12 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [OrWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...7)( + (0...1), + (2...5), IntegerNode(6...7)(), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/while.txt b/test/yarp/snapshots/whitequark/while.txt index 7d95f466c8c0d5..7529b6e1bc1ec6 100644 --- a/test/yarp/snapshots/whitequark/while.txt +++ b/test/yarp/snapshots/whitequark/while.txt @@ -3,6 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [WhileNode(0...21)( (0...5), + (18...21), CallNode(6...9)(nil, nil, (6...9), nil, nil, nil, nil, 2, "foo"), StatementsNode(13...17)( [CallNode(13...17)( @@ -21,6 +22,7 @@ ProgramNode(0...42)( ), WhileNode(23...42)( (23...28), + (39...42), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "foo"), StatementsNode(34...38)( [CallNode(34...38)( diff --git a/test/yarp/snapshots/whitequark/while_mod.txt b/test/yarp/snapshots/whitequark/while_mod.txt index b1a0203da3d19f..2f0ade7006b5fc 100644 --- a/test/yarp/snapshots/whitequark/while_mod.txt +++ b/test/yarp/snapshots/whitequark/while_mod.txt @@ -3,6 +3,7 @@ ProgramNode(0...14)( StatementsNode(0...14)( [WhileNode(0...14)( (5...10), + nil, CallNode(11...14)(nil, nil, (11...14), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...4)( [CallNode(0...4)(nil, nil, (0...4), nil, nil, nil, nil, 2, "meth")] diff --git a/test/yarp/snapshots/whitequark/while_post.txt b/test/yarp/snapshots/whitequark/while_post.txt index 28a423e1562b6e..1b2169107a160f 100644 --- a/test/yarp/snapshots/whitequark/while_post.txt +++ b/test/yarp/snapshots/whitequark/while_post.txt @@ -3,6 +3,7 @@ ProgramNode(0...24)( StatementsNode(0...24)( [WhileNode(0...24)( (15...20), + nil, CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...14)( [BeginNode(0...14)( diff --git a/test/yarp/snapshots/wrapping_heredoc.txt b/test/yarp/snapshots/wrapping_heredoc.txt deleted file mode 100644 index 674db56ed12546..00000000000000 --- a/test/yarp/snapshots/wrapping_heredoc.txt +++ /dev/null @@ -1,80 +0,0 @@ -ProgramNode(165...298)( - [], - StatementsNode(165...298)( - [CallNode(165...193)( - nil, - nil, - (165...167), - nil, - ArgumentsNode(168...193)( - [CallNode(168...193)( - InterpolatedStringNode(168...172)( - (168...172), - [StringNode(182...184)(nil, (182...184), nil, "a\n")], - (184...186) - ), - (172...173), - (173...177), - (177...178), - ArgumentsNode(178...192)( - [InterpolatedRegularExpressionNode(178...188)( - (178...179), - [StringNode(179...182)(nil, (179...182), nil, "b"), - StringNode(186...187)(nil, (186...187), nil, "c")], - (187...188), - 0 - ), - StringNode(190...192)( - (190...191), - (191...191), - (191...192), - "" - )] - ), - (192...193), - nil, - 0, - "gsub" - )] - ), - nil, - nil, - 0, - "pp" - ), - CallNode(278...298)( - nil, - nil, - (278...280), - nil, - ArgumentsNode(281...298)( - [CallNode(281...298)( - InterpolatedStringNode(281...285)( - (281...285), - [StringNode(292...294)(nil, (292...294), nil, "d\n")], - (294...296) - ), - nil, - (286...287), - nil, - ArgumentsNode(288...298)( - [InterpolatedStringNode(288...298)( - (288...289), - [StringNode(289...292)(nil, (289...292), nil, "e"), - StringNode(296...297)(nil, (296...297), nil, "f")], - (297...298) - )] - ), - nil, - nil, - 0, - "+" - )] - ), - nil, - nil, - 0, - "pp" - )] - ) -) diff --git a/test/yarp/yarp_test_helper.rb b/test/yarp/test_helper.rb similarity index 72% rename from test/yarp/yarp_test_helper.rb rename to test/yarp/test_helper.rb index 0be0f51651f03b..b79adf4b166e91 100644 --- a/test/yarp/yarp_test_helper.rb +++ b/test/yarp/test_helper.rb @@ -8,8 +8,15 @@ puts "Using YARP backend: #{YARP::BACKEND}" if ENV["YARP_FFI_BACKEND"] +# It is useful to have a diff even if the strings to compare are big +# However, ruby/ruby does not have a version of Test::Unit with access to +# max_diff_target_string_size +if defined?(Test::Unit::Assertions::AssertionMessage) + Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 +end + module YARP - module Assertions + class TestCase < ::Test::Unit::TestCase private def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) @@ -31,18 +38,17 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::SourceFileNode + when SourceFileNode deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys - # Filepaths can be different if test suites were run - # on different machines. - # We accommodate for this by comparing the basenames, - # and not the absolute filepaths + # Filepaths can be different if test suites were run on different + # machines. We accommodate for this by comparing the basenames, and not + # the absolute filepaths. assert_equal deconstructed_expected.except(:filepath), deconstructed_actual.except(:filepath) assert_equal File.basename(deconstructed_expected[:filepath]), File.basename(deconstructed_actual[:filepath]) - when YARP::Node + when Node deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys @@ -55,10 +61,11 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::Location + when Location assert_operator actual.start_offset, :<=, actual.end_offset, -> { "start_offset > end_offset for #{actual.inspect}, parent is #{parent.pretty_inspect}" } + if compare_location assert_equal( expected.start_offset, @@ -71,30 +78,10 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) actual.end_offset, -> { "End locations were different. Parent: #{parent.pretty_inspect}" } ) - end else assert_equal expected, actual end end - - def assert_valid_locations(value, parent: nil) - case value - when Array - value.each do |element| - assert_valid_locations(element, parent: value) - end - when YARP::Node - value.deconstruct_keys(nil).each_value do |field| - assert_valid_locations(field, parent: value) - end - when YARP::Location - assert_operator value.start_offset, :<=, value.end_offset, -> { - "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" - } - end - end end end - -Test::Unit::TestCase.include(YARP::Assertions) diff --git a/test/yarp/unescape_test.rb b/test/yarp/unescape_test.rb index 3b9e6652481a92..f39bdd0e394ee3 100644 --- a/test/yarp/unescape_test.rb +++ b/test/yarp/unescape_test.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI -module UnescapeTest - class UnescapeNoneTest < Test::Unit::TestCase +module YARP + class UnescapeNoneTest < TestCase def test_backslash assert_unescape_none("\\") end @@ -17,11 +17,11 @@ def test_single_quote private def assert_unescape_none(source) - assert_equal(source, YARP.const_get(:Debug).unescape_none(source)) + assert_equal(source, Debug.unescape_none(source)) end end - class UnescapeMinimalTest < Test::Unit::TestCase + class UnescapeMinimalTest < TestCase def test_backslash assert_unescape_minimal("\\", "\\\\") end @@ -37,11 +37,11 @@ def test_single_char private def assert_unescape_minimal(expected, source) - assert_equal(expected, YARP.const_get(:Debug).unescape_minimal(source)) + assert_equal(expected, Debug.unescape_minimal(source)) end end - class UnescapeAllTest < Test::Unit::TestCase + class UnescapeAllTest < TestCase def test_backslash assert_unescape_all("\\", "\\\\") end @@ -139,7 +139,7 @@ def test_escaping_normal_characters private def unescape_all(source) - YARP.const_get(:Debug).unescape_all(source) + Debug.unescape_all(source) end def assert_unescape_all(expected, source, forced_encoding = nil) diff --git a/test/yarp/version_test.rb b/test/yarp/version_test.rb index f431157ae9264c..6011eb695d7267 100644 --- a/test/yarp/version_test.rb +++ b/test/yarp/version_test.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" -class VersionTest < Test::Unit::TestCase - def test_version_is_set - refute_nil YARP::VERSION +module YARP + class VersionTest < TestCase + def test_version_is_set + refute_nil VERSION + end end end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 4887d944c5cc38..23de1f7e50ea67 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,6 +1,43 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions def assert_raises(*exp, &b) diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 68d2ab471e1cb9..23dfbd8f0f027d 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -24,42 +24,6 @@ def warn(message, category: nil, **kwargs) # See Test::Unit module Test - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] - - unless $DEBUG then - bt.each do |line| - break if pattern.match?(line) - new_bt << line - end - - new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. module Unit diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama index 7988afe50754bb..5e1ee582cf5681 100755 --- a/tool/lrama/exe/lrama +++ b/tool/lrama/exe/lrama @@ -1,6 +1,5 @@ #!/usr/bin/env ruby - $LOAD_PATH << File.join(__dir__, "../lib") require "lrama" diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb index 9fceb55fe05568..45670ae405c0c6 100644 --- a/tool/lrama/lib/lrama/command.rb +++ b/tool/lrama/lib/lrama/command.rb @@ -5,7 +5,6 @@ class Command def initialize(argv) @argv = argv - @version = nil @skeleton = "bison/yacc.c" @header = false @header_file = nil @@ -23,15 +22,11 @@ def initialize(argv) def run parse_option - if @version - puts Lrama::VERSION - exit 0 - end - Report::Duration.enable if @trace_opts[:time] warning = Lrama::Warning.new grammar = Lrama::Parser.new(@y.read).parse + @y.close if @y != STDIN states = Lrama::States.new(grammar, warning, trace_state: (@trace_opts[:automaton] || @trace_opts[:closure])) states.compute context = Lrama::Context.new(states) @@ -112,7 +107,7 @@ def parse_option opt = OptionParser.new # opt.on('-h') {|v| p v } - opt.on('-V', '--version') {|v| @version = true } + opt.on('-V', '--version') {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } # Tuning the Parser opt.on('-S', '--skeleton=FILE') {|v| @skeleton = v } diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb index 861c4e1137d5eb..da70bf15422990 100644 --- a/tool/lrama/lib/lrama/context.rb +++ b/tool/lrama/lib/lrama/context.rb @@ -401,7 +401,6 @@ def debug_sorted_actions end print sprintf("]\n\n") - print sprintf("width [\n") vectors_count.times do |i| print sprintf("%d, ", ary[i] ? ary[i][3] : 0) @@ -409,7 +408,6 @@ def debug_sorted_actions end print sprintf("]\n\n") - print sprintf("tally [\n") vectors_count.times do |i| print sprintf("%d, ", ary[i] ? ary[i][2] : 0) diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb index a5d62a0c7c3321..5019257dc3421a 100644 --- a/tool/lrama/lib/lrama/counterexamples.rb +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -205,7 +205,7 @@ def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, confli end def build_paths_from_state_items(state_items) - paths = state_items.zip([nil] + state_items).map do |si, prev_si| + state_items.zip([nil] + state_items).map do |si, prev_si| case when prev_si.nil? StartPath.new(si) @@ -215,8 +215,6 @@ def build_paths_from_state_items(state_items) TransitionPath.new(prev_si, si) end end - - paths end def shortest_path(conflict_state, conflict_reduce_item, conflict_term) diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb index 25f1a49170cd9c..81df3996820256 100644 --- a/tool/lrama/lib/lrama/grammar.rb +++ b/tool/lrama/lib/lrama/grammar.rb @@ -103,6 +103,10 @@ def add_right(sym, precedence) set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) end + def add_precedence(sym, precedence) + set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + end + def set_precedence(sym, precedence) raise "" if sym.nterm? sym.precedence = precedence @@ -310,7 +314,6 @@ def find_nterm_by_id!(id) end || (raise "Nterm not found: #{id}") end - def append_special_symbols # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) @@ -512,7 +515,7 @@ def fill_symbol_number sym.token_id = 11 when "\"" sym.token_id = 34 - when "\'" + when "'" sym.token_id = 39 when "\\\\" sym.token_id = 92 diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb index 0f61ebb58e4cf1..712cb1ad5a95cb 100644 --- a/tool/lrama/lib/lrama/grammar/code.rb +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -50,7 +50,6 @@ def translated_printer_code(tag) end alias :translated_error_token_code :translated_printer_code - private # * ($1) yyvsp[i] diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb index c1b5c8fe4edd05..c591684a05e621 100644 --- a/tool/lrama/lib/lrama/lexer.rb +++ b/tool/lrama/lib/lrama/lexer.rb @@ -30,7 +30,6 @@ def initialize(text) @grammar_rules = [] @epilogue = [] - # @bison_declarations_tokens = [] @grammar_rules_tokens = [] @@ -155,6 +154,8 @@ def lex_common(lines, tokens) tokens << create_token(Token::P_left, ss[0], line, ss.pos - column) when ss.scan(/%right/) tokens << create_token(Token::P_right, ss[0], line, ss.pos - column) + when ss.scan(/%precedence/) + tokens << create_token(Token::P_precedence, ss[0], line, ss.pos - column) when ss.scan(/%prec/) tokens << create_token(Token::P_prec, ss[0], line, ss.pos - column) when ss.scan(/{/) @@ -223,7 +224,7 @@ def lex_user_code(ss, line, column, lines) references << [:dollar, ss[2], tag, str.length, str.length + ss[0].length - 1] when ss.scan(/@\$/) # @$ references << [:at, "$", nil, str.length, str.length + ss[0].length - 1] - when ss.scan(/@(\d)+/) # @1 + when ss.scan(/@(\d+)/) # @1 references << [:at, Integer(ss[1]), nil, str.length, str.length + ss[0].length - 1] when ss.scan(/{/) brace_count += 1 @@ -314,8 +315,6 @@ def lex_line_comment(ss, line, str) str << ss.getch next end - - str << ss[0] end line # Reach to end of input diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb index aa49bfe47d7471..4eca6f60077bbb 100644 --- a/tool/lrama/lib/lrama/lexer/token.rb +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -61,6 +61,7 @@ def self.define_type(name) define_type(:P_nonassoc) # %nonassoc define_type(:P_left) # %left define_type(:P_right) # %right + define_type(:P_precedence) # %precedence define_type(:P_prec) # %prec define_type(:User_code) # { ... } define_type(:Tag) # diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb index 757d16e25ae3ff..2dcdae7b422a33 100644 --- a/tool/lrama/lib/lrama/output.rb +++ b/tool/lrama/lib/lrama/output.rb @@ -252,7 +252,7 @@ def user_args end def extract_param_name(param) - /\A(\W*)([a-zA-Z0-9_]+)\z/.match(param.split.last)[2] + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] end def parse_param_name diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index a12c8bbb34e33f..18e2ef033eb400 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -159,6 +159,14 @@ def parse_bison_declarations(ts, grammar) grammar.add_right(sym, precedence_number) end precedence_number += 1 + when T::P_precedence + # %precedence (ident|char|string)+ + ts.next + while (id = ts.consume(T::Ident, T::Char, T::String)) do + sym = grammar.add_term(id: id) + grammar.add_precedence(sym, precedence_number) + end + precedence_number += 1 when nil # end of input raise "Reach to end of input within declarations" diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb index 6632bafeccb43e..1b406402157346 100644 --- a/tool/lrama/lib/lrama/state.rb +++ b/tool/lrama/lib/lrama/state.rb @@ -62,7 +62,6 @@ def set_items_to_state(items, next_state) @items_to_state[items] = next_state end - # def set_look_ahead(rule, look_ahead) reduce = reduces.find do |r| r.rule == rule diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb index d27c5411ea9828..290e996b822b08 100644 --- a/tool/lrama/lib/lrama/states.rb +++ b/tool/lrama/lib/lrama/states.rb @@ -455,6 +455,11 @@ def compute_shift_reduce_conflicts # shift_prec == reduce_prec, then check associativity case sym.precedence.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next when :right # Shift is selected state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) @@ -515,9 +520,9 @@ def compute_default_reduction state.default_reduction_rule = state.reduces.map do |r| [r.rule, r.rule.id, (r.look_ahead || []).count] - end.sort_by do |rule, rule_id, count| + end.min_by do |rule, rule_id, count| [-count, rule_id] - end.first.first + end.first end end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb index 0d805829eb13f4..8be0f71e40dddf 100644 --- a/tool/lrama/lib/lrama/states_reporter.rb +++ b/tool/lrama/lib/lrama/states_reporter.rb @@ -110,7 +110,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report shifts tmp = state.term_transitions.select do |shift, _| !shift.not_selected @@ -123,7 +122,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - # Report error caused by %nonassoc nl = false tmp = state.resolved_conflicts.select do |resolved| @@ -138,7 +136,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - # Report reduces nl = false max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 @@ -171,7 +168,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if nl - # Report nonterminal transitions tmp = [] max_len = 0 @@ -189,7 +185,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - if solved # Report conflict resolutions state.resolved_conflicts.each do |resolved| @@ -202,13 +197,13 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) # Report counterexamples examples = cex.compute(state) examples.each do |example| - label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" + label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" - io << " #{example.path1_item.to_s}\n" - io << " #{example.path2_item.to_s}\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" io << " #{label1}\n" example.derivations1.render_strings_for_report.each do |str| io << " #{str}\n" @@ -234,7 +229,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report reads_relation io << " [Reads Relation]\n" @states.nterms.each do |nterm| @@ -248,7 +242,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report read_sets io << " [Read sets]\n" read_sets = @states.read_sets @@ -263,7 +256,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report includes_relation io << " [Includes Relation]\n" @states.nterms.each do |nterm| @@ -277,7 +269,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report lookback_relation io << " [Lookback Relation]\n" @states.rules.each do |rule| @@ -286,12 +277,11 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) a.each do |state_id2, nterm_id2| n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (Rule: #{rule.to_s}) -> (State #{state_id2}, #{n.id.s_value})\n" + io << " (Rule: #{rule}) -> (State #{state_id2}, #{n.id.s_value})\n" end end io << "\n" - # Report follow_sets io << " [Follow sets]\n" follow_sets = @states.follow_sets @@ -306,7 +296,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report LA io << " [Look-Ahead Sets]\n" tmp = [] @@ -326,7 +315,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) io << "\n" if !tmp.empty? end - # End of Report State io << "\n" end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 3f2eb3b9d4f5a3..41215c1aee8b18 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,3 +1,3 @@ module Lrama - VERSION = "0.5.4".freeze + VERSION = "0.5.5".freeze end diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index aefdb23a792bef..7a67e156ef55fe 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -24,6 +24,7 @@ test_collection_update(RBS::CliTest) running tests without Bundler test_subtract(RBS::CliTest) running tests without Bundler test_subtract_several_subtrahends(RBS::CliTest) running tests without Bundler test_subtract_write(RBS::CliTest) running tests without Bundler +test_subtract_write_removes_definition_if_empty(RBS::CliTest) runnint tests without Bundler test_aref(FiberSingletonTest) the method should not accept String keys diff --git a/tool/runruby.rb b/tool/runruby.rb index 1efe38fd1305e7..c6724f53d3f7b3 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -134,6 +134,9 @@ def File.realpath(*args) env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 29107fb48f317a..2879ff60812380 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -11,83 +11,83 @@ module SyncDefaultGems module_function REPOSITORIES = { - rubygems: 'rubygems/rubygems', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', "io-console": 'ruby/io-console', "io-nonblock": 'ruby/io-nonblock', "io-wait": 'ruby/io-wait', + "net-http": "ruby/net-http", + "net-protocol": "ruby/net-protocol", + "open-uri": "ruby/open-uri", + "resolv-replace": "ruby/resolv-replace", + English: "ruby/English", + abbrev: "ruby/abbrev", + base64: "ruby/base64", + benchmark: "ruby/benchmark", + bigdecimal: "ruby/bigdecimal", + cgi: "ruby/cgi", csv: 'ruby/csv', - etc: 'ruby/etc', date: 'ruby/date', - zlib: 'ruby/zlib', + delegate: "ruby/delegate", + did_you_mean: "ruby/did_you_mean", + digest: "ruby/digest", + drb: "ruby/drb", + erb: "ruby/erb", + error_highlight: "ruby/error_highlight", + etc: 'ruby/etc', fcntl: 'ruby/fcntl', - strscan: 'ruby/strscan', + fiddle: 'ruby/fiddle', + fileutils: 'ruby/fileutils', + find: "ruby/find", + forwardable: "ruby/forwardable", + getoptlong: "ruby/getoptlong", ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - ostruct: 'ruby/ostruct', irb: 'ruby/irb', - forwardable: "ruby/forwardable", + json: 'flori/json', + logger: 'ruby/logger', mutex_m: "ruby/mutex_m", - singleton: "ruby/singleton", - open3: "ruby/open3", - getoptlong: "ruby/getoptlong", - pstore: "ruby/pstore", - delegate: "ruby/delegate", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - readline: "ruby/readline", + nkf: "ruby/nkf", observer: "ruby/observer", - timeout: "ruby/timeout", - yaml: "ruby/yaml", - uri: "ruby/uri", + open3: "ruby/open3", openssl: "ruby/openssl", - did_you_mean: "ruby/did_you_mean", - weakref: "ruby/weakref", - tempfile: "ruby/tempfile", - tmpdir: "ruby/tmpdir", - English: "ruby/English", - "net-protocol": "ruby/net-protocol", - "net-http": "ruby/net-http", - bigdecimal: "ruby/bigdecimal", optparse: "ruby/optparse", - set: "ruby/set", - find: "ruby/find", + ostruct: 'ruby/ostruct', + pathname: "ruby/pathname", + pp: "ruby/pp", + prettyprint: "ruby/prettyprint", + pstore: "ruby/pstore", + psych: 'ruby/psych', + rdoc: 'ruby/rdoc', + readline: "ruby/readline", + reline: 'ruby/reline', + resolv: "ruby/resolv", rinda: "ruby/rinda", - erb: "ruby/erb", - nkf: "ruby/nkf", - tsort: "ruby/tsort", - abbrev: "ruby/abbrev", + rubygems: 'rubygems/rubygems', + securerandom: "ruby/securerandom", + set: "ruby/set", shellwords: "ruby/shellwords", - base64: "ruby/base64", + singleton: "ruby/singleton", + stringio: 'ruby/stringio', + strscan: 'ruby/strscan', + syntax_suggest: ["ruby/syntax_suggest", "main"], syslog: "ruby/syslog", - "open-uri": "ruby/open-uri", - securerandom: "ruby/securerandom", - resolv: "ruby/resolv", - "resolv-replace": "ruby/resolv-replace", + tempfile: "ruby/tempfile", time: "ruby/time", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - drb: "ruby/drb", - pathname: "ruby/pathname", - digest: "ruby/digest", - error_highlight: "ruby/error_highlight", - syntax_suggest: ["ruby/syntax_suggest", "main"], + timeout: "ruby/timeout", + tmpdir: "ruby/tmpdir", + tsort: "ruby/tsort", un: "ruby/un", + uri: "ruby/uri", + weakref: "ruby/weakref", win32ole: "ruby/win32ole", + yaml: "ruby/yaml", yarp: ["ruby/yarp", "main"], - } + zlib: 'ruby/zlib', + }.transform_keys(&:to_s) CLASSICAL_DEFAULT_BRANCH = "master" class << REPOSITORIES def [](gem) - repo, branch = super + repo, branch = super(gem) return repo, branch || CLASSICAL_DEFAULT_BRANCH end @@ -122,16 +122,17 @@ def replace_rdoc_ref(file) end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status porcelain -z -- *.c *.rb *.rdoc") + result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! + return if result.empty? result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem.to_sym] + repo, = REPOSITORIES[gem] puts "Sync #{repo}" upstream = File.join("..", "..", repo) @@ -399,13 +400,15 @@ def sync_default_gems(gem) when "yarp" # We don't want to remove yarp_init.c, so we temporarily move it # out of the yarp dir, wipe the yarp dir, and then put it back - mv("yarp/yarp_init.c", ".") if File.exist? "yarp/yarp_init.c" + mv("yarp/yarp_init.c", ".") + mv("yarp/yarp_compiler.c", ".") + mv("test/yarp/compiler_test.rb", ".") rm_rf(%w[test/yarp yarp]) # Run the YARP templating scripts cp_r("#{upstream}/ext/yarp", "yarp") cp_r("#{upstream}/lib/.", "lib") - cp_r("#{upstream}/test", "test/yarp") + cp_r("#{upstream}/test/yarp", "test") cp_r("#{upstream}/src/.", "yarp") cp_r("#{upstream}/yarp.gemspec", "lib/yarp") @@ -414,38 +417,43 @@ def sync_default_gems(gem) cp_r("#{upstream}/config.yml", "yarp/") cp_r("#{upstream}/templates", "yarp/") + rm_rf("yarp/templates/java") - rm_f("yarp/config.h") - File.write("yarp/config.h", "#include \"ruby/config.h\"\n") rm("yarp/extconf.rb") - mv("yarp_init.c", "yarp/") + mv("yarp_compiler.c", "yarp/") + mv("compiler_test.rb", "test/yarp/") else sync_lib gem, upstream end replace_rdoc_ref_all end - IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |Gemfile.lock - |bin\/.* - |rakelib\/.* - |test\/lib\/.* - )\z/mx - - YARP_IGNORE_FILE_PATTERN = - /\A(?:Makefile\.in - |configure\.ac - |fuzz\/.* - |rust\/.* - |tasks\/.* - |ext\/yarp\/extconf\.rb - )\z/mx + def ignore_file_pattern_for(gem) + patterns = [] + + # Common patterns + patterns << %r[\A(?: + [A-Z]\w*\.(?:md|txt) + |[^/]+\.yml + |\.git.* + |[A-Z]\w+file + |COPYING + |Gemfile.lock + |bin/.* + |rakelib/.* + |test/lib/.* + )\z]mx + + # Gem-specific patterns + case gem + when nil + end&.tap do |pattern| + patterns << pattern + end + + Regexp.union(*patterns) + end def message_filter(repo, sha, input: ARGF) log = input.read @@ -482,22 +490,8 @@ def message_filter(repo, sha, input: ARGF) puts subject, "\n", log end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh - # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array] "before..after". Note that it will NOT sync "before" (but commits after that). - # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. - def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem.to_sym] - puts "Sync #{repo} with commit history." - - # Fetch the repository to be synchronized - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} https://github.com/#{repo}.git` - end - end - system(*%W"git fetch --no-tags #{gem}") - + # Returns commit list as array of [commit_hash, subject]. + def commits_in_ranges(gem, repo, default_branch, ranges) # If -a is given, discover all commits since the last picked commit if ranges == true pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" @@ -506,7 +500,7 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) end # Parse a given range with git log - commits = ranges.flat_map do |range| + ranges.flat_map do |range| unless range.include?("..") range = "#{range}~1..#{range}" end @@ -515,15 +509,188 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} end end + end + + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Skip this commit if everything has been removed as `ignored_paths`. + changes = pipe_readlines(%W"git status --porcelain -z") + if changes.empty? + puts "Skip empty commit #{sha}" + return false + end + + # We want to skip DD: deleted by both. + deleted = changes.grep(/^DD /) {$'} + system(*%W"git rm -f --", *deleted) unless deleted.empty? + + # Import UA: added by them + added = changes.grep(/^UA /) {$'} + system(*%W"git add --", *added) unless added.empty? + + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + return system(*%w"git add --", *conflict) + end + end + return false + end - # Ignore Merge commits and already-merged commits. - ignore_file_pattern = IGNORE_FILE_PATTERN - if gem == "yarp" - ignore_file_pattern = Regexp.union(ignore_file_pattern, YARP_IGNORE_FILE_PATTERN) + return true + end + + def filter_pickup_files(changed, ignore_file_pattern, base) + toplevels = {} + remove = [] + ignore = [] + changed = changed.reject do |f| + case + when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { + remove << top unless + toplevels[top] = system(*%w"git cat-file -e", "#{base}:#{top}", err: File::NULL) + } + # Remove any new top-level directories. + true + when !f.include?("/"), + f.start_with?("test/fixtures/", "test/lib/", "tool/") + # Forcibly reset any top-level entries, and any changes under + # /test/fixtures, /test/lib, or /tool. + ignore << f + when ignore_file_pattern.match?(f) + # Forcibly reset any changes matching ignore_file_pattern. + ignore << f + end + end + return changed, remove, ignore + end + + def pickup_files(gem, changed, picked) + # Forcibly remove any files that we don't want to copy to this + # repository. + + ignore_file_pattern = ignore_file_pattern_for(gem) + + base = picked ? "HEAD~" : "HEAD" + changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) + + unless remove.empty? + puts "Remove added files: #{remove.join(', ')}" + system(*%w"git rm -fr --", *remove) + if picked + system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + end + end + + unless ignore.empty? + puts "Reset ignored files: #{ignore.join(', ')}" + system(*%W"git checkout -f", base, "--", *ignore) + end + + if changed.empty? + return nil + end + + return changed + end + + def pickup_commit(gem, sha, edit) + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{sha}", &:read) + picked = $?.success? + if result =~ /nothing\ to\ commit/ + `git reset` + puts "Skip empty commit #{sha}" + return false + end + + # Skip empty commits + if result.empty? + return false + end + + if picked + changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") + else + changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") + end + + # Pick up files to merge. + unless changed = pickup_files(gem, changed, picked) + puts "Skip commit #{sha} only for tools or toplevel" + if picked + `git reset --hard HEAD~` + else + `git cherry-pick --abort` + end + return false end + + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless picked or resolve_conflicts(gem, sha, edit) + `git reset` && `git checkout .` && `git clean -fd` + return picked || nil # Fail unless cherry-picked + end + + # Commit cherry-picked commit + if picked + system(*%w"git commit --amend --no-edit") + else + system(*%w"git cherry-pick --continue --no-edit") + end or return nil + + # Amend the commit if RDoc references need to be replaced + head = `git log --format=%H -1 HEAD`.chomp + system(*%w"git reset --quiet HEAD~ --") + amend = replace_rdoc_ref_all + system(*%W"git reset --quiet #{head} --") + if amend + `git commit --amend --no-edit --all` + end + + return true + end + + # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + repo, default_branch = REPOSITORIES[gem] + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags #{gem}") + + commits = commits_in_ranges(gem, repo, default_branch, ranges) + + # Ignore Merge commits and already-merged commits. commits.delete_if do |sha, subject| - files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") - subject.start_with?("Merge", "Auto Merge") or files.all?(ignore_file_pattern) + subject.start_with?("Merge", "Auto Merge") end if commits.empty? @@ -545,152 +712,14 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) ] commits.each do |sha, subject| puts "Pick #{sha} from #{repo}." - - # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", &:read) - if result =~ /nothing\ to\ commit/ - `git reset` - puts "Skip empty commit #{sha}" + case pickup_commit(gem, sha, edit) + when false next - end - - # Skip empty commits or deal with conflicts - skipped = false - if result.empty? - skipped = true - elsif /^CONFLICT/ =~ result - # Forcibly remove any files that we don't want to copy to this repository. - # We also ignore them as new `toplevels` even when they don't conflict. - ignored_paths = [] - case gem - when "rubygems" - # We don't copy any vcr_cassettes to this repository. Because the directory does not - # exist, rename detection doesn't work. So it starts with the original path `bundler/`. - ignored_paths += %w[bundler/spec/support/artifice/vcr_cassettes] - when "yarp" - # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs - # since ruby/ruby/doc is not something owned by YARP. - ignored_paths += %w[docs/] - end - ignored_paths.each do |path| - if File.exist?(path) - puts "Removing: #{path}" - system("git", "reset", path) - rm_rf(path) - end - end - - # git has inexact rename detection, so they follow directory renames even for new files. - # However, new files are considered a `CONFLICT (file location)`, so you need to git-add them here. - # We hope that they are not other kinds of conflicts, assuming we don't modify them in this repository. - case gem - when "rubygems" - system(*%w[git add spec/bundler]) - when "yarp" - system(*%w[git add lib/yarp]) - system(*%w[git add test/yarp]) - system(*%w[git add yarp]) - end - - # Skip this commit if everything has been removed as `ignored_paths`. - changes = pipe_readlines(%W"git status --porcelain -z") - if changes.empty? - `git reset` && `git checkout .` && `git clean -fd` - puts "Skip empty commit #{sha}" - next - end - - # For YARP, we want to skip DD: deleted by both. - if gem == "yarp" - deleted = changes.grep(/^DD /) - deleted.map! { |line| line.delete_prefix("DD ") } - system(*%W"git rm -f --", *deleted) unless deleted.empty? - end - - # Discover unmerged files - # AU: unmerged, added by us - # DU: unmerged, deleted by us - # UU: unmerged, both modified - # UA: unmerged, added by them - # AA: unmerged, both added - unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} - unmerged.compact! - ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} - # Reset ignored files if they conflict - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/\A.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? - end - # If -e option is given, open each conflicted file with an editor - unless conflict.empty? - if edit - case - when (editor = ENV["GIT_EDITOR"] and !editor.empty?) - when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) - end - if editor - system([editor, conflict].join(' ')) - end - end - end - # Attempt to commit the cherry-pick - skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") - end - - # Skip the commit if it's empty or the cherry-pick attempt failed - if skipped + when nil failed_commits << sha - `git reset` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" next end - # Forcibly remove any new top-level entries, and any changes under - # /test/fixtures, /test/lib, or /tool. - changed = pipe_readlines(%W"git diff --name-only -z HEAD~..HEAD --") - toplevels = changed.map {|f| f[%r[\A(?!tool/)[^/]+/?]]}.compact - toplevels.delete_if do |top| - if system(*%w"git checkout -f HEAD~ --", top, err: File::NULL) - # previously existent path - system(*%w"git checkout -f HEAD --", top, out: File::NULL) - true - end - end - unless toplevels.empty? - puts "Remove files added to toplevel: #{toplevels.join(', ')}" - system(*%w"git rm -r --", *toplevels) - end - tools = changed.select {|f|f.start_with?("test/fixtures/", "test/lib/", "tool/")} - unless tools.empty? - system(*%W"git rm -r --", *tools) - system(*%W"git checkout HEAD~ --", *tools) - end - unless toplevels.empty? and tools.empty? - clean = toplevels + tools - if system(*%W"git diff --quiet HEAD~") - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` - puts "Skip commit #{sha} only for tools or toplevel" - next - end - unless system(*%W"git commit --amend --no-edit --", *clean) - failed_commits << sha - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" - next - end - end - - # Amend the commit if RDoc references need to be replaced - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` - end - puts "Update commit message: #{sha}" # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter @@ -733,7 +762,7 @@ def sync_lib(repo, upstream = nil) def update_default_gems(gem, release: false) - repository, default_branch = REPOSITORIES[gem.to_sym] + repository, default_branch = REPOSITORIES[gem] author, repository = repository.split('/') puts "Update #{author}/#{repository}" @@ -771,16 +800,16 @@ def update_default_gems(gem, release: false) if ARGV[1] update_default_gems(ARGV[1]) else - REPOSITORIES.each_key {|gem| update_default_gems(gem.to_s)} + REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" if ARGV[1] == "release" REPOSITORIES.each_key do |gem| - update_default_gems(gem.to_s, release: true) - sync_default_gems(gem.to_s) + update_default_gems(gem, release: true) + sync_default_gems(gem) end else - REPOSITORIES.each_key {|gem| sync_default_gems(gem.to_s)} + REPOSITORIES.each_key {|gem| sync_default_gems(gem)} end when "list" ARGV.shift diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index e6c654ed55d914..a73fc65d6e6b48 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -87,7 +87,7 @@ def setup system(*%W"git config --global user.name", "Ruby") system(*%W"git config --global init.defaultBranch default") @target = "sync-test" - SyncDefaultGems::REPOSITORIES[@target.to_sym] = ["ruby/#{@target}", "default"] + SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] @sha = {} @origdir = Dir.pwd Dir.chdir(@testdir) @@ -113,7 +113,7 @@ def setup def teardown if @target Dir.chdir(@origdir) - SyncDefaultGems::REPOSITORIES.delete(@target.to_sym) + SyncDefaultGems::REPOSITORIES.delete(@target) ENV.update(@git_config) FileUtils.rm_rf(@testdir) end diff --git a/tool/update-deps b/tool/update-deps index 5662188deec402..90107e50c83758 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -156,6 +156,7 @@ FILES_NEED_VPATH = %w[ yarp/prettyprint.c yarp/serialize.c yarp/token_type.c + yarp/version.h ] # Multiple files with same filename. diff --git a/util.c b/util.c index 1030c3ecbede16..3c08879ce58698 100644 --- a/util.c +++ b/util.c @@ -13,6 +13,10 @@ # define MINGW_HAS_SECURE_API 1 #endif +#ifndef __STDC_WANT_LIB_EXT1__ +#define __STDC_WANT_LIB_EXT1__ 1 /* for qsort_s() */ +#endif + #include "ruby/internal/config.h" #include diff --git a/vm.c b/vm.c index f838c94022e8cf..14ddb2d3ab9d23 100644 --- a/vm.c +++ b/vm.c @@ -2745,6 +2745,8 @@ rb_vm_update_references(void *ptr) rb_gc_update_tbl_refs(vm->overloaded_cme_table); + rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd); + if (vm->coverages) { vm->coverages = rb_gc_location(vm->coverages); vm->me2counter = rb_gc_location(vm->me2counter); diff --git a/yarp/config.h b/yarp/config.h deleted file mode 100644 index 5a5987fa4499f5..00000000000000 --- a/yarp/config.h +++ /dev/null @@ -1 +0,0 @@ -#include "ruby/config.h" diff --git a/yarp/config.yml b/yarp/config.yml index 3662c176098a50..e65177302f455b 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -361,7 +361,7 @@ flags: comment: "o - only interpolates values into the regular expression once" nodes: - name: AliasNode - child_nodes: + fields: - name: new_name type: node - name: old_name @@ -374,7 +374,7 @@ nodes: alias foo bar ^^^^^^^^^^^^^ - name: AlternationPatternNode - child_nodes: + fields: - name: left type: node - name: right @@ -387,7 +387,7 @@ nodes: foo => bar | baz ^^^^^^^^^ - name: AndNode - child_nodes: + fields: - name: left type: node - name: right @@ -399,21 +399,8 @@ nodes: left and right ^^^^^^^^^^^^^^ - - name: AndWriteNode - child_nodes: - - name: target - type: node - - name: value - type: node - - name: operator_loc - type: location - comment: | - Represents the use of the `&&=` operator. - - target &&= value - ^^^^^^^^^^^^^^^^ - name: ArgumentsNode - child_nodes: + fields: - name: arguments type: node[] comment: | @@ -422,7 +409,7 @@ nodes: return foo, bar, baz ^^^^^^^^^^^^^ - name: ArrayNode - child_nodes: + fields: - name: elements type: node[] - name: opening_loc @@ -436,7 +423,7 @@ nodes: [1, 2, 3] ^^^^^^^^^ - name: ArrayPatternNode - child_nodes: + fields: - name: constant type: node? - name: requireds @@ -467,7 +454,7 @@ nodes: foo in Bar[1, 2, 3] ^^^^^^^^^^^^^^^^^^^ - name: AssocNode - child_nodes: + fields: - name: key type: node - name: value @@ -480,7 +467,7 @@ nodes: { a => b } ^^^^^^ - name: AssocSplatNode - child_nodes: + fields: - name: value type: node? - name: operator_loc @@ -497,7 +484,7 @@ nodes: $' ^^ - name: BeginNode - child_nodes: + fields: - name: begin_keyword_loc type: location? - name: statements @@ -523,7 +510,7 @@ nodes: end ^^^^^ - name: BlockArgumentNode - child_nodes: + fields: - name: expression type: node? - name: operator_loc @@ -534,7 +521,7 @@ nodes: bar(&args) ^^^^^^^^^^ - name: BlockNode - child_nodes: + fields: - name: locals type: constant[] - name: parameters @@ -552,7 +539,7 @@ nodes: [1, 2, 3].each { |i| puts x } ^^^^^^^^^^^^^^ - name: BlockParameterNode - child_nodes: + fields: - name: name_loc type: location? - name: operator_loc @@ -564,7 +551,7 @@ nodes: ^^ end - name: BlockParametersNode - child_nodes: + fields: - name: parameters type: node? kind: ParametersNode @@ -584,7 +571,7 @@ nodes: ^^^^^^^^^^^^^^^^^ end - name: BreakNode - child_nodes: + fields: - name: arguments type: node? kind: ArgumentsNode @@ -596,7 +583,7 @@ nodes: break foo ^^^^^^^^^ - name: CallNode - child_nodes: + fields: - name: receiver type: node? - name: operator_loc @@ -639,7 +626,7 @@ nodes: foo&.bar ^^^^^^^^ - name: CallOperatorAndWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -653,7 +640,7 @@ nodes: foo.bar &&= value ^^^^^^^^^^^^^^^^^ - name: CallOperatorOrWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -667,7 +654,7 @@ nodes: foo.bar ||= value ^^^^^^^^^^^^^^^^^ - name: CallOperatorWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -675,7 +662,7 @@ nodes: type: location - name: value type: node - - name: operator_id + - name: operator type: constant comment: | Represents the use of an assignment operator on a call. @@ -683,7 +670,7 @@ nodes: foo.bar += baz ^^^^^^^^^^^^^^ - name: CapturePatternNode - child_nodes: + fields: - name: value type: node - name: target @@ -696,7 +683,7 @@ nodes: foo => [bar => baz] ^^^^^^^^^^^^ - name: CaseNode - child_nodes: + fields: - name: predicate type: node? - name: conditions @@ -716,7 +703,7 @@ nodes: when false end - name: ClassNode - child_nodes: + fields: - name: locals type: constant[] - name: class_keyword_loc @@ -738,14 +725,75 @@ nodes: class Foo end ^^^^^^^^^^^^^ + - name: ClassVariableAndWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a class variable. + + @@target &&= value + ^^^^^^^^^^^^^^^^ + - name: ClassVariableOperatorWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a class variable using an operator that isn't `=`. + + @@target += value + ^^^^^^^^^^^^^^^^^ + - name: ClassVariableOrWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a class variable. + + @@target ||= value + ^^^^^^^^^^^^^^^^^^ - name: ClassVariableReadNode + fields: + - name: name + type: constant comment: | Represents referencing a class variable. @@foo ^^^^^ + - name: ClassVariableTargetNode + fields: + - name: name + type: constant + comment: | + Represents writing to a class variable in a context that doesn't have an explicit value. + + @@foo, @@bar = baz + ^^^^^ ^^^^^ - name: ClassVariableWriteNode - child_nodes: + fields: + - name: name + type: constant - name: name_loc type: location - name: value @@ -757,8 +805,63 @@ nodes: @@foo = 1 ^^^^^^^^^ + - name: ConstantAndWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a constant. + + Target &&= value + ^^^^^^^^^^^^^^^^ + - name: ConstantOperatorWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a constant using an operator that isn't `=`. + + Target += value + ^^^^^^^^^^^^^^^ + - name: ConstantOrWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a constant. + + Target ||= value + ^^^^^^^^^^^^^^^^ + - name: ConstantPathAndWriteNode + fields: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a constant path. + + Parent::Child &&= value + ^^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathNode - child_nodes: + fields: - name: parent type: node? - name: child @@ -770,15 +873,58 @@ nodes: Foo::Bar ^^^^^^^^ - - name: ConstantPathWriteNode - child_nodes: + - name: ConstantPathOperatorWriteNode + fields: - name: target type: node kind: ConstantPathNode - name: operator_loc - type: location? + type: location - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a constant path using an operator that isn't `=`. + + Parent::Child += value + ^^^^^^^^^^^^^^^^^^^^^^ + - name: ConstantPathOrWriteNode + fields: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a constant path. + + Parent::Child ||= value + ^^^^^^^^^^^^^^^^^^^^^^^ + - name: ConstantPathTargetNode + fields: + - name: parent type: node? + - name: child + type: node + - name: delimiter_loc + type: location + comment: | + Represents writing to a constant path in a context that doesn't have an explicit value. + + Foo::Foo, Bar::Bar = baz + ^^^^^^^^ ^^^^^^^^ + - name: ConstantPathWriteNode + fields: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node comment: | Represents writing to a constant path. @@ -796,21 +942,27 @@ nodes: Foo ^^^ + - name: ConstantTargetNode + comment: | + Represents writing to a constant in a context that doesn't have an explicit value. + + Foo, Bar = baz + ^^^ ^^^ - name: ConstantWriteNode - child_nodes: + fields: - name: name_loc type: location - name: value - type: node? + type: node - name: operator_loc - type: location? + type: location comment: | Represents writing to a constant. Foo = 1 ^^^^^^^ - name: DefNode - child_nodes: + fields: - name: name_loc type: location - name: receiver @@ -841,7 +993,7 @@ nodes: end ^^^^^^^^^^ - name: DefinedNode - child_nodes: + fields: - name: lparen_loc type: location? - name: value @@ -856,7 +1008,7 @@ nodes: defined?(a) ^^^^^^^^^^^ - name: ElseNode - child_nodes: + fields: - name: else_keyword_loc type: location - name: statements @@ -870,7 +1022,7 @@ nodes: if a then b else c end ^^^^^^^^^^ - name: EmbeddedStatementsNode - child_nodes: + fields: - name: opening_loc type: location - name: statements @@ -884,7 +1036,7 @@ nodes: "foo #{bar}" ^^^^^^ - name: EmbeddedVariableNode - child_nodes: + fields: - name: operator_loc type: location - name: variable @@ -895,7 +1047,7 @@ nodes: "foo #@bar" ^^^^^ - name: EnsureNode - child_nodes: + fields: - name: ensure_keyword_loc type: location - name: statements @@ -919,7 +1071,7 @@ nodes: false ^^^^^ - name: FindPatternNode - child_nodes: + fields: - name: constant type: node? - name: left @@ -944,7 +1096,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: FlipFlopNode - child_nodes: + fields: - name: left type: node? - name: right @@ -966,7 +1118,7 @@ nodes: 1.0 ^^^ - name: ForNode - child_nodes: + fields: - name: index type: node - name: collection @@ -1003,7 +1155,7 @@ nodes: ^^^ end - name: ForwardingSuperNode - child_nodes: + fields: - name: block type: node? kind: BlockNode @@ -1012,27 +1164,74 @@ nodes: super ^^^^^ + - name: GlobalVariableAndWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a global variable. + + $target &&= value + ^^^^^^^^^^^^^^^^^ + - name: GlobalVariableOperatorWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a global variable using an operator that isn't `=`. + + $target += value + ^^^^^^^^^^^^^^^^ + - name: GlobalVariableOrWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a global variable. + + $target ||= value + ^^^^^^^^^^^^^^^^^ - name: GlobalVariableReadNode comment: | Represents referencing a global variable. $foo ^^^^ + - name: GlobalVariableTargetNode + comment: | + Represents writing to a global variable in a context that doesn't have an explicit value. + + $foo, $bar = baz + ^^^^ ^^^^ - name: GlobalVariableWriteNode - child_nodes: + fields: - name: name_loc type: location - - name: operator_loc - type: location? - name: value - type: node? + type: node + - name: operator_loc + type: location comment: | Represents writing to a global variable. $foo = 1 ^^^^^^^^ - name: HashNode - child_nodes: + fields: - name: opening_loc type: location - name: elements @@ -1045,7 +1244,7 @@ nodes: { a => b } ^^^^^^^^^^ - name: HashPatternNode - child_nodes: + fields: - name: constant type: node? - name: assocs @@ -1065,7 +1264,7 @@ nodes: foo => { a: 1, b: 2, **c } ^^^^^^^^^^^^^^^^^^^ - name: IfNode - child_nodes: + fields: - name: if_keyword_loc type: location? - name: predicate @@ -1087,7 +1286,7 @@ nodes: if foo then bar end ^^^^^^^^^^^^^^^^^^^ - name: ImaginaryNode - child_nodes: + fields: - name: numeric type: node comment: | @@ -1096,7 +1295,7 @@ nodes: 1.0i ^^^^ - name: InNode - child_nodes: + fields: - name: pattern type: node - name: statements @@ -1111,20 +1310,81 @@ nodes: case a; in b then c end ^^^^^^^^^^^ + - name: InstanceVariableAndWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to an instance variable. + + @target &&= value + ^^^^^^^^^^^^^^^^^ + - name: InstanceVariableOperatorWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to an instance variable using an operator that isn't `=`. + + @target += value + ^^^^^^^^^^^^^^^^ + - name: InstanceVariableOrWriteNode + fields: + - name: name + type: constant + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to an instance variable. + + @target ||= value + ^^^^^^^^^^^^^^^^^ - name: InstanceVariableReadNode + fields: + - name: name + type: constant comment: | Represents referencing an instance variable. @foo ^^^^ + - name: InstanceVariableTargetNode + fields: + - name: name + type: constant + comment: | + Represents writing to an instance variable in a context that doesn't have an explicit value. + + @foo, @bar = baz + ^^^^ ^^^^ - name: InstanceVariableWriteNode - child_nodes: + fields: + - name: name + type: constant - name: name_loc type: location - name: value - type: node? + type: node - name: operator_loc - type: location? + type: location comment: | Represents writing to an instance variable. @@ -1137,7 +1397,7 @@ nodes: 1 ^ - name: InterpolatedRegularExpressionNode - child_nodes: + fields: - name: opening_loc type: location - name: parts @@ -1154,7 +1414,7 @@ nodes: /foo #{bar} baz/ ^^^^^^^^^^^^^^^^ - name: InterpolatedStringNode - child_nodes: + fields: - name: opening_loc type: location? - name: parts @@ -1168,7 +1428,7 @@ nodes: "foo #{bar} baz" ^^^^^^^^^^^^^^^^ - name: InterpolatedSymbolNode - child_nodes: + fields: - name: opening_loc type: location? - name: parts @@ -1182,7 +1442,7 @@ nodes: :"foo #{bar} baz" ^^^^^^^^^^^^^^^^^ - name: InterpolatedXStringNode - child_nodes: + fields: - name: opening_loc type: location - name: parts @@ -1196,7 +1456,7 @@ nodes: `foo #{bar} baz` ^^^^^^^^^^^^^^^^ - name: KeywordHashNode - child_nodes: + fields: - name: elements type: node[] comment: | @@ -1205,7 +1465,7 @@ nodes: foo(a: b) ^^^^ - name: KeywordParameterNode - child_nodes: + fields: - name: name_loc type: location - name: value @@ -1221,7 +1481,7 @@ nodes: ^^^^ end - name: KeywordRestParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: name_loc @@ -1233,11 +1493,15 @@ nodes: ^^^ end - name: LambdaNode - child_nodes: + fields: - name: locals type: constant[] + - name: operator_loc + type: location - name: opening_loc type: location + - name: closing_loc + type: location - name: parameters type: node? kind: BlockParametersNode @@ -1248,9 +1512,62 @@ nodes: ->(value) { value * 2 } ^^^^^^^^^^^^^^^^^^^^^^^ + - name: LocalVariableAndWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: name + type: constant + - name: depth + type: uint32 + comment: | + Represents the use of the `&&=` operator for assignment to a local variable. + + target &&= value + ^^^^^^^^^^^^^^^^ + - name: LocalVariableOperatorWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: name + type: constant + - name: operator + type: constant + - name: depth + type: uint32 + comment: | + Represents assigning to a local variable using an operator that isn't `=`. + + target += value + ^^^^^^^^^^^^^^^ + - name: LocalVariableOrWriteNode + fields: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: name + type: constant + - name: depth + type: uint32 + comment: | + Represents the use of the `||=` operator for assignment to a local variable. + + target ||= value + ^^^^^^^^^^^^^^^^ - name: LocalVariableReadNode - child_nodes: - - name: constant_id + fields: + - name: name type: constant - name: depth type: uint32 @@ -1261,25 +1578,36 @@ nodes: foo ^^^ + - name: LocalVariableTargetNode + fields: + - name: name + type: constant + - name: depth + type: uint32 + comment: | + Represents writing to a local variable in a context that doesn't have an explicit value. + + foo, bar = baz + ^^^ ^^^ - name: LocalVariableWriteNode - child_nodes: - - name: constant_id + fields: + - name: name type: constant - name: depth type: uint32 - - name: value - type: node? - name: name_loc type: location + - name: value + type: node - name: operator_loc - type: location? + type: location comment: | Represents writing to a local variable. foo = 1 ^^^^^^^ - name: MatchPredicateNode - child_nodes: + fields: - name: value type: node - name: pattern @@ -1292,7 +1620,7 @@ nodes: foo in bar ^^^^^^^^^^ - name: MatchRequiredNode - child_nodes: + fields: - name: value type: node - name: pattern @@ -1309,7 +1637,7 @@ nodes: Represents a node that is missing from the source and results in a syntax error. - name: ModuleNode - child_nodes: + fields: - name: locals type: constant[] - name: module_keyword_loc @@ -1328,7 +1656,7 @@ nodes: module Foo end ^^^^^^^^^^^^^^ - name: MultiWriteNode - child_nodes: + fields: - name: targets type: node[] - name: operator_loc @@ -1345,7 +1673,7 @@ nodes: a, b, c = 1, 2, 3 ^^^^^^^^^^^^^^^^^ - name: NextNode - child_nodes: + fields: - name: arguments type: node? kind: ArgumentsNode @@ -1363,7 +1691,7 @@ nodes: nil ^^^ - name: NoKeywordsParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: keyword_loc @@ -1375,29 +1703,17 @@ nodes: ^^^^^ end - name: NumberedReferenceReadNode + fields: + - name: number + type: uint32 comment: | Represents reading a numbered reference to a capture in the previous match. $1 ^^ - - name: OperatorWriteNode - child_nodes: - - name: target - type: node - - name: operator_loc - type: location - - name: operator - type: constant - - name: value - type: node - comment: | - Represents the use of an operator on a write. - - target += value - ^^^^^^^^^^^^^^^ - name: OptionalParameterNode - child_nodes: - - name: constant_id + fields: + - name: name type: constant - name: name_loc type: location @@ -1412,7 +1728,7 @@ nodes: ^^^^^ end - name: OrNode - child_nodes: + fields: - name: left type: node - name: right @@ -1424,21 +1740,8 @@ nodes: left or right ^^^^^^^^^^^^^ - - name: OrWriteNode - child_nodes: - - name: target - type: node - - name: value - type: node - - name: operator_loc - type: location - comment: | - Represents the use of the `||=` operator. - - target ||= value - ^^^^^^^^^^^^^^^^ - name: ParametersNode - child_nodes: + fields: - name: requireds type: node[] - name: optionals @@ -1462,7 +1765,7 @@ nodes: ^^^^^^^ end - name: ParenthesesNode - child_nodes: + fields: - name: body type: node? - name: opening_loc @@ -1476,7 +1779,7 @@ nodes: (10 + 34) ^^^^^^^^^ - name: PinnedExpressionNode - child_nodes: + fields: - name: expression type: node - name: operator_loc @@ -1492,7 +1795,7 @@ nodes: foo in ^(bar) ^^^^^^ - name: PinnedVariableNode - child_nodes: + fields: - name: variable type: node - name: operator_loc @@ -1504,7 +1807,7 @@ nodes: foo in ^bar ^^^^ - name: PostExecutionNode - child_nodes: + fields: - name: statements type: node? kind: StatementsNode @@ -1520,7 +1823,7 @@ nodes: END { foo } ^^^^^^^^^^^ - name: PreExecutionNode - child_nodes: + fields: - name: statements type: node? kind: StatementsNode @@ -1536,7 +1839,7 @@ nodes: BEGIN { foo } ^^^^^^^^^^^^^ - name: ProgramNode - child_nodes: + fields: - name: locals type: constant[] - name: statements @@ -1544,7 +1847,7 @@ nodes: kind: StatementsNode comment: The top level node of any parse tree. - name: RangeNode - child_nodes: + fields: - name: left type: node? - name: right @@ -1563,7 +1866,7 @@ nodes: c if a =~ /left/ ... b =~ /right/ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: RationalNode - child_nodes: + fields: - name: numeric type: node comment: | @@ -1578,7 +1881,7 @@ nodes: redo ^^^^ - name: RegularExpressionNode - child_nodes: + fields: - name: opening_loc type: location - name: content_loc @@ -1596,7 +1899,7 @@ nodes: /foo/i ^^^^^^ - name: RequiredDestructuredParameterNode - child_nodes: + fields: - name: parameters type: node[] - name: opening_loc @@ -1610,8 +1913,8 @@ nodes: ^^^^^^^^^^ end - name: RequiredParameterNode - child_nodes: - - name: constant_id + fields: + - name: name type: constant comment: | Represents a required parameter to a method, block, or lambda definition. @@ -1620,7 +1923,7 @@ nodes: ^ end - name: RescueModifierNode - child_nodes: + fields: - name: expression type: node - name: keyword_loc @@ -1634,7 +1937,7 @@ nodes: foo rescue nil ^^^^^^^^^^^^^^ - name: RescueNode - child_nodes: + fields: - name: keyword_loc type: location - name: exceptions @@ -1661,7 +1964,7 @@ nodes: `Foo, *splat, Bar` are in the `exceptions` field. `ex` is in the `exception` field. - name: RestParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: name_loc @@ -1679,7 +1982,7 @@ nodes: retry ^^^^^ - name: ReturnNode - child_nodes: + fields: - name: keyword_loc type: location - name: arguments @@ -1697,7 +2000,7 @@ nodes: self ^^^^ - name: SingletonClassNode - child_nodes: + fields: - name: locals type: constant[] - name: class_keyword_loc @@ -1722,7 +2025,7 @@ nodes: __ENCODING__ ^^^^^^^^^^^^ - name: SourceFileNode - child_nodes: + fields: - name: filepath type: string comment: | @@ -1737,7 +2040,7 @@ nodes: __LINE__ ^^^^^^^^ - name: SplatNode - child_nodes: + fields: - name: operator_loc type: location - name: expression @@ -1748,7 +2051,7 @@ nodes: [*a] ^^ - name: StatementsNode - child_nodes: + fields: - name: body type: node[] comment: | @@ -1757,7 +2060,7 @@ nodes: foo; bar; baz ^^^^^^^^^^^^^ - name: StringConcatNode - child_nodes: + fields: - name: left type: node - name: right @@ -1768,7 +2071,7 @@ nodes: "foo" "bar" ^^^^^^^^^^^ - name: StringNode - child_nodes: + fields: - name: opening_loc type: location? - name: content_loc @@ -1790,7 +2093,7 @@ nodes: "foo #{bar} baz" ^^^^ ^^^^ - name: SuperNode - child_nodes: + fields: - name: keyword_loc type: location - name: lparen_loc @@ -1812,11 +2115,11 @@ nodes: super foo, bar ^^^^^^^^^^^^^^ - name: SymbolNode - child_nodes: + fields: - name: opening_loc type: location? - name: value_loc - type: location + type: location? - name: closing_loc type: location? - name: unescaped @@ -1836,7 +2139,7 @@ nodes: true ^^^^ - name: UndefNode - child_nodes: + fields: - name: names type: node[] - name: keyword_loc @@ -1847,7 +2150,7 @@ nodes: undef :foo, :bar, :baz ^^^^^^^^^^^^^^^^^^^^^^ - name: UnlessNode - child_nodes: + fields: - name: keyword_loc type: location - name: predicate @@ -1870,9 +2173,11 @@ nodes: unless foo then bar end ^^^^^^^^^^^^^^^^^^^^^^^ - name: UntilNode - child_nodes: + fields: - name: keyword_loc type: location + - name: closing_loc + type: location? - name: predicate type: node - name: statements @@ -1891,7 +2196,7 @@ nodes: until foo do bar end ^^^^^^^^^^^^^^^^^^^^ - name: WhenNode - child_nodes: + fields: - name: keyword_loc type: location - name: conditions @@ -1907,9 +2212,11 @@ nodes: ^^^^^^^^^ end - name: WhileNode - child_nodes: + fields: - name: keyword_loc type: location + - name: closing_loc + type: location? - name: predicate type: node - name: statements @@ -1928,7 +2235,7 @@ nodes: while foo do bar end ^^^^^^^^^^^^^^^^^^^^ - name: XStringNode - child_nodes: + fields: - name: opening_loc type: location - name: content_loc @@ -1943,7 +2250,7 @@ nodes: `foo` ^^^^^ - name: YieldNode - child_nodes: + fields: - name: keyword_loc type: location - name: lparen_loc diff --git a/yarp/defines.h b/yarp/defines.h index c08d578422dd5b..5fe3530d914fe4 100644 --- a/yarp/defines.h +++ b/yarp/defines.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,6 @@ # define snprintf _snprintf #endif -int yp_strncasecmp(const char *string1, const char *string2, size_t length); +int yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length); #endif diff --git a/yarp/diagnostic.c b/yarp/diagnostic.c index 8bd888e379bcea..b216d96a330974 100644 --- a/yarp/diagnostic.c +++ b/yarp/diagnostic.c @@ -2,7 +2,7 @@ // Append an error to the given list of diagnostic. bool -yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message) { +yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message) { yp_diagnostic_t *diagnostic = (yp_diagnostic_t *) malloc(sizeof(yp_diagnostic_t)); if (diagnostic == NULL) return false; diff --git a/yarp/diagnostic.h b/yarp/diagnostic.h index bcbee5380ca98e..58228d8493ad31 100644 --- a/yarp/diagnostic.h +++ b/yarp/diagnostic.h @@ -10,13 +10,13 @@ // This struct represents a diagnostic found during parsing. typedef struct { yp_list_node_t node; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; const char *message; } yp_diagnostic_t; // Append a diagnostic to the given list of diagnostics. -bool yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message); +bool yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message); // Deallocate the internal state of the given diagnostic list. void yp_diagnostic_list_free(yp_list_t *list); diff --git a/yarp/enc/yp_big5.c b/yarp/enc/yp_big5.c index a33f5ce50493ac..a7c879cd0abb16 100644 --- a/yarp/enc/yp_big5.c +++ b/yarp/enc/yp_big5.c @@ -1,69 +1,42 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_big5_codepoint_t; - -static yp_big5_codepoint_t -yp_big5_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_big5_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. - if ((n > 1) && (uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xFE)) { - *width = 2; - return (yp_big5_codepoint_t) (uc[0] << 8 | uc[1]); + if ((n > 1) && (b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xFE)) { + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_big5_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_big5_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_big5_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_big5_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_big5_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_big5_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_big5_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/enc/yp_encoding.h b/yarp/enc/yp_encoding.h index 7c4ce28c9428b7..9e8e7e01f60dba 100644 --- a/yarp/enc/yp_encoding.h +++ b/yarp/enc/yp_encoding.h @@ -16,22 +16,22 @@ typedef struct { // Return the number of bytes that the next character takes if it is valid // in the encoding. Does not read more than n bytes. It is assumed that n is // at least 1. - size_t (*char_width)(const char *c, ptrdiff_t n); + size_t (*char_width)(const uint8_t *b, ptrdiff_t n); // Return the number of bytes that the next character takes if it is valid // in the encoding and is alphabetical. Does not read more than n bytes. It // is assumed that n is at least 1. - size_t (*alpha_char)(const char *c, ptrdiff_t n); + size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n); // Return the number of bytes that the next character takes if it is valid // in the encoding and is alphanumeric. Does not read more than n bytes. It // is assumed that n is at least 1. - size_t (*alnum_char)(const char *c, ptrdiff_t n); + size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n); // Return true if the next character is valid in the encoding and is an // uppercase character. Does not read more than n bytes. It is assumed that // n is at least 1. - bool (*isupper_char)(const char *c, ptrdiff_t n); + bool (*isupper_char)(const uint8_t *b, ptrdiff_t n); // The name of the encoding. This should correspond to a value that can be // passed to Encoding.find in Ruby. @@ -49,18 +49,18 @@ typedef struct { // These functions are reused by some other encodings, so they are defined here // so they can be shared. -size_t yp_encoding_ascii_alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); -size_t yp_encoding_ascii_alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); -bool yp_encoding_ascii_isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +size_t yp_encoding_ascii_alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +size_t yp_encoding_ascii_alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +bool yp_encoding_ascii_isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); // These functions are shared between the actual encoding and the fast path in // the parser so they need to be internally visible. -size_t yp_encoding_utf_8_alpha_char(const char *c, ptrdiff_t n); -size_t yp_encoding_utf_8_alnum_char(const char *c, ptrdiff_t n); +size_t yp_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n); +size_t yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n); // This lookup table is referenced in both the UTF-8 encoding file and the // parser directly in order to speed up the default encoding processing. -extern unsigned char yp_encoding_unicode_table[256]; +extern uint8_t yp_encoding_unicode_table[256]; // These are the encodings that are supported by the parser. They are defined in // their own files in the src/enc directory. diff --git a/yarp/enc/yp_euc_jp.c b/yarp/enc/yp_euc_jp.c index ebcd6a784992df..f6f80d528bcde0 100644 --- a/yarp/enc/yp_euc_jp.c +++ b/yarp/enc/yp_euc_jp.c @@ -1,75 +1,48 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_euc_jp_codepoint_t; - -static yp_euc_jp_codepoint_t -yp_euc_jp_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_euc_jp_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. if ( (n > 1) && ( - ((uc[0] == 0x8E) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || - ((uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) + ((b[0] == 0x8E) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || + ((b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0xA1 && b[1] <= 0xFE)) ) ) { - *width = 2; - return (yp_euc_jp_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_euc_jp_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_euc_jp_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_euc_jp_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_euc_jp_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_euc_jp_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_euc_jp_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_euc_jp_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return 0; } diff --git a/yarp/enc/yp_gbk.c b/yarp/enc/yp_gbk.c index 31e88756db5865..71de318612d45d 100644 --- a/yarp/enc/yp_gbk.c +++ b/yarp/enc/yp_gbk.c @@ -1,78 +1,51 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_gbk_codepoint_t; - -static yp_gbk_codepoint_t -yp_gbk_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_gbk_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. if ( (n > 1) && ( - ((uc[0] >= 0xA1 && uc[0] <= 0xA9) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/1 - ((uc[0] >= 0xB0 && uc[0] <= 0xF7) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/2 - ((uc[0] >= 0x81 && uc[0] <= 0xA0) && (uc[1] >= 0x40 && uc[1] <= 0xFE) && (uc[1] != 0x7F)) || // GBK/3 - ((uc[0] >= 0xAA && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) || // GBK/4 - ((uc[0] >= 0xA8 && uc[0] <= 0xA9) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) // GBK/5 + ((b[0] >= 0xA1 && b[0] <= 0xA9) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/1 + ((b[0] >= 0xB0 && b[0] <= 0xF7) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/2 + ((b[0] >= 0x81 && b[0] <= 0xA0) && (b[1] >= 0x40 && b[1] <= 0xFE) && (b[1] != 0x7F)) || // GBK/3 + ((b[0] >= 0xAA && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) || // GBK/4 + ((b[0] >= 0xA8 && b[0] <= 0xA9) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) // GBK/5 ) ) { - *width = 2; - return (yp_gbk_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_gbk_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_gbk_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_gbk_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_gbk_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_gbk_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_gbk_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_gbk_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/enc/yp_shift_jis.c b/yarp/enc/yp_shift_jis.c index 1f361b9e705cd2..e6ca10d1fdc5b1 100644 --- a/yarp/enc/yp_shift_jis.c +++ b/yarp/enc/yp_shift_jis.c @@ -1,73 +1,46 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_shift_jis_codepoint_t; - -static yp_shift_jis_codepoint_t -yp_shift_jis_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_shift_jis_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80 || (*uc >= 0xA1 && *uc <= 0xDF)) { - *width = 1; - return *uc; + if (*b < 0x80 || (*b >= 0xA1 && *b <= 0xDF)) { + return 1; } // These are the double byte characters. if ( (n > 1) && - ((uc[0] >= 0x81 && uc[0] <= 0x9F) || (uc[0] >= 0xE0 && uc[0] <= 0xFC)) && - (uc[1] >= 0x40 && uc[1] <= 0xFC) + ((b[0] >= 0x81 && b[0] <= 0x9F) || (b[0] >= 0xE0 && b[0] <= 0xFC)) && + (b[1] >= 0x40 && b[1] <= 0xFC) ) { - *width = 2; - return (yp_shift_jis_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_shift_jis_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_shift_jis_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_shift_jis_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_shift_jis_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_shift_jis_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_shift_jis_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_shift_jis_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return 0; } diff --git a/yarp/enc/yp_tables.c b/yarp/enc/yp_tables.c index 057f2b3f818259..5504cd5419ac93 100644 --- a/yarp/enc/yp_tables.c +++ b/yarp/enc/yp_tables.c @@ -2,7 +2,7 @@ // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ASCII character. -static unsigned char yp_encoding_ascii_table[256] = { +static uint8_t yp_encoding_ascii_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -24,7 +24,7 @@ static unsigned char yp_encoding_ascii_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-1 character. -static unsigned char yp_encoding_iso_8859_1_table[256] = { +static uint8_t yp_encoding_iso_8859_1_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -46,7 +46,7 @@ static unsigned char yp_encoding_iso_8859_1_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-2 character. -static unsigned char yp_encoding_iso_8859_2_table[256] = { +static uint8_t yp_encoding_iso_8859_2_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -68,7 +68,7 @@ static unsigned char yp_encoding_iso_8859_2_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-3 character. -static unsigned char yp_encoding_iso_8859_3_table[256] = { +static uint8_t yp_encoding_iso_8859_3_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -90,7 +90,7 @@ static unsigned char yp_encoding_iso_8859_3_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-4 character. -static unsigned char yp_encoding_iso_8859_4_table[256] = { +static uint8_t yp_encoding_iso_8859_4_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -112,7 +112,7 @@ static unsigned char yp_encoding_iso_8859_4_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-5 character. -static unsigned char yp_encoding_iso_8859_5_table[256] = { +static uint8_t yp_encoding_iso_8859_5_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -134,7 +134,7 @@ static unsigned char yp_encoding_iso_8859_5_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-6 character. -static unsigned char yp_encoding_iso_8859_6_table[256] = { +static uint8_t yp_encoding_iso_8859_6_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -156,7 +156,7 @@ static unsigned char yp_encoding_iso_8859_6_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-7 character. -static unsigned char yp_encoding_iso_8859_7_table[256] = { +static uint8_t yp_encoding_iso_8859_7_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -178,7 +178,7 @@ static unsigned char yp_encoding_iso_8859_7_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-8 character. -static unsigned char yp_encoding_iso_8859_8_table[256] = { +static uint8_t yp_encoding_iso_8859_8_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -200,7 +200,7 @@ static unsigned char yp_encoding_iso_8859_8_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-9 character. -static unsigned char yp_encoding_iso_8859_9_table[256] = { +static uint8_t yp_encoding_iso_8859_9_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -222,7 +222,7 @@ static unsigned char yp_encoding_iso_8859_9_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-10 character. -static unsigned char yp_encoding_iso_8859_10_table[256] = { +static uint8_t yp_encoding_iso_8859_10_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -244,7 +244,7 @@ static unsigned char yp_encoding_iso_8859_10_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-11 character. -static unsigned char yp_encoding_iso_8859_11_table[256] = { +static uint8_t yp_encoding_iso_8859_11_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -266,7 +266,7 @@ static unsigned char yp_encoding_iso_8859_11_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-13 character. -static unsigned char yp_encoding_iso_8859_13_table[256] = { +static uint8_t yp_encoding_iso_8859_13_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -288,7 +288,7 @@ static unsigned char yp_encoding_iso_8859_13_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-14 character. -static unsigned char yp_encoding_iso_8859_14_table[256] = { +static uint8_t yp_encoding_iso_8859_14_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -310,7 +310,7 @@ static unsigned char yp_encoding_iso_8859_14_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-15 character. -static unsigned char yp_encoding_iso_8859_15_table[256] = { +static uint8_t yp_encoding_iso_8859_15_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -332,7 +332,7 @@ static unsigned char yp_encoding_iso_8859_15_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-16 character. -static unsigned char yp_encoding_iso_8859_16_table[256] = { +static uint8_t yp_encoding_iso_8859_16_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -354,7 +354,7 @@ static unsigned char yp_encoding_iso_8859_16_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding KOI8-R character. -static unsigned char yp_encoding_koi8_r_table[256] = { +static uint8_t yp_encoding_koi8_r_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -376,7 +376,7 @@ static unsigned char yp_encoding_koi8_r_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding windows-1251 character. -static unsigned char yp_encoding_windows_1251_table[256] = { +static uint8_t yp_encoding_windows_1251_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -398,7 +398,7 @@ static unsigned char yp_encoding_windows_1251_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding windows-1252 character. -static unsigned char yp_encoding_windows_1252_table[256] = { +static uint8_t yp_encoding_windows_1252_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -419,34 +419,32 @@ static unsigned char yp_encoding_windows_1252_table[256] = { }; static size_t -yp_encoding_ascii_char_width(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - const unsigned char v = (const unsigned char) *c; - return v < 0x80 ? 1 : 0; +yp_encoding_ascii_char_width(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return *b < 0x80 ? 1 : 0; } size_t -yp_encoding_ascii_alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_ALPHABETIC_BIT); +yp_encoding_ascii_alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_ALPHABETIC_BIT); } size_t -yp_encoding_ascii_alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; +yp_encoding_ascii_alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; } bool -yp_encoding_ascii_isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_UPPERCASE_BIT); +yp_encoding_ascii_isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_UPPERCASE_BIT); } static size_t -yp_encoding_koi8_r_char_width(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - const unsigned char v = (const unsigned char) *c; - return ((v >= 0x20 && v <= 0x7E) || (v >= 0x80)) ? 1 : 0; +yp_encoding_koi8_r_char_width(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return ((*b >= 0x20 && *b <= 0x7E) || (*b >= 0x80)) ? 1 : 0; } static size_t -yp_encoding_single_char_width(YP_ATTRIBUTE_UNUSED const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { +yp_encoding_single_char_width(YP_ATTRIBUTE_UNUSED const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { return 1; } @@ -469,14 +467,14 @@ yp_encoding_t yp_encoding_ascii_8bit = { }; #define YP_ENCODING_TABLE(s, i, w) \ - static size_t yp_encoding_ ##i ## _alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_ALPHABETIC_BIT); \ + static size_t yp_encoding_ ##i ## _alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_ALPHABETIC_BIT); \ } \ - static size_t yp_encoding_ ##i ## _alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; \ + static size_t yp_encoding_ ##i ## _alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; \ } \ - static bool yp_encoding_ ##i ## _isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_UPPERCASE_BIT); \ + static bool yp_encoding_ ##i ## _isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_UPPERCASE_BIT); \ } \ yp_encoding_t yp_encoding_ ##i = { \ .name = s, \ diff --git a/yarp/enc/yp_unicode.c b/yarp/enc/yp_unicode.c index fc2f0336fb7bf3..bb4e041309baa7 100644 --- a/yarp/enc/yp_unicode.c +++ b/yarp/enc/yp_unicode.c @@ -10,7 +10,7 @@ typedef uint32_t yp_unicode_codepoint_t; // this table is different from other encodings where we used a lookup table // because the indices of those tables are the byte representations, not the // codepoints themselves. -unsigned char yp_encoding_unicode_table[256] = { +uint8_t yp_encoding_unicode_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -2220,7 +2220,7 @@ static const uint8_t yp_utf_8_dfa[] = { }; static yp_unicode_codepoint_t -yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { +yp_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) { assert(n >= 1); size_t maximum = (size_t) n; @@ -2228,7 +2228,7 @@ yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { uint32_t state = 0; for (size_t index = 0; index < 4 && index < maximum; index++) { - uint32_t byte = c[index]; + uint32_t byte = b[index]; uint32_t type = yp_utf_8_dfa[byte]; codepoint = (state != 0) ? @@ -2247,60 +2247,55 @@ yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { } static size_t -yp_encoding_utf_8_char_width(const char *c, ptrdiff_t n) { +yp_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n) { size_t width; - const unsigned char *v = (const unsigned char *) c; - - yp_utf_8_codepoint(v, n, &width); + yp_utf_8_codepoint(b, n, &width); return width; } size_t -yp_encoding_utf_8_alpha_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & YP_ENCODING_ALPHABETIC_BIT) ? 1 : 0; +yp_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHABETIC_BIT) ? 1 : 0; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & YP_ENCODING_ALPHABETIC_BIT) ? width : 0; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & YP_ENCODING_ALPHABETIC_BIT) ? width : 0; } else { return yp_unicode_codepoint_match(codepoint, unicode_alpha_codepoints, UNICODE_ALPHA_CODEPOINTS_LENGTH) ? width : 0; } } size_t -yp_encoding_utf_8_alnum_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? 1 : 0; +yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? 1 : 0; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? width : 0; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? width : 0; } else { return yp_unicode_codepoint_match(codepoint, unicode_alnum_codepoints, UNICODE_ALNUM_CODEPOINTS_LENGTH) ? width : 0; } } static bool -yp_encoding_utf_8_isupper_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & YP_ENCODING_UPPERCASE_BIT) ? true : false; +yp_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_UPPERCASE_BIT) ? true : false; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & YP_ENCODING_UPPERCASE_BIT) ? true : false; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & YP_ENCODING_UPPERCASE_BIT) ? true : false; } else { return yp_unicode_codepoint_match(codepoint, unicode_isupper_codepoints, UNICODE_ISUPPER_CODEPOINTS_LENGTH) ? true : false; } diff --git a/yarp/enc/yp_windows_31j.c b/yarp/enc/yp_windows_31j.c index 7062d7c39c3a90..0d346395353e1e 100644 --- a/yarp/enc/yp_windows_31j.c +++ b/yarp/enc/yp_windows_31j.c @@ -1,73 +1,46 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_windows_31j_codepoint_t; - -static yp_windows_31j_codepoint_t -yp_windows_31j_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_windows_31j_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80 || (*uc >= 0xA1 && *uc <= 0xDF)) { - *width = 1; - return *uc; + if (*b < 0x80 || (*b >= 0xA1 && *b <= 0xDF)) { + return 1; } // These are the double byte characters. if ( (n > 1) && - ((uc[0] >= 0x81 && uc[0] <= 0x9F) || (uc[0] >= 0xE0 && uc[0] <= 0xFC)) && - (uc[1] >= 0x40 && uc[1] <= 0xFC) + ((b[0] >= 0x81 && b[0] <= 0x9F) || (b[0] >= 0xE0 && b[0] <= 0xFC)) && + (b[1] >= 0x40 && b[1] <= 0xFC) ) { - *width = 2; - return (yp_windows_31j_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_windows_31j_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_windows_31j_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_windows_31j_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_windows_31j_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_windows_31j_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_windows_31j_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_windows_31j_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/extension.c b/yarp/extension.c index 8aef456c0010a9..de925f1509d7fd 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -83,7 +83,21 @@ dump(int argc, VALUE *argv, VALUE self) { yp_string_t input; input_load_string(&input, string); - return dump_input(&input, check_string(filepath)); + +#ifdef YARP_DEBUG_MODE_BUILD + size_t length = yp_string_length(&input); + char* dup = malloc(length); + memcpy(dup, yp_string_source(&input), length); + yp_string_constant_init(&input, dup, length); +#endif + + VALUE value = dump_input(&input, check_string(filepath)); + +#ifdef YARP_DEBUG_MODE_BUILD + free(dup); +#endif + + return value; } // Dump the AST corresponding to the given file to a string. @@ -198,66 +212,67 @@ typedef struct { VALUE source; VALUE tokens; rb_encoding *encoding; -} lex_data_t; +} parse_lex_data_t; // This is passed as a callback to the parser. It gets called every time a new // token is found. Once found, we initialize a new instance of Token and push it // onto the tokens array. static void -lex_token(void *data, yp_parser_t *parser, yp_token_t *token) { - lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; +parse_lex_token(void *data, yp_parser_t *parser, yp_token_t *token) { + parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data; VALUE yields = rb_ary_new_capa(2); - rb_ary_push(yields, yp_token_new(parser, token, lex_data->encoding, lex_data->source)); + rb_ary_push(yields, yp_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source)); rb_ary_push(yields, INT2FIX(parser->lex_state)); - rb_ary_push(lex_data->tokens, yields); + rb_ary_push(parse_lex_data->tokens, yields); } // This is called whenever the encoding changes based on the magic comment at // the top of the file. We use it to update the encoding that we are using to // create tokens. static void -lex_encoding_changed_callback(yp_parser_t *parser) { - lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; - lex_data->encoding = rb_enc_find(parser->encoding.name); +parse_lex_encoding_changed_callback(yp_parser_t *parser) { + parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data; + parse_lex_data->encoding = rb_enc_find(parser->encoding.name); - // Since we got a new encoding, we need to go back and change the encoding - // of the tokens that we've already lexed. This should be a tiny amount - // since encoding magic comments need to be the first or second line of the + // Since the encoding changed, we need to go back and change the encoding of + // the tokens that were already lexed. This is only going to end up being + // one or two tokens, since the encoding can only change at the top of the // file. - VALUE tokens = lex_data->tokens; + VALUE tokens = parse_lex_data->tokens; for (long index = 0; index < RARRAY_LEN(tokens); index++) { VALUE yields = rb_ary_entry(tokens, index); VALUE token = rb_ary_entry(yields, 0); VALUE value = rb_ivar_get(token, rb_intern("@value")); - rb_enc_associate(value, lex_data->encoding); + rb_enc_associate(value, parse_lex_data->encoding); ENC_CODERANGE_CLEAR(value); } } -// Return an array of tokens corresponding to the given source. +// Parse the given input and return a ParseResult containing just the tokens or +// the nodes and tokens. static VALUE -lex_input(yp_string_t *input, const char *filepath) { +parse_lex_input(yp_string_t *input, const char *filepath, bool return_nodes) { yp_parser_t parser; yp_parser_init(&parser, yp_string_source(input), yp_string_length(input), filepath); - yp_parser_register_encoding_changed_callback(&parser, lex_encoding_changed_callback); + yp_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback); VALUE offsets = rb_ary_new(); - VALUE source_argv[] = { rb_str_new(yp_string_source(input), yp_string_length(input)), offsets }; + VALUE source_argv[] = { rb_str_new((const char *) yp_string_source(input), yp_string_length(input)), offsets }; VALUE source = rb_class_new_instance(2, source_argv, rb_cYARPSource); - lex_data_t lex_data = { + parse_lex_data_t parse_lex_data = { .source = source, .tokens = rb_ary_new(), .encoding = rb_utf8_encoding() }; - lex_data_t *data = &lex_data; + parse_lex_data_t *data = &parse_lex_data; yp_lex_callback_t lex_callback = (yp_lex_callback_t) { .data = (void *) data, - .callback = lex_token, + .callback = parse_lex_token, }; parser.lex_callback = &lex_callback; @@ -270,20 +285,26 @@ lex_input(yp_string_t *input, const char *filepath) { rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index])); } + VALUE value; + if (return_nodes) { + value = rb_ary_new_capa(2); + rb_ary_push(value, yp_ast_new(&parser, node, parse_lex_data.encoding)); + rb_ary_push(value, parse_lex_data.tokens); + } else { + value = parse_lex_data.tokens; + } + VALUE result_argv[] = { - lex_data.tokens, + value, parser_comments(&parser, source), - parser_errors(&parser, lex_data.encoding, source), - parser_warnings(&parser, lex_data.encoding, source), + parser_errors(&parser, parse_lex_data.encoding, source), + parser_warnings(&parser, parse_lex_data.encoding, source), source }; - VALUE result = rb_class_new_instance(5, result_argv, rb_cYARPParseResult); - yp_node_destroy(&parser, node); yp_parser_free(&parser); - - return result; + return rb_class_new_instance(5, result_argv, rb_cYARPParseResult); } // Return an array of tokens corresponding to the given string. @@ -295,7 +316,8 @@ lex(int argc, VALUE *argv, VALUE self) { yp_string_t input; input_load_string(&input, string); - return lex_input(&input, check_string(filepath)); + + return parse_lex_input(&input, check_string(filepath), false); } // Return an array of tokens corresponding to the given file. @@ -306,7 +328,7 @@ lex_file(VALUE self, VALUE filepath) { const char *checked = check_string(filepath); if (!yp_string_mapped_init(&input, checked)) return Qnil; - VALUE value = lex_input(&input, checked); + VALUE value = parse_lex_input(&input, checked, false); yp_string_free(&input); return value; @@ -382,6 +404,32 @@ parse_file(VALUE self, VALUE filepath) { return value; } +// Parse the given string and return a ParseResult instance. +static VALUE +parse_lex(int argc, VALUE *argv, VALUE self) { + VALUE string; + VALUE filepath; + rb_scan_args(argc, argv, "11", &string, &filepath); + + yp_string_t input; + input_load_string(&input, string); + return parse_lex_input(&input, check_string(filepath), true); +} + +// Parse and lex the given file and return a ParseResult instance. +static VALUE +parse_lex_file(VALUE self, VALUE filepath) { + yp_string_t input; + + const char *checked = check_string(filepath); + if (!yp_string_mapped_init(&input, checked)) return Qnil; + + VALUE value = parse_lex_input(&input, checked, true); + yp_string_free(&input); + + return value; +} + /******************************************************************************/ /* Utility functions exposed to make testing easier */ /******************************************************************************/ @@ -394,7 +442,7 @@ named_captures(VALUE self, VALUE source) { yp_string_list_t string_list; yp_string_list_init(&string_list); - if (!yp_regexp_named_capture_group_names(RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, &yp_encoding_utf_8)) { + if (!yp_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, &yp_encoding_utf_8)) { yp_string_list_free(&string_list); return Qnil; } @@ -402,7 +450,7 @@ named_captures(VALUE self, VALUE source) { VALUE names = rb_ary_new(); for (size_t index = 0; index < string_list.length; index++) { const yp_string_t *string = &string_list.strings[index]; - rb_ary_push(names, rb_str_new(yp_string_source(string), yp_string_length(string))); + rb_ary_push(names, rb_str_new((const char *) yp_string_source(string), yp_string_length(string))); } yp_string_list_free(&string_list); @@ -415,8 +463,8 @@ static VALUE unescape(VALUE source, yp_unescape_type_t unescape_type) { yp_string_t result; - if (yp_unescape_string(RSTRING_PTR(source), RSTRING_LEN(source), unescape_type, &result)) { - VALUE str = rb_str_new(yp_string_source(&result), yp_string_length(&result)); + if (yp_unescape_string((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), unescape_type, &result)) { + VALUE str = rb_str_new((const char *) yp_string_source(&result), yp_string_length(&result)); yp_string_free(&result); return str; } else { @@ -450,7 +498,7 @@ static VALUE memsize(VALUE self, VALUE string) { yp_parser_t parser; size_t length = RSTRING_LEN(string); - yp_parser_init(&parser, RSTRING_PTR(string), length, NULL); + yp_parser_init(&parser, (const uint8_t *) RSTRING_PTR(string), length, NULL); yp_node_t *node = yp_parse(&parser); yp_memsize_t memsize; @@ -535,7 +583,6 @@ Init_yarp(void) { // Define the version string here so that we can use the constants defined // in yarp.h. rb_define_const(rb_cYARP, "VERSION", rb_str_new2(EXPECTED_YARP_VERSION)); - rb_define_const(rb_cYARP, "BACKEND", ID2SYM(rb_intern("CExtension"))); // First, the functions that have to do with lexing and parsing. @@ -545,6 +592,8 @@ Init_yarp(void) { rb_define_singleton_method(rb_cYARP, "lex_file", lex_file, 1); rb_define_singleton_method(rb_cYARP, "parse", parse, -1); rb_define_singleton_method(rb_cYARP, "parse_file", parse_file, 1); + rb_define_singleton_method(rb_cYARP, "parse_lex", parse_lex, -1); + rb_define_singleton_method(rb_cYARP, "parse_lex_file", parse_lex_file, 1); // Next, the functions that will be called by the parser to perform various // internal tasks. We expose these to make them easier to test. diff --git a/yarp/extension.h b/yarp/extension.h index 75d5cc84a7d77b..fc24add420f2f1 100644 --- a/yarp/extension.h +++ b/yarp/extension.h @@ -1,12 +1,12 @@ #ifndef YARP_EXT_NODE_H #define YARP_EXT_NODE_H +#define EXPECTED_YARP_VERSION "0.9.0" + #include #include #include "yarp.h" -#define EXPECTED_YARP_VERSION "0.8.0" - VALUE yp_source_new(yp_parser_t *parser); VALUE yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALUE source); VALUE yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding); diff --git a/yarp/parser.h b/yarp/parser.h index 2091be7fd92c8f..0ae01f78da6e53 100644 --- a/yarp/parser.h +++ b/yarp/parser.h @@ -109,14 +109,14 @@ typedef struct yp_lex_mode { // When lexing a list, it takes into account balancing the // terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the list literal. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the list. - char breakpoints[11]; + uint8_t breakpoints[11]; } list; struct { @@ -125,14 +125,14 @@ typedef struct yp_lex_mode { // When lexing a regular expression, it takes into account balancing // the terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the regular expression. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the regular expression. - char breakpoints[6]; + uint8_t breakpoints[6]; } regexp; struct { @@ -149,21 +149,21 @@ typedef struct yp_lex_mode { // When lexing a string, it takes into account balancing the // terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the string. It is typically either a // single or double quote. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the string. - char breakpoints[6]; + uint8_t breakpoints[6]; } string; struct { // These pointers point to the beginning and end of the heredoc // identifier. - const char *ident_start; + const uint8_t *ident_start; size_t ident_length; yp_heredoc_quote_t quote; @@ -171,7 +171,7 @@ typedef struct yp_lex_mode { // This is the pointer to the character where lexing should resume // once the heredoc has been completely processed. - const char *next_start; + const uint8_t *next_start; } heredoc; } as; @@ -239,8 +239,8 @@ typedef enum { // This is a node in the linked list of comments that we've found while parsing. typedef struct yp_comment { yp_list_node_t node; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; yp_comment_type_t type; } yp_comment_t; @@ -252,7 +252,7 @@ typedef void (*yp_encoding_changed_callback_t)(yp_parser_t *parser); // the ability here to call out to a user-defined function to get an encoding // struct. If the function returns something that isn't NULL, we set that to // our encoding and use it to parse identifiers. -typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const char *name, size_t width); +typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const uint8_t *name, size_t width); // When you are lexing through a file, the lexer needs all of the information // that the parser additionally provides (for example, the local table). So if @@ -316,21 +316,21 @@ struct yp_parser { size_t index; // the current index into the lexer mode stack } lex_modes; - const char *start; // the pointer to the start of the source - const char *end; // the pointer to the end of the source + const uint8_t *start; // the pointer to the start of the source + const uint8_t *end; // the pointer to the end of the source yp_token_t previous; // the previous token we were considering yp_token_t current; // the current token we're considering // This is a special field set on the parser when we need the parser to jump // to a specific location when lexing the next token, as opposed to just // using the end of the previous token. Normally this is NULL. - const char *next_start; + const uint8_t *next_start; // This field indicates the end of a heredoc whose identifier was found on // the current line. If another heredoc is found on the same line, then this // will be moved forward to the end of that heredoc. If no heredocs are // found on a line then this is NULL. - const char *heredoc_end; + const uint8_t *heredoc_end; yp_list_t comment_list; // the list of comments that have been found while parsing yp_list_t warning_list; // the list of warnings that have been found while parsing @@ -361,7 +361,7 @@ struct yp_parser { // This pointer indicates where a comment must start if it is to be // considered an encoding comment. - const char *encoding_comment_start; + const uint8_t *encoding_comment_start; // This is an optional callback that can be attached to the parser that will // be called whenever a new token is lexed by the parser. diff --git a/yarp/regexp.c b/yarp/regexp.c index 4d6b67ebe633a4..2aeadc1bfcfef0 100644 --- a/yarp/regexp.c +++ b/yarp/regexp.c @@ -2,9 +2,9 @@ // This is the parser that is going to handle parsing regular expressions. typedef struct { - const char *start; - const char *cursor; - const char *end; + const uint8_t *start; + const uint8_t *cursor; + const uint8_t *end; yp_string_list_t *named_captures; bool encoding_changed; yp_encoding_t *encoding; @@ -12,7 +12,7 @@ typedef struct { // This initializes a new parser with the given source. static void -yp_regexp_parser_init(yp_regexp_parser_t *parser, const char *start, const char *end, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { +yp_regexp_parser_init(yp_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { *parser = (yp_regexp_parser_t) { .start = start, .cursor = start, @@ -25,7 +25,7 @@ yp_regexp_parser_init(yp_regexp_parser_t *parser, const char *start, const char // This appends a new string to the list of named captures. static void -yp_regexp_parser_named_capture(yp_regexp_parser_t *parser, const char *start, const char *end) { +yp_regexp_parser_named_capture(yp_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_string_t string; yp_string_shared_init(&string, start, end); yp_string_list_append(parser->named_captures, &string); @@ -40,7 +40,7 @@ yp_regexp_char_is_eof(yp_regexp_parser_t *parser) { // Optionally accept a char and consume it if it exists. static inline bool -yp_regexp_char_accept(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_accept(yp_regexp_parser_t *parser, uint8_t value) { if (!yp_regexp_char_is_eof(parser) && *parser->cursor == value) { parser->cursor++; return true; @@ -50,7 +50,7 @@ yp_regexp_char_accept(yp_regexp_parser_t *parser, char value) { // Expect a character to be present and consume it. static inline bool -yp_regexp_char_expect(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_expect(yp_regexp_parser_t *parser, uint8_t value) { if (!yp_regexp_char_is_eof(parser) && *parser->cursor == value) { parser->cursor++; return true; @@ -60,12 +60,12 @@ yp_regexp_char_expect(yp_regexp_parser_t *parser, char value) { // This advances the current token to the next instance of the given character. static bool -yp_regexp_char_find(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_find(yp_regexp_parser_t *parser, uint8_t value) { if (yp_regexp_char_is_eof(parser)) { return false; } - const char *end = (const char *) yp_memchr(parser->cursor, value, (size_t) (parser->end - parser->cursor), parser->encoding_changed, parser->encoding); + const uint8_t *end = (const uint8_t *) yp_memchr(parser->cursor, value, (size_t) (parser->end - parser->cursor), parser->encoding_changed, parser->encoding); if (end == NULL) { return false; } @@ -107,7 +107,7 @@ yp_regexp_char_find(yp_regexp_parser_t *parser, char value) { // consumed so we're in the start state. static bool yp_regexp_parse_range_quantifier(yp_regexp_parser_t *parser) { - const char *savepoint = parser->cursor; + const uint8_t *savepoint = parser->cursor; enum { YP_REGEXP_RANGE_QUANTIFIER_STATE_START, @@ -252,7 +252,7 @@ yp_regexp_parse_character_set(yp_regexp_parser_t *parser) { // A left bracket can either mean a POSIX class or a character set. static bool yp_regexp_parse_lbracket(yp_regexp_parser_t *parser) { - const char *reset = parser->cursor; + const uint8_t *reset = parser->cursor; if ((parser->cursor + 2 < parser->end) && parser->cursor[0] == '[' && parser->cursor[1] == ':') { parser->cursor++; @@ -287,7 +287,7 @@ typedef enum { // This is the set of options that are configurable on the regular expression. typedef struct { - unsigned char values[YP_REGEXP_OPTION_STATE_SLOTS]; + uint8_t values[YP_REGEXP_OPTION_STATE_SLOTS]; } yp_regexp_options_t; // Initialize a new set of options to their default values. @@ -305,9 +305,9 @@ yp_regexp_options_init(yp_regexp_options_t *options) { // Attempt to add the given option to the set of options. Returns true if it was // added, false if it was already present. static bool -yp_regexp_options_add(yp_regexp_options_t *options, unsigned char key) { +yp_regexp_options_add(yp_regexp_options_t *options, uint8_t key) { if (key >= YP_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= YP_REGEXP_OPTION_STATE_SLOT_MAXIMUM) { - key = (unsigned char) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); + key = (uint8_t) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); switch (options->values[key]) { case YP_REGEXP_OPTION_STATE_INVALID: @@ -328,9 +328,9 @@ yp_regexp_options_add(yp_regexp_options_t *options, unsigned char key) { // Attempt to remove the given option from the set of options. Returns true if // it was removed, false if it was already absent. static bool -yp_regexp_options_remove(yp_regexp_options_t *options, unsigned char key) { +yp_regexp_options_remove(yp_regexp_options_t *options, uint8_t key) { if (key >= YP_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= YP_REGEXP_OPTION_STATE_SLOT_MAXIMUM) { - key = (unsigned char) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); + key = (uint8_t) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); switch (options->values[key]) { case YP_REGEXP_OPTION_STATE_INVALID: @@ -431,7 +431,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { parser->cursor++; break; default: { // named capture group - const char *start = parser->cursor; + const uint8_t *start = parser->cursor; if (!yp_regexp_char_find(parser, '>')) { return false; } @@ -441,7 +441,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { } break; case '\'': { // named capture group - const char *start = ++parser->cursor; + const uint8_t *start = ++parser->cursor; if (!yp_regexp_char_find(parser, '\'')) { return false; } @@ -456,7 +456,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { break; case 'i': case 'm': case 'x': case 'd': case 'a': case 'u': // options while (!yp_regexp_char_is_eof(parser) && *parser->cursor != '-' && *parser->cursor != ':' && *parser->cursor != ')') { - if (!yp_regexp_options_add(&options, (unsigned char) *parser->cursor)) { + if (!yp_regexp_options_add(&options, *parser->cursor)) { return false; } parser->cursor++; @@ -474,7 +474,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { case '-': parser->cursor++; while (!yp_regexp_char_is_eof(parser) && *parser->cursor != ':' && *parser->cursor != ')') { - if (!yp_regexp_options_remove(&options, (unsigned char) *parser->cursor)) { + if (!yp_regexp_options_remove(&options, *parser->cursor)) { return false; } parser->cursor++; @@ -573,7 +573,7 @@ yp_regexp_parse_pattern(yp_regexp_parser_t *parser) { // Parse a regular expression and extract the names of all of the named capture // groups. YP_EXPORTED_FUNCTION bool -yp_regexp_named_capture_group_names(const char *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { +yp_regexp_named_capture_group_names(const uint8_t *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { yp_regexp_parser_t parser; yp_regexp_parser_init(&parser, source, source + size, named_captures, encoding_changed, encoding); return yp_regexp_parse_pattern(&parser); diff --git a/yarp/regexp.h b/yarp/regexp.h index 5a2f13047e552f..6807c58398080b 100644 --- a/yarp/regexp.h +++ b/yarp/regexp.h @@ -14,6 +14,6 @@ // Parse a regular expression and extract the names of all of the named capture // groups. -YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const char *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding); +YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const uint8_t *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding); #endif diff --git a/yarp/templates/ext/yarp/api_node.c.erb b/yarp/templates/ext/yarp/api_node.c.erb index 599bf2e9ffb2b7..b8407350589711 100644 --- a/yarp/templates/ext/yarp/api_node.c.erb +++ b/yarp/templates/ext/yarp/api_node.c.erb @@ -12,7 +12,7 @@ static VALUE rb_cYARP<%= node.name %>; <%- end -%> static VALUE -yp_location_new(yp_parser_t *parser, const char *start, const char *end, VALUE source) { +yp_location_new(yp_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source) { VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(end - start) }; return rb_class_new_instance(3, argv, rb_cYARPLocation); } @@ -24,7 +24,7 @@ yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALU VALUE argv[] = { ID2SYM(type), - rb_enc_str_new(token->start, token->end - token->start, encoding), + rb_enc_str_new((const char *) token->start, token->end - token->start, encoding), location }; @@ -33,13 +33,13 @@ yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALU static VALUE yp_string_new(yp_string_t *string, rb_encoding *encoding) { - return rb_enc_str_new(yp_string_source(string), yp_string_length(string), encoding); + return rb_enc_str_new((const char *) yp_string_source(string), yp_string_length(string), encoding); } // Create a YARP::Source object from the given parser. VALUE yp_source_new(yp_parser_t *parser) { - VALUE source = rb_str_new(parser->start, parser->end - parser->start); + VALUE source = rb_str_new((const char *) parser->start, parser->end - parser->start); VALUE offsets = rb_ary_new_capa(parser->newline_list.size); for (size_t index = 0; index < parser->newline_list.size; index++) { @@ -85,7 +85,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { yp_constant_t constant = parser->constant_pool.constants[index]; if (constant.id != 0) { - constants[constant.id - 1] = rb_intern3(constant.start, constant.length, encoding); + constants[constant.id - 1] = rb_intern3((const char *) constant.start, constant.length, encoding); } } @@ -106,17 +106,17 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { switch (YP_NODE_TYPE(node)) { <%- nodes.each do |node| -%> - <%- if node.params.any? { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%> + <%- if node.fields.any? { |field| [YARP::NodeField, YARP::OptionalNodeField, YARP::NodeListField].include?(field.class) } -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeParam, OptionalNodeParam -%> - yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>); - <%- when NodeListParam -%> - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>.nodes[index]); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= field.name %>.nodes[index]); } <%- end -%> <%- end -%> @@ -135,53 +135,53 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.params.any? { |param| ![NodeParam, OptionalNodeParam].include?(param.class) } -%> + <%- if node.fields.any? { |field| ![YARP::NodeField, YARP::OptionalNodeField].include?(field.class) } -%> yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; <%- end -%> - VALUE argv[<%= node.params.length + 1 %>]; - <%- node.params.each_with_index do |param, index| -%> + VALUE argv[<%= node.fields.length + 1 %>]; + <%- node.fields.each_with_index do |field, index| -%> - // <%= param.name %> - <%- case param -%> - <%- when NodeParam, OptionalNodeParam -%> + // <%= field.name %> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> argv[<%= index %>] = rb_ary_pop(value_stack); - <%- when NodeListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { + <%- when YARP::NodeListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { rb_ary_push(argv[<%= index %>], rb_ary_pop(value_stack)); } - <%- when StringParam -%> - argv[<%= index %>] = yp_string_new(&cast-><%= param.name %>, encoding); - <%- when LocationListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - yp_location_t location = cast-><%= param.name %>.locations[index]; + <%- when YARP::StringField -%> + argv[<%= index %>] = yp_string_new(&cast-><%= field.name %>, encoding); + <%- when YARP::LocationListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + yp_location_t location = cast-><%= field.name %>.locations[index]; rb_ary_push(argv[<%= index %>], yp_location_new(parser, location.start, location.end, source)); } - <%- when ConstantParam -%> - argv[<%= index %>] = rb_id2sym(constants[cast-><%= param.name %> - 1]); - <%- when ConstantListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - rb_ary_push(argv[<%= index %>], rb_id2sym(constants[cast-><%= param.name %>.ids[index] - 1])); + <%- when YARP::ConstantField -%> + argv[<%= index %>] = rb_id2sym(constants[cast-><%= field.name %> - 1]); + <%- when YARP::ConstantListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + rb_ary_push(argv[<%= index %>], rb_id2sym(constants[cast-><%= field.name %>.ids[index] - 1])); } - <%- when LocationParam -%> - argv[<%= index %>] = yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source); - <%- when OptionalLocationParam -%> - argv[<%= index %>] = cast-><%= param.name %>.start == NULL ? Qnil : yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source); - <%- when UInt32Param -%> - argv[<%= index %>] = ULONG2NUM(cast-><%= param.name %>); - <%- when FlagsParam -%> - argv[<%= index %>] = ULONG2NUM(node->flags >> <%= COMMON_FLAGS %>); + <%- when YARP::LocationField -%> + argv[<%= index %>] = yp_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source); + <%- when YARP::OptionalLocationField -%> + argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : yp_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source); + <%- when YARP::UInt32Field -%> + argv[<%= index %>] = ULONG2NUM(cast-><%= field.name %>); + <%- when YARP::FlagsField -%> + argv[<%= index %>] = ULONG2NUM(node->flags >> <%= YARP::COMMON_FLAGS %>); <%- else -%> <%- raise -%> <%- end -%> <%- end -%> // location - argv[<%= node.params.length %>] = yp_location_new(parser, node->location.start, node->location.end, source); + argv[<%= node.fields.length %>] = yp_location_new(parser, node->location.start, node->location.end, source); - rb_ary_push(value_stack, rb_class_new_instance(<%= node.params.length + 1 %>, argv, rb_cYARP<%= node.name %>)); + rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 1 %>, argv, rb_cYARP<%= node.name %>)); break; } <%- end -%> diff --git a/yarp/templates/include/yarp/ast.h.erb b/yarp/templates/include/yarp/ast.h.erb index 6fe3bc2c2456bb..4325d403778da6 100644 --- a/yarp/templates/include/yarp/ast.h.erb +++ b/yarp/templates/include/yarp/ast.h.erb @@ -21,15 +21,15 @@ typedef enum yp_token_type { // type and location information. typedef struct { yp_token_type_t type; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; } yp_token_t; // This represents a range of bytes in the source string to which a node or // token corresponds. typedef struct { - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; } yp_location_t; typedef struct { @@ -83,17 +83,17 @@ typedef struct yp_node { // <%= node.name %> typedef struct yp_<%= node.human %> { yp_node_t base; -<%- node.params.grep_v(FlagsParam).each do |param| -%> - <%= case param - when NodeParam, OptionalNodeParam then "struct #{param.c_type} *#{param.name}" - when NodeListParam then "struct yp_node_list #{param.name}" - when LocationListParam then "yp_location_list_t #{param.name}" - when ConstantParam then "yp_constant_id_t #{param.name}" - when ConstantListParam then "yp_constant_id_list_t #{param.name}" - when StringParam then "yp_string_t #{param.name}" - when LocationParam, OptionalLocationParam then "yp_location_t #{param.name}" - when UInt32Param then "uint32_t #{param.name}" - else raise param.class.name +<%- node.fields.grep_v(YARP::FlagsField).each do |field| -%> + <%= case field + when YARP::NodeField, YARP::OptionalNodeField then "struct #{field.c_type} *#{field.name}" + when YARP::NodeListField then "struct yp_node_list #{field.name}" + when YARP::LocationListField then "yp_location_list_t #{field.name}" + when YARP::ConstantField then "yp_constant_id_t #{field.name}" + when YARP::ConstantListField then "yp_constant_id_list_t #{field.name}" + when YARP::StringField then "yp_string_t #{field.name}" + when YARP::LocationField, YARP::OptionalLocationField then "yp_location_t #{field.name}" + when YARP::UInt32Field then "uint32_t #{field.name}" + else raise field.class.name end %>; <%- end -%> @@ -103,7 +103,7 @@ typedef struct yp_<%= node.human %> { // <%= flag.name %> typedef enum { - <%- flag.values.each.with_index(COMMON_FLAGS) do |value, index| -%> + <%- flag.values.each.with_index(YARP::COMMON_FLAGS) do |value, index| -%> YP_<%= flag.human.upcase %>_<%= value.name %> = 1 << <%= index %>, <%- end -%> } yp_<%= flag.human %>_t; diff --git a/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb b/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb deleted file mode 100644 index fa9f65a84acbcb..00000000000000 --- a/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb +++ /dev/null @@ -1,14 +0,0 @@ -package org.yarp; - -// GENERATED BY <%= File.basename(__FILE__) %> -public abstract class AbstractNodeVisitor { - - protected abstract T defaultVisit(Nodes.Node node); - - <%- nodes.each do |node| -%> - public T visit<%= node.name -%>(Nodes.<%= node.name -%> node) { - return defaultVisit(node); - } - - <%- end -%> -} diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb deleted file mode 100644 index 312e232182592e..00000000000000 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ /dev/null @@ -1,297 +0,0 @@ -package org.yarp; - -import org.yarp.ParseResult; - -import java.lang.Short; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; - -// GENERATED BY <%= File.basename(__FILE__) %> -// @formatter:off -public class Loader { - - public static ParseResult load(byte[] serialized, Nodes.Source source) { - return new Loader(serialized, source).load(); - } - - private static final class ConstantPool { - - private final byte[] source; - private final int bufferOffset; - private final byte[][] cache; - - ConstantPool(byte[] source, int bufferOffset, int length) { - this.source = source; - this.bufferOffset = bufferOffset; - cache = new byte[length][]; - } - - byte[] get(ByteBuffer buffer, int oneBasedIndex) { - int index = oneBasedIndex - 1; - byte[] constant = cache[index]; - if (constant == null) { - int offset = bufferOffset + index * 8; - int start = buffer.getInt(offset); - int length = buffer.getInt(offset + 4); - - constant = new byte[length]; - System.arraycopy(source, start, constant, 0, length); - cache[index] = constant; - } - return constant; - } - - } - - private final ByteBuffer buffer; - private ConstantPool constantPool; - private final Nodes.Source source; - - private byte MAJOR_VERSION = (byte) 0; - private byte MINOR_VERSION = (byte) 8; - private byte PATCH_VERSION = (byte) 0; - - private Loader(byte[] serialized, Nodes.Source source) { - this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder()); - this.source = source; - } - - private ParseResult load() { - expect((byte) 'Y'); - expect((byte) 'A'); - expect((byte) 'R'); - expect((byte) 'P'); - - expect(MAJOR_VERSION); - expect(MINOR_VERSION); - expect(PATCH_VERSION); - - // This loads the name of the encoding. We don't actually do anything - // with it just yet. - int encodingLength = loadVarInt(); - byte[] encodingName = new byte[encodingLength]; - buffer.get(encodingName); - - ParseResult.Comment[] comments = loadComments(); - ParseResult.Error[] errors = loadSyntaxErrors(); - ParseResult.Warning[] warnings = loadWarnings(); - - int constantPoolBufferOffset = buffer.getInt(); - int constantPoolLength = loadVarInt(); - this.constantPool = new ConstantPool(source.bytes, constantPoolBufferOffset, constantPoolLength); - - Nodes.Node node = loadNode(); - - int left = constantPoolBufferOffset - buffer.position(); - if (left != 0) { - throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left"); - } - - boolean[] newlineMarked = new boolean[1 + source.getLineCount()]; - MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked); - node.accept(visitor); - - return new ParseResult(node, comments, errors, warnings); - } - - private byte[] loadEmbeddedString() { - int length = loadVarInt(); - byte[] string = new byte[length]; - buffer.get(string); - return string; - } - - private byte[] loadString() { - switch (buffer.get()) { - case 1: - int start = loadVarInt(); - int length = loadVarInt(); - byte[] string = new byte[length]; - System.arraycopy(source.bytes, start, string, 0, length); - return string; - case 2: - return loadEmbeddedString(); - default: - throw new Error("Expected 0 or 1 but was " + buffer.get()); - } - } - - private ParseResult.Comment[] loadComments() { - int count = loadVarInt(); - ParseResult.Comment[] comments = new ParseResult.Comment[count]; - - for (int i = 0; i < count; i++) { - ParseResult.CommentType type = ParseResult.CommentType.VALUES[buffer.get()]; - Nodes.Location location = loadLocation(); - - ParseResult.Comment comment = new ParseResult.Comment(type, location); - comments[i] = comment; - } - - return comments; - } - - private ParseResult.Error[] loadSyntaxErrors() { - int count = loadVarInt(); - ParseResult.Error[] errors = new ParseResult.Error[count]; - - // error messages only contain ASCII characters - for (int i = 0; i < count; i++) { - byte[] bytes = loadEmbeddedString(); - String message = new String(bytes, StandardCharsets.US_ASCII); - Nodes.Location location = loadLocation(); - - ParseResult.Error error = new ParseResult.Error(message, location); - errors[i] = error; - } - - return errors; - } - - private ParseResult.Warning[] loadWarnings() { - int count = loadVarInt(); - ParseResult.Warning[] warnings = new ParseResult.Warning[count]; - - // warning messages only contain ASCII characters - for (int i = 0; i < count; i++) { - byte[] bytes = loadEmbeddedString(); - String message = new String(bytes, StandardCharsets.US_ASCII); - Nodes.Location location = loadLocation(); - - ParseResult.Warning warning = new ParseResult.Warning(message, location); - warnings[i] = warning; - } - - return warnings; - } - - private Nodes.Node loadOptionalNode() { - if (buffer.get(buffer.position()) != 0) { - return loadNode(); - } else { - buffer.position(buffer.position() + 1); // continue after the 0 byte - return null; - } - } - - private Nodes.Location[] loadLocations() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.Location.EMPTY_ARRAY; - } - Nodes.Location[] locations = new Nodes.Location[length]; - for (int i = 0; i < length; i++) { - locations[i] = loadLocation(); - } - return locations; - } - - private byte[] loadConstant() { - return constantPool.get(buffer, loadVarInt()); - } - - private byte[][] loadConstants() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.EMPTY_BYTE_ARRAY_ARRAY; - } - byte[][] constants = new byte[length][]; - for (int i = 0; i < length; i++) { - constants[i] = constantPool.get(buffer, loadVarInt()); - } - return constants; - } - - private Nodes.Node[] loadNodes() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.Node.EMPTY_ARRAY; - } - Nodes.Node[] nodes = new Nodes.Node[length]; - for (int i = 0; i < length; i++) { - nodes[i] = loadNode(); - } - return nodes; - } - - private Nodes.Location loadLocation() { - return new Nodes.Location(loadVarInt(), loadVarInt()); - } - - private Nodes.Location loadOptionalLocation() { - if (buffer.get() != 0) { - return loadLocation(); - } else { - return null; - } - } - - // From https://github.com/protocolbuffers/protobuf/blob/v23.1/java/core/src/main/java/com/google/protobuf/BinaryReader.java#L1507 - private int loadVarInt() { - int x; - if ((x = buffer.get()) >= 0) { - return x; - } else if ((x ^= (buffer.get() << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (buffer.get() << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (buffer.get() << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - x ^= buffer.get() << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - } - return x; - } - - private short loadFlags() { - int flags = loadVarInt(); - assert flags >= 0 && flags <= Short.MAX_VALUE; - return (short) flags; - } - - private Nodes.Node loadNode() { - int type = buffer.get() & 0xFF; - int startOffset = loadVarInt(); - int length = loadVarInt(); - - switch (type) { - <%- nodes.each_with_index do |node, index| -%> - case <%= index + 1 %>: - <%- - params = node.needs_serialized_length? ? ["buffer.getInt()"] : [] - params.concat node.params.map { |param| - case param - when NodeParam then "#{param.java_cast}loadNode()" - when OptionalNodeParam then "#{param.java_cast}loadOptionalNode()" - when StringParam then "loadString()" - when NodeListParam then "loadNodes()" - when LocationListParam then "loadLocations()" - when ConstantParam then "loadConstant()" - when ConstantListParam then "loadConstants()" - when LocationParam then "loadLocation()" - when OptionalLocationParam then "loadOptionalLocation()" - when UInt32Param then "loadVarInt()" - when FlagsParam then "loadFlags()" - else raise - end - } - params.concat ["startOffset", "length"] - -%> - return new Nodes.<%= node.name %>(<%= params.join(", ") -%>); - <%- end -%> - default: - throw new Error("Unknown node type: " + type); - } - } - - private void expect(byte value) { - byte b = buffer.get(); - if (b != value) { - throw new Error("Expected " + value + " but was " + b + " at position " + buffer.position()); - } - } - -} -// @formatter:on diff --git a/yarp/templates/java/org/yarp/Nodes.java.erb b/yarp/templates/java/org/yarp/Nodes.java.erb deleted file mode 100644 index aa16f4729d3c1e..00000000000000 --- a/yarp/templates/java/org/yarp/Nodes.java.erb +++ /dev/null @@ -1,291 +0,0 @@ -package org.yarp; - -import java.lang.Override; -import java.lang.String; -import java.lang.StringBuilder; -import java.util.ArrayList; -import java.util.Arrays; - -// GENERATED BY <%= File.basename(__FILE__) %> -// @formatter:off -public abstract class Nodes { - - public static final byte[][] EMPTY_BYTE_ARRAY_ARRAY = {}; - - public static final class Location { - - public static final Location[] EMPTY_ARRAY = {}; - - public final int startOffset; - public final int length; - - public Location(int startOffset, int length) { - this.startOffset = startOffset; - this.length = length; - } - - public int endOffset() { - return startOffset + length; - } - } - - public static final class Source { - public final byte[] bytes; - private final int[] lineOffsets; - - public Source(byte[] bytes) { - this(bytes, computeLineOffsets(bytes)); - } - - public Source(byte[] bytes, int[] lineOffsets) { - assert lineOffsets[0] == 0; - this.bytes = bytes; - this.lineOffsets = lineOffsets; - } - - public static int[] computeLineOffsets(byte[] bytes) { - int[] lineOffsets = new int[8]; - int lineOffsetsSize = 0; - lineOffsets[lineOffsetsSize++] = 0; - - for (int i = 0; i < bytes.length; i++) { - if (bytes[i] == '\n') { - if (lineOffsetsSize == lineOffsets.length) { - lineOffsets = Arrays.copyOf(lineOffsets, lineOffsets.length * 2); - } - lineOffsets[lineOffsetsSize++] = i + 1; - } - } - return Arrays.copyOf(lineOffsets, lineOffsetsSize); - } - - public int line(int byteOffset) { - assert byteOffset >= 0 && byteOffset < bytes.length : byteOffset; - int index = Arrays.binarySearch(lineOffsets, byteOffset); - int line; - if (index < 0) { - line = -index - 1; - } else { - line = index + 1; - } - assert line >= 1 && line <= getLineCount() : line; - return line; - } - - public int getLineCount() { - return lineOffsets.length; - } - } - - public static abstract class Node { - - public static final Node[] EMPTY_ARRAY = {}; - - public final int startOffset; - public final int length; - private boolean newLineFlag = false; - - public Node(int startOffset, int length) { - this.startOffset = startOffset; - this.length = length; - } - - public final int endOffset() { - return startOffset + length; - } - - public final boolean hasNewLineFlag() { - return newLineFlag; - } - - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - int line = source.line(this.startOffset); - if (!newlineMarked[line]) { - newlineMarked[line] = true; - this.newLineFlag = true; - } - } - - public abstract T accept(AbstractNodeVisitor visitor); - - public abstract void visitChildNodes(AbstractNodeVisitor visitor); - - public abstract Node[] childNodes(); - - @Override - public String toString() { - return toString(""); - } - - private String toString(String indent) { - StringBuilder builder = new StringBuilder(); - builder.append(indent).append(this.getClass().getSimpleName()); - if (hasNewLineFlag()) { - builder.append("[Li]"); - } - builder.append('\n'); - for (Node child : childNodes()) { - if (child != null) { - builder.append(child.toString(indent + " ")); - } - } - return builder.toString(); - } - } -<%# FLAGS -%> - <%- flags.each do |group| -%> - - public static final class <%= group.name %> implements Comparable<<%= group.name %>> { - <%- group.values.each_with_index do |value, index| -%> - - // <%= value.comment %> - public static final short <%= value.name %> = 1 << <%= index %>; - <%- end -%> - - <%- group.values.each do |value| -%> - public static boolean is<%= value.camelcase %>(short flags) { - return (flags & <%= value.name %>) != 0; - } - - <%- end -%> - private final short flags; - - public <%= group.name %>(short flags) { - this.flags = flags; - } - - @Override - public int hashCode() { - return flags; - } - - @Override - public boolean equals(Object other) { - if (!(other instanceof <%= group.name %>)) { - return false; - } - - return flags == ((<%= group.name %>) other).flags; - } - - @Override - public int compareTo(<%= group.name %> other) { - return flags - other.flags; - } - - <%- group.values.each do |value| -%> - public boolean is<%= value.camelcase %>() { - return (flags & <%= value.name %>) != 0; - } - - <%- end -%> - } -<%- end -%> -<%# NODES -%> - <%- nodes.each do |node| -%> - - <%= "#{node.comment.split("\n").map { |line| "// #{line}" }.join("\n ")}\n" if node.comment -%> - public static final class <%= node.name -%> extends Node { - <%- if node.needs_serialized_length? -%> - public final int serializedLength; - <%- end -%> - <%- node.params.each do |param| -%> - public final <%= param.java_type %> <%= param.name %>;<%= ' // optional' if param.class.name.start_with?('Optional') %> - <%- end -%> - - <%- - params = node.needs_serialized_length? ? ["int serializedLength"] : [] - params.concat node.params.map { "#{_1.java_type} #{_1.name}" } - params.concat ["int startOffset", "int length"] - -%> - public <%=node.name -%>(<%= params.join(", ") %>) { - super(startOffset, length); - <%- if node.needs_serialized_length? -%> - this.serializedLength = serializedLength; - <%- end -%> - <%- node.params.each do |param| -%> - this.<%= param.name %> = <%= param.name %>; - <%- end -%> - } - <%# methods for flags -%> - <%- node.params.each do |param| -%> - <%- if param.is_a?(FlagsParam) -%> - <%- flags.find { |flag| flag.name == param.kind }.tap { raise "Expected to find #{param.kind}" unless _1 }.values.each do |value| -%> - - public boolean is<%= value.camelcase %>() { - return <%= param.kind %>.is<%= value.camelcase %>(this.<%= param.name %>); - } - <%- end -%> - <%- end -%> - <%- end -%> - <%# potential override of setNewLineFlag() -%> - <%- if node.newline == false -%> - - @Override - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - // Never mark <%= node.name %> with a newline flag, mark children instead - } - <%- elsif node.newline.is_a?(String) -%> - - @Override - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - <%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%> - <%- case param -%> - <%- when SingleNodeParam -%> - this.<%= param.name %>.setNewLineFlag(source, newlineMarked); - <%- when NodeListParam -%> - Node first = this.<%= param.name %>.length > 0 ? this.<%= param.name %>[0] : null; - if (first != null) { - first.setNewLineFlag(source, newlineMarked); - } - <%- else raise param.class.name -%> - <%- end -%> - } - <%- end -%> - - public void visitChildNodes(AbstractNodeVisitor visitor) { - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeListParam -%> - for (Nodes.Node child : this.<%= param.name %>) { - child.accept(visitor); - } - <%- when NodeParam -%> - this.<%= param.name %>.accept(visitor); - <%- when OptionalNodeParam -%> - if (this.<%= param.name %> != null) { - this.<%= param.name %>.accept(visitor); - } - <%- end -%> - <%- end -%> - } - - public Node[] childNodes() { - <%- if node.params.none?(NodeListParam) and node.params.none?(SingleNodeParam) -%> - return EMPTY_ARRAY; - <%- elsif node.params.one?(NodeListParam) and node.params.none?(SingleNodeParam) -%> - return this.<%= node.params.grep(NodeListParam).first.name %>; - <%- elsif node.params.none?(NodeListParam) -%> - return new Node[] { <%= node.params.grep(SingleNodeParam).map { "this.#{_1.name}" }.join(', ') %> }; - <%- else -%> - ArrayList childNodes = new ArrayList<>(); - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when SingleNodeParam -%> - childNodes.add(this.<%= param.name %>); - <%- when NodeListParam -%> - childNodes.addAll(Arrays.asList(this.<%= param.name %>)); - <%- end -%> - <%- end -%> - return childNodes.toArray(EMPTY_ARRAY); - <%- end -%> - } - - public T accept(AbstractNodeVisitor visitor) { - return visitor.visit<%= node.name -%>(this); - } - } - <%- end -%> - -} -// @formatter:on diff --git a/yarp/templates/lib/yarp/mutation_visitor.rb.erb b/yarp/templates/lib/yarp/mutation_visitor.rb.erb new file mode 100644 index 00000000000000..3bb2dfb4b27616 --- /dev/null +++ b/yarp/templates/lib/yarp/mutation_visitor.rb.erb @@ -0,0 +1,19 @@ +module YARP + # This visitor walks through the tree and copies each node as it is being + # visited. This is useful for consumers that want to mutate the tree, as you + # can change subtrees in place without effecting the rest of the tree. + class MutationVisitor < BasicVisitor + <%- nodes.each_with_index do |node, index| -%> +<%= "\n" if index != 0 -%> + # Copy a <%= node.name %> node + def visit_<%= node.human %>(node) + <%- fields = node.fields.select { |field| [YARP::NodeField, YARP::OptionalNodeField, YARP::NodeListField].include?(field.class) } -%> + <%- if fields.any? -%> + node.copy(<%= fields.map { |field| "#{field.name}: #{field.is_a?(YARP::NodeListField) ? "visit_all" : "visit"}(node.#{field.name})" }.join(", ") %>) + <%- else -%> + node.copy + <%- end -%> + end + <%- end -%> + end +end diff --git a/yarp/templates/lib/yarp/node.rb.erb b/yarp/templates/lib/yarp/node.rb.erb index c1f14f537af9ec..6241eabd12bd0a 100644 --- a/yarp/templates/lib/yarp/node.rb.erb +++ b/yarp/templates/lib/yarp/node.rb.erb @@ -1,15 +1,15 @@ module YARP <%- nodes.each do |node| -%> <%= "#{node.comment.split("\n").map { |line| line.empty? ? "#" : "# #{line}" }.join("\n ")}\n " if node.comment %>class <%= node.name -%> < Node - <%- node.params.each do |param| -%> - # attr_reader <%= param.name %>: <%= param.rbs_class %> - attr_reader :<%= param.name %> + <%- node.fields.each do |field| -%> + # attr_reader <%= field.name %>: <%= field.rbs_class %> + attr_reader :<%= field.name %> <%- end -%> - # def initialize: (<%= (node.params.map { |param| "#{param.name}: #{param.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void - def initialize(<%= (node.params.map(&:name) + ["location"]).join(", ") %>) - <%- node.params.each do |param| -%> - @<%= param.name %> = <%= param.name %> + # def initialize: (<%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void + def initialize(<%= (node.fields.map(&:name) + ["location"]).join(", ") %>) + <%- node.fields.each do |field| -%> + @<%= field.name %> = <%= field.name %> <%- end -%> @location = location end @@ -26,59 +26,68 @@ module YARP <%- elsif node.newline.is_a?(String) -%> def set_newline_flag(newline_marked) - <%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%> - <%- case param -%> - <%- when SingleNodeParam -%> - <%= param.name %>.set_newline_flag(newline_marked) - <%- when NodeListParam -%> - first = <%= param.name %>.first + <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + <%= field.name %>.set_newline_flag(newline_marked) + <%- when YARP::NodeListField -%> + first = <%= field.name %>.first first.set_newline_flag(newline_marked) if first - <%- else raise param.class.name -%> + <%- else raise field.class.name -%> <%- end -%> end <%- end -%> # def child_nodes: () -> Array[nil | Node] def child_nodes - [<%= node.params.map { |param| - case param - when SingleNodeParam then param.name - when NodeListParam then "*#{param.name}" + [<%= node.fields.map { |field| + case field + when YARP::NodeField, YARP::OptionalNodeField then field.name + when YARP::NodeListField then "*#{field.name}" end }.compact.join(", ") %>] end + # def copy: (**params) -> <%= node.name %> + def copy(**params) + <%= node.name %>.new( + <%- (node.fields.map(&:name) + ["location"]).map do |name| -%> + params.fetch(:<%= name %>) { <%= name %> }, + <%- end -%> + ) + end + # def deconstruct: () -> Array[nil | Node] alias deconstruct child_nodes # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location] def deconstruct_keys(keys) - { <%= (node.params.map { |param| "#{param.name}: #{param.name}" } + ["location: location"]).join(", ") %> } + { <%= (node.fields.map { |field| "#{field.name}: #{field.name}" } + ["location: location"]).join(", ") %> } end - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when LocationParam -%> - <%- raise unless param.name.end_with?("_loc") -%> - <%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%> - - # def <%= param.name.delete_suffix("_loc") %>: () -> String - def <%= param.name.delete_suffix("_loc") %> - <%= param.name %>.slice + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::LocationField -%> + <%- raise unless field.name.end_with?("_loc") -%> + <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> + + # def <%= field.name.delete_suffix("_loc") %>: () -> String + def <%= field.name.delete_suffix("_loc") %> + <%= field.name %>.slice end - <%- when OptionalLocationParam -%> - <%- raise unless param.name.end_with?("_loc") -%> - <%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%> + <%- when YARP::OptionalLocationField -%> + <%- raise unless field.name.end_with?("_loc") -%> + <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> - # def <%= param.name.delete_suffix("_loc") %>: () -> String? - def <%= param.name.delete_suffix("_loc") %> - <%= param.name %>&.slice + # def <%= field.name.delete_suffix("_loc") %>: () -> String? + def <%= field.name.delete_suffix("_loc") %> + <%= field.name %>&.slice end - <%- when FlagsParam -%> - <%- flags.find { |flag| flag.name == param.kind }.tap { |flag| raise "Expected to find #{param.kind}" unless flag }.values.each do |value| -%> + <%- when YARP::FlagsField -%> + <%- flags.find { |flag| flag.name == field.kind }.tap { |flag| raise "Expected to find #{field.kind}" unless flag }.values.each do |value| -%> # def <%= value.name.downcase %>?: () -> bool def <%= value.name.downcase %>? - <%= param.name %>.anybits?(<%= param.kind %>::<%= value.name %>) + <%= field.name %>.anybits?(<%= field.kind %>::<%= value.name %>) end <%- end -%> <%- end -%> @@ -114,8 +123,8 @@ module YARP <%- nodes.each do |node| -%> # Create a new <%= node.name %> node - def <%= node.name %>(<%= (node.params.map(&:name) + ["location = Location()"]).join(", ") %>) - <%= node.name %>.new(<%= (node.params.map(&:name) + ["location"]).join(", ") %>) + def <%= node.name %>(<%= (node.fields.map(&:name) + ["location = Location()"]).join(", ") %>) + <%= node.name %>.new(<%= (node.fields.map(&:name) + ["location"]).join(", ") %>) end <%- end -%> end diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 8ee072c0b18a19..d54c3119543d99 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -14,15 +14,15 @@ end module YARP module Serialize MAJOR_VERSION = 0 - MINOR_VERSION = 8 + MINOR_VERSION = 9 PATCH_VERSION = 0 def self.load(input, serialized) - Loader.new(Source.new(input), serialized).load + Loader.new(Source.new(input), serialized).load_result end def self.load_tokens(source, serialized) - Loader.new(source, serialized).load_tokens + Loader.new(source, serialized).load_tokens_result end class Loader @@ -43,6 +43,17 @@ module YARP @source = source end + def load_encoding + Encoding.find(io.read(load_varint)) + end + + def load_metadata + comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } + errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } + warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + [comments, errors, warnings] + end + def load_tokens tokens = [] while type = TOKEN_TYPES.fetch(load_varint) @@ -53,32 +64,40 @@ module YARP tokens << [YARP::Token.new(type, location.slice, location), lex_state] end - comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } - errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + tokens + end - raise "Expected to consume all bytes while deserializing" unless @io.eof? + def load_tokens_result + tokens = load_tokens + encoding = load_encoding + comments, errors, warnings = load_metadata + + if encoding != @encoding + tokens.each { |token,| token.value.force_encoding(encoding) } + end + raise "Expected to consume all bytes while deserializing" unless @io.eof? YARP::ParseResult.new(tokens, comments, errors, warnings, @source) end - def load + def load_nodes raise "Invalid serialization" if io.read(4) != "YARP" raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] - @encoding = Encoding.find(io.read(load_varint)) + @encoding = load_encoding @input = input.force_encoding(@encoding).freeze - comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) } - errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + comments, errors, warnings = load_metadata @constant_pool_offset = io.read(4).unpack1("L") @constant_pool = Array.new(load_varint, nil) - ast = load_node + [load_node, comments, errors, warnings] + end - YARP::ParseResult.new(ast, comments, errors, warnings, @source) + def load_result + node, comments, errors, warnings = load_nodes + YARP::ParseResult.new(node, comments, errors, warnings, @source) end private @@ -160,18 +179,18 @@ module YARP <%- if node.needs_serialized_length? -%> load_serialized_length <%- end -%> - <%= node.name %>.new(<%= (node.params.map { |param| - case param - when NodeParam then "load_node" - when OptionalNodeParam then "load_optional_node" - when StringParam then "load_string" - when NodeListParam then "Array.new(load_varint) { load_node }" - when LocationListParam then "Array.new(load_varint) { load_location }" - when ConstantParam then "load_constant" - when ConstantListParam then "Array.new(load_varint) { load_constant }" - when LocationParam then "load_location" - when OptionalLocationParam then "load_optional_location" - when UInt32Param, FlagsParam then "load_varint" + <%= node.name %>.new(<%= (node.fields.map { |field| + case field + when YARP::NodeField then "load_node" + when YARP::OptionalNodeField then "load_optional_node" + when YARP::StringField then "load_string" + when YARP::NodeListField then "Array.new(load_varint) { load_node }" + when YARP::LocationListField then "Array.new(load_varint) { load_location }" + when YARP::ConstantField then "load_constant" + when YARP::ConstantListField then "Array.new(load_varint) { load_constant }" + when YARP::LocationField then "load_location" + when YARP::OptionalLocationField then "load_optional_location" + when YARP::UInt32Field, YARP::FlagsField then "load_varint" else raise end } + ["location"]).join(", ") -%>) diff --git a/yarp/templates/src/node.c.erb b/yarp/templates/src/node.c.erb index d288628fffa666..aa756ed4f9a81f 100644 --- a/yarp/templates/src/node.c.erb +++ b/yarp/templates/src/node.c.erb @@ -78,29 +78,33 @@ yp_node_destroy(yp_parser_t *parser, yp_node_t *node) { switch (YP_NODE_TYPE(node)) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - case <%= node.type %>: - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam -%> - <%- when NodeParam -%> - yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) { - yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); + case <%= node.type %>: { + <%- if node.fields.any? { |field| ![YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField].include?(field.class) } -%> + yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; + <%- end -%> + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField -%> + <%- when YARP::NodeField -%> + yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>); + <%- when YARP::OptionalNodeField -%> + if (cast-><%= field.name %> != NULL) { + yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>); } - <%- when StringParam -%> - yp_string_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when NodeListParam -%> - yp_node_list_free(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when LocationListParam -%> - yp_location_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when ConstantListParam -%> - yp_constant_id_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + <%- when YARP::StringField -%> + yp_string_free(&cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + yp_node_list_free(parser, &cast-><%= field.name %>); + <%- when YARP::LocationListField -%> + yp_location_list_free(&cast-><%= field.name %>); + <%- when YARP::ConstantListField -%> + yp_constant_id_list_free(&cast-><%= field.name %>); <%- else -%> <%- raise -%> <%- end -%> <%- end -%> break; + } <%- end -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" default: @@ -122,24 +126,25 @@ yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { - memsize->memsize += sizeof(yp_<%= node.human %>_t); - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when ConstantParam, UInt32Param, FlagsParam, LocationParam, OptionalLocationParam -%> - <%- when NodeParam -%> - yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) { - yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); + yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; + memsize->memsize += sizeof(*cast); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::ConstantField, YARP::UInt32Field, YARP::FlagsField, YARP::LocationField, YARP::OptionalLocationField -%> + <%- when YARP::NodeField -%> + yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize); + <%- when YARP::OptionalNodeField -%> + if (cast-><%= field.name %> != NULL) { + yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize); } - <%- when StringParam -%> - memsize->memsize += yp_string_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when NodeListParam -%> - yp_node_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); - <%- when LocationListParam -%> - memsize->memsize += yp_location_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when ConstantListParam -%> - memsize->memsize += yp_constant_id_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + <%- when YARP::StringField -%> + memsize->memsize += yp_string_memsize(&cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + yp_node_list_memsize(&cast-><%= field.name %>, memsize); + <%- when YARP::LocationListField -%> + memsize->memsize += yp_location_list_memsize(&cast-><%= field.name %>); + <%- when YARP::ConstantListField -%> + memsize->memsize += yp_constant_id_list_memsize(&cast-><%= field.name %>); <%- else -%> <%- raise -%> <%- end -%> @@ -168,5 +173,5 @@ yp_node_type_to_str(yp_node_type_t node_type) return "<%= node.type %>"; <%- end -%> } - return "\0"; + return ""; } diff --git a/yarp/templates/src/prettyprint.c.erb b/yarp/templates/src/prettyprint.c.erb index cf2f12f2aeff73..4c1f49fdd2f017 100644 --- a/yarp/templates/src/prettyprint.c.erb +++ b/yarp/templates/src/prettyprint.c.erb @@ -23,64 +23,64 @@ prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) { <%- nodes.each do |node| -%> case <%= node.type %>: { yp_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>); - <%- node.params.each_with_index do |param, index| -%> + <%- node.fields.each_with_index do |field, index| -%> <%= "yp_buffer_append_str(buffer, \", \", 2);" if index != 0 -%> - <%- case param -%> - <%- when NodeParam -%> - prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) { + <%- case field -%> + <%- when YARP::NodeField -%> + prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::OptionalNodeField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) { yp_buffer_append_str(buffer, "nil", 3); } else { - prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); + prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>); } - <%- when StringParam -%> + <%- when YARP::StringField -%> yp_buffer_append_str(buffer, "\"", 1); - yp_buffer_append_str(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>)); + yp_buffer_append_bytes(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= field.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= field.name %>)); yp_buffer_append_str(buffer, "\"", 1); - <%- when NodeListParam -%> + <%- when YARP::NodeListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= param.name %>.nodes[index]); + prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= field.name %>.nodes[index]); } yp_buffer_append_str(buffer, "]", 1); - <%- when LocationListParam -%> + <%- when YARP::LocationListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index]); + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>.locations[index]); } yp_buffer_append_str(buffer, "]", 1); - <%- when ConstantParam -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); - <%- when ConstantListParam -%> + <%- when YARP::ConstantField -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); + <%- when YARP::ConstantListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index]); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); } yp_buffer_append_str(buffer, "]", 1); - <%- when LocationParam -%> - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when OptionalLocationParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) { + <%- when YARP::LocationField -%> + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::OptionalLocationField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { yp_buffer_append_str(buffer, "nil", 3); } else { - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>); } - <%- when UInt32Param -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); - <%- when FlagsParam -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", node->flags >> <%= COMMON_FLAGS %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); + <%- when YARP::UInt32Field -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); + <%- when YARP::FlagsField -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", node->flags >> <%= YARP::COMMON_FLAGS %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); <%- else -%> <%- raise -%> <%- end -%> diff --git a/yarp/templates/src/serialize.c.erb b/yarp/templates/src/serialize.c.erb index 9b49540566c435..b1049ba11668bf 100644 --- a/yarp/templates/src/serialize.c.erb +++ b/yarp/templates/src/serialize.c.erb @@ -38,7 +38,7 @@ yp_serialize_string(yp_parser_t *parser, yp_string_t *string, yp_buffer_t *buffe uint32_t length = yp_sizet_to_u32(yp_string_length(string)); yp_buffer_append_u8(buffer, 2); yp_buffer_append_u32(buffer, length); - yp_buffer_append_str(buffer, yp_string_source(string), length); + yp_buffer_append_bytes(buffer, yp_string_source(string), length); break; } case YP_STRING_MAPPED: @@ -68,51 +68,51 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { size_t length_offset = buffer->length; yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */ <%- end -%> - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeParam -%> - yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) { + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeField -%> + yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::OptionalNodeField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) { yp_buffer_append_u8(buffer, 0); } else { - yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } - <%- when StringParam -%> - yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when NodeListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= param.name %>.nodes[index], buffer); + <%- when YARP::StringField -%> + yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::NodeListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer); } - <%- when LocationListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index], buffer); + <%- when YARP::LocationListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>.locations[index], buffer); } - <%- when ConstantParam -%> - yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>)); - <%- when ConstantListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index])); + <%- when YARP::ConstantField -%> + yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>)); + <%- when YARP::ConstantListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index])); } - <%- when LocationParam -%> - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when OptionalLocationParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) { + <%- when YARP::LocationField -%> + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::OptionalLocationField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { yp_buffer_append_u8(buffer, 0); } else { yp_buffer_append_u8(buffer, 1); - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } - <%- when UInt32Param -%> - yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when FlagsParam -%> - yp_buffer_append_u32(buffer, node->flags >> <%= COMMON_FLAGS %>); + <%- when YARP::UInt32Field -%> + yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::FlagsField -%> + yp_buffer_append_u32(buffer, node->flags >> <%= YARP::COMMON_FLAGS %>); <%- else -%> <%- raise -%> <%- end -%> @@ -170,14 +170,17 @@ yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t * } } +static void +yp_serialize_encoding(yp_encoding_t *encoding, yp_buffer_t *buffer) { + size_t encoding_length = strlen(encoding->name); + yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length)); + yp_buffer_append_str(buffer, encoding->name, encoding_length); +} + #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { - // First, serialize the encoding of the parser. - size_t encoding_length = strlen(parser->encoding.name); - yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length)); - yp_buffer_append_str(buffer, parser->encoding.name, encoding_length); - + yp_serialize_encoding(&parser->encoding, buffer); yp_serialize_comment_list(parser, &parser->comment_list, buffer); yp_serialize_diagnostic_list(parser, &parser->error_list, buffer); yp_serialize_diagnostic_list(parser, &parser->warning_list, buffer); @@ -231,7 +234,7 @@ serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) { } YP_EXPORTED_FUNCTION void -yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer) { +yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer) { yp_parser_t parser; yp_parser_init(&parser, source, size, filepath); @@ -246,6 +249,7 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe // Append 0 to mark end of tokens yp_buffer_append_u8(buffer, 0); + yp_serialize_encoding(&parser.encoding, buffer); yp_serialize_comment_list(&parser, &parser.comment_list, buffer); yp_serialize_diagnostic_list(&parser, &parser.error_list, buffer); yp_serialize_diagnostic_list(&parser, &parser.warning_list, buffer); @@ -253,3 +257,26 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe yp_node_destroy(&parser, node); yp_parser_free(&parser); } + +// Parse and serialize both the AST and the tokens represented by the given +// source to the given buffer. +YP_EXPORTED_FUNCTION void +yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) { + yp_parser_t parser; + yp_parser_init(&parser, source, size, NULL); + if (metadata) yp_parser_metadata(&parser, metadata); + + yp_lex_callback_t lex_callback = (yp_lex_callback_t) { + .data = (void *) buffer, + .callback = serialize_token, + }; + + parser.lex_callback = &lex_callback; + yp_node_t *node = yp_parse(&parser); + + yp_buffer_append_u8(buffer, 0); + yp_serialize(&parser, node, buffer); + + yp_node_destroy(&parser, node); + yp_parser_free(&parser); +} diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 8b8a175172ffea..3969645643e581 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,330 +4,343 @@ require "fileutils" require "yaml" -COMMON_FLAGS = 1 +module YARP + COMMON_FLAGS = 1 -class Param - attr_reader :name, :options + # This represents a field on a node. It contains all of the necessary + # information to template out the code for that field. + class Field + attr_reader :name, :options - def initialize(name:, type:, **options) - @name, @type, @options = name, type, options + def initialize(name:, type:, **options) + @name, @type, @options = name, type, options + end end -end -module KindTypes - def c_type - if options[:kind] - "yp_#{options[:kind].gsub(/(?<=.)[A-Z]/, "_\\0").downcase}" - else - "yp_node" + # Some node fields can be specialized if they point to a specific kind of + # node and not just a generic node. + class NodeKindField < Field + def c_type + if options[:kind] + "yp_#{options[:kind].gsub(/(?<=.)[A-Z]/, "_\\0").downcase}" + else + "yp_node" + end + end + + def ruby_type + options[:kind] || "Node" end - end - def ruby_type - options[:kind] || "Node" + def java_type + options[:kind] || "Node" + end + + def java_cast + if options[:kind] + "(Nodes.#{options[:kind]}) " + else + "" + end + end end - def java_type - options[:kind] || "Node" + # This represents a field on a node that is itself a node. We pass them as + # references and store them as references. + class NodeField < NodeKindField + def rbs_class + ruby_type + end end - def java_cast - if options[:kind] - "(Nodes.#{options[:kind]}) " - else - "" + # This represents a field on a node that is itself a node and can be + # optionally null. We pass them as references and store them as references. + class OptionalNodeField < NodeKindField + def rbs_class + "#{ruby_type}?" end end -end -# This represents a parameter to a node that is itself a node. We pass them as -# references and store them as references. -class NodeParam < Param - include KindTypes + # This represents a field on a node that is a list of nodes. We pass them as + # references and store them directly on the struct. + class NodeListField < Field + def rbs_class + "Array[Node]" + end - def rbs_class - ruby_type + def java_type + "Node[]" + end end -end -# This represents a parameter to a node that is itself a node and can be -# optionally null. We pass them as references and store them as references. -class OptionalNodeParam < Param - include KindTypes + # This represents a field on a node that is a list of locations. + class LocationListField < Field + def rbs_class + "Array[Location]" + end - def rbs_class - "#{ruby_type}?" + def java_type + "Location[]" + end end -end -SingleNodeParam = -> (node) { NodeParam === node or OptionalNodeParam === node } + # This represents a field on a node that is the ID of a string interned + # through the parser's constant pool. + class ConstantField < Field + def rbs_class + "Symbol" + end -# This represents a parameter to a node that is a list of nodes. We pass them as -# references and store them as references. -class NodeListParam < Param - def rbs_class - "Array[Node]" + def java_type + "byte[]" + end end - def java_type - "Node[]" - end -end + # This represents a field on a node that is a list of IDs that are associated + # with strings interned through the parser's constant pool. + class ConstantListField < Field + def rbs_class + "Array[Symbol]" + end -# This represents a parameter to a node that is a list of locations. -class LocationListParam < Param - def rbs_class - "Array[Location]" + def java_type + "byte[][]" + end end - def java_type - "Location[]" - end -end + # This represents a field on a node that is a string. + class StringField < Field + def rbs_class + "String" + end -# This represents a parameter to a node that is the ID of a string interned -# through the parser's constant pool. -class ConstantParam < Param - def rbs_class - "Symbol" + def java_type + "byte[]" + end end - def java_type - "byte[]" - end -end + # This represents a field on a node that is a location. + class LocationField < Field + def rbs_class + "Location" + end -# This represents a parameter to a node that is a list of IDs that are -# associated with strings interned through the parser's constant pool. -class ConstantListParam < Param - def rbs_class - "Array[Symbol]" + def java_type + "Location" + end end - def java_type - "byte[][]" - end -end + # This represents a field on a node that is a location that is optional. + class OptionalLocationField < Field + def rbs_class + "Location?" + end -# This represents a parameter to a node that is a string. -class StringParam < Param - def rbs_class - "String" + def java_type + "Location" + end end - def java_type - "byte[]" - end -end + # This represents an integer field. + class UInt32Field < Field + def rbs_class + "Integer" + end -# This represents a parameter to a node that is a location. -class LocationParam < Param - def rbs_class - "Location" + def java_type + "int" + end end - def java_type - "Location" - end -end + # This represents a set of flags. It is very similar to the UInt32Field, but + # can be directly embedded into the flags field on the struct and provides + # convenient methods for checking if a flag is set. + class FlagsField < Field + def rbs_class + "Integer" + end -# This represents a parameter to a node that is a location that is optional. -class OptionalLocationParam < Param - def rbs_class - "Location?" - end + def java_type + "short" + end - def java_type - "Location" + def kind + options.fetch(:kind) + end end -end -# This represents an integer parameter. -class UInt32Param < Param - def rbs_class - "Integer" - end + # This class represents a node in the tree, configured by the config.yml file in + # YAML format. It contains information about the name of the node and the + # various child nodes it contains. + class NodeType + attr_reader :name, :type, :human, :fields, :newline, :comment - def java_type - "int" - end -end + def initialize(config) + @name = config.fetch("name") -# This represents a set of flags. It is very similar to the UInt32Param, but can -# be directly embedded into the flags field on the struct and provides -# convenient methods for checking if a flag is set. -class FlagsParam < Param - def rbs_class - "Integer" - end + type = @name.gsub(/(?<=.)[A-Z]/, "_\\0") + @type = "YP_NODE_#{type.upcase}" + @human = type.downcase - def java_type - "short" - end + @fields = + config.fetch("fields", []).map do |field| + field_type_for(field.fetch("type")).new(**field.transform_keys(&:to_sym)) + end - def kind - options.fetch(:kind) - end -end + @newline = config.fetch("newline", true) + @comment = config.fetch("comment") + end -PARAM_TYPES = { - "node" => NodeParam, - "node?" => OptionalNodeParam, - "node[]" => NodeListParam, - "string" => StringParam, - "location[]" => LocationListParam, - "constant" => ConstantParam, - "constant[]" => ConstantListParam, - "location" => LocationParam, - "location?" => OptionalLocationParam, - "uint32" => UInt32Param, - "flags" => FlagsParam -} - -# This class represents a node in the tree, configured by the config.yml file in -# YAML format. It contains information about the name of the node and the -# various child nodes it contains. -class NodeType - attr_reader :name, :type, :human, :params, :newline, :comment - - def initialize(config) - @name = config.fetch("name") - - type = @name.gsub(/(?<=.)[A-Z]/, "_\\0") - @type = "YP_NODE_#{type.upcase}" - @human = type.downcase - @params = config.fetch("child_nodes", []).map do |param| - param_type = PARAM_TYPES[param.fetch("type")] || - raise("Unknown param type: #{param["type"].inspect}") - param_type.new(**param.transform_keys(&:to_sym)) - end - @newline = config.fetch("newline", true) - @comment = config.fetch("comment") - end + # Should emit serialized length of node so implementations can skip + # the node to enable lazy parsing. + def needs_serialized_length? + name == "DefNode" + end - # Should emit serialized length of node so implementations can skip - # the node to enable lazy parsing. - def needs_serialized_length? - @name == "DefNode" + private + + def field_type_for(name) + case name + when "node" then NodeField + when "node?" then OptionalNodeField + when "node[]" then NodeListField + when "string" then StringField + when "location[]" then LocationListField + when "constant" then ConstantField + when "constant[]" then ConstantListField + when "location" then LocationField + when "location?" then OptionalLocationField + when "uint32" then UInt32Field + when "flags" then FlagsField + else raise("Unknown field type: #{name.inspect}") + end + end end -end -# This represents a token in the lexer. They are configured through the -# config.yml file for now, but this will probably change as we transition to -# storing semantic strings instead of the lexer tokens. -class Token - attr_reader :name, :value, :comment + # This represents a token in the lexer. + class Token + attr_reader :name, :value, :comment - def initialize(config) - @name = config.fetch("name") - @value = config["value"] - @comment = config.fetch("comment") - end + def initialize(config) + @name = config.fetch("name") + @value = config["value"] + @comment = config.fetch("comment") + end - def declaration - output = [] - output << "YP_TOKEN_#{name}" - output << " = #{value}" if value - output << ", // #{comment}" - output.join + def declaration + output = [] + output << "YP_TOKEN_#{name}" + output << " = #{value}" if value + output << ", // #{comment}" + output.join + end end -end -# Represents a set of flags that should be internally represented with an enum. -class Flags - attr_reader :name, :human, :values + # Represents a set of flags that should be internally represented with an enum. + class Flags + # Represents an individual flag within a set of flags. + class Flag + attr_reader :name, :camelcase, :comment - def initialize(config) - @name = config.fetch("name") - @human = @name.gsub(/(?<=.)[A-Z]/, "_\\0").downcase - @values = config.fetch("values").map { |flag| Flag.new(flag) } - end -end + def initialize(config) + @name = config.fetch("name") + @camelcase = @name.split("_").map(&:capitalize).join + @comment = config.fetch("comment") + end + end -class Flag - attr_reader :name, :camelcase, :comment + attr_reader :name, :human, :values - def initialize(config) - @name = config.fetch("name") - @camelcase = @name.split("_").map(&:capitalize).join - @comment = config.fetch("comment") + def initialize(config) + @name = config.fetch("name") + @human = @name.gsub(/(?<=.)[A-Z]/, "_\\0").downcase + @values = config.fetch("values").map { |flag| Flag.new(flag) } + end end -end -# This templates out a file using ERB with the given locals. The locals are -# derived from the config.yml file. -def template(name, locals, write_to: nil) - filepath = "templates/#{name}.erb" - template = File.expand_path("../#{filepath}", __dir__) - write_to ||= File.expand_path("../#{name}", __dir__) + class << self + # This templates out a file using ERB with the given locals. The locals are + # derived from the config.yml file. + def template(name, write_to: nil) + filepath = "templates/#{name}.erb" + template = File.expand_path("../#{filepath}", __dir__) + write_to ||= File.expand_path("../#{name}", __dir__) + + if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ + erb = ERB.new(File.read(template), trim_mode: "-") + else + erb = ERB.new(File.read(template), nil, "-") + end + erb.filename = template + + non_ruby_heading = <<~HEADING + /******************************************************************************/ + /* This file is generated by the templates/template.rb script and should not */ + /* be modified manually. See */ + /* #{filepath + " " * (74 - filepath.size) } */ + /* if you are looking to modify the */ + /* template */ + /******************************************************************************/ + HEADING + + ruby_heading = <<~HEADING + # frozen_string_literal: true + =begin + This file is generated by the templates/template.rb script and should not be + modified manually. See #{filepath} + if you are looking to modify the template + =end + + HEADING + + heading = if File.extname(filepath.gsub(".erb", "")) == ".rb" + ruby_heading + else + non_ruby_heading + end + + contents = heading + erb.result_with_hash(locals) + FileUtils.mkdir_p(File.dirname(write_to)) + File.write(write_to, contents) + end - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: "-") - else - erb = ERB.new(File.read(template), nil, "-") - end - erb.filename = template - - non_ruby_heading = <<~HEADING - /******************************************************************************/ - /* This file is generated by the templates/template.rb script and should not */ - /* be modified manually. See */ - /* #{filepath + " " * (74 - filepath.size) } */ - /* if you are looking to modify the */ - /* template */ - /******************************************************************************/ - HEADING - - ruby_heading = <<~HEADING - # frozen_string_literal: true - =begin - This file is generated by the templates/template.rb script and should not be - modified manually. See #{filepath} - if you are looking to modify the template - =end - - HEADING - - heading = if File.extname(filepath.gsub(".erb", "")) == ".rb" - ruby_heading - else - non_ruby_heading - end - - contents = heading + erb.result_with_hash(locals) - FileUtils.mkdir_p(File.dirname(write_to)) - File.write(write_to, contents) -end + private -def locals - config = YAML.load_file(File.expand_path("../config.yml", __dir__)) + def locals + @locals ||= + begin + config = YAML.load_file(File.expand_path("../config.yml", __dir__)) - { - nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name), - tokens: config.fetch("tokens").map { |token| Token.new(token) }, - flags: config.fetch("flags").map { |flags| Flags.new(flags) } - } -end + { + nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name), + tokens: config.fetch("tokens").map { |token| Token.new(token) }, + flags: config.fetch("flags").map { |flags| Flags.new(flags) } + } + end + end + end -TEMPLATES = [ - "ext/yarp/api_node.c", - "include/yarp/ast.h", - "java/org/yarp/Loader.java", - "java/org/yarp/Nodes.java", - "java/org/yarp/AbstractNodeVisitor.java", - "lib/yarp/node.rb", - "lib/yarp/serialize.rb", - "src/node.c", - "src/prettyprint.c", - "src/serialize.c", - "src/token_type.c" -] + TEMPLATES = [ + "ext/yarp/api_node.c", + "include/yarp/ast.h", + "java/org/yarp/Loader.java", + "java/org/yarp/Nodes.java", + "java/org/yarp/AbstractNodeVisitor.java", + "lib/yarp/mutation_visitor.rb", + "lib/yarp/node.rb", + "lib/yarp/serialize.rb", + "src/node.c", + "src/prettyprint.c", + "src/serialize.c", + "src/token_type.c" + ] +end if __FILE__ == $0 if ARGV.empty? - TEMPLATES.each { |f| template(f, locals) } + YARP::TEMPLATES.each { |filepath| YARP.template(filepath) } else name, write_to = ARGV - template(name, locals, write_to: write_to) + YARP.template(name, write_to: write_to) end end diff --git a/yarp/unescape.c b/yarp/unescape.c index 7cf2631b9b943f..830c5996ae380d 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -5,9 +5,9 @@ /******************************************************************************/ static inline bool -yp_char_is_hexadecimal_digits(const char *c, size_t length) { +yp_char_is_hexadecimal_digits(const uint8_t *string, size_t length) { for (size_t index = 0; index < length; index++) { - if (!yp_char_is_hexadecimal_digit(c[index])) { + if (!yp_char_is_hexadecimal_digit(string[index])) { return false; } } @@ -18,10 +18,8 @@ yp_char_is_hexadecimal_digits(const char *c, size_t length) { // expensive to go through the indirection of the function pointer. Instead we // provide a fast path that will check if we can just return 1. static inline size_t -yp_char_width(yp_parser_t *parser, const char *start, const char *end) { - const unsigned char *uc = (const unsigned char *) start; - - if (parser->encoding_changed || (*uc >= 0x80)) { +yp_char_width(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { + if (parser->encoding_changed || (*start >= 0x80)) { return parser->encoding.char_width(start, end - start); } else { return 1; @@ -33,7 +31,7 @@ yp_char_width(yp_parser_t *parser, const char *start, const char *end) { /******************************************************************************/ // This is a lookup table for unescapes that only take up a single character. -static const unsigned char unescape_chars[] = { +static const uint8_t unescape_chars[] = { ['\''] = '\'', ['\\'] = '\\', ['a'] = '\a', @@ -60,9 +58,8 @@ static const bool ascii_printable_chars[] = { }; static inline bool -char_is_ascii_printable(const char c) { - unsigned char v = (unsigned char) c; - return (v < 0x80) && ascii_printable_chars[v]; +char_is_ascii_printable(const uint8_t b) { + return (b < 0x80) && ascii_printable_chars[b]; } /******************************************************************************/ @@ -72,37 +69,39 @@ char_is_ascii_printable(const char c) { // Scan the 1-3 digits of octal into the value. Returns the number of digits // scanned. static inline size_t -unescape_octal(const char *backslash, unsigned char *value) { - *value = (unsigned char) (backslash[1] - '0'); - if (!yp_char_is_octal_digit(backslash[2])) { +unescape_octal(const uint8_t *backslash, uint8_t *value, const uint8_t *end) { + *value = (uint8_t) (backslash[1] - '0'); + if (backslash + 2 >= end || !yp_char_is_octal_digit(backslash[2])) { return 2; } - - *value = (unsigned char) ((*value << 3) | (backslash[2] - '0')); - if (!yp_char_is_octal_digit(backslash[3])) { + *value = (uint8_t) ((*value << 3) | (backslash[2] - '0')); + if (backslash + 3 >= end || !yp_char_is_octal_digit(backslash[3])) { return 3; } - - *value = (unsigned char) ((*value << 3) | (backslash[3] - '0')); + *value = (uint8_t) ((*value << 3) | (backslash[3] - '0')); return 4; } // Convert a hexadecimal digit into its equivalent value. -static inline unsigned char -unescape_hexadecimal_digit(const char value) { - return (unsigned char) ((value <= '9') ? (value - '0') : (value & 0x7) + 9); +static inline uint8_t +unescape_hexadecimal_digit(const uint8_t value) { + return (uint8_t) ((value <= '9') ? (value - '0') : (value & 0x7) + 9); } // Scan the 1-2 digits of hexadecimal into the value. Returns the number of // digits scanned. static inline size_t -unescape_hexadecimal(const char *backslash, unsigned char *value) { +unescape_hexadecimal(const uint8_t *backslash, uint8_t *value, const uint8_t *end, yp_list_t *error_list) { + *value = 0; + if (backslash + 2 >= end || !yp_char_is_hexadecimal_digit(backslash[2])) { + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid hex escape."); + return 2; + } *value = unescape_hexadecimal_digit(backslash[2]); - if (!yp_char_is_hexadecimal_digit(backslash[3])) { + if (backslash + 3 >= end || !yp_char_is_hexadecimal_digit(backslash[3])) { return 3; } - - *value = (unsigned char) ((*value << 4) | unescape_hexadecimal_digit(backslash[3])); + *value = (uint8_t) ((*value << 4) | unescape_hexadecimal_digit(backslash[3])); return 4; } @@ -110,7 +109,7 @@ unescape_hexadecimal(const char *backslash, unsigned char *value) { // digits scanned. This function assumes that the characters have already been // validated. static inline void -unescape_unicode(const char *string, size_t length, uint32_t *value) { +unescape_unicode(const uint8_t *string, size_t length, uint32_t *value) { *value = 0; for (size_t index = 0; index < length; index++) { if (index != 0) *value <<= 4; @@ -122,27 +121,25 @@ unescape_unicode(const char *string, size_t length, uint32_t *value) { // 32-bit value to write. Writes the UTF-8 representation of the value to the // string and returns the number of bytes written. static inline size_t -unescape_unicode_write(char *dest, uint32_t value, const char *start, const char *end, yp_list_t *error_list) { - unsigned char *bytes = (unsigned char *) dest; - +unescape_unicode_write(uint8_t *dest, uint32_t value, const uint8_t *start, const uint8_t *end, yp_list_t *error_list) { if (value <= 0x7F) { // 0xxxxxxx - bytes[0] = (unsigned char) value; + dest[0] = (uint8_t) value; return 1; } if (value <= 0x7FF) { // 110xxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xC0 | (value >> 6)); - bytes[1] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xC0 | (value >> 6)); + dest[1] = (uint8_t) (0x80 | (value & 0x3F)); return 2; } if (value <= 0xFFFF) { // 1110xxxx 10xxxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xE0 | (value >> 12)); - bytes[1] = (unsigned char) (0x80 | ((value >> 6) & 0x3F)); - bytes[2] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xE0 | (value >> 12)); + dest[1] = (uint8_t) (0x80 | ((value >> 6) & 0x3F)); + dest[2] = (uint8_t) (0x80 | (value & 0x3F)); return 3; } @@ -150,20 +147,20 @@ unescape_unicode_write(char *dest, uint32_t value, const char *start, const char // the input is invalid. if (value <= 0x10FFFF) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xF0 | (value >> 18)); - bytes[1] = (unsigned char) (0x80 | ((value >> 12) & 0x3F)); - bytes[2] = (unsigned char) (0x80 | ((value >> 6) & 0x3F)); - bytes[3] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xF0 | (value >> 18)); + dest[1] = (uint8_t) (0x80 | ((value >> 12) & 0x3F)); + dest[2] = (uint8_t) (0x80 | ((value >> 6) & 0x3F)); + dest[3] = (uint8_t) (0x80 | (value & 0x3F)); return 4; } // If we get here, then the value is too big. This is an error, but we don't // want to just crash, so instead we'll add an error to the error list and put // in a replacement character instead. - yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence."); - bytes[0] = 0xEF; - bytes[1] = 0xBF; - bytes[2] = 0xBD; + if (error_list) yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence."); + dest[0] = 0xEF; + dest[1] = 0xBF; + dest[2] = 0xBD; return 3; } @@ -175,24 +172,30 @@ typedef enum { } yp_unescape_flag_t; // Unescape a single character value based on the given flags. -static inline unsigned char -unescape_char(const unsigned char value, const unsigned char flags) { - unsigned char unescaped = value; - +static inline uint8_t +unescape_char(uint8_t value, const uint8_t flags) { if (flags & YP_UNESCAPE_FLAG_CONTROL) { - unescaped &= 0x1f; + value &= 0x1f; } if (flags & YP_UNESCAPE_FLAG_META) { - unescaped |= 0x80; + value |= 0x80; } - return unescaped; + return value; } // Read a specific escape sequence into the given destination. -static const char * -unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backslash, const char *end, const unsigned char flags, bool write_to_str) { +static const uint8_t * +unescape( + yp_parser_t *parser, + uint8_t *dest, + size_t *dest_length, + const uint8_t *backslash, + const uint8_t *end, + const uint8_t flags, + yp_list_t *error_list +) { switch (backslash[1]) { case 'a': case 'b': @@ -203,28 +206,28 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs case 's': case 't': case 'v': - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(unescape_chars[(unsigned char) backslash[1]], flags); + if (dest) { + dest[(*dest_length)++] = unescape_char(unescape_chars[backslash[1]], flags); } return backslash + 2; // \nnn octal bit pattern, where nnn is 1-3 octal digits ([0-7]) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - unsigned char value; - const char *cursor = backslash + unescape_octal(backslash, &value); + uint8_t value; + const uint8_t *cursor = backslash + unescape_octal(backslash, &value, end); - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(value, flags); + if (dest) { + dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; } // \xnn hexadecimal bit pattern, where nn is 1-2 hexadecimal digits ([0-9a-fA-F]) case 'x': { - unsigned char value; - const char *cursor = backslash + unescape_hexadecimal(backslash, &value); + uint8_t value; + const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value, end, error_list); - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(value, flags); + if (dest) { + dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; } @@ -232,28 +235,28 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs // \unnnn Unicode character, where nnnn is exactly 4 hexadecimal digits ([0-9a-fA-F]) case 'u': { if ((flags & YP_UNESCAPE_FLAG_CONTROL) | (flags & YP_UNESCAPE_FLAG_META)) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); return backslash + 2; } if ((backslash + 3) < end && backslash[2] == '{') { - const char *unicode_cursor = backslash + 3; - const char *extra_codepoints_start = NULL; + const uint8_t *unicode_cursor = backslash + 3; + const uint8_t *extra_codepoints_start = NULL; int codepoints_count = 0; unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); - while ((*unicode_cursor != '}') && (unicode_cursor < end)) { - const char *unicode_start = unicode_cursor; + while ((unicode_cursor < end) && (*unicode_cursor != '}')) { + const uint8_t *unicode_start = unicode_cursor; size_t hexadecimal_length = yp_strspn_hexadecimal_digit(unicode_cursor, end - unicode_cursor); // \u{nnnn} character literal allows only 1-6 hexadecimal digits - if (hexadecimal_length > 6) - yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); - + if (hexadecimal_length > 6) { + if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); + } // there are not hexadecimal characters - if (hexadecimal_length == 0) { - yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); + else if (hexadecimal_length == 0) { + if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); return unicode_cursor; } @@ -265,31 +268,37 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs uint32_t value; unescape_unicode(unicode_start, (size_t) (unicode_cursor - unicode_start), &value); - if (write_to_str) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, &parser->error_list); + if (dest) { + *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, error_list); } unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); } // ?\u{nnnn} character literal should contain only one codepoint and cannot be like ?\u{nnnn mmmm} - if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) - yp_diagnostic_list_append(&parser->error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); + if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) { + if (error_list) yp_diagnostic_list_append(error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); + } - return unicode_cursor + 1; - } + if (unicode_cursor < end && *unicode_cursor == '}') { + unicode_cursor++; + } else { + if (error_list) yp_diagnostic_list_append(error_list, backslash, unicode_cursor, "invalid Unicode escape."); + } - if ((backslash + 2) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { + return unicode_cursor; + } + else if ((backslash + 5) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { uint32_t value; unescape_unicode(backslash + 2, 4, &value); - if (write_to_str) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, &parser->error_list); + if (dest) { + *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, error_list); } return backslash + 6; } - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); return backslash + 2; } // \c\M-x meta control character, where x is an ASCII printable character @@ -297,31 +306,31 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs // \cx control character, where x is an ASCII printable character case 'c': if (backslash + 2 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } switch (backslash[2]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, error_list); case '?': - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); + if (dest) { + dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 3; default: { if (!char_is_ascii_printable(backslash[2])) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[2], flags | YP_UNESCAPE_FLAG_CONTROL); + if (dest) { + dest[(*dest_length)++] = unescape_char(backslash[2], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 3; } @@ -330,36 +339,36 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs // \C-? delete, ASCII 7Fh (DEL) case 'C': if (backslash + 3 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } switch (backslash[3]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, error_list); case '?': - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); + if (dest) { + dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 4; default: if (!char_is_ascii_printable(backslash[3])) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid control escape sequence"); return backslash + 2; } - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[3], flags | YP_UNESCAPE_FLAG_CONTROL); + if (dest) { + dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 4; } @@ -368,32 +377,32 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs // \M-x meta character, where x is an ASCII printable character case 'M': { if (backslash + 3 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_META) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 2; } if (backslash[3] == '\\') { - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META, error_list); } if (char_is_ascii_printable(backslash[3])) { - if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[3], flags | YP_UNESCAPE_FLAG_META); + if (dest) { + dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_META); } return backslash + 4; } - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 3; } // \n @@ -409,7 +418,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs default: { size_t width = yp_char_width(parser, backslash + 1, end); - if (write_to_str) { + if (dest) { memcpy(dest + *dest_length, backslash + 1, width); *dest_length += width; } @@ -447,14 +456,14 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs // \c\M-x same as above // \c? or \C-? delete, ASCII 7Fh (DEL) // -YP_EXPORTED_FUNCTION void -yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { +static void +yp_unescape_manipulate_string_or_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { if (unescape_type == YP_UNESCAPE_NONE) { // If we're not unescaping then we can reference the source directly. return; } - const char *backslash = yp_memchr(string->source, '\\', string->length, parser->encoding_changed, &parser->encoding); + const uint8_t *backslash = yp_memchr(string->source, '\\', string->length, parser->encoding_changed, &parser->encoding); if (backslash == NULL) { // Here there are no escapes, so we can reference the source directly. @@ -463,21 +472,21 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // Here we have found an escape character, so we need to handle all escapes // within the string. - char *allocated = malloc(string->length); + uint8_t *allocated = malloc(string->length); if (allocated == NULL) { yp_diagnostic_list_append(&parser->error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping."); return; } // This is the memory address where we're putting the unescaped string. - char *dest = allocated; + uint8_t *dest = allocated; size_t dest_length = 0; // This is the current position in the source string that we're looking at. // It's going to move along behind the backslash so that we can copy each // segment of the string that doesn't contain an escape. - const char *cursor = string->source; - const char *end = string->source + string->length; + const uint8_t *cursor = string->source; + const uint8_t *end = string->source + string->length; // For each escape found in the source string, we will handle it and update // the moving cursor->backslash window. @@ -496,7 +505,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc switch (backslash[1]) { case '\\': case '\'': - dest[dest_length++] = (char) unescape_chars[(unsigned char) backslash[1]]; + dest[dest_length++] = unescape_chars[backslash[1]]; cursor = backslash + 2; break; default: @@ -510,7 +519,13 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // This is the only type of unescaping left. In this case we need to // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - cursor = unescape(parser, dest, &dest_length, backslash, end, YP_UNESCAPE_FLAG_NONE, true); + + uint8_t flags = YP_UNESCAPE_FLAG_NONE; + if (expect_single_codepoint) { + flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; + } + + cursor = unescape(parser, dest, &dest_length, backslash, end, flags, &parser->error_list); break; } @@ -538,13 +553,27 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc yp_string_owned_init(string, allocated, dest_length + ((size_t) (end - cursor))); } +YP_EXPORTED_FUNCTION void +yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { + yp_unescape_manipulate_string_or_char_literal(parser, string, unescape_type, false); +} + +void +yp_unescape_manipulate_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { + yp_unescape_manipulate_string_or_char_literal(parser, string, unescape_type, true); +} + // This function is similar to yp_unescape_manipulate_string, except it doesn't // actually perform any string manipulations. Instead, it calculates how long // the unescaped character is, and returns that value size_t -yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { +yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { assert(unescape_type != YP_UNESCAPE_NONE); + if (backslash + 1 >= parser->end) { + return 0; + } + switch (backslash[1]) { case '\\': case '\'': @@ -558,11 +587,12 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_ // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - unsigned char flags = YP_UNESCAPE_FLAG_NONE; - if (expect_single_codepoint) + uint8_t flags = YP_UNESCAPE_FLAG_NONE; + if (expect_single_codepoint) { flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; + } - const char *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, false); + const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, NULL); assert(cursor > backslash); return (size_t) (cursor - backslash); @@ -574,7 +604,7 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_ // string, a type of unescaping, and a pointer to a result string. It returns a // boolean indicating whether or not the unescaping was successful. YP_EXPORTED_FUNCTION bool -yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { +yp_unescape_string(const uint8_t *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { yp_parser_t parser; yp_parser_init(&parser, start, length, NULL); diff --git a/yarp/unescape.h b/yarp/unescape.h index 30c433febd525f..fb0df9fcb08145 100644 --- a/yarp/unescape.h +++ b/yarp/unescape.h @@ -29,16 +29,16 @@ typedef enum { YP_UNESCAPE_ALL } yp_unescape_type_t; -// Unescape the contents of the given token into the given string using the -// given unescape mode. +// Unescape the contents of the given token into the given string using the given unescape mode. YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type); +void yp_unescape_manipulate_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type); // Accepts a source string and a type of unescaping and returns the unescaped version. // The caller must yp_string_free(result); after calling this function. -YP_EXPORTED_FUNCTION bool yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result); +YP_EXPORTED_FUNCTION bool yp_unescape_string(const uint8_t *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result); // Returns the number of bytes that encompass the first escape sequence in the // given string. -size_t yp_unescape_calculate_difference(yp_parser_t *parser, const char *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint); +size_t yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint); #endif diff --git a/yarp/util/yp_buffer.c b/yarp/util/yp_buffer.c index c9f06ae1c6dc0e..15cdef74f88073 100644 --- a/yarp/util/yp_buffer.c +++ b/yarp/util/yp_buffer.c @@ -63,8 +63,13 @@ yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length) { // Append a string to the buffer. void yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length) { - const void *source = value; - yp_buffer_append(buffer, source, length); + yp_buffer_append(buffer, value, length); +} + +// Append a list of bytes to the buffer. +void +yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length) { + yp_buffer_append(buffer, (const char *) value, length); } // Append a single byte to the buffer. diff --git a/yarp/util/yp_buffer.h b/yarp/util/yp_buffer.h index 095f62a833c565..c388e8d5ce0000 100644 --- a/yarp/util/yp_buffer.h +++ b/yarp/util/yp_buffer.h @@ -36,6 +36,9 @@ void yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length); // Append a string to the buffer. void yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length); +// Append a list of bytes to the buffer. +void yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length); + // Append a single byte to the buffer. void yp_buffer_append_u8(yp_buffer_t *buffer, uint8_t value); diff --git a/yarp/util/yp_char.c b/yarp/util/yp_char.c index d27a04104ec62e..e9f1ef45c20997 100644 --- a/yarp/util/yp_char.c +++ b/yarp/util/yp_char.c @@ -13,8 +13,8 @@ #define YP_NUMBER_BIT_HEXADECIMAL_DIGIT (1 << 6) #define YP_NUMBER_BIT_HEXADECIMAL_NUMBER (1 << 7) -static const unsigned char yp_char_table[256] = { -//0 1 2 3 4 5 6 7 8 9 A B C D E F +static const uint8_t yp_byte_table[256] = { +// 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x @@ -33,7 +33,7 @@ static const unsigned char yp_char_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx }; -static const unsigned char yp_number_table[256] = { +static const uint8_t yp_number_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 1x @@ -54,20 +54,20 @@ static const unsigned char yp_number_table[256] = { }; static inline size_t -yp_strspn_char_kind(const char *string, ptrdiff_t length, unsigned char kind) { +yp_strspn_char_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_char_table[(unsigned char) string[size]] & kind)) size++; + while (size < maximum && (yp_byte_table[string[size]] & kind)) size++; return size; } // Returns the number of characters at the start of the string that are // whitespace. Disallows searching past the given maximum number of characters. size_t -yp_strspn_whitespace(const char *string, ptrdiff_t length) { +yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_WHITESPACE); } @@ -75,13 +75,13 @@ yp_strspn_whitespace(const char *string, ptrdiff_t length) { // whitespace while also tracking the location of each newline. Disallows // searching past the given maximum number of characters. size_t -yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) { +yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_char_table[(unsigned char) string[size]] & YP_CHAR_BIT_WHITESPACE)) { + while (size < maximum && (yp_byte_table[string[size]] & YP_CHAR_BIT_WHITESPACE)) { if (string[size] == '\n') { if (stop_at_newline) { return size + 1; @@ -100,42 +100,42 @@ yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_l // Returns the number of characters at the start of the string that are inline // whitespace. Disallows searching past the given maximum number of characters. size_t -yp_strspn_inline_whitespace(const char *string, ptrdiff_t length) { +yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_INLINE_WHITESPACE); } // Returns the number of characters at the start of the string that are regexp // options. Disallows searching past the given maximum number of characters. size_t -yp_strspn_regexp_option(const char *string, ptrdiff_t length) { +yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_REGEXP_OPTION); } static inline bool -yp_char_is_char_kind(const char c, unsigned char kind) { - return (yp_char_table[(unsigned char) c] & kind) != 0; +yp_char_is_char_kind(const uint8_t b, uint8_t kind) { + return (yp_byte_table[b] & kind) != 0; } // Returns true if the given character is a whitespace character. bool -yp_char_is_whitespace(const char c) { - return yp_char_is_char_kind(c, YP_CHAR_BIT_WHITESPACE); +yp_char_is_whitespace(const uint8_t b) { + return yp_char_is_char_kind(b, YP_CHAR_BIT_WHITESPACE); } // Returns true if the given character is an inline whitespace character. bool -yp_char_is_inline_whitespace(const char c) { - return yp_char_is_char_kind(c, YP_CHAR_BIT_INLINE_WHITESPACE); +yp_char_is_inline_whitespace(const uint8_t b) { + return yp_char_is_char_kind(b, YP_CHAR_BIT_INLINE_WHITESPACE); } static inline size_t -yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind) { +yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_number_table[(unsigned char) string[size]] & kind)) size++; + while (size < maximum && (yp_number_table[string[size]] & kind)) size++; return size; } @@ -143,7 +143,7 @@ yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind) // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_binary_number(const char *string, ptrdiff_t length) { +yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_BINARY_NUMBER); } @@ -151,14 +151,14 @@ yp_strspn_binary_number(const char *string, ptrdiff_t length) { // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_octal_number(const char *string, ptrdiff_t length) { +yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_OCTAL_NUMBER); } // Returns the number of characters at the start of the string that are decimal // digits. Disallows searching past the given maximum number of characters. size_t -yp_strspn_decimal_digit(const char *string, ptrdiff_t length) { +yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_DIGIT); } @@ -166,7 +166,7 @@ yp_strspn_decimal_digit(const char *string, ptrdiff_t length) { // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_decimal_number(const char *string, ptrdiff_t length) { +yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_NUMBER); } @@ -174,7 +174,7 @@ yp_strspn_decimal_number(const char *string, ptrdiff_t length) { // hexadecimal digits. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) { +yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); } @@ -182,37 +182,37 @@ yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) { // hexadecimal digits or underscores. Disallows searching past the given maximum // number of characters. size_t -yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length) { +yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_NUMBER); } static inline bool -yp_char_is_number_kind(const char c, unsigned char kind) { - return (yp_number_table[(unsigned char) c] & kind) != 0; +yp_char_is_number_kind(const uint8_t b, uint8_t kind) { + return (yp_number_table[b] & kind) != 0; } // Returns true if the given character is a binary digit. bool -yp_char_is_binary_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_BINARY_DIGIT); +yp_char_is_binary_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_BINARY_DIGIT); } // Returns true if the given character is an octal digit. bool -yp_char_is_octal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_OCTAL_DIGIT); +yp_char_is_octal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_OCTAL_DIGIT); } // Returns true if the given character is a decimal digit. bool -yp_char_is_decimal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_DECIMAL_DIGIT); +yp_char_is_decimal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_DECIMAL_DIGIT); } // Returns true if the given character is a hexadecimal digit. bool -yp_char_is_hexadecimal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); +yp_char_is_hexadecimal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); } #undef YP_CHAR_BIT_WHITESPACE diff --git a/yarp/util/yp_char.h b/yarp/util/yp_char.h index 010d34d6693c56..67ba31d34d687a 100644 --- a/yarp/util/yp_char.h +++ b/yarp/util/yp_char.h @@ -9,67 +9,67 @@ // Returns the number of characters at the start of the string that are // whitespace. Disallows searching past the given maximum number of characters. -size_t yp_strspn_whitespace(const char *string, ptrdiff_t length); +size_t yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // whitespace while also tracking the location of each newline. Disallows // searching past the given maximum number of characters. size_t -yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool); +yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline); // Returns the number of characters at the start of the string that are inline // whitespace. Disallows searching past the given maximum number of characters. -size_t yp_strspn_inline_whitespace(const char *string, ptrdiff_t length); +size_t yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are decimal // digits. Disallows searching past the given maximum number of characters. -size_t yp_strspn_decimal_digit(const char *string, ptrdiff_t length); +size_t yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // hexadecimal digits. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length); +size_t yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are octal // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_octal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are decimal // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_decimal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // hexadecimal digits or underscores. Disallows searching past the given maximum // number of characters. -size_t yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are regexp // options. Disallows searching past the given maximum number of characters. -size_t yp_strspn_regexp_option(const char *string, ptrdiff_t length); +size_t yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are binary // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_binary_number(const char *string, ptrdiff_t length); +size_t yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length); // Returns true if the given character is a whitespace character. -bool yp_char_is_whitespace(const char c); +bool yp_char_is_whitespace(const uint8_t b); // Returns true if the given character is an inline whitespace character. -bool yp_char_is_inline_whitespace(const char c); +bool yp_char_is_inline_whitespace(const uint8_t b); // Returns true if the given character is a binary digit. -bool yp_char_is_binary_digit(const char c); +bool yp_char_is_binary_digit(const uint8_t b); // Returns true if the given character is an octal digit. -bool yp_char_is_octal_digit(const char c); +bool yp_char_is_octal_digit(const uint8_t b); // Returns true if the given character is a decimal digit. -bool yp_char_is_decimal_digit(const char c); +bool yp_char_is_decimal_digit(const uint8_t b); // Returns true if the given character is a hexadecimal digit. -bool yp_char_is_hexadecimal_digit(const char c); +bool yp_char_is_hexadecimal_digit(const uint8_t b); #endif diff --git a/yarp/util/yp_constant_pool.c b/yarp/util/yp_constant_pool.c index fdece2dabb25b0..3ad241a9d1bb54 100644 --- a/yarp/util/yp_constant_pool.c +++ b/yarp/util/yp_constant_pool.c @@ -48,12 +48,12 @@ yp_constant_id_list_free(yp_constant_id_list_t *list) { // A relatively simple hash function (djb2) that is used to hash strings. We are // optimizing here for simplicity and speed. static inline size_t -yp_constant_pool_hash(const char *start, size_t length) { +yp_constant_pool_hash(const uint8_t *start, size_t length) { // This is a prime number used as the initial value for the hash function. size_t value = 5381; for (size_t index = 0; index < length; index++) { - value = ((value << 5) + value) + ((unsigned char) start[index]); + value = ((value << 5) + value) + start[index]; } return value; @@ -109,7 +109,7 @@ yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity) { // Insert a constant into a constant pool. Returns the id of the constant, or 0 // if any potential calls to resize fail. yp_constant_id_t -yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length) { +yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length) { if (pool->size >= (pool->capacity / 4 * 3)) { if (!yp_constant_pool_resize(pool)) return 0; } @@ -122,7 +122,7 @@ yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t leng // If there is a collision, then we need to check if the content is the // same as the content we are trying to insert. If it is, then we can // return the id of the existing constant. - if ((constant->length == length) && strncmp(constant->start, start, length) == 0) { + if ((constant->length == length) && memcmp(constant->start, start, length) == 0) { return pool->constants[index].id; } diff --git a/yarp/util/yp_constant_pool.h b/yarp/util/yp_constant_pool.h index 3726ecc44a28d6..1ac23cf88bbf28 100644 --- a/yarp/util/yp_constant_pool.h +++ b/yarp/util/yp_constant_pool.h @@ -40,7 +40,7 @@ void yp_constant_id_list_free(yp_constant_id_list_t *list); typedef struct { yp_constant_id_t id; - const char *start; + const uint8_t *start; size_t length; size_t hash; } yp_constant_t; @@ -59,7 +59,7 @@ bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity); // Insert a constant into a constant pool. Returns the id of the constant, or 0 // if any potential calls to resize fail. -yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length); +yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length); // Free the memory associated with a constant pool. void yp_constant_pool_free(yp_constant_pool_t *pool); diff --git a/yarp/util/yp_memchr.c b/yarp/util/yp_memchr.c index c323f37a6658e7..af9c14397e0b37 100644 --- a/yarp/util/yp_memchr.c +++ b/yarp/util/yp_memchr.c @@ -8,7 +8,7 @@ void * yp_memchr(const void *memory, int character, size_t number, bool encoding_changed, yp_encoding_t *encoding) { if (encoding_changed && encoding->multibyte && character >= YP_MEMCHR_TRAILING_BYTE_MINIMUM) { - const char *source = (const char *) memory; + const uint8_t *source = (const uint8_t *) memory; size_t index = 0; while (index < number) { diff --git a/yarp/util/yp_newline_list.c b/yarp/util/yp_newline_list.c index de353acf62a869..0a2050df059dd5 100644 --- a/yarp/util/yp_newline_list.c +++ b/yarp/util/yp_newline_list.c @@ -3,7 +3,7 @@ // Initialize a new newline list with the given capacity. Returns true if the // allocation of the offsets succeeds, otherwise returns false. bool -yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity) { +yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity) { list->offsets = (size_t *) calloc(capacity, sizeof(size_t)); if (list->offsets == NULL) return false; @@ -23,7 +23,7 @@ yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity // Append a new offset to the newline list. Returns true if the reallocation of // the offsets succeeds (if one was necessary), otherwise returns false. bool -yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor) { if (list->size == list->capacity) { list->capacity = (list->capacity * 3) / 2; list->offsets = (size_t *) realloc(list->offsets, list->capacity * sizeof(size_t)); @@ -33,6 +33,7 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { assert(*cursor == '\n'); assert(cursor >= list->start); size_t newline_offset = (size_t) (cursor - list->start + 1); + assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]); list->offsets[list->size++] = newline_offset; @@ -41,7 +42,7 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { // Conditionally append a new offset to the newline list, if the value passed in is a newline. bool -yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor) { if (*cursor != '\n') { return true; } @@ -105,7 +106,7 @@ yp_newline_list_line_column_scan(yp_newline_list_t *list, size_t offset) { // list, the line and column of the closest offset less than the given offset // are returned. yp_line_column_t -yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor) { assert(cursor >= list->start); size_t offset = (size_t) (cursor - list->start); yp_line_column_t result; diff --git a/yarp/util/yp_newline_list.h b/yarp/util/yp_newline_list.h index b7c8c1f3aace57..92313050084dc3 100644 --- a/yarp/util/yp_newline_list.h +++ b/yarp/util/yp_newline_list.h @@ -19,7 +19,7 @@ // A list of offsets of newlines in a string. The offsets are assumed to be // sorted/inserted in ascending order. typedef struct { - const char *start; + const uint8_t *start; size_t *offsets; size_t size; @@ -41,19 +41,19 @@ typedef struct { // Initialize a new newline list with the given capacity. Returns true if the // allocation of the offsets succeeds, otherwise returns false. -bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity); +bool yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity); // Append a new offset to the newline list. Returns true if the reallocation of // the offsets succeeds (if one was necessary), otherwise returns false. -bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor); +bool yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor); // Conditionally append a new offset to the newline list, if the value passed in is a newline. -bool yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor); +bool yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor); // Returns the line and column of the given offset. If the offset is not in the // list, the line and column of the closest offset less than the given offset // are returned. -yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor); +yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor); // Free the internal memory allocated for the newline list. void yp_newline_list_free(yp_newline_list_t *list); diff --git a/yarp/util/yp_string.c b/yarp/util/yp_string.c index bdd001d2b0f4f3..9ee25155a3bdfb 100644 --- a/yarp/util/yp_string.c +++ b/yarp/util/yp_string.c @@ -12,18 +12,19 @@ // Initialize a shared string that is based on initial input. void -yp_string_shared_init(yp_string_t *string, const char *start, const char *end) { +yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end) { assert(start <= end); + *string = (yp_string_t) { .type = YP_STRING_SHARED, - .source = (char*) start, + .source = start, .length = (size_t) (end - start) }; } // Initialize an owned string that is responsible for freeing allocated memory. void -yp_string_owned_init(yp_string_t *string, char *source, size_t length) { +yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_OWNED, .source = source, @@ -36,13 +37,13 @@ void yp_string_constant_init(yp_string_t *string, const char *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_CONSTANT, - .source = (char*) source, + .source = (const uint8_t *) source, .length = length }; } static void -yp_string_mapped_init_internal(yp_string_t *string, char *source, size_t length) { +yp_string_mapped_init_internal(yp_string_t *string, uint8_t *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_MAPPED, .source = source, @@ -67,13 +68,13 @@ yp_string_ensure_owned(yp_string_t *string) { if (string->type == YP_STRING_OWNED) return; size_t length = yp_string_length(string); - const char *source = yp_string_source(string); + const uint8_t *source = yp_string_source(string); - char *memory = malloc(length); + uint8_t *memory = malloc(length); if (!memory) return; yp_string_owned_init(string, memory, length); - memcpy(string->source, source, length); + memcpy((void *) string->source, source, length); } // Returns the length associated with the string. @@ -83,7 +84,7 @@ yp_string_length(const yp_string_t *string) { } // Returns the start pointer associated with the string. -YP_EXPORTED_FUNCTION const char * +YP_EXPORTED_FUNCTION const uint8_t * yp_string_source(const yp_string_t *string) { return string->source; } @@ -91,15 +92,16 @@ yp_string_source(const yp_string_t *string) { // Free the associated memory of the given string. YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string) { + void *memory = (void *) string->source; + if (string->type == YP_STRING_OWNED) { - free(string->source); + free(memory); } else if (string->type == YP_STRING_MAPPED && string->length) { - void *memory = (void *) string->source; - #if defined(_WIN32) +#if defined(_WIN32) UnmapViewOfFile(memory); - #else +#else munmap(memory, string->length); - #endif +#endif } } @@ -126,8 +128,8 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { // the source to a constant empty string and return. if (file_size == 0) { CloseHandle(file); - char empty_string[] = ""; - yp_string_mapped_init_internal(string, empty_string, 0); + uint8_t empty[] = ""; + yp_string_mapped_init_internal(string, empty, 0); return true; } @@ -140,7 +142,7 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { } // Map the file into memory. - char *source = (char *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); CloseHandle(mapping); CloseHandle(file); @@ -169,12 +171,12 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { // mmap the file descriptor to virtually get the contents size_t size = (size_t) sb.st_size; - char *source = NULL; + uint8_t *source = NULL; if (size == 0) { close(fd); - char empty_string[] = ""; - yp_string_mapped_init_internal(string, empty_string, 0); + uint8_t empty[] = ""; + yp_string_mapped_init_internal(string, empty, 0); return true; } diff --git a/yarp/util/yp_string.h b/yarp/util/yp_string.h index 8d5a925232df59..bcdf8b66d99c60 100644 --- a/yarp/util/yp_string.h +++ b/yarp/util/yp_string.h @@ -12,17 +12,17 @@ // This struct represents a string value. typedef struct { enum { YP_STRING_SHARED, YP_STRING_OWNED, YP_STRING_CONSTANT, YP_STRING_MAPPED } type; - char *source; + const uint8_t *source; size_t length; } yp_string_t; #define YP_EMPTY_STRING ((yp_string_t) { .type = YP_STRING_CONSTANT, .source = NULL, .length = 0 }) // Initialize a shared string that is based on initial input. -void yp_string_shared_init(yp_string_t *string, const char *start, const char *end); +void yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end); // Initialize an owned string that is responsible for freeing allocated memory. -void yp_string_owned_init(yp_string_t *string, char *source, size_t length); +void yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length); // Initialize a constant string that doesn't own its memory source. void yp_string_constant_init(yp_string_t *string, const char *source, size_t length); @@ -49,7 +49,7 @@ void yp_string_ensure_owned(yp_string_t *string); YP_EXPORTED_FUNCTION size_t yp_string_length(const yp_string_t *string); // Returns the start pointer associated with the string. -YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string); +YP_EXPORTED_FUNCTION const uint8_t * yp_string_source(const yp_string_t *string); // Free the associated memory of the given string. YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string); diff --git a/yarp/util/yp_string_list.c b/yarp/util/yp_string_list.c index 74822729ff2e22..b03a3d259b1d45 100644 --- a/yarp/util/yp_string_list.c +++ b/yarp/util/yp_string_list.c @@ -1,11 +1,5 @@ #include "yarp/util/yp_string_list.h" -// Allocate a new yp_string_list_t. -yp_string_list_t * -yp_string_list_alloc(void) { - return (yp_string_list_t *) malloc(sizeof(yp_string_list_t)); -} - // Initialize a yp_string_list_t with its default values. void yp_string_list_init(yp_string_list_t *string_list) { diff --git a/yarp/util/yp_string_list.h b/yarp/util/yp_string_list.h index ae252eb5d5b3e1..0009a27a60cce4 100644 --- a/yarp/util/yp_string_list.h +++ b/yarp/util/yp_string_list.h @@ -13,9 +13,6 @@ typedef struct { size_t capacity; } yp_string_list_t; -// Allocate a new yp_string_list_t. -yp_string_list_t * yp_string_list_alloc(void); - // Initialize a yp_string_list_t with its default values. YP_EXPORTED_FUNCTION void yp_string_list_init(yp_string_list_t *string_list); diff --git a/yarp/util/yp_strncasecmp.c b/yarp/util/yp_strncasecmp.c index 899bba4eaa633f..1cbaf904f49c73 100644 --- a/yarp/util/yp_strncasecmp.c +++ b/yarp/util/yp_strncasecmp.c @@ -1,18 +1,15 @@ #include #include +#include int -yp_strncasecmp(const char *string1, const char *string2, size_t length) { +yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) { size_t offset = 0; int difference = 0; while (offset < length && string1[offset] != '\0') { if (string2[offset] == '\0') return string1[offset]; - - unsigned char left = (unsigned char) string1[offset]; - unsigned char right = (unsigned char) string2[offset]; - - if ((difference = tolower(left) - tolower(right)) != 0) return difference; + if ((difference = tolower(string1[offset]) - tolower(string2[offset])) != 0) return difference; offset++; } diff --git a/yarp/util/yp_strpbrk.c b/yarp/util/yp_strpbrk.c index 14a032f3f59708..7c0015d2890fcf 100644 --- a/yarp/util/yp_strpbrk.c +++ b/yarp/util/yp_strpbrk.c @@ -1,12 +1,12 @@ #include "yarp/util/yp_strpbrk.h" // This is the slow path that does care about the encoding. -static inline const char * -yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *charset, size_t maximum) { +static inline const uint8_t * +yp_strpbrk_multi_byte(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) { size_t index = 0; while (index < maximum) { - if (strchr(charset, source[index]) != NULL) { + if (strchr((const char *) charset, source[index]) != NULL) { return source + index; } @@ -22,12 +22,12 @@ yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *chars } // This is the fast path that does not care about the encoding. -static inline const char * -yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum) { +static inline const uint8_t * +yp_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t maximum) { size_t index = 0; while (index < maximum) { - if (strchr(charset, source[index]) != NULL) { + if (strchr((const char *) charset, source[index]) != NULL) { return source + index; } @@ -54,8 +54,8 @@ yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum) // characters that are trailing bytes of multi-byte characters. For example, in // Shift-JIS, the backslash character can be a trailing byte. In that case we // need to take a slower path and iterate one multi-byte character at a time. -const char * -yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length) { +const uint8_t * +yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) { if (length <= 0) { return NULL; } else if (parser->encoding_changed && parser->encoding.multibyte) { diff --git a/yarp/util/yp_strpbrk.h b/yarp/util/yp_strpbrk.h index 7a664d5452115f..d0bdd5bec000cc 100644 --- a/yarp/util/yp_strpbrk.h +++ b/yarp/util/yp_strpbrk.h @@ -24,6 +24,6 @@ // characters that are trailing bytes of multi-byte characters. For example, in // Shift-JIS, the backslash character can be a trailing byte. In that case we // need to take a slower path and iterate one multi-byte character at a time. -const char * yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length); +const uint8_t * yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length); #endif diff --git a/yarp/version.h b/yarp/version.h index 179543f54d751b..a364aec2474f03 100644 --- a/yarp/version.h +++ b/yarp/version.h @@ -1,4 +1,4 @@ #define YP_VERSION_MAJOR 0 -#define YP_VERSION_MINOR 8 +#define YP_VERSION_MINOR 9 #define YP_VERSION_PATCH 0 -#define YP_VERSION "0.8.0" +#define YP_VERSION "0.9.0" diff --git a/yarp/yarp.c b/yarp/yarp.c index 012f8136e56c05..7e098daa48322c 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1,5 +1,4 @@ #include "yarp.h" -#include "yarp/version.h" // The YARP version and the serialization format. const char * @@ -162,14 +161,18 @@ debug_token(yp_token_t * token) { #endif +/* Macros for min/max. */ +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) + /******************************************************************************/ /* Lex mode manipulations */ /******************************************************************************/ // Returns the incrementor character that should be used to increment the // nesting count if one is possible. -static inline char -lex_mode_incrementor(const char start) { +static inline uint8_t +lex_mode_incrementor(const uint8_t start) { switch (start) { case '(': case '[': @@ -183,8 +186,8 @@ lex_mode_incrementor(const char start) { // Returns the matching character that should be used to terminate a list // beginning with the given character. -static inline char -lex_mode_terminator(const char start) { +static inline uint8_t +lex_mode_terminator(const uint8_t start) { switch (start) { case '(': return ')'; @@ -222,9 +225,9 @@ lex_mode_push(yp_parser_t *parser, yp_lex_mode_t lex_mode) { // Push on a new list lex mode. static inline bool -lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { - char incrementor = lex_mode_incrementor(delimiter); - char terminator = lex_mode_terminator(delimiter); +lex_mode_push_list(yp_parser_t *parser, bool interpolation, uint8_t delimiter) { + uint8_t incrementor = lex_mode_incrementor(delimiter); + uint8_t terminator = lex_mode_terminator(delimiter); yp_lex_mode_t lex_mode = { .mode = YP_LEX_LIST, @@ -238,7 +241,7 @@ lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { // These are the places where we need to split up the content of the list. // We'll use strpbrk to find the first of these characters. - char *breakpoints = lex_mode.as.list.breakpoints; + uint8_t *breakpoints = lex_mode.as.list.breakpoints; memcpy(breakpoints, "\\ \t\f\r\v\n\0\0\0", sizeof(lex_mode.as.list.breakpoints)); // Now we'll add the terminator to the list of breakpoints. @@ -261,7 +264,7 @@ lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { // Push on a new regexp lex mode. static inline bool -lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { +lex_mode_push_regexp(yp_parser_t *parser, uint8_t incrementor, uint8_t terminator) { yp_lex_mode_t lex_mode = { .mode = YP_LEX_REGEXP, .as.regexp = { @@ -274,7 +277,7 @@ lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { // These are the places where we need to split up the content of the // regular expression. We'll use strpbrk to find the first of these // characters. - char *breakpoints = lex_mode.as.regexp.breakpoints; + uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; memcpy(breakpoints, "\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); // First we'll add the terminator. @@ -290,7 +293,7 @@ lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { // Push on a new string lex mode. static inline bool -lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed, char incrementor, char terminator) { +lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed, uint8_t incrementor, uint8_t terminator) { yp_lex_mode_t lex_mode = { .mode = YP_LEX_STRING, .as.string = { @@ -304,7 +307,7 @@ lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. - char *breakpoints = lex_mode.as.string.breakpoints; + uint8_t *breakpoints = lex_mode.as.string.breakpoints; memcpy(breakpoints, "\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); // Now add in the terminator. @@ -381,6 +384,9 @@ lex_state_arg_p(yp_parser_t *parser) { static inline bool lex_state_spcarg_p(yp_parser_t *parser, bool space_seen) { + if (parser->current.end >= parser->end) { + return false; + } return lex_state_arg_p(parser) && space_seen && !yp_char_is_whitespace(*parser->current.end); } @@ -421,7 +427,7 @@ debug_lex_state_set(yp_parser_t *parser, yp_lex_state_t state, char const * call // Retrieve the constant pool id for the given location. static inline yp_constant_id_t -yp_parser_constant_id_location(yp_parser_t *parser, const char *start, const char *end) { +yp_parser_constant_id_location(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { return yp_constant_pool_insert(&parser->constant_pool, start, (size_t) (end - start)); } @@ -607,13 +613,45 @@ yp_scope_node_init(yp_node_t *node, yp_scope_node_t *scope) { /* Node creation functions */ /******************************************************************************/ +// Parse the decimal number represented by the range of bytes. returns +// UINT32_MAX if the number fails to parse. This function assumes that the range +// of bytes has already been validated to contain only decimal digits. +static uint32_t +parse_decimal_number(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { + ptrdiff_t diff = end - start; + assert(diff > 0 && ((unsigned long) diff < SIZE_MAX)); + size_t length = (size_t) diff; + + char *digits = calloc(length + 1, sizeof(char)); + memcpy(digits, start, length); + digits[length] = '\0'; + + char *endptr; + errno = 0; + unsigned long value = strtoul(digits, &endptr, 10); + + if ((digits == endptr) || (*endptr != '\0') || (errno == ERANGE)) { + yp_diagnostic_list_append(&parser->error_list, start, end, "invalid decimal number"); + value = UINT32_MAX; + } + + free(digits); + + if (value > UINT32_MAX) { + yp_diagnostic_list_append(&parser->error_list, start, end, "invalid decimal number"); + value = UINT32_MAX; + } + + return (uint32_t) value; +} + // Parse out the options for a regular expression. static inline yp_node_flags_t yp_regular_expression_flags_create(const yp_token_t *closing) { yp_node_flags_t flags = 0; if (closing->type == YP_TOKEN_REGEXP_END) { - for (const char *flag = closing->start + 1; flag < closing->end; flag++) { + for (const uint8_t *flag = closing->start + 1; flag < closing->end; flag++) { switch (*flag) { case 'i': flags |= YP_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE; break; case 'm': flags |= YP_REGULAR_EXPRESSION_FLAGS_MULTI_LINE; break; @@ -655,7 +693,7 @@ yp_alloc_node(YP_ATTRIBUTE_UNUSED yp_parser_t *parser, size_t size) { // Allocate a new MissingNode node. static yp_missing_node_t * -yp_missing_node_create(yp_parser_t *parser, const char *start, const char *end) { +yp_missing_node_create(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_missing_node_t *node = YP_ALLOC_NODE(parser, yp_missing_node_t); *node = (yp_missing_node_t) {{ .type = YP_NODE_MISSING_NODE, .location = { .start = start, .end = end } }}; return node; @@ -725,27 +763,6 @@ yp_and_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *opera return node; } -// Allocate and initialize a new AndWriteNode. -static yp_and_write_node_t * -yp_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_and_write_node_t); - - *node = (yp_and_write_node_t) { - { - .type = YP_NODE_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate an initialize a new arguments node. static yp_arguments_node_t * yp_arguments_node_create(yp_parser_t *parser) { @@ -945,7 +962,7 @@ yp_array_pattern_node_requireds_append(yp_array_pattern_node_t *node, yp_node_t static yp_assoc_node_t * yp_assoc_node_create(yp_parser_t *parser, yp_node_t *key, const yp_token_t *operator, yp_node_t *value) { yp_assoc_node_t *node = YP_ALLOC_NODE(parser, yp_assoc_node_t); - const char *end; + const uint8_t *end; if (value != NULL) { end = value->location.end; @@ -1129,7 +1146,7 @@ static yp_block_parameters_node_t * yp_block_parameters_node_create(yp_parser_t *parser, yp_parameters_node_t *parameters, const yp_token_t *opening) { yp_block_parameters_node_t *node = YP_ALLOC_NODE(parser, yp_block_parameters_node_t); - const char *start; + const uint8_t *start; if (opening->type != YP_TOKEN_NOT_PROVIDED) { start = opening->start; } else if (parameters != NULL) { @@ -1138,7 +1155,7 @@ yp_block_parameters_node_create(yp_parser_t *parser, yp_parameters_node_t *param start = NULL; } - const char *end; + const uint8_t *end; if (parameters != NULL) { end = parameters->base.location.end; } else if (opening->type != YP_TOKEN_NOT_PROVIDED) { @@ -1218,7 +1235,7 @@ yp_call_node_create(yp_parser_t *parser) { }, .receiver = NULL, .operator_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, - .message_loc = YP_LOCATION_NULL_VALUE(parser), + .message_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .opening_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .arguments = NULL, .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, @@ -1259,8 +1276,8 @@ static yp_call_node_t * yp_call_node_binary_create(yp_parser_t *parser, yp_node_t *receiver, yp_token_t *operator, yp_node_t *argument) { yp_call_node_t *node = yp_call_node_create(parser); - node->base.location.start = receiver->location.start; - node->base.location.end = argument->location.end; + node->base.location.start = MIN(receiver->location.start, argument->location.start); + node->base.location.end = MAX(receiver->location.end, argument->location.end); node->receiver = receiver; node->message_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator); @@ -1456,7 +1473,7 @@ yp_call_operator_write_node_create(yp_parser_t *parser, yp_call_node_t *target, .target = target, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator_id = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -1575,18 +1592,95 @@ yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const y return node; } +// Allocate and initialize a new ClassVariableAndWriteNode node. +static yp_class_variable_and_write_node_t * +yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_class_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_and_write_node_t); + + *node = (yp_class_variable_and_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ClassVariableOperatorWriteNode node. +static yp_class_variable_operator_write_node_t * +yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_class_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_write_node_t); + + *node = (yp_class_variable_operator_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ClassVariableOrWriteNode node. +static yp_class_variable_or_write_node_t * +yp_class_variable_or_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_class_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_or_write_node_t); + + *node = (yp_class_variable_or_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ClassVariableReadNode node. static yp_class_variable_read_node_t * yp_class_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { assert(token->type == YP_TOKEN_CLASS_VARIABLE); yp_class_variable_read_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_read_node_t); - *node = (yp_class_variable_read_node_t) {{ .type = YP_NODE_CLASS_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }}; + + *node = (yp_class_variable_read_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_READ_NODE, + .location = YP_LOCATION_TOKEN_VALUE(token) + }, + .name = yp_parser_constant_id_location(parser, token->start, token->end) + }; + return node; } // Initialize a new ClassVariableWriteNode node from a ClassVariableRead node. static yp_class_variable_write_node_t * -yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp_class_variable_read_node_t *read_node, yp_token_t *operator, yp_node_t *value) { +yp_class_variable_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *read_node, yp_token_t *operator, yp_node_t *value) { yp_class_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_write_node_t); *node = (yp_class_variable_write_node_t) { @@ -1594,10 +1688,11 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp .type = YP_NODE_CLASS_VARIABLE_WRITE_NODE, .location = { .start = read_node->base.location.start, - .end = value != NULL ? value->location.end : read_node->base.location.end + .end = value->location.end }, }, - .name_loc = YP_LOCATION_NODE_VALUE((yp_node_t *)read_node), + .name = read_node->name, + .name_loc = YP_LOCATION_NODE_VALUE((yp_node_t *) read_node), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -1605,6 +1700,72 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp return node; } +// Allocate and initialize a new ConstantPathAndWriteNode node. +static yp_constant_path_and_write_node_t * +yp_constant_path_and_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_constant_path_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_and_write_node_t); + + *node = (yp_constant_path_and_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_AND_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ConstantPathOperatorWriteNode node. +static yp_constant_path_operator_write_node_t * +yp_constant_path_operator_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_constant_path_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_operator_write_node_t); + + *node = (yp_constant_path_operator_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_OPERATOR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ConstantPathOrWriteNode node. +static yp_constant_path_or_write_node_t * +yp_constant_path_or_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_constant_path_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_or_write_node_t); + + *node = (yp_constant_path_or_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_OR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ConstantPathNode node. static yp_constant_path_node_t * yp_constant_path_node_create(yp_parser_t *parser, yp_node_t *parent, const yp_token_t *delimiter, yp_node_t *child) { @@ -1636,7 +1797,7 @@ yp_constant_path_write_node_create(yp_parser_t *parser, yp_constant_path_node_t .type = YP_NODE_CONSTANT_PATH_WRITE_NODE, .location = { .start = target->base.location.start, - .end = (value == NULL ? target->base.location.end : value->location.end) + .end = value->location.end }, }, .target = target, @@ -1647,6 +1808,74 @@ yp_constant_path_write_node_create(yp_parser_t *parser, yp_constant_path_node_t return node; } +// Allocate and initialize a new ConstantAndWriteNode node. +static yp_constant_and_write_node_t * +yp_constant_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_constant_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_and_write_node_t); + + *node = (yp_constant_and_write_node_t) { + { + .type = YP_NODE_CONSTANT_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ConstantOperatorWriteNode node. +static yp_constant_operator_write_node_t * +yp_constant_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_constant_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_operator_write_node_t); + + *node = (yp_constant_operator_write_node_t) { + { + .type = YP_NODE_CONSTANT_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ConstantOrWriteNode node. +static yp_constant_or_write_node_t * +yp_constant_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_constant_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_or_write_node_t); + + *node = (yp_constant_or_write_node_t) { + { + .type = YP_NODE_CONSTANT_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ConstantReadNode node. static yp_constant_read_node_t * yp_constant_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -1667,7 +1896,7 @@ yp_constant_write_node_create(yp_parser_t *parser, yp_location_t *name_loc, cons .type = YP_NODE_CONSTANT_WRITE_NODE, .location = { .start = name_loc->start, - .end = value != NULL ? value->location.end : name_loc->end + .end = value->location.end }, }, .name_loc = *name_loc, @@ -1695,7 +1924,7 @@ yp_def_node_create( const yp_token_t *end_keyword ) { yp_def_node_t *node = YP_ALLOC_NODE(parser, yp_def_node_t); - const char *end; + const uint8_t *end; if (end_keyword->type == YP_TOKEN_NOT_PROVIDED) { end = body->location.end; @@ -1750,7 +1979,7 @@ yp_defined_node_create(yp_parser_t *parser, const yp_token_t *lparen, yp_node_t static yp_else_node_t * yp_else_node_create(yp_parser_t *parser, const yp_token_t *else_keyword, yp_statements_node_t *statements, const yp_token_t *end_keyword) { yp_else_node_t *node = YP_ALLOC_NODE(parser, yp_else_node_t); - const char *end = NULL; + const uint8_t *end = NULL; if ((end_keyword->type == YP_TOKEN_NOT_PROVIDED) && (statements != NULL)) { end = statements->base.location.end; } else { @@ -2081,6 +2310,74 @@ yp_hash_pattern_node_node_list_create(yp_parser_t *parser, yp_node_list_t *assoc return node; } +// Allocate and initialize a new GlobalVariableAndWriteNode node. +static yp_global_variable_and_write_node_t * +yp_global_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_global_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_and_write_node_t); + + *node = (yp_global_variable_and_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new GlobalVariableOperatorWriteNode node. +static yp_global_variable_operator_write_node_t * +yp_global_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_global_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_operator_write_node_t); + + *node = (yp_global_variable_operator_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new GlobalVariableOrWriteNode node. +static yp_global_variable_or_write_node_t * +yp_global_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_global_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_or_write_node_t); + + *node = (yp_global_variable_or_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate a new GlobalVariableReadNode node. static yp_global_variable_read_node_t * yp_global_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -2106,7 +2403,7 @@ yp_global_variable_write_node_create(yp_parser_t *parser, const yp_location_t *n .type = YP_NODE_GLOBAL_VARIABLE_WRITE_NODE, .location = { .start = name_loc->start, - .end = (value == NULL ? name_loc->end : value->location.end) + .end = value->location.end }, }, .name_loc = *name_loc, @@ -2162,7 +2459,7 @@ yp_if_node_create(yp_parser_t *parser, yp_flip_flop(predicate); yp_if_node_t *node = YP_ALLOC_NODE(parser, yp_if_node_t); - const char *end; + const uint8_t *end; if (end_keyword->type != YP_TOKEN_NOT_PROVIDED) { end = end_keyword->end; } else if (consequent != NULL) { @@ -2345,7 +2642,7 @@ static yp_in_node_t * yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t *statements, const yp_token_t *in_keyword, const yp_token_t *then_keyword) { yp_in_node_t *node = YP_ALLOC_NODE(parser, yp_in_node_t); - const char *end; + const uint8_t *end; if (statements != NULL) { end = statements->base.location.end; } else if (then_keyword->type != YP_TOKEN_NOT_PROVIDED) { @@ -2371,15 +2668,88 @@ yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t return node; } +// Allocate and initialize a new InstanceVariableAndWriteNode node. +static yp_instance_variable_and_write_node_t * +yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_instance_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_and_write_node_t); + + *node = (yp_instance_variable_and_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new InstanceVariableOperatorWriteNode node. +static yp_instance_variable_operator_write_node_t * +yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_instance_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_write_node_t); + + *node = (yp_instance_variable_operator_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new InstanceVariableOrWriteNode node. +static yp_instance_variable_or_write_node_t * +yp_instance_variable_or_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_instance_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_or_write_node_t); + + *node = (yp_instance_variable_or_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .name = target->name, + .name_loc = target->base.location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new InstanceVariableReadNode node. static yp_instance_variable_read_node_t * yp_instance_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { assert(token->type == YP_TOKEN_INSTANCE_VARIABLE); yp_instance_variable_read_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_read_node_t); - *node = (yp_instance_variable_read_node_t) {{ - .type = YP_NODE_INSTANCE_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) - }}; + *node = (yp_instance_variable_read_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_READ_NODE, + .location = YP_LOCATION_TOKEN_VALUE(token) + }, + .name = yp_parser_constant_id_location(parser, token->start, token->end) + }; return node; } @@ -2393,9 +2763,10 @@ yp_instance_variable_write_node_create(yp_parser_t *parser, yp_instance_variable .type = YP_NODE_INSTANCE_VARIABLE_WRITE_NODE, .location = { .start = read_node->base.location.start, - .end = value == NULL ? read_node->base.location.end : value->location.end + .end = value->location.end } }, + .name = read_node->name, .name_loc = YP_LOCATION_NODE_BASE_VALUE(read_node), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value @@ -2427,8 +2798,13 @@ yp_interpolated_regular_expression_node_create(yp_parser_t *parser, const yp_tok static inline void yp_interpolated_regular_expression_node_append(yp_interpolated_regular_expression_node_t *node, yp_node_t *part) { + if (node->base.location.start > part->location.start) { + node->base.location.start = part->location.start; + } + if (node->base.location.end < part->location.end) { + node->base.location.end = part->location.end; + } yp_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; } static inline void @@ -2500,17 +2876,12 @@ yp_interpolated_symbol_node_create(yp_parser_t *parser, const yp_token_t *openin static inline void yp_interpolated_symbol_node_append(yp_interpolated_symbol_node_t *node, yp_node_t *part) { - yp_node_list_append(&node->parts, part); - if (!node->base.location.start) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { node->base.location.start = part->location.start; } - node->base.location.end = part->location.end; -} -static inline void -yp_interpolated_symbol_node_closing_set(yp_interpolated_symbol_node_t *node, const yp_token_t *closing) { - node->closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; + yp_node_list_append(&node->parts, part); + node->base.location.end = part->location.end; } // Allocate a new InterpolatedXStringNode node. @@ -2620,10 +2991,11 @@ static yp_lambda_node_t * yp_lambda_node_create( yp_parser_t *parser, yp_constant_id_list_t *locals, + const yp_token_t *operator, const yp_token_t *opening, + const yp_token_t *closing, yp_block_parameters_node_t *parameters, - yp_node_t *body, - const yp_token_t *closing + yp_node_t *body ) { yp_lambda_node_t *node = YP_ALLOC_NODE(parser, yp_lambda_node_t); @@ -2631,12 +3003,14 @@ yp_lambda_node_create( { .type = YP_NODE_LAMBDA_NODE, .location = { - .start = opening->start, + .start = operator->start, .end = closing->end }, }, .locals = *locals, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), + .closing_loc = YP_LOCATION_TOKEN_VALUE(closing), .parameters = parameters, .body = body }; @@ -2644,6 +3018,80 @@ yp_lambda_node_create( return node; } +// Allocate and initialize a new LocalVariableAndWriteNode node. +static yp_local_variable_and_write_node_t * +yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { + assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_local_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_and_write_node_t); + + *node = (yp_local_variable_and_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .name = name, + .depth = depth + }; + + return node; +} + +// Allocate and initialize a new LocalVariableOperatorWriteNode node. +static yp_local_variable_operator_write_node_t * +yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { + yp_local_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_write_node_t); + + *node = (yp_local_variable_operator_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .name = name, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), + .depth = depth + }; + + return node; +} + +// Allocate and initialize a new LocalVariableOrWriteNode node. +static yp_local_variable_or_write_node_t * +yp_local_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { + assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_local_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_or_write_node_t); + + *node = (yp_local_variable_or_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .name = name, + .depth = depth + }; + + return node; +} + // Allocate a new LocalVariableReadNode node. static yp_local_variable_read_node_t * yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, uint32_t depth) { @@ -2654,7 +3102,7 @@ yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, .type = YP_NODE_LOCAL_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(name) }, - .constant_id = yp_parser_constant_id_token(parser, name), + .name = yp_parser_constant_id_token(parser, name), .depth = depth }; @@ -2663,7 +3111,7 @@ yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, // Allocate and initialize a new LocalVariableWriteNode node. static yp_local_variable_write_node_t * -yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t constant_id, uint32_t depth, yp_node_t *value, const yp_location_t *name_loc, const yp_token_t *operator) { +yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t name, uint32_t depth, yp_node_t *value, const yp_location_t *name_loc, const yp_token_t *operator) { yp_local_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_write_node_t); *node = (yp_local_variable_write_node_t) { @@ -2671,10 +3119,10 @@ yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t consta .type = YP_NODE_LOCAL_VARIABLE_WRITE_NODE, .location = { .start = name_loc->start, - .end = value == NULL ? name_loc->end : value->location.end + .end = value->location.end } }, - .constant_id = constant_id, + .name = name, .depth = depth, .value = value, .name_loc = *name_loc, @@ -2684,21 +3132,18 @@ yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t consta return node; } -// Allocate and initialize a new LocalVariableWriteNode node without an operator or target. -static yp_local_variable_write_node_t * +// Allocate and initialize a new LocalVariableTargetNode node. +static yp_local_variable_target_node_t * yp_local_variable_target_node_create(yp_parser_t *parser, const yp_token_t *name) { - yp_local_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_write_node_t); + yp_local_variable_target_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_target_node_t); - *node = (yp_local_variable_write_node_t) { + *node = (yp_local_variable_target_node_t) { { - .type = YP_NODE_LOCAL_VARIABLE_WRITE_NODE, + .type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE, .location = YP_LOCATION_TOKEN_VALUE(name) }, - .constant_id = yp_parser_constant_id_token(parser, name), - .depth = 0, - .value = NULL, - .name_loc = YP_LOCATION_TOKEN_VALUE(name), - .operator_loc = { .start = NULL, .end = NULL } + .name = yp_parser_constant_id_token(parser, name), + .depth = 0 }; return node; @@ -2779,7 +3224,10 @@ yp_multi_write_node_create(yp_parser_t *parser, const yp_token_t *operator, yp_n *node = (yp_multi_write_node_t) { { .type = YP_NODE_MULTI_WRITE_NODE, - .location = { .start = NULL, .end = NULL }, + .location = { + .start = lparen_loc->start, + .end = value == NULL ? rparen_loc->end : value->location.end + }, }, .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value, @@ -2873,29 +3321,8 @@ yp_numbered_reference_read_node_create(yp_parser_t *parser, const yp_token_t *na { .type = YP_NODE_NUMBERED_REFERENCE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(name), - } - }; - - return node; -} - -// Allocate and initialize a new OperatorWriteNode. -static yp_operator_write_node_t * -yp_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_operator_write_node_t); - - *node = (yp_operator_write_node_t) { - { - .type = YP_NODE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), - .value = value + .number = parse_decimal_number(parser, name->start + 1, name->end) }; return node; @@ -2914,7 +3341,7 @@ yp_optional_parameter_node_create(yp_parser_t *parser, const yp_token_t *name, c .end = value->location.end } }, - .constant_id = yp_parser_constant_id_token(parser, name), + .name = yp_parser_constant_id_token(parser, name), .name_loc = YP_LOCATION_TOKEN_VALUE(name), .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value @@ -2944,27 +3371,6 @@ yp_or_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *operat return node; } -// Allocate and initialize a new OrWriteNode. -static yp_or_write_node_t * -yp_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_or_write_node_t); - - *node = (yp_or_write_node_t) { - { - .type = YP_NODE_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate and initialize a new ParametersNode node. static yp_parameters_node_t * yp_parameters_node_create(yp_parser_t *parser) { @@ -3232,8 +3638,8 @@ yp_regular_expression_node_create(yp_parser_t *parser, const yp_token_t *opening .type = YP_NODE_REGULAR_EXPRESSION_NODE, .flags = yp_regular_expression_flags_create(closing), .location = { - .start = opening->start, - .end = closing->end + .start = MIN(opening->start, closing->start), + .end = MAX(opening->end, closing->end) } }, .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), @@ -3286,7 +3692,7 @@ yp_required_parameter_node_create(yp_parser_t *parser, const yp_token_t *token) .type = YP_NODE_REQUIRED_PARAMETER_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }, - .constant_id = yp_parser_constant_id_token(parser, token) + .name = yp_parser_constant_id_token(parser, token) }; return node; @@ -3537,19 +3943,21 @@ yp_statements_node_body_length(yp_statements_node_t *node) { // Set the location of the given StatementsNode. static void -yp_statements_node_location_set(yp_statements_node_t *node, const char *start, const char *end) { +yp_statements_node_location_set(yp_statements_node_t *node, const uint8_t *start, const uint8_t *end) { node->base.location = (yp_location_t) { .start = start, .end = end }; } // Append a new node to the given StatementsNode node's body. static void yp_statements_node_body_append(yp_statements_node_t *node, yp_node_t *statement) { - if (yp_statements_node_body_length(node) == 0) { + if (yp_statements_node_body_length(node) == 0 || statement->location.start < node->base.location.start) { node->base.location.start = statement->location.start; } + if (statement->location.end > node->base.location.end) { + node->base.location.end = statement->location.end; + } yp_node_list_append(&node->body, statement); - node->base.location.end = statement->location.end; // Every statement gets marked as a place where a newline can occur. statement->flags |= YP_NODE_FLAG_NEWLINE; @@ -3603,7 +4011,7 @@ yp_super_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_argument assert(keyword->type == YP_TOKEN_KEYWORD_SUPER); yp_super_node_t *node = YP_ALLOC_NODE(parser, yp_super_node_t); - const char *end; + const uint8_t *end; if (arguments->block != NULL) { end = arguments->block->base.location.end; } else if (arguments->closing_loc.start != NULL) { @@ -3694,7 +4102,7 @@ yp_symbol_node_label_create(yp_parser_t *parser, const yp_token_t *token) { // Check if the given node is a label in a hash. static bool yp_symbol_node_label_p(yp_node_t *node) { - const char *end = NULL; + const uint8_t *end = NULL; switch (YP_NODE_TYPE(node)) { case YP_NODE_SYMBOL_NODE: @@ -3712,20 +4120,20 @@ yp_symbol_node_label_p(yp_node_t *node) { // Convert the given StringNode node to a SymbolNode node. static yp_symbol_node_t * -yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node) { +yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node, const yp_token_t *opening, const yp_token_t *closing) { yp_symbol_node_t *new_node = YP_ALLOC_NODE(parser, yp_symbol_node_t); *new_node = (yp_symbol_node_t) { { .type = YP_NODE_SYMBOL_NODE, .location = { - .start = node->base.location.start - 2, - .end = node->base.location.end + 1 + .start = opening->start, + .end = closing->end } }, - .opening_loc = node->opening_loc, + .opening_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(opening), .value_loc = node->content_loc, - .closing_loc = node->closing_loc, + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), .unescaped = node->unescaped }; @@ -3802,7 +4210,7 @@ yp_unless_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t yp_flip_flop(predicate); yp_unless_node_t *node = YP_ALLOC_NODE(parser, yp_unless_node_t); - const char *end; + const uint8_t *end; if (statements != NULL) { end = statements->base.location.end; } else { @@ -3864,34 +4272,43 @@ yp_unless_node_end_keyword_loc_set(yp_unless_node_t *node, const yp_token_t *end // Allocate a new UntilNode node. static yp_until_node_t * -yp_until_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { +yp_until_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_token_t *closing, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { yp_until_node_t *node = YP_ALLOC_NODE(parser, yp_until_node_t); - bool has_statements = (statements != NULL) && (statements->body.size != 0); - const char *start = NULL; - if (has_statements && (keyword->start > statements->base.location.start)) { - start = statements->base.location.start; - } else { - start = keyword->start; - } + *node = (yp_until_node_t) { + { + .type = YP_NODE_UNTIL_NODE, + .flags = flags, + .location = { + .start = keyword->start, + .end = closing->end, + }, + }, + .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .predicate = predicate, + .statements = statements + }; - const char *end = NULL; - if (has_statements && (predicate->location.end < statements->base.location.end)) { - end = statements->base.location.end; - } else { - end = predicate->location.end; - } + return node; +} + +// Allocate a new UntilNode node. +static yp_until_node_t * +yp_until_node_modifier_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { + yp_until_node_t *node = YP_ALLOC_NODE(parser, yp_until_node_t); *node = (yp_until_node_t) { { .type = YP_NODE_UNTIL_NODE, .flags = flags, .location = { - .start = start, - .end = end, + .start = statements->base.location.start, + .end = predicate->location.end, }, }, .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .predicate = predicate, .statements = statements }; @@ -3939,34 +4356,43 @@ yp_when_node_statements_set(yp_when_node_t *node, yp_statements_node_t *statemen // Allocate a new WhileNode node. static yp_while_node_t * -yp_while_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { +yp_while_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_token_t *closing, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { yp_while_node_t *node = YP_ALLOC_NODE(parser, yp_while_node_t); - const char *start = NULL; - bool has_statements = (statements != NULL) && (statements->body.size != 0); - if (has_statements && (keyword->start > statements->base.location.start)) { - start = statements->base.location.start; - } else { - start = keyword->start; - } + *node = (yp_while_node_t) { + { + .type = YP_NODE_WHILE_NODE, + .flags = flags, + .location = { + .start = keyword->start, + .end = closing->end + }, + }, + .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .predicate = predicate, + .statements = statements + }; - const char *end = NULL; - if (has_statements && (predicate->location.end < statements->base.location.end)) { - end = statements->base.location.end; - } else { - end = predicate->location.end; - } + return node; +} + +// Allocate a new WhileNode node. +static yp_while_node_t * +yp_while_node_modifier_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { + yp_while_node_t *node = YP_ALLOC_NODE(parser, yp_while_node_t); *node = (yp_while_node_t) { { .type = YP_NODE_WHILE_NODE, .flags = flags, .location = { - .start = start, - .end = end, + .start = statements->base.location.start, + .end = predicate->location.end }, }, .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .predicate = predicate, .statements = statements }; @@ -4001,7 +4427,7 @@ static yp_yield_node_t * yp_yield_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_location_t *lparen_loc, yp_arguments_node_t *arguments, const yp_location_t *rparen_loc) { yp_yield_node_t *node = YP_ALLOC_NODE(parser, yp_yield_node_t); - const char *end; + const uint8_t *end; if (rparen_loc->start != NULL) { end = rparen_loc->end; } else if (arguments != NULL) { @@ -4074,13 +4500,15 @@ yp_parser_local_depth(yp_parser_t *parser, yp_token_t *token) { } // Add a local variable from a location to the current scope. -static void -yp_parser_local_add_location(yp_parser_t *parser, const char *start, const char *end) { +static yp_constant_id_t +yp_parser_local_add_location(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, start, end); if (!yp_constant_id_list_includes(&parser->current_scope->locals, constant_id)) { yp_constant_id_list_append(&parser->current_scope->locals, constant_id); } + + return constant_id; } // Add a local variable from a token to the current scope. @@ -4122,15 +4550,13 @@ yp_parser_scope_pop(yp_parser_t *parser) { // reason we have the encoding_changed boolean to check if we need to go through // the function pointer or can just directly use the UTF-8 functions. static inline size_t -char_is_identifier_start(yp_parser_t *parser, const char *c) { - const unsigned char uc = (unsigned char) *c; - +char_is_identifier_start(yp_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { - return parser->encoding.alpha_char(c, parser->end - c) || (uc == '_') || (uc >= 0x80); - } else if (uc < 0x80) { - return (yp_encoding_unicode_table[uc] & YP_ENCODING_ALPHABETIC_BIT ? 1 : 0) || (uc == '_'); + return parser->encoding.alpha_char(b, parser->end - b) || (*b == '_') || (*b >= 0x80); + } else if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHABETIC_BIT ? 1 : 0) || (*b == '_'); } else { - return (size_t) (yp_encoding_utf_8_alpha_char(c, parser->end - c) || 1u); + return (size_t) (yp_encoding_utf_8_alpha_char(b, parser->end - b) || 1u); } } @@ -4138,15 +4564,13 @@ char_is_identifier_start(yp_parser_t *parser, const char *c) { // the identifiers in a source file once the first character has been found. So // it's important that it be as fast as possible. static inline size_t -char_is_identifier(yp_parser_t *parser, const char *c) { - const unsigned char uc = (unsigned char) *c; - +char_is_identifier(yp_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { - return parser->encoding.alnum_char(c, parser->end - c) || (uc == '_') || (uc >= 0x80); - } else if (uc < 0x80) { - return (yp_encoding_unicode_table[uc] & YP_ENCODING_ALPHANUMERIC_BIT ? 1 : 0) || (uc == '_'); + return parser->encoding.alnum_char(b, parser->end - b) || (*b == '_') || (*b >= 0x80); + } else if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHANUMERIC_BIT ? 1 : 0) || (*b == '_'); } else { - return (size_t) (yp_encoding_utf_8_alnum_char(c, parser->end - c) || 1u); + return (size_t) (yp_encoding_utf_8_alnum_char(b, parser->end - b) || 1u); } } @@ -4168,15 +4592,15 @@ const unsigned int yp_global_name_punctuation_hash[(0x7e - 0x20 + 31) / 32] = { #undef PUNCT static inline bool -char_is_global_name_punctuation(const char c) { - const unsigned int i = (const unsigned int) c; +char_is_global_name_punctuation(const uint8_t b) { + const unsigned int i = (const unsigned int) b; if (i <= 0x20 || 0x7e < i) return false; - return (yp_global_name_punctuation_hash[(i - 0x20) / 32] >> (c % 32)) & 1; + return (yp_global_name_punctuation_hash[(i - 0x20) / 32] >> (i % 32)) & 1; } static inline bool -token_is_numbered_parameter(const char *start, const char *end) { +token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { return (end - start == 2) && (start[0] == '_') && (start[1] != '0') && (yp_char_is_decimal_digit(start[1])); } @@ -4230,8 +4654,8 @@ yp_do_loop_stack_p(yp_parser_t *parser) { // Get the next character in the source starting from +cursor+. If that position // is beyond the end of the source then return '\0'. -static inline char -peek_at(yp_parser_t *parser, const char *cursor) { +static inline uint8_t +peek_at(yp_parser_t *parser, const uint8_t *cursor) { if (cursor < parser->end) { return *cursor; } else { @@ -4242,33 +4666,33 @@ peek_at(yp_parser_t *parser, const char *cursor) { // Get the next character in the source starting from parser->current.end and // adding the given offset. If that position is beyond the end of the source // then return '\0'. -static inline char +static inline uint8_t peek_offset(yp_parser_t *parser, ptrdiff_t offset) { return peek_at(parser, parser->current.end + offset); } // Get the next character in the source starting from parser->current.end. If // that position is beyond the end of the source then return '\0'. -static inline char +static inline uint8_t peek(yp_parser_t *parser) { return peek_at(parser, parser->current.end); } // Get the next string of length len in the source starting from parser->current.end. // If the string extends beyond the end of the source, return the empty string "" -static inline const char* +static inline const uint8_t * peek_string(yp_parser_t *parser, size_t len) { if (parser->current.end + len <= parser->end) { return parser->current.end; } else { - return ""; + return (const uint8_t *) ""; } } // If the character to be read matches the given value, then returns true and // advanced the current pointer. static inline bool -match(yp_parser_t *parser, char value) { +match(yp_parser_t *parser, uint8_t value) { if (peek(parser) == value) { parser->current.end++; return true; @@ -4279,7 +4703,7 @@ match(yp_parser_t *parser, char value) { // Return the length of the line ending string starting at +cursor+, or 0 if it // is not a line ending. This function is intended to be CRLF/LF agnostic. static inline size_t -match_eol_at(yp_parser_t *parser, const char *cursor) { +match_eol_at(yp_parser_t *parser, const uint8_t *cursor) { if (peek_at(parser, cursor) == '\n') { return 1; } @@ -4306,8 +4730,8 @@ match_eol(yp_parser_t *parser) { } // Skip to the next newline character or NUL byte. -static inline const char * -next_newline(const char *cursor, ptrdiff_t length) { +static inline const uint8_t * +next_newline(const uint8_t *cursor, ptrdiff_t length) { assert(length >= 0); // Note that it's okay for us to use memchr here to look for \n because none @@ -4318,17 +4742,17 @@ next_newline(const char *cursor, ptrdiff_t length) { // Find the start of the encoding comment. This is effectively an inlined // version of strnstr with some modifications. -static inline const char * -parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdiff_t remaining) { +static inline const uint8_t * +parser_lex_encoding_comment_start(yp_parser_t *parser, const uint8_t *cursor, ptrdiff_t remaining) { assert(remaining >= 0); size_t length = (size_t) remaining; size_t key_length = strlen("coding:"); if (key_length > length) return NULL; - const char *cursor_limit = cursor + length - key_length + 1; + const uint8_t *cursor_limit = cursor + length - key_length + 1; while ((cursor = yp_memchr(cursor, 'c', (size_t) (cursor_limit - cursor), parser->encoding_changed, &parser->encoding)) != NULL) { - if (strncmp(cursor, "coding", key_length - 1) == 0) { + if (memcmp(cursor, "coding", key_length - 1) == 0) { size_t whitespace_after_coding = yp_strspn_inline_whitespace(cursor + key_length - 1, parser->end - (cursor + key_length - 1)); size_t cur_pos = key_length + whitespace_after_coding; @@ -4347,13 +4771,13 @@ parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdi // actions are necessary for it here. static void parser_lex_encoding_comment(yp_parser_t *parser) { - const char *start = parser->current.start + 1; - const char *end = next_newline(start, parser->end - start); + const uint8_t *start = parser->current.start + 1; + const uint8_t *end = next_newline(start, parser->end - start); if (end == NULL) end = parser->end; // These are the patterns we're going to match to find the encoding comment. // This is definitely not complete or even really correct. - const char *encoding_start = parser_lex_encoding_comment_start(parser, start, end - start); + const uint8_t *encoding_start = parser_lex_encoding_comment_start(parser, start, end - start); // If we didn't find anything that matched our patterns, then return. Note // that this does a _very_ poor job of actually finding the encoding, and @@ -4366,7 +4790,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Now determine the end of the encoding string. This is either the end of // the line, the first whitespace character, or a punctuation mark. - const char *encoding_end = yp_strpbrk(parser, encoding_start, " \t\f\r\v\n;,", end - encoding_start); + const uint8_t *encoding_end = yp_strpbrk(parser, encoding_start, (const uint8_t *) " \t\f\r\v\n;,", end - encoding_start); encoding_end = encoding_end == NULL ? end : encoding_end; // Finally, we can determine the width of the encoding string. @@ -4388,7 +4812,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Extensions like utf-8 can contain extra encoding details like, // utf-8-dos, utf-8-linux, utf-8-mac. We treat these all as utf-8 should // treat any encoding starting utf-8 as utf-8. - if ((encoding_start + 5 <= parser->end) && (yp_strncasecmp(encoding_start, "utf-8", 5) == 0)) { + if ((encoding_start + 5 <= parser->end) && (yp_strncasecmp(encoding_start, (const uint8_t *) "utf-8", 5) == 0)) { // We don't need to do anything here because the default encoding is // already UTF-8. We'll just return. return; @@ -4397,7 +4821,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Next, we're going to loop through each of the encodings that we handle // explicitly. If we found one that we understand, we'll use that value. #define ENCODING(value, prebuilt) \ - if (width == sizeof(value) - 1 && encoding_start + width <= parser->end && yp_strncasecmp(encoding_start, value, width) == 0) { \ + if (width == sizeof(value) - 1 && encoding_start + width <= parser->end && yp_strncasecmp(encoding_start, (const uint8_t *) value, width) == 0) { \ parser->encoding = prebuilt; \ parser->encoding_changed |= true; \ if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser); \ @@ -4537,14 +4961,9 @@ context_push(yp_parser_t *parser, yp_context_t context) { static void context_pop(yp_parser_t *parser) { - if (parser->current_context->prev == NULL) { - free(parser->current_context); - parser->current_context = NULL; - } else { - yp_context_node_t *prev = parser->current_context->prev; - free(parser->current_context); - parser->current_context = prev; - } + yp_context_node_t *prev = parser->current_context->prev; + free(parser->current_context); + parser->current_context = prev; } static bool @@ -4628,7 +5047,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0d1111 is a decimal number case 'd': case 'D': - if (yp_char_is_decimal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_decimal_digit(peek(parser))) { parser->current.end += yp_strspn_decimal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid decimal number."); @@ -4639,7 +5059,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0b1111 is a binary number case 'b': case 'B': - if (yp_char_is_binary_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_binary_digit(peek(parser))) { parser->current.end += yp_strspn_binary_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid binary number."); @@ -4650,7 +5071,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0o1111 is an octal number case 'o': case 'O': - if (yp_char_is_octal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_octal_digit(peek(parser))) { parser->current.end += yp_strspn_octal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid octal number."); @@ -4674,7 +5096,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0x1111 is a hexadecimal number case 'x': case 'X': - if (yp_char_is_hexadecimal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_hexadecimal_digit(peek(parser))) { parser->current.end += yp_strspn_hexadecimal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid hexadecimal number."); @@ -4720,7 +5143,7 @@ lex_numeric(yp_parser_t *parser) { if (parser->current.end < parser->end) { type = lex_numeric_prefix(parser); - const char *end = parser->current.end; + const uint8_t *end = parser->current.end; yp_token_type_t suffix_type = type; if (type == YP_TOKEN_INTEGER) { @@ -4745,8 +5168,8 @@ lex_numeric(yp_parser_t *parser) { } } - const unsigned char uc = (const unsigned char) peek(parser); - if (uc != '\0' && (uc >= 0x80 || ((uc >= 'a' && uc <= 'z') || (uc >= 'A' && uc <= 'Z')) || uc == '_')) { + const uint8_t b = peek(parser); + if (b != '\0' && (b >= 0x80 || ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')) || b == '_')) { parser->current.end = end; } else { type = suffix_type; @@ -4758,6 +5181,11 @@ lex_numeric(yp_parser_t *parser) { static yp_token_type_t lex_global_variable(yp_parser_t *parser) { + if (parser->current.end >= parser->end) { + yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid global variable."); + return YP_TOKEN_GLOBAL_VARIABLE; + } + switch (*parser->current.end) { case '~': // $~: match-data case '*': // $*: argv @@ -4846,7 +5274,7 @@ lex_keyword(yp_parser_t *parser, const char *value, yp_lex_state_t state, yp_tok yp_lex_state_t last_state = parser->lex_state; const size_t vlen = strlen(value); - if (parser->current.start + vlen <= parser->end && strncmp(parser->current.start, value, vlen) == 0) { + if (parser->current.start + vlen <= parser->end && memcmp(parser->current.start, value, vlen) == 0) { if (parser->lex_state & YP_LEX_STATE_FNAME) { lex_state_set(parser, YP_LEX_STATE_ENDFN); } else { @@ -5012,7 +5440,7 @@ current_token_starts_line(yp_parser_t *parser) { // this token type. // static yp_token_type_t -lex_interpolation(yp_parser_t *parser, const char *pound) { +lex_interpolation(yp_parser_t *parser, const uint8_t *pound) { // If there is no content following this #, then we're at the end of // the string and we can safely return string content. if (pound + 1 >= parser->end) { @@ -5033,7 +5461,7 @@ lex_interpolation(yp_parser_t *parser, const char *pound) { // If we're looking at a @ and there's another @, then we'll skip past the // second @. - const char *variable = pound + 2; + const uint8_t *variable = pound + 2; if (*variable == '@' && pound + 3 < parser->end) variable++; if (char_is_identifier_start(parser, variable)) { @@ -5069,7 +5497,7 @@ lex_interpolation(yp_parser_t *parser, const char *pound) { // This is the character that we're going to check to see if it is the // start of an identifier that would indicate that this is a global // variable. - const char *check = pound + 2; + const uint8_t *check = pound + 2; if (pound[2] == '-') { if (pound + 3 >= parser->end) { @@ -5260,7 +5688,7 @@ parser_comment(yp_parser_t *parser, yp_comment_type_t type) { static yp_token_type_t lex_embdoc(yp_parser_t *parser) { // First, lex out the EMBDOC_BEGIN token. - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5283,9 +5711,9 @@ lex_embdoc(yp_parser_t *parser) { // If we've hit the end of the embedded documentation then we'll return that // token here. - if (strncmp(parser->current.end, "=end", 4) == 0 && + if (memcmp(parser->current.end, "=end", 4) == 0 && (parser->current.end + 4 == parser->end || yp_char_is_whitespace(parser->current.end[4]))) { - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5305,7 +5733,7 @@ lex_embdoc(yp_parser_t *parser) { // Otherwise, we'll parse until the end of the line and return a line of // embedded documentation. - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5455,7 +5883,7 @@ parser_lex(yp_parser_t *parser) { LEX(YP_TOKEN_EOF); case '#': { // comments - const char *ending = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *ending = next_newline(parser->current.end, parser->end - parser->current.end); parser->current.end = ending == NULL ? parser->end : ending + 1; parser->current.type = YP_TOKEN_COMMENT; @@ -5524,7 +5952,7 @@ parser_lex(yp_parser_t *parser) { // (either . or &.) that starts the next line. If there is, then this // is going to become an ignored newline and we're going to instead // return the call operator. - const char *next_content = parser->next_start == NULL ? parser->current.end : parser->next_start; + const uint8_t *next_content = parser->next_start == NULL ? parser->current.end : parser->next_start; next_content += yp_strspn_inline_whitespace(next_content, parser->end - next_content); if (next_content < parser->end) { @@ -5535,15 +5963,15 @@ parser_lex(yp_parser_t *parser) { // Otherwise we'll return a regular newline. if (next_content[0] == '#') { // Here we look for a "." or "&." following a "\n". - const char *following = next_newline(next_content, parser->end - next_content); + const uint8_t *following = next_newline(next_content, parser->end - next_content); - while (following && (following < parser->end)) { + while (following && (following + 1 < parser->end)) { following++; following += yp_strspn_inline_whitespace(following, parser->end - following); // If this is not followed by a comment, then we can break out // of this loop. - if (*following != '#') break; + if (peek_at(parser, following) != '#') break; // If there is a comment, then we need to find the end of the // comment and continue searching from there. @@ -5786,7 +6214,7 @@ parser_lex(yp_parser_t *parser) { // = => =~ == === =begin case '=': - if (current_token_starts_line(parser) && strncmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_offset(parser, 5))) { + if (current_token_starts_line(parser) && memcmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_offset(parser, 5))) { yp_token_type_t type = lex_embdoc(parser); if (type == YP_TOKEN_EOF) { @@ -5824,7 +6252,7 @@ parser_lex(yp_parser_t *parser) { !lex_state_end_p(parser) && (!lex_state_p(parser, YP_LEX_STATE_ARG_ANY) || lex_state_p(parser, YP_LEX_STATE_LABELED) || space_seen) ) { - const char *end = parser->current.end; + const uint8_t *end = parser->current.end; yp_heredoc_quote_t quote = YP_HEREDOC_QUOTE_NONE; yp_heredoc_indent_t indent = YP_HEREDOC_INDENT_NONE; @@ -5846,7 +6274,7 @@ parser_lex(yp_parser_t *parser) { quote = YP_HEREDOC_QUOTE_SINGLE; } - const char *ident_start = parser->current.end; + const uint8_t *ident_start = parser->current.end; size_t width = 0; if (parser->current.end >= parser->end) { @@ -5869,7 +6297,7 @@ parser_lex(yp_parser_t *parser) { } size_t ident_length = (size_t) (parser->current.end - ident_start); - if (quote != YP_HEREDOC_QUOTE_NONE && !match(parser, (char) quote)) { + if (quote != YP_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) { // TODO: handle unterminated heredoc } @@ -5885,7 +6313,7 @@ parser_lex(yp_parser_t *parser) { }); if (parser->heredoc_end == NULL) { - const char *body_start = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *body_start = next_newline(parser->current.end, parser->end - parser->current.end); if (body_start == NULL) { // If there is no newline after the heredoc identifier, then @@ -6210,7 +6638,7 @@ parser_lex(yp_parser_t *parser) { LEX(YP_TOKEN_COLON_COLON); } - if (lex_state_end_p(parser) || yp_char_is_whitespace(*parser->current.end) || peek(parser) == '#') { + if (lex_state_end_p(parser) || yp_char_is_whitespace(peek(parser)) || peek(parser) == '#') { lex_state_set(parser, YP_LEX_STATE_BEG); LEX(YP_TOKEN_COLON); } @@ -6451,7 +6879,7 @@ parser_lex(yp_parser_t *parser) { if ( ((parser->current.end - parser->current.start) == 7) && current_token_starts_line(parser) && - (strncmp(parser->current.start, "__END__", 7) == 0) && + (memcmp(parser->current.start, "__END__", 7) == 0) && (parser->current.end == parser->end || match_eol(parser)) ) { @@ -6527,8 +6955,8 @@ parser_lex(yp_parser_t *parser) { // Here we'll get a list of the places where strpbrk should break, // and then find the first one. yp_lex_mode_t *lex_mode = parser->lex_modes.current; - const char *breakpoints = lex_mode->as.list.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = lex_mode->as.list.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit a null byte, skip directly past it. @@ -6576,10 +7004,25 @@ parser_lex(yp_parser_t *parser) { if (*breakpoint == '\\') { yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + continue; + } - // If the result is an escaped newline, then we need to - // track that newline. - yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); + // If the result is an escaped newline ... + if (breakpoint[difference - 1] == '\n') { + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, flush the heredoc and + // continue parsing after heredoc_end. + parser->current.end = breakpoint + difference; + parser_flush_heredoc_end(parser); + LEX(YP_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + } + } breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); continue; @@ -6634,8 +7077,8 @@ parser_lex(yp_parser_t *parser) { // These are the places where we need to split up the content of the // regular expression. We'll use strpbrk to find the first of these // characters. - const char *breakpoints = lex_mode->as.regexp.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = lex_mode->as.regexp.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit a null byte, skip directly past it. @@ -6698,9 +7141,14 @@ parser_lex(yp_parser_t *parser) { // and find the next breakpoint. if (*breakpoint == '\\') { size_t difference = yp_unescape_calculate_difference(parser, breakpoint, YP_UNESCAPE_ALL, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + continue; + } // If the result is an escaped newline ... - if (*(breakpoint + difference - 1) == '\n') { + if (breakpoint[difference - 1] == '\n') { if (parser->heredoc_end) { // ... if we are on the same line as a heredoc, flush the heredoc and // continue parsing after heredoc_end. @@ -6762,8 +7210,8 @@ parser_lex(yp_parser_t *parser) { // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. - const char *breakpoints = parser->lex_modes.current->as.string.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = parser->lex_modes.current->as.string.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit the incrementor, then we'll increment then nesting and @@ -6848,9 +7296,14 @@ parser_lex(yp_parser_t *parser) { // find the next breakpoint. yp_unescape_type_t unescape_type = parser->lex_modes.current->as.string.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + break; + } // If the result is an escaped newline ... - if (*(breakpoint + difference - 1) == '\n') { + if (breakpoint[difference - 1] == '\n') { if (parser->heredoc_end) { // ... if we are on the same line as a heredoc, flush the heredoc and // continue parsing after heredoc_end. @@ -6908,18 +7361,18 @@ parser_lex(yp_parser_t *parser) { // Now let's grab the information about the identifier off of the current // lex mode. - const char *ident_start = parser->lex_modes.current->as.heredoc.ident_start; + const uint8_t *ident_start = parser->lex_modes.current->as.heredoc.ident_start; size_t ident_length = parser->lex_modes.current->as.heredoc.ident_length; // If we are immediately following a newline and we have hit the // terminator, then we need to return the ending of the heredoc. if (current_token_starts_line(parser)) { - const char *start = parser->current.start; + const uint8_t *start = parser->current.start; if (parser->lex_modes.current->as.heredoc.indent != YP_HEREDOC_INDENT_NONE) { start += yp_strspn_inline_whitespace(start, parser->end - start); } - if ((start + ident_length <= parser->end) && (strncmp(start, ident_start, ident_length) == 0)) { + if ((start + ident_length <= parser->end) && (memcmp(start, ident_start, ident_length) == 0)) { bool matched = true; bool at_end = false; @@ -6954,14 +7407,14 @@ parser_lex(yp_parser_t *parser) { // Otherwise we'll be parsing string content. These are the places where // we need to split up the content of the heredoc. We'll use strpbrk to // find the first of these characters. - char breakpoints[] = "\n\\#"; + uint8_t breakpoints[] = "\n\\#"; yp_heredoc_quote_t quote = parser->lex_modes.current->as.heredoc.quote; if (quote == YP_HEREDOC_QUOTE_SINGLE) { breakpoints[2] = '\0'; } - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { switch (*breakpoint) { @@ -6978,7 +7431,7 @@ parser_lex(yp_parser_t *parser) { yp_newline_list_append(&parser->newline_list, breakpoint); - const char *start = breakpoint + 1; + const uint8_t *start = breakpoint + 1; if (parser->lex_modes.current->as.heredoc.indent != YP_HEREDOC_INDENT_NONE) { start += yp_strspn_inline_whitespace(start, parser->end - start); } @@ -6989,7 +7442,7 @@ parser_lex(yp_parser_t *parser) { // again and return the end of the heredoc. if ( (start + ident_length <= parser->end) && - (strncmp(start, ident_start, ident_length) == 0) + (memcmp(start, ident_start, ident_length) == 0) ) { // Heredoc terminators must be followed by a newline, CRLF, or EOF to be valid. if ( @@ -7019,6 +7472,11 @@ parser_lex(yp_parser_t *parser) { } else { yp_unescape_type_t unescape_type = (quote == YP_HEREDOC_QUOTE_SINGLE) ? YP_UNESCAPE_MINIMAL : YP_UNESCAPE_ALL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + break; + } yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); @@ -7089,6 +7547,17 @@ yp_symbol_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *openin return node; } +static yp_string_node_t * +yp_char_literal_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *content, const yp_token_t *closing, yp_unescape_type_t unescape_type) { + yp_string_node_t *node = yp_string_node_create(parser, opening, content, closing); + + assert((content->end - content->start) >= 0); + yp_string_shared_init(&node->unescaped, content->start, content->end); + + yp_unescape_manipulate_char_literal(parser, &node->unescaped, unescape_type); + return node; +} + static yp_string_node_t * yp_string_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *content, const yp_token_t *closing, yp_unescape_type_t unescape_type) { yp_string_node_t *node = yp_string_node_create(parser, opening, content, closing); @@ -7444,27 +7913,162 @@ token_begins_expression_p(yp_token_type_t type) { } } -// Parse an expression with the given binding power that may be optionally -// prefixed by the * operator. -static yp_node_t * -parse_starred_expression(yp_parser_t *parser, yp_binding_power_t binding_power, const char *message) { - if (accept(parser, YP_TOKEN_USTAR)) { - yp_token_t operator = parser->previous; - yp_node_t *expression = parse_expression(parser, binding_power, "Expected expression after `*'."); - return (yp_node_t *) yp_splat_node_create(parser, &operator, expression); - } +// Parse an expression with the given binding power that may be optionally +// prefixed by the * operator. +static yp_node_t * +parse_starred_expression(yp_parser_t *parser, yp_binding_power_t binding_power, const char *message) { + if (accept(parser, YP_TOKEN_USTAR)) { + yp_token_t operator = parser->previous; + yp_node_t *expression = parse_expression(parser, binding_power, "Expected expression after `*'."); + return (yp_node_t *) yp_splat_node_create(parser, &operator, expression); + } + + return parse_expression(parser, binding_power, message); +} + +// Convert the given node into a valid target node. +static yp_node_t * +parse_target(yp_parser_t *parser, yp_node_t *target) { + switch (YP_NODE_TYPE(target)) { + case YP_NODE_MISSING_NODE: + return target; + case YP_NODE_CLASS_VARIABLE_READ_NODE: + assert(sizeof(yp_class_variable_target_node_t) == sizeof(yp_class_variable_read_node_t)); + target->type = YP_NODE_CLASS_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_CONSTANT_PATH_NODE: + assert(sizeof(yp_constant_path_target_node_t) == sizeof(yp_constant_path_node_t)); + target->type = YP_NODE_CONSTANT_PATH_TARGET_NODE; + return target; + case YP_NODE_CONSTANT_READ_NODE: + assert(sizeof(yp_constant_target_node_t) == sizeof(yp_constant_read_node_t)); + target->type = YP_NODE_CONSTANT_TARGET_NODE; + return target; + case YP_NODE_BACK_REFERENCE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_back_reference_read_node_t)); + /* fallthrough */ + case YP_NODE_NUMBERED_REFERENCE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_numbered_reference_read_node_t)); + yp_diagnostic_list_append(&parser->error_list, target->location.start, target->location.end, "Can't set variable"); + /* fallthrough */ + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_global_variable_read_node_t)); + target->type = YP_NODE_GLOBAL_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_LOCAL_VARIABLE_READ_NODE: + assert(sizeof(yp_local_variable_target_node_t) == sizeof(yp_local_variable_read_node_t)); + target->type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + assert(sizeof(yp_instance_variable_target_node_t) == sizeof(yp_instance_variable_read_node_t)); + target->type = YP_NODE_INSTANCE_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_MULTI_WRITE_NODE: + return target; + case YP_NODE_SPLAT_NODE: { + yp_splat_node_t *splat = (yp_splat_node_t *) target; + + if (splat->expression != NULL) { + splat->expression = parse_target(parser, splat->expression); + } + + yp_token_t operator = not_provided(parser); + yp_location_t location = { .start = NULL, .end = NULL }; + + yp_multi_write_node_t *multi_write = yp_multi_write_node_create(parser, &operator, NULL, &location, &location); + yp_multi_write_node_targets_append(multi_write, (yp_node_t *) splat); + + return (yp_node_t *) multi_write; + } + case YP_NODE_CALL_NODE: { + yp_call_node_t *call = (yp_call_node_t *) target; + + // If we have no arguments to the call node and we need this to be a + // target then this is either a method call or a local variable write. + if ( + (call->opening_loc.start == NULL) && + (call->arguments == NULL) && + (call->block == NULL) + ) { + if (call->receiver == NULL) { + // When we get here, we have a local variable write, because it + // was previously marked as a method call but now we have an =. + // This looks like: + // + // foo = 1 + // + // When it was parsed in the prefix position, foo was seen as a + // method call with no receiver and no arguments. Now we have an + // =, so we know it's a local variable write. + const yp_location_t message = call->message_loc; - return parse_expression(parser, binding_power, message); + yp_parser_local_add_location(parser, message.start, message.end); + yp_node_destroy(parser, target); + + const yp_token_t name = { .type = YP_TOKEN_IDENTIFIER, .start = message.start, .end = message.end }; + target = (yp_node_t *) yp_local_variable_read_node_create(parser, &name, 0); + + assert(sizeof(yp_local_variable_target_node_t) == sizeof(yp_local_variable_read_node_t)); + target->type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE; + + if (token_is_numbered_parameter(message.start, message.end)) { + yp_diagnostic_list_append(&parser->error_list, message.start, message.end, "reserved for numbered parameter"); + } + + return target; + } + + // The method name needs to change. If we previously had foo, we now + // need foo=. In this case we'll allocate a new owned string, copy + // the previous method name in, and append an =. + size_t length = yp_string_length(&call->name); + + uint8_t *name = calloc(length + 1, sizeof(uint8_t)); + if (name == NULL) return NULL; + + memcpy(name, yp_string_source(&call->name), length); + name[length] = '='; + + // Now switch the name to the new string. + yp_string_free(&call->name); + yp_string_owned_init(&call->name, name, length + 1); + + return target; + } + + // If there is no call operator and the message is "[]" then this is + // an aref expression, and we can transform it into an aset + // expression. + if ( + (call->operator_loc.start == NULL) && + (call->message_loc.start[0] == '[') && + (call->message_loc.end[-1] == ']') && + (call->block == NULL) + ) { + // Free the previous name and replace it with "[]=". + yp_string_free(&call->name); + yp_string_constant_init(&call->name, "[]=", 3); + return target; + } + } + /* fallthrough */ + default: + // In this case we have a node that we don't know how to convert + // into a target. We need to treat it as an error. For now, we'll + // mark it as an error and just skip right past it. + yp_diagnostic_list_append(&parser->error_list, target->location.start, target->location.end, "Unexpected write target."); + return target; + } } -// Convert the given node into a valid target node. +// Convert the given node into a valid write node. static yp_node_t * -parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_node_t *value) { +parse_write(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_node_t *value) { switch (YP_NODE_TYPE(target)) { case YP_NODE_MISSING_NODE: return target; case YP_NODE_CLASS_VARIABLE_READ_NODE: { - yp_class_variable_write_node_t *write_node = yp_class_variable_read_node_to_class_variable_write_node(parser, (yp_class_variable_read_node_t *) target, operator, value); + yp_class_variable_write_node_t *write_node = yp_class_variable_write_node_create(parser, (yp_class_variable_read_node_t *) target, operator, value); yp_node_destroy(parser, target); return (yp_node_t *) write_node; } @@ -7489,7 +8093,7 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no case YP_NODE_LOCAL_VARIABLE_READ_NODE: { yp_local_variable_read_node_t *local_read = (yp_local_variable_read_node_t *) target; - yp_constant_id_t constant_id = local_read->constant_id; + yp_constant_id_t constant_id = local_read->name; uint32_t depth = local_read->depth; yp_location_t name_loc = target->location; @@ -7506,18 +8110,15 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no yp_multi_write_node_t *multi_write = (yp_multi_write_node_t *) target; yp_multi_write_node_operator_loc_set(multi_write, operator); - if (value != NULL) { - multi_write->value = value; - multi_write->base.location.end = value->location.end; - } - + multi_write->value = value; + multi_write->base.location.end = value->location.end; return (yp_node_t *) multi_write; } case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat = (yp_splat_node_t *) target; if (splat->expression != NULL) { - splat->expression = parse_target(parser, splat->expression, operator, value); + splat->expression = parse_write(parser, splat->expression, operator, value); } yp_location_t location = { .start = NULL, .end = NULL }; @@ -7570,22 +8171,21 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no // method call with no arguments. Now we have an =, so we know it's // a method call with an argument. In this case we will create the // arguments node, parse the argument, and add it to the list. - if (value) { - yp_arguments_node_t *arguments = yp_arguments_node_create(parser); - call->arguments = arguments; - yp_arguments_node_arguments_append(arguments, value); - target->location.end = arguments->base.location.end; - } + yp_arguments_node_t *arguments = yp_arguments_node_create(parser); + call->arguments = arguments; + yp_arguments_node_arguments_append(arguments, value); + target->location.end = arguments->base.location.end; // The method name needs to change. If we previously had foo, we now // need foo=. In this case we'll allocate a new owned string, copy // the previous method name in, and append an =. size_t length = yp_string_length(&call->name); - char *name = calloc(length + 2, sizeof(char)); + uint8_t *name = calloc(length + 1, sizeof(uint8_t)); if (name == NULL) return NULL; - snprintf(name, length + 2, "%.*s=", (int) length, yp_string_source(&call->name)); + memcpy(name, yp_string_source(&call->name), length); + name[length] = '='; // Now switch the name to the new string. yp_string_free(&call->name); @@ -7603,15 +8203,13 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no (call->message_loc.end[-1] == ']') && (call->block == NULL) ) { - if (value != NULL) { - if (call->arguments == NULL) { - call->arguments = yp_arguments_node_create(parser); - } - - yp_arguments_node_arguments_append(call->arguments, value); - target->location.end = value->location.end; + if (call->arguments == NULL) { + call->arguments = yp_arguments_node_create(parser); } + yp_arguments_node_arguments_append(call->arguments, value); + target->location.end = value->location.end; + // Free the previous name and replace it with "[]=". yp_string_free(&call->name); yp_string_constant_init(&call->name, "[]=", 3); @@ -7623,9 +8221,7 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no // syntax error. In this case we'll fall through to our default // handling. We need to free the value that we parsed because there // is no way for us to attach it to the tree at this point. - if (value != NULL) { - yp_node_destroy(parser, value); - } + yp_node_destroy(parser, value); } /* fallthrough */ default: @@ -7653,7 +8249,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b // location that we know requires a multi write, as in the case of a for loop. // In this case we will set up the parsing loop slightly differently. if (first_target != NULL) { - first_target = parse_target(parser, first_target, &operator, NULL); + first_target = parse_target(parser, first_target); if (!match_type_p(parser, YP_TOKEN_COMMA)) { return first_target; @@ -7684,9 +8280,8 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b yp_node_t *name = NULL; if (token_begins_expression_p(parser->current.type)) { - yp_token_t operator = not_provided(parser); name = parse_expression(parser, binding_power, "Expected an expression after '*'."); - name = parse_target(parser, name, &operator, NULL); + name = parse_target(parser, name); } yp_node_t *splat = (yp_node_t *) yp_splat_node_create(parser, &star_operator, name); @@ -7716,6 +8311,8 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b if (YP_NODE_TYPE_P(child_target, YP_NODE_MULTI_WRITE_NODE)) { target = (yp_multi_write_node_t *) child_target; + target->base.location.start = lparen.start; + target->base.location.end = rparen.end; target->lparen_loc = (yp_location_t) { .start = lparen.start, .end = lparen.end }; target->rparen_loc = (yp_location_t) { .start = rparen.start, .end = rparen.end }; } else { @@ -7732,6 +8329,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b yp_multi_write_node_targets_append(target, child_target); } + target->base.location.start = lparen.start; target->base.location.end = rparen.end; yp_multi_write_node_targets_append(result, (yp_node_t *) target); } @@ -7754,7 +8352,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b } yp_node_t *target = parse_expression(parser, binding_power, "Expected another expression after ','."); - target = parse_target(parser, target, &operator, NULL); + target = parse_target(parser, target); yp_multi_write_node_targets_append(result, target); } @@ -8505,8 +9103,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_rescue_node_operator_set(rescue, &parser->previous); yp_node_t *reference = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected an exception variable after `=>` in rescue statement."); - yp_token_t operator = not_provided(parser); - reference = parse_target(parser, reference, &operator, NULL); + reference = parse_target(parser, reference); yp_rescue_node_reference_set(rescue, reference); break; @@ -8536,8 +9133,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_rescue_node_operator_set(rescue, &parser->previous); yp_node_t *reference = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected an exception variable after `=>` in rescue statement."); - yp_token_t operator = not_provided(parser); - reference = parse_target(parser, reference, &operator, NULL); + reference = parse_target(parser, reference); yp_rescue_node_reference_set(rescue, reference); break; @@ -8554,10 +9150,12 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { } if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_ELSE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); yp_statements_node_t *statements = parse_statements(parser, YP_CONTEXT_RESCUE); if (statements) { yp_rescue_node_statements_set(rescue, statements); } + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -8574,7 +9172,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { // since we won't know the end until we've found all consequent // clauses. This sets the end location on all rescues once we know it if (current) { - const char *end_to_set = current->base.location.end; + const uint8_t *end_to_set = current->base.location.end; current = parent_node->rescue_clause; while (current) { current->base.location.end = end_to_set; @@ -8588,7 +9186,9 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_statements_node_t *else_statements = NULL; if (!match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_END, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); else_statements = parse_statements(parser, YP_CONTEXT_RESCUE_ELSE); + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -8602,7 +9202,9 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_statements_node_t *ensure_statements = NULL; if (!match_type_p(parser, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); ensure_statements = parse_statements(parser, YP_CONTEXT_ENSURE); + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -8627,7 +9229,7 @@ parse_rescues_as_begin(yp_parser_t *parser, yp_statements_node_t *statements) { // All nodes within a begin node are optional, so we look // for the earliest possible node that we can use to set // the BeginNode's start location - const char * start = begin_node->base.location.start; + const uint8_t *start = begin_node->base.location.start; if (begin_node->statements) { start = begin_node->statements->base.location.start; } else if (begin_node->rescue_clause) { @@ -8712,7 +9314,9 @@ parse_block(yp_parser_t *parser) { } else { if (!match_type_p(parser, YP_TOKEN_KEYWORD_END)) { if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ELSE, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_BLOCK_KEYWORDS); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -9089,14 +9693,10 @@ parse_string_part(yp_parser_t *parser) { static yp_node_t * parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_state) { - bool lex_string = lex_mode->mode == YP_LEX_STRING; - bool can_be_interpolated = lex_string && lex_mode->as.string.interpolation; yp_token_t opening = parser->previous; - if (!lex_string) { - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (lex_mode->mode != YP_LEX_STRING) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); yp_token_t symbol; switch (parser->current.type) { @@ -9126,37 +9726,44 @@ parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_s return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &symbol, &closing, YP_UNESCAPE_ALL); } - if (can_be_interpolated) { - // Create a node_list first. We'll use this to check if it should be an InterpolatedSymbolNode - // or a SymbolNode + if (lex_mode->as.string.interpolation) { + // If we have the end of the symbol, then we can return an empty symbol. + if (match_type_p(parser, YP_TOKEN_STRING_END)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + yp_token_t content = not_provided(parser); + yp_token_t closing = parser->previous; + return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_NONE); + } + + // Now we can parse the first part of the symbol. + yp_node_t *part = parse_string_part(parser); + + // If we got a string part, then it's possible that we could transform + // what looks like an interpolated symbol into a regular symbol. + if (part && YP_NODE_TYPE_P(part, YP_NODE_STRING_NODE) && match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + return (yp_node_t *) yp_string_node_to_symbol_node(parser, (yp_string_node_t *) part, &opening, &parser->previous); + } + + // Create a node_list first. We'll use this to check if it should be an + // InterpolatedSymbolNode or a SymbolNode. yp_node_list_t node_list = YP_EMPTY_NODE_LIST; + if (part) yp_node_list_append(&node_list, part); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - yp_node_t *part = parse_string_part(parser); - if (part != NULL) { + if ((part = parse_string_part(parser)) != NULL) { yp_node_list_append(&node_list, part); } } - yp_node_t *res; - // If the only element on the node_list is a StringNode, we know this is a SymbolNode - // and not an InterpolatedSymbolNode - if (node_list.size == 1 && YP_NODE_TYPE_P(node_list.nodes[0], YP_NODE_STRING_NODE)) { - res = (yp_node_t *)yp_string_node_to_symbol_node(parser, (yp_string_node_t *)node_list.nodes[0]); - free(node_list.nodes); - } - else { - yp_interpolated_symbol_node_t *interpolated = yp_interpolated_symbol_node_create(parser, &opening, &node_list, &opening); - yp_interpolated_symbol_node_closing_set(interpolated, &parser->current); - res = (yp_node_t *) interpolated; - } - - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); expect(parser, YP_TOKEN_STRING_END, "Expected a closing delimiter for an interpolated symbol."); - return res; + return (yp_node_t *) yp_interpolated_symbol_node_create(parser, &opening, &node_list, &parser->previous); } yp_token_t content; @@ -9290,14 +9897,14 @@ parse_heredoc_common_whitespace(yp_parser_t *parser, yp_node_list_t *nodes) { yp_node_t *node = nodes->nodes[index]; if (!YP_NODE_TYPE_P(node, YP_NODE_STRING_NODE)) continue; - yp_location_t *content_loc = &((yp_string_node_t *) node)->content_loc; + const yp_location_t *content_loc = &((yp_string_node_t *) node)->content_loc; // If the previous node wasn't a string node, we don't want to trim // whitespace. This could happen after an interpolated expression or // variable. if (index == 0 || YP_NODE_TYPE_P(nodes->nodes[index - 1], YP_NODE_STRING_NODE)) { int cur_whitespace; - const char *cur_char = content_loc->start; + const uint8_t *cur_char = content_loc->start; while (cur_char && cur_char < content_loc->end) { // Any empty newlines aren't included in the minimum whitespace @@ -9388,15 +9995,15 @@ parse_heredoc_dedent(yp_parser_t *parser, yp_node_t *node, yp_heredoc_quote_t qu // destination to move bytes into. We'll also use it for bounds checking // since we don't require that these strings be null terminated. size_t dest_length = yp_string_length(string); - char *source_start = string->source; + uint8_t *source_start = (uint8_t *) string->source; - const char *source_cursor = source_start; - const char *source_end = source_cursor + dest_length; + const uint8_t *source_cursor = source_start; + const uint8_t *source_end = source_cursor + dest_length; // We're going to move bytes backward in the string when we get leading // whitespace, so we'll maintain a pointer to the current position in the // string that we're writing to. - char *dest_cursor = source_start; + uint8_t *dest_cursor = source_start; while (source_cursor < source_end) { // If we need to dedent the next element within the heredoc or the next @@ -9423,7 +10030,7 @@ parse_heredoc_dedent(yp_parser_t *parser, yp_node_t *node, yp_heredoc_quote_t qu // At this point we have dedented all that we need to, so we need to find // the next newline. - const char *breakpoint = next_newline(source_cursor, source_end - source_cursor); + const uint8_t *breakpoint = next_newline(source_cursor, source_end - source_cursor); if (breakpoint == NULL) { // If there isn't another newline, then we can just move the rest of the @@ -9635,7 +10242,7 @@ parse_pattern_hash(yp_parser_t *parser, yp_node_t *first_assoc) { yp_node_t *key = ((yp_assoc_node_t *) first_assoc)->key; if (YP_NODE_TYPE_P(key, YP_NODE_SYMBOL_NODE)) { - yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; + const yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; yp_parser_local_add_location(parser, value_loc->start, value_loc->end); } } @@ -9663,7 +10270,7 @@ parse_pattern_hash(yp_parser_t *parser, yp_node_t *first_assoc) { if (!match_any_type_p(parser, 7, YP_TOKEN_COMMA, YP_TOKEN_KEYWORD_THEN, YP_TOKEN_BRACE_RIGHT, YP_TOKEN_BRACKET_RIGHT, YP_TOKEN_PARENTHESIS_RIGHT, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON)) { value = parse_pattern(parser, false, "Expected a pattern expression after the key."); } else { - yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; + const yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; yp_parser_local_add_location(parser, value_loc->start, value_loc->end); } @@ -10203,10 +10810,8 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { } case YP_TOKEN_PARENTHESIS_LEFT: case YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES: { - yp_token_type_t current_token_type = parser->current.type; + yp_token_t opening = parser->current; parser_lex(parser); - - yp_token_t opening = parser->previous; while (accept_any(parser, 2, YP_TOKEN_SEMICOLON, YP_TOKEN_NEWLINE)); // If this is the end of the file or we match a right parenthesis, then @@ -10225,7 +10830,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // If we hit a right parenthesis, then we're done parsing the parentheses // node, and we can check which kind of node we should return. if (match_type_p(parser, YP_TOKEN_PARENTHESIS_RIGHT)) { - if (current_token_type == YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES) { + if (opening.type == YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES) { lex_state_set(parser, YP_LEX_STATE_ENDARG); } parser_lex(parser); @@ -10243,6 +10848,8 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { if (multi_statement->lparen_loc.start == NULL) { multi_write = (yp_multi_write_node_t *) statement; + multi_write->base.location.start = lparen_loc.start; + multi_write->base.location.end = rparen_loc.end; multi_write->lparen_loc = lparen_loc; multi_write->rparen_loc = rparen_loc; } else { @@ -10325,7 +10932,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_token_t closing = not_provided(parser); - return (yp_node_t *) yp_string_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_ALL); + return (yp_node_t *) yp_char_literal_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_ALL); } case YP_TOKEN_CLASS_VARIABLE: { parser_lex(parser); @@ -10870,7 +11477,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_node_t *statements = NULL; if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_SCLASS); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -11151,7 +11760,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_do_loop_stack_push(parser, false); if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_DEF); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -11413,12 +12024,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close `until` statement."); } - yp_until_node_t *until_node = yp_until_node_create(parser, &keyword, predicate, statements, 0); - if (parser->previous.type == YP_TOKEN_KEYWORD_END) { - until_node->base.location.end = parser->previous.end; - } - - return (yp_node_t *) until_node; + return (yp_node_t *) yp_until_node_create(parser, &keyword, &parser->previous, predicate, statements, 0); } case YP_TOKEN_KEYWORD_WHILE: { yp_do_loop_stack_push(parser, true); @@ -11439,25 +12045,16 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close `while` statement."); } - yp_while_node_t *while_node = yp_while_node_create(parser, &keyword, predicate, statements, 0); - if (parser->previous.type == YP_TOKEN_KEYWORD_END) { - while_node->base.location.end = parser->previous.end; - } - return (yp_node_t *) while_node; + return (yp_node_t *) yp_while_node_create(parser, &keyword, &parser->previous, predicate, statements, 0); } case YP_TOKEN_PERCENT_LOWER_I: { parser_lex(parser); yp_array_node_t *array = yp_array_node_create(parser, &parser->previous); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - if (yp_array_node_size(array) == 0) { - accept(parser, YP_TOKEN_WORDS_SEP); - } else { - expect(parser, YP_TOKEN_WORDS_SEP, "Expected a separator for the symbols in a `%i` list."); - if (match_type_p(parser, YP_TOKEN_STRING_END)) break; - } - + accept(parser, YP_TOKEN_WORDS_SEP); if (match_type_p(parser, YP_TOKEN_STRING_END)) break; + expect(parser, YP_TOKEN_STRING_CONTENT, "Expected a symbol in a `%i` list."); yp_token_t opening = not_provided(parser); @@ -11512,6 +12109,19 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // to the list of child nodes. yp_node_t *part = parse_string_part(parser); yp_interpolated_symbol_node_append((yp_interpolated_symbol_node_t *) current, part); + } else if (YP_NODE_TYPE_P(current, YP_NODE_SYMBOL_NODE)) { + // If we hit string content and the current node is a string node, + // then we need to convert the current node into an interpolated + // string and add the string content to the list of child nodes. + yp_token_t opening = not_provided(parser); + yp_token_t closing = not_provided(parser); + yp_interpolated_symbol_node_t *interpolated = + yp_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + yp_interpolated_symbol_node_append(interpolated, current); + + yp_node_t *part = parse_string_part(parser); + yp_interpolated_symbol_node_append(interpolated, part); + current = (yp_node_t *) interpolated; } else { assert(false && "unreachable"); } @@ -11614,12 +12224,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { accept(parser, YP_TOKEN_WORDS_SEP); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - if (yp_array_node_size(array) == 0) { - accept(parser, YP_TOKEN_WORDS_SEP); - } else { - expect(parser, YP_TOKEN_WORDS_SEP, "Expected a separator for the strings in a `%w` list."); - if (match_type_p(parser, YP_TOKEN_STRING_END)) break; - } + accept(parser, YP_TOKEN_WORDS_SEP); + if (match_type_p(parser, YP_TOKEN_STRING_END)) break; + expect(parser, YP_TOKEN_STRING_CONTENT, "Expected a string in a `%w` list."); yp_token_t opening = not_provided(parser); @@ -11669,6 +12276,19 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // to the list of child nodes. yp_node_t *part = parse_string_part(parser); yp_interpolated_string_node_append((yp_interpolated_string_node_t *) current, part); + } else if (YP_NODE_TYPE_P(current, YP_NODE_STRING_NODE)) { + // If we hit string content and the current node is a string node, + // then we need to convert the current node into an interpolated + // string and add the string content to the list of child nodes. + yp_token_t opening = not_provided(parser); + yp_token_t closing = not_provided(parser); + yp_interpolated_string_node_t *interpolated = + yp_interpolated_string_node_create(parser, &opening, NULL, &closing); + yp_interpolated_string_node_append(interpolated, current); + + yp_node_t *part = parse_string_part(parser); + yp_interpolated_string_node_append(interpolated, part); + current = (yp_node_t *) interpolated; } else { assert(false && "unreachable"); } @@ -11949,25 +12569,25 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_accepts_block_stack_push(parser, true); parser_lex(parser); - yp_token_t opening = parser->previous; + yp_token_t operator = parser->previous; yp_parser_scope_push(parser, false); yp_block_parameters_node_t *params; switch (parser->current.type) { case YP_TOKEN_PARENTHESIS_LEFT: { - yp_token_t block_parameters_opening = parser->current; + yp_token_t opening = parser->current; parser_lex(parser); if (match_type_p(parser, YP_TOKEN_PARENTHESIS_RIGHT)) { - params = yp_block_parameters_node_create(parser, NULL, &block_parameters_opening); + params = yp_block_parameters_node_create(parser, NULL, &opening); } else { - params = parse_block_parameters(parser, false, &block_parameters_opening, true); + params = parse_block_parameters(parser, false, &opening, true); } accept(parser, YP_TOKEN_NEWLINE); expect(parser, YP_TOKEN_PARENTHESIS_RIGHT, "Expected ')' after left parenthesis."); - yp_block_parameters_node_closing_set(params, &parser->previous); + yp_block_parameters_node_closing_set(params, &parser->previous); break; } case YP_CASE_PARAMETER: { @@ -11983,19 +12603,25 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { } } + yp_token_t opening; yp_node_t *body = NULL; parser->lambda_enclosure_nesting = previous_lambda_enclosure_nesting; if (accept(parser, YP_TOKEN_LAMBDA_BEGIN)) { + opening = parser->previous; + if (!accept(parser, YP_TOKEN_BRACE_RIGHT)) { body = (yp_node_t *) parse_statements(parser, YP_CONTEXT_LAMBDA_BRACES); expect(parser, YP_TOKEN_BRACE_RIGHT, "Expecting '}' to close lambda block."); } } else { expect(parser, YP_TOKEN_KEYWORD_DO, "Expected a 'do' keyword or a '{' to open lambda block."); + opening = parser->previous; if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_END, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); body = (yp_node_t *) parse_statements(parser, YP_CONTEXT_LAMBDA_DO_END); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -12009,7 +12635,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_constant_id_list_t locals = parser->current_scope->locals; yp_parser_scope_pop(parser); yp_accepts_block_stack_pop(parser); - return (yp_node_t *) yp_lambda_node_create(parser, &locals, &opening, params, body, &parser->previous); + return (yp_node_t *) yp_lambda_node_create(parser, &locals, &operator, &opening, &parser->previous, params, body); } case YP_TOKEN_UPLUS: { parser_lex(parser); @@ -12228,7 +12854,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_CASE_WRITABLE: { parser_lex(parser); yp_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, "Expected a value after =."); - return parse_target(parser, node, &token, value); + return parse_write(parser, node, &token, value); } case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat_node = (yp_splat_node_t *) node; @@ -12237,7 +12863,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_CASE_WRITABLE: parser_lex(parser); yp_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, "Expected a value after =."); - return parse_target(parser, (yp_node_t *) splat_node, &token, value); + return parse_write(parser, (yp_node_t *) splat_node, &token, value); default: break; } @@ -12259,19 +12885,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_global_variable_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_class_variable_and_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + return (yp_node_t *) yp_constant_path_and_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_constant_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_instance_variable_and_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; + parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, cast->name, cast->depth); + + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12281,25 +12945,22 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, constant_id, 0); + + yp_node_destroy(parser, node); + return result; } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + node = parse_target(parser, node); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); return (yp_node_t *) yp_call_operator_and_write_node_create(parser, (yp_call_node_t *) node, &token, value); @@ -12325,19 +12986,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_global_variable_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_class_variable_or_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + return (yp_node_t *) yp_constant_path_or_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_constant_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_instance_variable_or_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; + parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, cast->name, cast->depth); + + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12347,25 +13046,22 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, constant_id, 0); + + yp_node_destroy(parser, node); + return result; } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + node = parse_target(parser, node); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); return (yp_node_t *) yp_call_operator_or_write_node_create(parser, (yp_call_node_t *) node, &token, value); @@ -12401,19 +13097,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_global_variable_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_class_variable_operator_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + return (yp_node_t *) yp_constant_path_operator_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_constant_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); + + yp_node_destroy(parser, node); + return result; + } case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, cast->name, cast->depth); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator"); - return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12423,25 +13157,23 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, constant_id, 0); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); + yp_node_destroy(parser, node); + return result; } - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - + node = parse_target(parser, node); parser_lex(parser); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); return (yp_node_t *) yp_call_operator_write_node_create(parser, (yp_call_node_t *) node, &token, value); } @@ -12490,7 +13222,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_string_list_t named_captures; yp_string_list_init(&named_captures); - yp_location_t *content_loc = &((yp_regular_expression_node_t *) node)->content_loc; + const yp_location_t *content_loc = &((yp_regular_expression_node_t *) node)->content_loc; if (yp_regexp_named_capture_group_names(content_loc->start, (size_t) (content_loc->end - content_loc->start), &named_captures, parser->encoding_changed, &parser->encoding)) { for (size_t index = 0; index < named_captures.length; index++) { @@ -12610,7 +13342,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_statements_node_body_append(statements, node); yp_node_t *predicate = parse_expression(parser, binding_power, "Expected a predicate after 'until'"); - return (yp_node_t *) yp_until_node_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); + return (yp_node_t *) yp_until_node_modifier_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); } case YP_TOKEN_KEYWORD_WHILE_MODIFIER: { parser_lex(parser); @@ -12618,7 +13350,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_statements_node_body_append(statements, node); yp_node_t *predicate = parse_expression(parser, binding_power, "Expected a predicate after 'while'"); - return (yp_node_t *) yp_while_node_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); + return (yp_node_t *) yp_while_node_modifier_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); } case YP_TOKEN_QUESTION_MARK: { parser_lex(parser); @@ -12888,7 +13620,7 @@ yp_metadata_read_u32(const char *ptr) { // ]* // ] // ``` -static void +void yp_parser_metadata(yp_parser_t *parser, const char *metadata) { uint32_t filepath_size = yp_metadata_read_u32(metadata); metadata += 4; @@ -12914,7 +13646,7 @@ yp_parser_metadata(yp_parser_t *parser, const char *metadata) { uint32_t local_size = yp_metadata_read_u32(metadata); metadata += 4; - yp_parser_local_add_location(parser, metadata, metadata + local_size); + yp_parser_local_add_location(parser, (const uint8_t *) metadata, (const uint8_t *) (metadata + local_size)); metadata += local_size; } } @@ -12926,7 +13658,7 @@ yp_parser_metadata(yp_parser_t *parser, const char *metadata) { // Initialize a parser with the given start and end pointers. YP_EXPORTED_FUNCTION void -yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath) { +yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const char *filepath) { assert(source != NULL); // Set filepath to the file that was passed @@ -12998,7 +13730,7 @@ yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char yp_newline_list_init(&parser->newline_list, source, newline_size < 4 ? 4 : newline_size); // Skip past the UTF-8 BOM if it exists. - if (size >= 3 && (unsigned char) source[0] == 0xef && (unsigned char) source[1] == 0xbb && (unsigned char) source[2] == 0xbf) { + if (size >= 3 && source[0] == 0xef && source[1] == 0xbb && source[2] == 0xbf) { parser->current.end += 3; parser->encoding_comment_start += 3; } @@ -13006,7 +13738,7 @@ yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char // If the first two bytes of the source are a shebang, then we'll indicate // that the encoding comment is at the end of the shebang. if (peek(parser) == '#' && peek_offset(parser, 1) == '!') { - const char *encoding_comment_start = next_newline(source, (ptrdiff_t) size); + const uint8_t *encoding_comment_start = next_newline(source, (ptrdiff_t) size); if (encoding_comment_start) { parser->encoding_comment_start = encoding_comment_start + 1; } @@ -13078,7 +13810,7 @@ yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { // Parse and serialize the AST represented by the given source to the given // buffer. YP_EXPORTED_FUNCTION void -yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata) { +yp_parse_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) { yp_parser_t parser; yp_parser_init(&parser, source, size, NULL); if (metadata) yp_parser_metadata(&parser, metadata); diff --git a/yarp/yarp.h b/yarp/yarp.h index 078483193d58a5..378efe0c93746d 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -13,8 +13,10 @@ #include "yarp/util/yp_char.h" #include "yarp/util/yp_memchr.h" #include "yarp/util/yp_strpbrk.h" +#include "yarp/version.h" #include +#include #include #include #include @@ -30,6 +32,8 @@ void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buf void yp_print_node(yp_parser_t *parser, yp_node_t *node); +void yp_parser_metadata(yp_parser_t *parser, const char *metadata); + // Generate a scope node from the given node. void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest); @@ -37,7 +41,7 @@ void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest); YP_EXPORTED_FUNCTION const char * yp_version(void); // Initialize a parser with the given start and end pointers. -YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath); +YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const char *filepath); // Register a callback that will be called whenever YARP changes the encoding it // is using to parse based on the magic comment. @@ -63,10 +67,14 @@ YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, y YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer); // Parse the given source to the AST and serialize the AST to the given buffer. -YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata); +YP_EXPORTED_FUNCTION void yp_parse_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata); // Lex the given source and serialize to the given buffer. -YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer); +YP_EXPORTED_FUNCTION void yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer); + +// Parse and serialize both the AST and the tokens represented by the given +// source to the given buffer. +YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata); // Returns a string representation of the given token type. YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type); diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c new file mode 100644 index 00000000000000..cba528d2caaa44 --- /dev/null +++ b/yarp/yarp_compiler.c @@ -0,0 +1,1452 @@ +#include "yarp.h" + +#define OLD_ISEQ NEW_ISEQ +#undef NEW_ISEQ + +#define NEW_ISEQ(node, name, type, line_no) \ + yp_new_child_iseq(iseq, (node), parser, rb_fstring(name), 0, (type), (line_no)) + +#define OLD_CHILD_ISEQ NEW_CHILD_ISEQ +#undef NEW_CHILD_ISEQ + +#define NEW_CHILD_ISEQ(node, name, type, line_no) \ + yp_new_child_iseq(iseq, (node), parser, rb_fstring(name), iseq, (type), (line_no)) + +rb_iseq_t * +yp_iseq_new_with_opt(yp_scope_node_t *node, yp_parser_t *parser, VALUE name, VALUE path, VALUE realpath, + int first_lineno, const rb_iseq_t *parent, int isolated_depth, + enum rb_iseq_type type, const rb_compile_option_t *option); + +static VALUE +parse_number(const yp_node_t *node) { + const uint8_t *start = node->location.start; + const uint8_t *end = node->location.end; + size_t length = end - start; + + char *buffer = malloc(length + 1); + memcpy(buffer, start, length); + + buffer[length] = '\0'; + VALUE number = rb_cstr_to_inum(buffer, -10, Qfalse); + + free(buffer); + return number; +} + +static inline VALUE +parse_string(yp_string_t *string) { + return rb_str_new((const char *) yp_string_source(string), yp_string_length(string)); +} + +static inline ID +parse_symbol(const uint8_t *start, const uint8_t *end) { + return rb_intern2((const char *) start, end - start); +} + +static inline ID +parse_node_symbol(yp_node_t *node) { + return parse_symbol(node->location.start, node->location.end); +} + +static inline ID +parse_string_symbol(yp_string_t *string) { + const uint8_t *start = yp_string_source(string); + return parse_symbol(start, start + yp_string_length(string)); +} + +static inline ID +parse_location_symbol(yp_location_t *location) { + return parse_symbol(location->start, location->end); +} + +static int +yp_optimizable_range_item_p(yp_node_t *node) +{ + return (!node || node->type == YP_NODE_INTEGER_NODE || node->type == YP_NODE_NIL_NODE); +} + +static bool +yp_static_node_literal_p(yp_node_t *node) +{ + switch(node->type) { + case YP_NODE_FALSE_NODE: + case YP_NODE_FLOAT_NODE: + case YP_NODE_IMAGINARY_NODE: + case YP_NODE_INTEGER_NODE: + case YP_NODE_NIL_NODE: + case YP_NODE_RATIONAL_NODE: + case YP_NODE_SELF_NODE: + case YP_NODE_STRING_NODE: + case YP_NODE_SOURCE_ENCODING_NODE: + case YP_NODE_SOURCE_FILE_NODE: + case YP_NODE_SOURCE_LINE_NODE: + case YP_NODE_SYMBOL_NODE: + case YP_NODE_TRUE_NODE: + return true; + default: + return false; + } +} + +static inline VALUE +yp_static_literal_value(yp_node_t *node) +{ + switch(node->type) { + case YP_NODE_NIL_NODE: + return Qnil; + case YP_NODE_TRUE_NODE: + return Qtrue; + case YP_NODE_FALSE_NODE: + return Qfalse; + // TODO: Implement this method for the other literal nodes described above + default: + rb_bug("This node type doesn't have a literal value"); + } +} + +static void +yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context); + +static void +yp_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) +{ + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int)yp_newline_list_line_column(&newline_list, cond->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + DECL_ANCHOR(seq); + INIT_ANCHOR(seq); + LABEL *label = NEW_LABEL(lineno); + if (!then_label) then_label = label; + else if (!else_label) else_label = label; + + yp_compile_branch_condition(iseq, seq, cond, then_label, else_label, src, popped, compile_context); + + if (LIST_INSN_SIZE_ONE(seq)) { + INSN *insn = (INSN *)ELEM_FIRST_INSN(FIRST_ELEMENT(seq)); + if (insn->insn_id == BIN(jump) && (LABEL *)(insn->operands[0]) == label) + return; + } + if (!label->refcnt) { + ADD_INSN(seq, &dummy_line_node, putnil); + } + else { + ADD_LABEL(seq, label); + } + ADD_SEQ(ret, seq); + return; +} + +static void yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *context); + +static void +yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) +{ + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int) yp_newline_list_line_column(&newline_list, cond->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + +again: + switch (YP_NODE_TYPE(cond)) { + case YP_NODE_AND_NODE: { + yp_and_node_t *and_node = (yp_and_node_t *)cond; + yp_compile_logical(iseq, ret, and_node->left, NULL, else_label, src, popped, compile_context); + cond = and_node->right; + goto again; + } + case YP_NODE_OR_NODE: { + yp_or_node_t *or_node = (yp_or_node_t *)cond; + yp_compile_logical(iseq, ret, or_node->left, then_label, NULL, src, popped, compile_context); + cond = or_node->right; + goto again; + } + case YP_NODE_FALSE_NODE: + case YP_NODE_NIL_NODE: + ADD_INSNL(ret, &dummy_line_node, jump, else_label); + return; + case YP_NODE_FLOAT_NODE: + case YP_NODE_IMAGINARY_NODE: + case YP_NODE_INTEGER_NODE: + case YP_NODE_LAMBDA_NODE: + case YP_NODE_RATIONAL_NODE: + case YP_NODE_REGULAR_EXPRESSION_NODE: + case YP_NODE_STRING_NODE: + case YP_NODE_SYMBOL_NODE: + case YP_NODE_TRUE_NODE: + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + return; + // TODO: Several more nodes in this case statement + default: + { + DECL_ANCHOR(cond_seq); + INIT_ANCHOR(cond_seq); + + yp_compile_node(iseq, cond, cond_seq, src, false, compile_context); + ADD_SEQ(ret, cond_seq); + } + break; + } + ADD_INSNL(ret, &dummy_line_node, branchunless, else_label); + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + return; +} + +static void +yp_compile_if(rb_iseq_t *iseq, const int line, yp_statements_node_t *node_body, yp_node_t *node_else, yp_node_t *predicate, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { + NODE line_node = generate_dummy_line_node(line, line); + + DECL_ANCHOR(cond_seq); + + LABEL *then_label, *else_label, *end_label; + + INIT_ANCHOR(cond_seq); + then_label = NEW_LABEL(line); + else_label = NEW_LABEL(line); + end_label = 0; + + yp_compile_branch_condition(iseq, cond_seq, predicate, then_label, else_label, src, popped, compile_context); + ADD_SEQ(ret, cond_seq); + + if (then_label->refcnt) { + ADD_LABEL(ret, then_label); + + DECL_ANCHOR(then_seq); + INIT_ANCHOR(then_seq); + if (node_body) { + yp_compile_node(iseq, (yp_node_t *)node_body, then_seq, src, popped, compile_context); + } + else { + if (!popped) { + ADD_INSN(ret, &line_node, putnil); + } + } + + if (else_label->refcnt) { + end_label = NEW_LABEL(line); + ADD_INSNL(then_seq, &line_node, jump, end_label); + if (!popped) { + ADD_INSN(then_seq, &line_node, pop); + } + } + ADD_SEQ(ret, then_seq); + } + + if (else_label->refcnt) { + ADD_LABEL(ret, else_label); + + DECL_ANCHOR(else_seq); + INIT_ANCHOR(else_seq); + if (node_else) { + yp_compile_node(iseq, (yp_node_t *)(((yp_else_node_t *)node_else)->statements), else_seq, src, popped, compile_context); + } + else { + if (!popped) { + ADD_INSN(ret, &line_node, putnil); + } + } + + ADD_SEQ(ret, else_seq); + } + + if (end_label) { + ADD_LABEL(ret, end_label); + } + + return; +} + +static void +yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_node_type type, yp_statements_node_t *statements, yp_node_t *predicate, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) +{ + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + LABEL *prev_start_label = ISEQ_COMPILE_DATA(iseq)->start_label; + LABEL *prev_end_label = ISEQ_COMPILE_DATA(iseq)->end_label; + LABEL *prev_redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label; + + // TODO: Deal with ensures in here + LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ + LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ + LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ + LABEL *end_label = NEW_LABEL(lineno); + LABEL *adjust_label = NEW_LABEL(lineno); + + LABEL *next_catch_label = NEW_LABEL(lineno); + LABEL *tmp_label = NULL; + + // begin; end while true + if (flags & YP_LOOP_FLAGS_BEGIN_MODIFIER) { + tmp_label = NEW_LABEL(lineno); + ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); + } + else { + // while true; end + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + } + + ADD_LABEL(ret, adjust_label); + ADD_INSN(ret, &dummy_line_node, putnil); + ADD_LABEL(ret, next_catch_label); + ADD_INSN(ret, &dummy_line_node, pop); + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + if (tmp_label) ADD_LABEL(ret, tmp_label); + + ADD_LABEL(ret, redo_label); + if (statements) { + yp_compile_node(iseq, (yp_node_t *)statements, ret, src, true, compile_context); + } + + ADD_LABEL(ret, next_label); + + if (type == YP_NODE_WHILE_NODE) { + yp_compile_branch_condition(iseq, ret, predicate, redo_label, end_label, src, popped, compile_context); + } + else if (type == YP_NODE_UNTIL_NODE) { + yp_compile_branch_condition(iseq, ret, predicate, end_label, redo_label, src, popped, compile_context); + } + + ADD_LABEL(ret, end_label); + ADD_ADJUST_RESTORE(ret, adjust_label); + + ADD_INSN(ret, &dummy_line_node, putnil); + + ADD_LABEL(ret, break_label); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + ADD_CATCH_ENTRY(CATCH_TYPE_BREAK, redo_label, break_label, NULL, + break_label); + ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, redo_label, break_label, NULL, + next_catch_label); + ADD_CATCH_ENTRY(CATCH_TYPE_REDO, redo_label, break_label, NULL, + ISEQ_COMPILE_DATA(iseq)->redo_label); + + ISEQ_COMPILE_DATA(iseq)->start_label = prev_start_label; + ISEQ_COMPILE_DATA(iseq)->end_label = prev_end_label; + ISEQ_COMPILE_DATA(iseq)->redo_label = prev_redo_label; + return; +} + +static rb_iseq_t * +yp_new_child_iseq(rb_iseq_t *iseq, yp_scope_node_t * node, yp_parser_t *parser, + VALUE name, const rb_iseq_t *parent, enum rb_iseq_type type, int line_no) +{ + debugs("[new_child_iseq]> ---------------------------------------\n"); + int isolated_depth = ISEQ_COMPILE_DATA(iseq)->isolated_depth; + rb_iseq_t * ret_iseq = yp_iseq_new_with_opt(node, parser, name, + rb_iseq_path(iseq), rb_iseq_realpath(iseq), + line_no, parent, + isolated_depth ? isolated_depth + 1 : 0, + type, ISEQ_COMPILE_DATA(iseq)->option); + debugs("[new_child_iseq]< ---------------------------------------\n"); + return ret_iseq; +} + + +static int +yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t *constant_path_node, const NODE *line_node, const uint8_t * src, bool popped, yp_compile_context_t *compile_context) +{ + if (constant_path_node->type == YP_NODE_CONSTANT_PATH_NODE) { + yp_node_t *parent = ((yp_constant_path_node_t *)constant_path_node)->parent; + if (parent) { + /* Bar::Foo */ + yp_compile_node(iseq, parent, ret, src, popped, compile_context); + return VM_DEFINECLASS_FLAG_SCOPED; + } + else { + /* toplevel class ::Foo */ + ADD_INSN1(ret, line_node, putobject, rb_cObject); + return VM_DEFINECLASS_FLAG_SCOPED; + } + } + else { + /* class at cbase Foo */ + ADD_INSN1(ret, line_node, putspecialobject, + INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + return 0; + } +} + +/* + * Compiles a YARP node into instruction sequences + * + * iseq - The current instruction sequence object (used for locals) + * node - The yarp node to compile + * ret - The linked list of instruction sequences to append instructions onto + * popped - True if compiling something with no side effects, so instructions don't + * need to be added + * compile_context - Stores parser and local information + */ +static void +yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) +{ + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int)yp_newline_list_line_column(&newline_list, node->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + switch (YP_NODE_TYPE(node)) { + case YP_NODE_ALIAS_NODE: { + yp_alias_node_t *alias_node = (yp_alias_node_t *) node; + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_VMCORE); + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_CBASE); + + yp_compile_node(iseq, alias_node->new_name, ret, src, popped, compile_context); + yp_compile_node(iseq, alias_node->old_name, ret, src, popped, compile_context); + + ADD_SEND(ret, &dummy_line_node, id_core_set_method_alias, INT2FIX(3)); + return; + } + case YP_NODE_AND_NODE: { + yp_and_node_t *and_node = (yp_and_node_t *) node; + + LABEL *end_label = NEW_LABEL(lineno); + yp_compile_node(iseq, and_node->left, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ADD_INSNL(ret, &dummy_line_node, branchunless, end_label); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + yp_compile_node(iseq, and_node->right, ret, src, popped, compile_context); + ADD_LABEL(ret, end_label); + return; + } + case YP_NODE_ARGUMENTS_NODE: { + yp_arguments_node_t *arguments_node = (yp_arguments_node_t *) node; + yp_node_list_t node_list = arguments_node->arguments; + for (size_t index = 0; index < node_list.size; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); + } + return; + } + case YP_NODE_ARRAY_NODE: { + yp_array_node_t *array_node = (yp_array_node_t *) node; + yp_node_list_t elements = array_node->elements; + if (elements.size == 1 && yp_static_node_literal_p(elements.nodes[0])) { + VALUE ary = rb_ary_hidden_new(1); + rb_ary_push(ary, yp_static_literal_value(elements.nodes[0])); + OBJ_FREEZE(ary); + + ADD_INSN1(ret, &dummy_line_node, duparray, ary); + } + else { + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(elements.size)); + } + } + + return; + } + case YP_NODE_ASSOC_NODE: { + yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) node; + yp_compile_node(iseq, assoc_node->key, ret, src, popped, compile_context); + if (assoc_node->value) { + yp_compile_node(iseq, assoc_node->value, ret, src, popped, compile_context); + } + return; + } + case YP_NODE_ASSOC_SPLAT_NODE: { + yp_assoc_splat_node_t *assoc_splat_node = (yp_assoc_splat_node_t *)node; + yp_compile_node(iseq, assoc_splat_node->value, ret, src, popped, compile_context); + + // TODO: Not sure this is accurate, look at FLUSH_CHUNK in the compiler + ADD_INSN1(ret, &dummy_line_node, newarraykwsplat, INT2FIX(0)); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_BEGIN_NODE: { + yp_begin_node_t *begin_node = (yp_begin_node_t *) node; + if (begin_node->statements) { + yp_compile_node(iseq, (yp_node_t *)begin_node->statements, ret, src, popped, compile_context); + } + return; + } + case YP_NODE_BREAK_NODE: { + yp_break_node_t *break_node = (yp_break_node_t *) node; + if (break_node->arguments) { + yp_compile_node(iseq, (yp_node_t *)break_node->arguments, ret, src, Qfalse, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->end_label); + + return; + } + case YP_NODE_CALL_NODE: { + yp_call_node_t *call_node = (yp_call_node_t *) node; + + ID method_id = parse_string_symbol(&call_node->name); + int flags = 0; + int orig_argc = 0; + + if (call_node->receiver == NULL) { + ADD_INSN(ret, &dummy_line_node, putself); + } else { + yp_compile_node(iseq, call_node->receiver, ret, src, false, compile_context); + } + + if (call_node->arguments == NULL) { + if (flags & VM_CALL_FCALL) { + flags |= VM_CALL_VCALL; + } + } else { + yp_arguments_node_t *arguments = call_node->arguments; + yp_compile_node(iseq, (yp_node_t *) arguments, ret, src, false, compile_context); + orig_argc = (int)arguments->arguments.size; + } + + VALUE block_iseq = Qnil; + if (call_node->block != NULL) { + // Scope associated with the block + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)call_node->block, &scope_node); + + const rb_iseq_t *block_iseq = NEW_CHILD_ISEQ(&scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + ISEQ_COMPILE_DATA(iseq)->current_block = block_iseq; + ADD_SEND_WITH_BLOCK(ret, &dummy_line_node, method_id, INT2FIX(orig_argc), block_iseq); + } + else { + if (block_iseq == Qnil && flags == 0) { + flags |= VM_CALL_ARGS_SIMPLE; + } + + if (call_node->receiver == NULL) { + flags |= VM_CALL_FCALL; + + if (block_iseq == Qnil && call_node->arguments == NULL) { + flags |= VM_CALL_VCALL; + } + } + + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, method_id, INT2NUM(orig_argc), INT2FIX(flags)); + } + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_CLASS_NODE: { + yp_class_node_t *class_node = (yp_class_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)class_node, &scope_node); + + ID class_id = parse_string_symbol(&class_node->name); + + VALUE class_name = rb_str_freeze(rb_sprintf("", rb_id2str(class_id))); + + const rb_iseq_t *class_iseq = NEW_CHILD_ISEQ(&scope_node, class_name, ISEQ_TYPE_CLASS, lineno); + + // TODO: Once we merge constant path nodes correctly, fix this flag + const int flags = VM_DEFINECLASS_TYPE_CLASS | + (class_node->superclass ? VM_DEFINECLASS_FLAG_HAS_SUPERCLASS : 0) | + yp_compile_class_path(ret, iseq, class_node->constant_path, &dummy_line_node, src, popped, compile_context); + + if (class_node->superclass) { + yp_compile_node(iseq, class_node->superclass, ret, src, popped, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSN3(ret, &dummy_line_node, defineclass, ID2SYM(class_id), class_iseq, INT2FIX(flags)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)class_iseq); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: + if (!popped) { + ID cvar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN2( + ret, + &dummy_line_node, + getclassvariable, + ID2SYM(cvar_name), + get_cvar_ic_value(iseq, cvar_name) + ); + } + return; + case YP_NODE_CLASS_VARIABLE_WRITE_NODE: { + yp_class_variable_write_node_t *write_node = (yp_class_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ID cvar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN2(ret, &dummy_line_node, setclassvariable, ID2SYM(cvar_name), get_cvar_ic_value(iseq, cvar_name)); + return; + } + case YP_NODE_CONSTANT_PATH_NODE: { + yp_constant_path_node_t *constant_path_node = (yp_constant_path_node_t*) node; + if (constant_path_node->parent) { + yp_compile_node(iseq, constant_path_node->parent, ret, src, popped, compile_context); + } + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_INSN1(ret, &dummy_line_node, getconstant, ID2SYM(parse_node_symbol((yp_node_t *)constant_path_node->child))); + return; + } + case YP_NODE_CONSTANT_PATH_WRITE_NODE: { + yp_constant_path_write_node_t *constant_path_write_node = (yp_constant_path_write_node_t*) node; + yp_compile_node(iseq, constant_path_write_node->value, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ID constant_var_name = parse_location_symbol(&constant_path_write_node->target->base.location); + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + ADD_INSN1(ret, &dummy_line_node, setconstant, ID2SYM(constant_var_name)); + return; + } + + case YP_NODE_CONSTANT_READ_NODE: { + yp_constant_read_node_t *constant_read_node = (yp_constant_read_node_t *) node; + ADD_INSN(ret, &dummy_line_node, putnil); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSN1(ret, &dummy_line_node, getconstant, ID2SYM(parse_node_symbol((yp_node_t *)constant_read_node))); + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_CONSTANT_WRITE_NODE: { + yp_constant_write_node_t *constant_write_node = (yp_constant_write_node_t *) node; + yp_compile_node(iseq, constant_write_node->value, ret, src, false, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + ID constant_name = parse_location_symbol(&constant_write_node->name_loc); + ADD_INSN1(ret, &dummy_line_node, setconstant, ID2SYM(constant_name)); + return; + } + case YP_NODE_DEF_NODE: { + yp_def_node_t *def_node = (yp_def_node_t *) node; + ID method_name = parse_location_symbol(&def_node->name_loc); + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)def_node, &scope_node); + rb_iseq_t *method_iseq = NEW_ISEQ(&scope_node, rb_id2str(method_name), ISEQ_TYPE_METHOD, lineno); + + ADD_INSN2(ret, &dummy_line_node, definemethod, ID2SYM(method_name), method_iseq); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)method_iseq); + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, ID2SYM(method_name)); + } + return; + } + case YP_NODE_DEFINED_NODE: { + ADD_INSN(ret, &dummy_line_node, putself); + yp_defined_node_t *defined_node = (yp_defined_node_t *)node; + // TODO: Correct defined_type + enum defined_type dtype = DEFINED_CONST; + VALUE sym; + + sym = parse_number(defined_node->value); + + ADD_INSN3(ret, &dummy_line_node, defined, INT2FIX(dtype), sym, rb_iseq_defined_string(dtype)); + return; + } + case YP_NODE_EMBEDDED_STATEMENTS_NODE: { + yp_embedded_statements_node_t *embedded_statements_node = (yp_embedded_statements_node_t *)node; + + if (embedded_statements_node->statements) + yp_compile_node(iseq, (yp_node_t *) (embedded_statements_node->statements), ret, src, popped, compile_context); + // TODO: Concatenate the strings that exist here + return; + } + case YP_NODE_EMBEDDED_VARIABLE_NODE: { + yp_embedded_variable_node_t *embedded_node = (yp_embedded_variable_node_t *)node; + yp_compile_node(iseq, embedded_node->variable, ret, src, popped, compile_context); + return; + } + case YP_NODE_FALSE_NODE: + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + } + return; + case YP_NODE_FLIP_FLOP_NODE: { + // TODO: The labels here are wrong, figure out why..... + yp_flip_flop_node_t *flip_flop_node = (yp_flip_flop_node_t *)node; + + LABEL *lend = NEW_LABEL(lineno); + LABEL *then_label = NEW_LABEL(lineno); + LABEL *else_label = NEW_LABEL(lineno); + //TODO: int again = type == NODE_FLIP2; + int again = 0; + + rb_num_t cnt = ISEQ_FLIP_CNT_INCREMENT(ISEQ_BODY(iseq)->local_iseq) + + VM_SVAR_FLIPFLOP_START; + VALUE key = INT2FIX(cnt); + + ADD_INSN2(ret, &dummy_line_node, getspecial, key, INT2FIX(0)); + ADD_INSNL(ret, &dummy_line_node, branchif, lend); + + yp_compile_node(iseq, flip_flop_node->left, ret, src, popped, compile_context); + /* *flip == 0 */ + ADD_INSNL(ret, &dummy_line_node, branchunless, else_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSN1(ret, &dummy_line_node, setspecial, key); + if (!again) { + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + } + + /* *flip == 1 */ + ADD_LABEL(ret, lend); + yp_compile_node(iseq, flip_flop_node->right, ret, src, popped, compile_context); + ADD_INSNL(ret, &dummy_line_node, branchunless, then_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_INSN1(ret, &dummy_line_node, setspecial, key); + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + ADD_LABEL(ret, then_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSNL(ret, &dummy_line_node, jump, lend); + ADD_LABEL(ret, else_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_LABEL(ret, lend); + return; + } + case YP_NODE_FLOAT_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + if (!popped) { + ID gvar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN1(ret, &dummy_line_node, getglobal, ID2SYM(gvar_name)); + } + return; + case YP_NODE_GLOBAL_VARIABLE_WRITE_NODE: { + yp_global_variable_write_node_t *write_node = (yp_global_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ID ivar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN1(ret, &dummy_line_node, setglobal, ID2SYM(ivar_name)); + return; + } + case YP_NODE_HASH_NODE: { + yp_hash_node_t *hash_node = (yp_hash_node_t *) node; + yp_node_list_t elements = hash_node->elements; + + if (elements.size == 1) { + assert(elements.nodes[0]->type == YP_NODE_ASSOC_NODE); + yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) elements.nodes[0]; + + if (yp_static_node_literal_p(assoc_node->key) && + yp_static_node_literal_p(assoc_node->value)) { + VALUE hash = rb_hash_new_with_size(1); + hash = rb_obj_hide(hash); + OBJ_FREEZE(hash); + ADD_INSN1(ret, &dummy_line_node, duphash, hash); + return; + } + } + + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(elements.size * 2)); + } + return; + } + case YP_NODE_IF_NODE: { + const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; + yp_if_node_t *if_node = (yp_if_node_t *)node; + yp_statements_node_t *node_body = if_node->statements; + yp_node_t *node_else = if_node->consequent; + yp_node_t *predicate = if_node->predicate; + + yp_compile_if(iseq, line, node_body, node_else, predicate, ret, src, popped, compile_context); + return; + } + case YP_NODE_IMAGINARY_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + if (!popped) { + ID ivar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN2(ret, &dummy_line_node, getinstancevariable, + ID2SYM(ivar_name), + get_ivar_ic_value(iseq, ivar_name)); + } + return; + } + case YP_NODE_INSTANCE_VARIABLE_WRITE_NODE: { + yp_instance_variable_write_node_t *write_node = (yp_instance_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ID ivar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN2(ret, &dummy_line_node, setinstancevariable, + ID2SYM(ivar_name), + get_ivar_ic_value(iseq, ivar_name)); + return; + } + case YP_NODE_INTEGER_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_INTERPOLATED_STRING_NODE: { + yp_interpolated_string_node_t *interp_string_node= (yp_interpolated_string_node_t *) node; + + for (size_t index = 0; index < interp_string_node->parts.size; index++) { + yp_node_t *part = interp_string_node->parts.nodes[index]; + + switch (part->type) { + case YP_NODE_STRING_NODE: { + yp_string_node_t *string_node = (yp_string_node_t *) part; + ADD_INSN1(ret, &dummy_line_node, putobject,parse_string(&string_node->unescaped)); + break; + } + default: + yp_compile_node(iseq, part, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + break; + } + } + + if (interp_string_node->parts.size > 1) { + ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX((int)(interp_string_node->parts.size))); + } + return; + } + case YP_NODE_INTERPOLATED_SYMBOL_NODE: { + yp_interpolated_symbol_node_t *interp_symbol_node= (yp_interpolated_symbol_node_t *) node; + + for (size_t index = 0; index < interp_symbol_node->parts.size; index++) { + yp_node_t *part = interp_symbol_node->parts.nodes[index]; + + switch (part->type) { + case YP_NODE_STRING_NODE: { + yp_string_node_t *string_node = (yp_string_node_t *) part; + ADD_INSN1(ret, &dummy_line_node, putobject, parse_string(&string_node->unescaped)); + break; + } + default: + yp_compile_node(iseq, part, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE, NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + break; + } + } + + if (interp_symbol_node->parts.size > 1) { + ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX((int)(interp_symbol_node->parts.size))); + } + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, intern); + } + else { + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_KEYWORD_HASH_NODE: { + yp_keyword_hash_node_t *keyword_hash_node = (yp_keyword_hash_node_t *) node; + yp_node_list_t elements = keyword_hash_node->elements; + + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(elements.size * 2)); + return; + } + case YP_NODE_LAMBDA_NODE: { + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)node, &scope_node); + + const rb_iseq_t *block = NEW_CHILD_ISEQ(&scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + VALUE argc = INT2FIX(0); + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + ADD_CALL_WITH_BLOCK(ret, &dummy_line_node, idLambda, argc, block); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)block); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *local_read_node = (yp_local_variable_read_node_t *) node; + + yp_constant_id_t constant_id = local_read_node->name; + st_data_t local_index; + + for(uint32_t i = 0; i < local_read_node->depth; i++) { + compile_context = compile_context->previous; + iseq = (rb_iseq_t *)ISEQ_BODY(iseq)->parent_iseq; + } + + int num_params = ISEQ_BODY(iseq)->param.size; + + if (!st_lookup(compile_context->index_lookup_table, constant_id, &local_index)) { + rb_bug("This local does not exist"); + } + + int index = num_params - (int)local_index; + + if (!popped) { + ADD_GETLOCAL(ret, &dummy_line_node, index, local_read_node->depth); + } + return; + } + case YP_NODE_LOCAL_VARIABLE_WRITE_NODE: { + yp_local_variable_write_node_t *local_write_node = (yp_local_variable_write_node_t *) node; + yp_compile_node(iseq, local_write_node->value, ret, src, false, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + yp_constant_id_t constant_id = local_write_node->name; + size_t stack_index; + + if (!st_lookup(compile_context->index_lookup_table, constant_id, &stack_index)) { + rb_bug("This local doesn't exist"); + } + + unsigned int num_params = ISEQ_BODY(iseq)->param.size; + size_t index = num_params - stack_index; + + ADD_SETLOCAL(ret, &dummy_line_node, (int)index, local_write_node->depth); + return; + } + case YP_NODE_MISSING_NODE: { + rb_bug("A yp_missing_node_t should not exist in YARP's AST."); + return; + } + case YP_NODE_MODULE_NODE: { + yp_module_node_t *module_node = (yp_module_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)module_node, &scope_node); + + ID module_id = parse_string_symbol(&module_node->name); + VALUE module_name = rb_str_freeze(rb_sprintf("", rb_id2str(module_id))); + + const rb_iseq_t *module_iseq = NEW_CHILD_ISEQ(&scope_node, module_name, ISEQ_TYPE_CLASS, lineno); + + const int flags = VM_DEFINECLASS_TYPE_MODULE | + yp_compile_class_path(ret, iseq, module_node->constant_path, &dummy_line_node, src, popped, compile_context); + + ADD_INSN (ret, &dummy_line_node, putnil); + ADD_INSN3(ret, &dummy_line_node, defineclass, ID2SYM(module_id), module_iseq, INT2FIX(flags)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)module_iseq); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_MULTI_WRITE_NODE: { + yp_multi_write_node_t *multi_write_node = (yp_multi_write_node_t *)node; + yp_compile_node(iseq, multi_write_node->value, ret, src, popped, compile_context); + + // TODO: int flag = 0x02 | (NODE_NAMED_REST_P(restn) ? 0x01 : 0x00); + int flag = 0x00; + + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN2(ret, &dummy_line_node, expandarray, INT2FIX(multi_write_node->targets.size), INT2FIX(flag)); + yp_node_list_t node_list = multi_write_node->targets; + + for (size_t index = 0; index < node_list.size; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); + } + + return; + } + case YP_NODE_NEXT_NODE: { + yp_next_node_t *next_node = (yp_next_node_t *) node; + if (next_node->arguments) { + yp_compile_node(iseq, (yp_node_t *)next_node->arguments, ret, src, Qfalse, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSN(ret, &dummy_line_node, pop); + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->start_label); + + return; + } + case YP_NODE_NIL_NODE: + if (!popped) { + ADD_INSN(ret, &dummy_line_node, putnil); + } + return; + case YP_NODE_OR_NODE: { + yp_or_node_t *or_node = (yp_or_node_t *) node; + + LABEL *end_label = NEW_LABEL(lineno); + yp_compile_node(iseq, or_node->left, ret, src, popped, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ADD_INSNL(ret, &dummy_line_node, branchif, end_label); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + yp_compile_node(iseq, or_node->right, ret, src, popped, compile_context); + ADD_LABEL(ret, end_label); + + return; + } + case YP_NODE_OPTIONAL_PARAMETER_NODE: { + yp_optional_parameter_node_t *optional_parameter_node = (yp_optional_parameter_node_t *)node; + yp_compile_node(iseq, optional_parameter_node->value, ret, src, false, compile_context); + + yp_constant_id_t constant_id = optional_parameter_node->name; + + size_t param_number; + if (!st_lookup(compile_context->index_lookup_table, constant_id, ¶m_number)) { + rb_bug("This local doesn't exist"); + } + + unsigned int num_params = ISEQ_BODY(iseq)->param.size; + int index = (int) (num_params - param_number); + + ADD_SETLOCAL(ret, &dummy_line_node, index, 0); + + return; + } + case YP_NODE_PARENTHESES_NODE: { + yp_parentheses_node_t *parentheses_node = (yp_parentheses_node_t *) node; + + if (parentheses_node->body == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, parentheses_node->body, ret, src, popped, compile_context); + } + + return; + } + case YP_NODE_PROGRAM_NODE: { + yp_program_node_t *program_node = (yp_program_node_t *) node; + + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)node, &scope_node); + if (program_node->statements->body.size == 0) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_scope_node_t *res_node = &scope_node; + yp_compile_node(iseq, (yp_node_t *) res_node, ret, src, popped, compile_context); + } + + return; + } + case YP_NODE_RANGE_NODE: { + yp_range_node_t *range_node = (yp_range_node_t *) node; + bool exclusive = (range_node->operator_loc.end - range_node->operator_loc.start) == 3; + + if (yp_optimizable_range_item_p(range_node->left) && yp_optimizable_range_item_p(range_node->right)) { + if (!popped) { + yp_node_t *left = range_node->left; + yp_node_t *right = range_node->right; + VALUE val = rb_range_new( + left && left->type == YP_NODE_INTEGER_NODE ? parse_number(left) : Qnil, + right && right->type == YP_NODE_INTEGER_NODE ? parse_number(right) : Qnil, + exclusive + ); + ADD_INSN1(ret, &dummy_line_node, putobject, val); + RB_OBJ_WRITTEN(iseq, Qundef, val); + } + } + else { + if (range_node->left == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, range_node->left, ret, src, popped, compile_context); + } + + if (range_node->right == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, range_node->right, ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newrange, INT2FIX(exclusive)); + } + } + return; + } + case YP_NODE_REDO_NODE: { + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->redo_label); + return; + } + case YP_NODE_RETURN_NODE: { + yp_arguments_node_t *arguments = ((yp_return_node_t *)node)->arguments; + + if (arguments) { + yp_compile_node(iseq, (yp_node_t *)arguments, ret, src, popped, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_TRACE(ret, RUBY_EVENT_RETURN); + ADD_INSN(ret, &dummy_line_node, leave); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, putnil); + } + return; + } + case YP_NODE_SCOPE_NODE: { + yp_scope_node_t *scope_node = (yp_scope_node_t *)node; + yp_constant_id_list_t locals = scope_node->locals; + + yp_parameters_node_t *parameters_node = (yp_parameters_node_t *)scope_node->parameters; + yp_node_list_t requireds_list = YP_EMPTY_NODE_LIST; + yp_node_list_t optionals_list = YP_EMPTY_NODE_LIST; + + + if (parameters_node) { + requireds_list = parameters_node->requireds; + optionals_list = parameters_node->optionals; + } + + size_t size = locals.size; + + // Index lookup table buffer size is only the number of the locals + st_table *index_lookup_table = st_init_numtable(); + + VALUE idtmp = 0; + rb_ast_id_table_t *tbl = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + size * sizeof(ID)); + tbl->size = (int)size; + + // First param gets 0, second param 1, param n... + // Calculate the local index for all locals + for (size_t i = 0; i < size; i++) { + yp_constant_id_t constant_id = locals.ids[i]; + ID local = compile_context->constants[constant_id - 1]; + tbl->ids[i] = local; + st_insert(index_lookup_table, constant_id, i); + } + + yp_compile_context_t scope_compile_context = { + .parser = parser, + .previous = compile_context, + .constants = compile_context->constants, + .index_lookup_table = index_lookup_table + }; + + ISEQ_BODY(iseq)->param.lead_num = (int)requireds_list.size; + ISEQ_BODY(iseq)->param.opt_num = (int)optionals_list.size; + // TODO: Set all the other nums (good comment by lead_num illustrating what they are) + ISEQ_BODY(iseq)->param.size = (unsigned int)size; + + if (optionals_list.size) { + LABEL **opt_table = (LABEL **)ALLOC_N(VALUE, optionals_list.size + 1); + LABEL *label; + + // TODO: Should we make an api for NEW_LABEL where you can pass + // a pointer to the label it should fill out? We already + // have a list of labels allocated above so it seems wasteful + // to do the copies. + for (size_t i = 0; i < optionals_list.size; i++) { + label = NEW_LABEL(lineno); + opt_table[i] = label; + ADD_LABEL(ret, label); + yp_node_t *optional_node = optionals_list.nodes[i]; + yp_compile_node(iseq, optional_node, ret, src, false, &scope_compile_context); + } + + // Set the last label + label = NEW_LABEL(lineno); + opt_table[optionals_list.size] = label; + ADD_LABEL(ret, label); + + ISEQ_BODY(iseq)->param.flags.has_opt = TRUE; + ISEQ_BODY(iseq)->param.opt_table = (const VALUE *)opt_table; + } + + iseq_set_local_table(iseq, tbl); + + switch (ISEQ_BODY(iseq)->type) { + case ISEQ_TYPE_BLOCK: + { + LABEL *start = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(0); + LABEL *end = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(0); + + start->rescued = LABEL_RESCUE_BEG; + end->rescued = LABEL_RESCUE_END; + + ADD_TRACE(ret, RUBY_EVENT_B_CALL); + NODE dummy_line_node = generate_dummy_line_node(ISEQ_BODY(iseq)->location.first_lineno, -1); + ADD_INSN (ret, &dummy_line_node, nop); + ADD_LABEL(ret, start); + + if (scope_node->body) { + yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + } + + ADD_LABEL(ret, end); + ADD_TRACE(ret, RUBY_EVENT_B_RETURN); + ISEQ_COMPILE_DATA(iseq)->last_line = ISEQ_BODY(iseq)->location.code_location.end_pos.lineno; + + /* wide range catch handler must put at last */ + ADD_CATCH_ENTRY(CATCH_TYPE_REDO, start, end, NULL, start); + ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, start, end, NULL, end); + break; + } + default: + if (scope_node->body) { + yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + } + + free(index_lookup_table); + + ADD_INSN(ret, &dummy_line_node, leave); + return; + } + case YP_NODE_SELF_NODE: + ADD_INSN(ret, &dummy_line_node, putself); + return; + case YP_NODE_SINGLETON_CLASS_NODE: { + yp_singleton_class_node_t *singleton_class_node = (yp_singleton_class_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)singleton_class_node, &scope_node); + + const rb_iseq_t *singleton_class = NEW_ISEQ(&scope_node, rb_fstring_lit("singleton class"), + ISEQ_TYPE_CLASS, lineno); + + yp_compile_node(iseq, singleton_class_node->expression, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, putnil); + ID singletonclass; + CONST_ID(singletonclass, "singletonclass"); + + ADD_INSN3(ret, &dummy_line_node, defineclass, + ID2SYM(singletonclass), singleton_class, + INT2FIX(VM_DEFINECLASS_TYPE_SINGLETON_CLASS)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)singleton_class); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_SOURCE_ENCODING_NODE: { + const char *encoding = compile_context->parser->encoding.name; + if (!popped) { + rb_encoding *enc = rb_find_encoding(rb_str_new_cstr(encoding)); + if (!enc) { + rb_bug("Encoding not found!"); + } + ADD_INSN1(ret, &dummy_line_node, putobject, rb_enc_from_encoding(enc)); + } + return; + } + case YP_NODE_SOURCE_FILE_NODE: { + yp_source_file_node_t *source_file_node = (yp_source_file_node_t *)node; + + if (!popped) { + VALUE filepath; + if (source_file_node->filepath.length == 0) { + filepath = rb_fstring_lit(""); + } + else { + filepath = parse_string(&source_file_node->filepath); + } + + ADD_INSN1(ret, &dummy_line_node, putstring, filepath); + } + return; + } + case YP_NODE_SOURCE_LINE_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, INT2FIX(lineno)); + } + return; + } + case YP_NODE_SPLAT_NODE: { + yp_splat_node_t *splat_node = (yp_splat_node_t *)node; + yp_compile_node(iseq, splat_node->expression, ret, src, popped, compile_context); + + ADD_INSN1(ret, &dummy_line_node, splatarray, Qtrue); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_STATEMENTS_NODE: { + yp_statements_node_t *statements_node = (yp_statements_node_t *) node; + yp_node_list_t node_list = statements_node->body; + for (size_t index = 0; index < node_list.size - 1; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, true, compile_context); + } + yp_compile_node(iseq, node_list.nodes[node_list.size - 1], ret, src, false, compile_context); + return; + } + case YP_NODE_STRING_CONCAT_NODE: { + yp_string_concat_node_t *str_concat_node = (yp_string_concat_node_t *)node; + yp_compile_node(iseq, str_concat_node->left, ret, src, popped, compile_context); + yp_compile_node(iseq, str_concat_node->right, ret, src, popped, compile_context); + return; + } + case YP_NODE_STRING_NODE: { + if (!popped) { + yp_string_node_t *string_node = (yp_string_node_t *) node; + ADD_INSN1(ret, &dummy_line_node, putstring, parse_string(&string_node->unescaped)); + } + return; + } + case YP_NODE_SYMBOL_NODE: { + yp_symbol_node_t *symbol_node = (yp_symbol_node_t *) node; + ADD_INSN1(ret, &dummy_line_node, putobject, ID2SYM(parse_string_symbol(&symbol_node->unescaped))); + return; + } + case YP_NODE_TRUE_NODE: + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + } + return; + case YP_NODE_UNDEF_NODE: { + yp_undef_node_t *undef_node = (yp_undef_node_t *) node; + + for (size_t index = 0; index < undef_node->names.size; index++) { + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_VMCORE); + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_CBASE); + + yp_compile_node(iseq, undef_node->names.nodes[index], ret, src, popped, compile_context); + + ADD_SEND(ret, &dummy_line_node, rb_intern("core#undef_method"), INT2NUM(2)); + + if (index < undef_node->names.size - 1) + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_UNLESS_NODE: { + const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; + yp_unless_node_t *unless_node = (yp_unless_node_t *)node; + yp_statements_node_t *node_body = unless_node->statements; + yp_node_t *node_else = (yp_node_t *)(unless_node->consequent); + yp_node_t *predicate = unless_node->predicate; + + yp_compile_if(iseq, line, node_body, node_else, predicate, ret, src, popped, compile_context); + return; + } + case YP_NODE_UNTIL_NODE: { + yp_until_node_t *until_node = (yp_until_node_t *)node; + yp_statements_node_t *statements = until_node->statements; + yp_node_t *predicate = until_node->predicate; + yp_node_flags_t flags = node->flags; + + yp_compile_while(iseq, lineno, flags, node->type, statements, predicate, ret, src, popped, compile_context); + return; + } + case YP_NODE_WHILE_NODE: { + yp_while_node_t *while_node = (yp_while_node_t *)node; + yp_statements_node_t *statements = while_node->statements; + yp_node_t *predicate = while_node->predicate; + yp_node_flags_t flags = node->flags; + + yp_compile_while(iseq, lineno, flags, node->type, statements, predicate, ret, src, popped, compile_context); + return; + } + case YP_NODE_X_STRING_NODE: { + yp_x_string_node_t *xstring_node = (yp_x_string_node_t *) node; + ADD_INSN(ret, &dummy_line_node, putself); + ADD_INSN1(ret, &dummy_line_node, putobject, parse_string(&xstring_node->unescaped)); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, rb_intern("`"), INT2NUM(1), INT2FIX(VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE)); + return; + } + case YP_NODE_YIELD_NODE: { + unsigned int flag = 0; + struct rb_callinfo_kwarg *keywords = NULL; + + VALUE argc = INT2FIX(0); + + ADD_INSN1(ret, &dummy_line_node, invokeblock, new_callinfo(iseq, 0, FIX2INT(argc), flag, keywords, FALSE)); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + int level = 0; + const rb_iseq_t *tmp_iseq = iseq; + for (; tmp_iseq != ISEQ_BODY(iseq)->local_iseq; level++ ) { + tmp_iseq = ISEQ_BODY(tmp_iseq)->parent_iseq; + } + + if (level > 0) access_outer_variables(iseq, level, rb_intern("yield"), true); + + return; + } + default: + rb_raise(rb_eNotImpError, "node type %s not implemented", yp_node_type_to_str(node->type)); + return; + } +} + +static VALUE +rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_compile_context_t *compile_context) +{ + RUBY_ASSERT(ISEQ_COMPILE_DATA(iseq)); + RUBY_ASSERT(node->type == YP_NODE_PROGRAM_NODE || node->type == YP_NODE_SCOPE_NODE); + + yp_compile_node(iseq, node, ret, node->location.start, false, compile_context); + iseq_set_sequence(iseq, ret); + return Qnil; +} + +#undef NEW_ISEQ +#define NEW_ISEQ OLD_ISEQ + +#undef NEW_CHILD_ISEQ +#define NEW_CHILD_ISEQ OLD_CHILD_ISEQ diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 6ccb14264bdf9f..b51704b2c606f0 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -342,12 +342,14 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { // Verify stack operand types let top_idx = cmp::min(ctx.get_stack_size(), MAX_TEMP_TYPES as u8); for i in 0..top_idx { - let (learned_mapping, learned_type) = ctx.get_opnd_mapping(StackOpnd(i)); + let learned_mapping = ctx.get_opnd_mapping(StackOpnd(i)); + let learned_type = ctx.get_opnd_type(StackOpnd(i)); + let stack_val = jit.peek_at_stack(ctx, i as isize); let val_type = Type::from(stack_val); - match learned_mapping { - TempMapping::MapToSelf => { + match learned_mapping.get_kind() { + TempMappingKind::MapToSelf => { if self_val != stack_val { panic!( "verify_ctx: stack value was mapped to self, but values did not match!\n stack: {}\n self: {}", @@ -356,8 +358,8 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMapping::MapToLocal(local_idx) => { - let local_idx: u8 = local_idx.into(); + TempMappingKind::MapToLocal => { + let local_idx: u8 = learned_mapping.get_local_idx().into(); let local_val = jit.peek_at_local(local_idx.into()); if local_val != stack_val { panic!( @@ -368,7 +370,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMapping::MapToStack => {} + TempMappingKind::MapToStack => {} } // If the actual type differs from the learned type @@ -1009,9 +1011,9 @@ fn gen_dup( _ocb: &mut OutlinedCb, ) -> Option { let dup_val = asm.stack_opnd(0); - let (mapping, tmp_type) = asm.ctx.get_opnd_mapping(dup_val.into()); + let mapping = asm.ctx.get_opnd_mapping(dup_val.into()); - let loc0 = asm.stack_push_mapping((mapping, tmp_type)); + let loc0 = asm.stack_push_mapping(mapping); asm.mov(loc0, dup_val); Some(KeepCompiling) @@ -1272,7 +1274,7 @@ fn gen_newarray( ); asm.stack_pop(n.as_usize()); - let stack_ret = asm.stack_push(Type::CArray); + let stack_ret = asm.stack_push(Type::TArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1295,7 +1297,7 @@ fn gen_duparray( vec![ary.into()], ); - let stack_ret = asm.stack_push(Type::CArray); + let stack_ret = asm.stack_push(Type::TArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1926,7 +1928,7 @@ fn gen_putstring( vec![EC, put_val.into()] ); - let stack_top = asm.stack_push(Type::CString); + let stack_top = asm.stack_push(Type::TString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -2327,7 +2329,7 @@ fn gen_setinstancevariable( return None; } - let (_, stack_type) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let stack_type = asm.ctx.get_opnd_type(StackOpnd(0)); // Check if the comptime class uses a custom allocator let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; @@ -2722,7 +2724,7 @@ fn gen_concatstrings( ); asm.stack_pop(n); - let stack_ret = asm.stack_push(Type::CString); + let stack_ret = asm.stack_push(Type::TString); asm.mov(stack_ret, return_value); Some(KeepCompiling) @@ -4170,9 +4172,14 @@ fn jit_guard_known_klass( jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); if known_klass == unsafe { rb_cString } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::CString); + // Upgrading to Type::CString here is incorrect. + // The guard we put only checks RBASIC_CLASS(obj), + // which adding a singleton class can change. We + // additionally need to know the string is frozen + // to claim Type::CString. + asm.ctx.upgrade_opnd_type(insn_opnd, Type::TString); } else if known_klass == unsafe { rb_cArray } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::CArray); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::TArray); } } } @@ -6196,7 +6203,7 @@ fn gen_send_iseq( let rest_param = if opts_missing == 0 { // All optionals are filled, the rest param goes at the top of the stack argc += 1; - asm.stack_push(Type::CArray) + asm.stack_push(Type::TArray) } else { // The top of the stack will be a missing optional, but the rest // parameter needs to be placed after all the missing optionals. @@ -8144,9 +8151,10 @@ fn gen_getblockparamproxy( // Peek at the block handler so we can check whether it's nil let comptime_handler = jit.peek_at_block_handler(level); - // Filter for the 3 cases we currently handle + // Filter for the 4 cases we currently handle if !(comptime_handler.as_u64() == 0 || // no block given comptime_handler.as_u64() & 0x3 == 0x1 || // iseq block (no associated GC managed object) + comptime_handler.as_u64() & 0x3 == 0x3 || // ifunc block (no associated GC managed object) unsafe { rb_obj_is_proc(comptime_handler) }.test() // block is a Proc ) { // Missing the symbol case, where we basically need to call Symbol#to_proc at runtime @@ -8174,7 +8182,7 @@ fn gen_getblockparamproxy( // Use block handler sample to guide specialization... // NOTE: we use jit_chain_guard() in this decision tree, and since - // there are only 3 cases, it should never reach the depth limit use + // there are only a few cases, it should never reach the depth limit use // the exit counter we pass to it. // // No block given @@ -8192,15 +8200,18 @@ fn gen_getblockparamproxy( ); jit_putobject(asm, Qnil); - } else if comptime_handler.as_u64() & 0x3 == 0x1 { - // Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P(). - let block_handler = asm.and(block_handler, 0x3.into()); - - // Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null. - asm.cmp(block_handler, 0x1.into()); - + } else if comptime_handler.as_u64() & 0x1 == 0x1 { + // This handles two cases which are nearly identical + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + // Procs are aligned heap pointers so testing the bit rejects them too. + + asm.test(block_handler, 0x1.into()); jit_chain_guard( - JCC_JNZ, + JCC_JZ, jit, asm, ocb, @@ -8963,8 +8974,8 @@ mod tests { let status = gen_swap(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); - let (_, tmp_type_next) = asm.ctx.get_opnd_mapping(StackOpnd(1)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); + let tmp_type_next = asm.ctx.get_opnd_type(StackOpnd(1)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Fixnum); @@ -8976,7 +8987,7 @@ mod tests { let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); let status = gen_putnil(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Nil); @@ -8995,7 +9006,7 @@ mod tests { let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::True); @@ -9015,7 +9026,7 @@ mod tests { let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Fixnum); @@ -9029,7 +9040,7 @@ mod tests { jit.opcode = YARVINSN_putobject_INT2FIX_0_.as_usize(); let status = gen_putobject_int2fix(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); // Right now we're not testing the generated machine code to make sure a literal 1 or 0 was pushed. I've checked locally. assert_eq!(status, Some(KeepCompiling)); diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 40646ebd342fdf..805b7dd68599ce 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -18,13 +18,14 @@ use std::cell::*; use std::collections::HashSet; use std::fmt; use std::mem; +use std::mem::transmute; use std::ops::Range; use std::rc::Rc; use mem::MaybeUninit; use std::ptr; use ptr::NonNull; use YARVOpnd::*; -use TempMapping::*; +use TempMappingKind::*; use crate::invariants::*; // Maximum number of temp value types we keep track of @@ -39,6 +40,7 @@ pub type IseqIdx = u16; // Represent the type of a value (local/stack/self) in YJIT #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] +#[repr(u8)] pub enum Type { Unknown, UnknownImm, @@ -57,7 +59,6 @@ pub enum Type { TString, // An object with the T_STRING flag set, possibly an rb_cString CString, // An un-subclassed string of type rb_cString (can have instance vars in some cases) TArray, // An object with the T_ARRAY flag set, possibly an rb_cArray - CArray, // An un-subclassed string of type rb_cArray (can have instance vars in some cases) TProc, // A proc object. Could be an instance of a subclass of ::rb_cProc @@ -95,13 +96,9 @@ impl Type { // Core.rs can't reference rb_cString because it's linked by Rust-only tests. // But CString vs TString is only an optimisation and shouldn't affect correctness. #[cfg(not(test))] - if val.class_of() == unsafe { rb_cString } { + if val.class_of() == unsafe { rb_cString } && val.is_frozen() { return Type::CString; } - #[cfg(not(test))] - if val.class_of() == unsafe { rb_cArray } { - return Type::CArray; - } // We likewise can't reference rb_block_param_proxy, but it's again an optimisation; // we can just treat it as a normal Object. #[cfg(not(test))] @@ -153,7 +150,6 @@ impl Type { match self { Type::UnknownHeap => true, Type::TArray => true, - Type::CArray => true, Type::Hash => true, Type::HeapSymbol => true, Type::TString => true, @@ -166,11 +162,7 @@ impl Type { /// Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) pub fn is_array(&self) -> bool { - match self { - Type::TArray => true, - Type::CArray => true, - _ => false, - } + matches!(self, Type::TArray) } /// Check if it's a T_STRING object (both TString and CString are T_STRING) @@ -190,7 +182,7 @@ impl Type { Type::False => Some(RUBY_T_FALSE), Type::Fixnum => Some(RUBY_T_FIXNUM), Type::Flonum => Some(RUBY_T_FLOAT), - Type::TArray | Type::CArray => Some(RUBY_T_ARRAY), + Type::TArray => Some(RUBY_T_ARRAY), Type::Hash => Some(RUBY_T_HASH), Type::ImmSymbol | Type::HeapSymbol => Some(RUBY_T_SYMBOL), Type::TString | Type::CString => Some(RUBY_T_STRING), @@ -211,7 +203,6 @@ impl Type { Type::Flonum => Some(rb_cFloat), Type::ImmSymbol | Type::HeapSymbol => Some(rb_cSymbol), Type::CString => Some(rb_cString), - Type::CArray => Some(rb_cArray), _ => None, } } @@ -266,11 +257,6 @@ impl Type { return TypeDiff::Compatible(1); } - // A CArray is also a TArray. - if self == Type::CArray && dst == Type::TArray { - return TypeDiff::Compatible(1); - } - // Specific heap type into unknown heap type is imperfect but valid if self.is_heap() && dst == Type::UnknownHeap { return TypeDiff::Compatible(1); @@ -302,63 +288,92 @@ pub enum TypeDiff { Incompatible, } -// Potential mapping of a value on the temporary stack to -// self, a local variable or constant so that we can track its type #[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)] -pub enum TempMapping { - MapToStack, // Normal stack value - MapToSelf, // Temp maps to the self operand - MapToLocal(LocalIndex), // Temp maps to a local variable with index - //ConstMapping, // Small constant (0, 1, 2, Qnil, Qfalse, Qtrue) +#[repr(u8)] +pub enum TempMappingKind +{ + MapToStack = 0, + MapToSelf = 1, + MapToLocal = 2, } -// Index used by MapToLocal. Using this instead of u8 makes TempMapping 1 byte. +// Potential mapping of a value on the temporary stack to +// self, a local variable or constant so that we can track its type +// +// The highest two bits represent TempMappingKind, and the rest of +// the bits are used differently across different kinds. +// * MapToStack: The lowest 5 bits are used for mapping Type. +// * MapToSelf: The remaining bits are not used; the type is stored in self_type. +// * MapToLocal: The lowest 3 bits store the index of a local variable. #[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)] -pub enum LocalIndex { - Local0, - Local1, - Local2, - Local3, - Local4, - Local5, - Local6, - Local7, -} +pub struct TempMapping(u8); -impl From for u8 { - fn from(idx: LocalIndex) -> Self { - match idx { - LocalIndex::Local0 => 0, - LocalIndex::Local1 => 1, - LocalIndex::Local2 => 2, - LocalIndex::Local3 => 3, - LocalIndex::Local4 => 4, - LocalIndex::Local5 => 5, - LocalIndex::Local6 => 6, - LocalIndex::Local7 => 7, - } +impl TempMapping { + pub fn map_to_stack(t: Type) -> TempMapping + { + let kind_bits = TempMappingKind::MapToStack as u8; + let type_bits = t as u8; + assert!(type_bits <= 0b11111); + let bits = (kind_bits << 6) | (type_bits & 0b11111); + TempMapping(bits) } -} -impl From for LocalIndex { - fn from(idx: u8) -> Self { - match idx { - 0 => LocalIndex::Local0, - 1 => LocalIndex::Local1, - 2 => LocalIndex::Local2, - 3 => LocalIndex::Local3, - 4 => LocalIndex::Local4, - 5 => LocalIndex::Local5, - 6 => LocalIndex::Local6, - 7 => LocalIndex::Local7, - _ => unreachable!("{idx} was larger than {MAX_LOCAL_TYPES}"), + pub fn map_to_self() -> TempMapping + { + let kind_bits = TempMappingKind::MapToSelf as u8; + let bits = kind_bits << 6; + TempMapping(bits) + } + + pub fn map_to_local(local_idx: u8) -> TempMapping + { + let kind_bits = TempMappingKind::MapToLocal as u8; + assert!(local_idx <= 0b111); + let bits = (kind_bits << 6) | (local_idx & 0b111); + TempMapping(bits) + } + + pub fn without_type(&self) -> TempMapping + { + if self.get_kind() != TempMappingKind::MapToStack { + return *self; } + + TempMapping::map_to_stack(Type::Unknown) + } + + pub fn get_kind(&self) -> TempMappingKind + { + // Take the two highest bits + let TempMapping(bits) = self; + let kind_bits = bits >> 6; + assert!(kind_bits <= 2); + unsafe { transmute::(kind_bits) } + } + + pub fn get_type(&self) -> Type + { + assert!(self.get_kind() == TempMappingKind::MapToStack); + + // Take the 5 lowest bits + let TempMapping(bits) = self; + let type_bits = bits & 0b11111; + unsafe { transmute::(type_bits) } + } + + pub fn get_local_idx(&self) -> u8 + { + assert!(self.get_kind() == TempMappingKind::MapToLocal); + + // Take the 3 lowest bits + let TempMapping(bits) = self; + bits & 0b111 } } impl Default for TempMapping { fn default() -> Self { - MapToStack + TempMapping::map_to_stack(Type::Unknown) } } @@ -441,9 +456,6 @@ pub struct Context { // Local variable types we keep track of local_types: [Type; MAX_LOCAL_TYPES], - // Temporary variable types we keep track of - temp_types: [Type; MAX_TEMP_TYPES], - // Type we track for self self_type: Type, @@ -1663,10 +1675,11 @@ impl Context { let mapping = self.temp_mapping[stack_idx]; - match mapping { + match mapping.get_kind() { MapToSelf => self.self_type, - MapToStack => self.temp_types[(self.stack_size - 1 - idx) as usize], - MapToLocal(idx) => { + MapToStack => mapping.get_type(), + MapToLocal => { + let idx = mapping.get_local_idx(); assert!((idx as usize) < MAX_LOCAL_TYPES); return self.local_types[idx as usize]; } @@ -1703,11 +1716,15 @@ impl Context { let mapping = self.temp_mapping[stack_idx]; - match mapping { + match mapping.get_kind() { MapToSelf => self.self_type.upgrade(opnd_type), - MapToStack => self.temp_types[stack_idx].upgrade(opnd_type), - MapToLocal(idx) => { - let idx = idx as usize; + MapToStack => { + let mut temp_type = mapping.get_type(); + temp_type.upgrade(opnd_type); + self.temp_mapping[stack_idx] = TempMapping::map_to_stack(temp_type); + } + MapToLocal => { + let idx = mapping.get_local_idx() as usize; assert!(idx < MAX_LOCAL_TYPES); self.local_types[idx].upgrade(opnd_type); } @@ -1721,29 +1738,29 @@ impl Context { This is can be used with stack_push_mapping or set_opnd_mapping to copy a stack value's type while maintaining the mapping. */ - pub fn get_opnd_mapping(&self, opnd: YARVOpnd) -> (TempMapping, Type) { + pub fn get_opnd_mapping(&self, opnd: YARVOpnd) -> TempMapping { let opnd_type = self.get_opnd_type(opnd); match opnd { - SelfOpnd => (MapToSelf, opnd_type), + SelfOpnd => TempMapping::map_to_self(), StackOpnd(idx) => { assert!(idx < self.stack_size); let stack_idx = (self.stack_size - 1 - idx) as usize; if stack_idx < MAX_TEMP_TYPES { - (self.temp_mapping[stack_idx], opnd_type) + self.temp_mapping[stack_idx] } else { // We can't know the source of this stack operand, so we assume it is // a stack-only temporary. type will be UNKNOWN assert!(opnd_type == Type::Unknown); - (MapToStack, opnd_type) + TempMapping::map_to_stack(opnd_type) } } } } /// Overwrite both the type and mapping of a stack operand. - pub fn set_opnd_mapping(&mut self, opnd: YARVOpnd, (mapping, opnd_type): (TempMapping, Type)) { + pub fn set_opnd_mapping(&mut self, opnd: YARVOpnd, mapping: TempMapping) { match opnd { SelfOpnd => unreachable!("self always maps to self"), StackOpnd(idx) => { @@ -1761,9 +1778,6 @@ impl Context { } self.temp_mapping[stack_idx] = mapping; - - // Only used when mapping == MAP_STACK - self.temp_types[stack_idx] = opnd_type; } } } @@ -1782,16 +1796,16 @@ impl Context { } // If any values on the stack map to this local we must detach them - for (i, mapping) in ctx.temp_mapping.iter_mut().enumerate() { - *mapping = match *mapping { - MapToStack => MapToStack, - MapToSelf => MapToSelf, - MapToLocal(idx) => { + for mapping in ctx.temp_mapping.iter_mut() { + *mapping = match mapping.get_kind() { + MapToStack => *mapping, + MapToSelf => *mapping, + MapToLocal => { + let idx = mapping.get_local_idx(); if idx as usize == local_idx { - ctx.temp_types[i] = ctx.local_types[idx as usize]; - MapToStack + TempMapping::map_to_stack(ctx.local_types[idx as usize]) } else { - MapToLocal(idx) + TempMapping::map_to_local(idx) } } } @@ -1805,14 +1819,10 @@ impl Context { pub fn clear_local_types(&mut self) { // When clearing local types we must detach any stack mappings to those // locals. Even if local values may have changed, stack values will not. - for (i, mapping) in self.temp_mapping.iter_mut().enumerate() { - *mapping = match *mapping { - MapToStack => MapToStack, - MapToSelf => MapToSelf, - MapToLocal(idx) => { - self.temp_types[i] = self.local_types[idx as usize]; - MapToStack - } + for mapping in self.temp_mapping.iter_mut() { + if mapping.get_kind() == MapToLocal { + let idx = mapping.get_local_idx(); + *mapping = TempMapping::map_to_stack(self.local_types[idx as usize]); } } @@ -1869,12 +1879,12 @@ impl Context { // For each value on the temp stack for i in 0..src.stack_size { - let (src_mapping, src_type) = src.get_opnd_mapping(StackOpnd(i)); - let (dst_mapping, dst_type) = dst.get_opnd_mapping(StackOpnd(i)); + let src_mapping = src.get_opnd_mapping(StackOpnd(i)); + let dst_mapping = dst.get_opnd_mapping(StackOpnd(i)); // If the two mappings aren't the same if src_mapping != dst_mapping { - if dst_mapping == MapToStack { + if dst_mapping.get_kind() == MapToStack { // We can safely drop information about the source of the temp // stack operand. diff += 1; @@ -1883,6 +1893,9 @@ impl Context { } } + let src_type = src.get_opnd_type(StackOpnd(i)); + let dst_type = dst.get_opnd_type(StackOpnd(i)); + diff += match src_type.diff(dst_type) { TypeDiff::Compatible(diff) => diff, TypeDiff::Incompatible => return TypeDiff::Incompatible, @@ -1912,10 +1925,10 @@ impl Context { impl Assembler { /// Push one new value on the temp stack with an explicit mapping /// Return a pointer to the new stack top - pub fn stack_push_mapping(&mut self, (mapping, temp_type): (TempMapping, Type)) -> Opnd { + pub fn stack_push_mapping(&mut self, mapping: TempMapping) -> Opnd { // If type propagation is disabled, store no types if get_option!(no_type_prop) { - return self.stack_push_mapping((mapping, Type::Unknown)); + return self.stack_push_mapping(mapping.without_type()); } let stack_size: usize = self.ctx.stack_size.into(); @@ -1923,9 +1936,9 @@ impl Assembler { // Keep track of the type and mapping of the value if stack_size < MAX_TEMP_TYPES { self.ctx.temp_mapping[stack_size] = mapping; - self.ctx.temp_types[stack_size] = temp_type; - if let MapToLocal(idx) = mapping { + if mapping.get_kind() == MapToLocal { + let idx = mapping.get_local_idx(); assert!((idx as usize) < MAX_LOCAL_TYPES); } } @@ -1944,12 +1957,12 @@ impl Assembler { /// Push one new value on the temp stack /// Return a pointer to the new stack top pub fn stack_push(&mut self, val_type: Type) -> Opnd { - return self.stack_push_mapping((MapToStack, val_type)); + return self.stack_push_mapping(TempMapping::map_to_stack(val_type)); } /// Push the self value on the stack pub fn stack_push_self(&mut self) -> Opnd { - return self.stack_push_mapping((MapToSelf, Type::Unknown)); + return self.stack_push_mapping(TempMapping::map_to_self()); } /// Push a local variable on the stack @@ -1958,7 +1971,7 @@ impl Assembler { return self.stack_push(Type::Unknown); } - return self.stack_push_mapping((MapToLocal((local_idx as u8).into()), Type::Unknown)); + return self.stack_push_mapping(TempMapping::map_to_local((local_idx as u8).into())); } // Pop N values off the stack @@ -1973,8 +1986,7 @@ impl Assembler { let idx: usize = (self.ctx.stack_size as usize) - i - 1; if idx < MAX_TEMP_TYPES { - self.ctx.temp_types[idx] = Type::Unknown; - self.ctx.temp_mapping[idx] = MapToStack; + self.ctx.temp_mapping[idx] = TempMapping::map_to_stack(Type::Unknown); } } @@ -1992,7 +2004,6 @@ impl Assembler { for i in method_name_index..(self.ctx.stack_size - 1) as usize { if i + 1 < MAX_TEMP_TYPES { - self.ctx.temp_types[i] = self.ctx.temp_types[i + 1]; self.ctx.temp_mapping[i] = self.ctx.temp_mapping[i + 1]; } } @@ -3169,6 +3180,31 @@ impl RefUnchecked for Cell { mod tests { use crate::core::*; + #[test] + fn tempmapping_size() { + assert_eq!(mem::size_of::(), 1); + } + + #[test] + fn tempmapping() { + let t = TempMapping::map_to_stack(Type::Unknown); + assert_eq!(t.get_kind(), MapToStack); + assert_eq!(t.get_type(), Type::Unknown); + + let t = TempMapping::map_to_stack(Type::TString); + assert_eq!(t.get_kind(), MapToStack); + assert_eq!(t.get_type(), Type::TString); + + let t = TempMapping::map_to_local(7); + assert_eq!(t.get_kind(), MapToLocal); + assert_eq!(t.get_local_idx(), 7); + } + + #[test] + fn context_size() { + assert_eq!(mem::size_of::(), 21); + } + #[test] fn types() { // Valid src => dst