diff --git a/.gdbinit b/.gdbinit index 9841c1466da357..7dd3336e286115 100644 --- a/.gdbinit +++ b/.gdbinit @@ -689,11 +689,6 @@ define nd_stts end -define nd_entry - printf "%su3.entry%s: ", $color_highlite, $color_end - p ($arg0).u3.entry -end - define nd_vid printf "%su1.id%s: ", $color_highlite, $color_end p ($arg0).u1.id diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml index 76489f1885263b..2f5ece83280727 100644 --- a/.github/workflows/annocheck.yml +++ b/.github/workflows/annocheck.yml @@ -45,6 +45,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} env: @@ -64,7 +65,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml index b5455f5bc9a885..50176b3eb0d8bc 100644 --- a/.github/workflows/baseruby.yml +++ b/.github/workflows/baseruby.yml @@ -41,6 +41,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} strategy: @@ -59,7 +60,7 @@ jobs: ruby-version: ${{ matrix.ruby }} bundler: none - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: ./.github/actions/setup/ubuntu diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml index eb82441c4cfda3..8c78e3905b1fda 100644 --- a/.github/workflows/bundled_gems.yml +++ b/.github/workflows/bundled_gems.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: ./.github/actions/setup/directories diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml index 65ba10b7a2ebd7..ac065faaa0165c 100644 --- a/.github/workflows/check_dependencies.yml +++ b/.github/workflows/check_dependencies.yml @@ -43,10 +43,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: ./.github/actions/setup/ubuntu if: ${{ contains(matrix.os, 'ubuntu') }} diff --git a/.github/workflows/check_misc.yml b/.github/workflows/check_misc.yml index acd863ce30dd60..e64a93189371bb 100644 --- a/.github/workflows/check_misc.yml +++ b/.github/workflows/check_misc.yml @@ -16,9 +16,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: ./.github/actions/setup/directories + with: + makeup: true - name: Check if C-sources are US-ASCII run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 28142e8e0e8ef4..04edf2364101a7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,6 +41,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} env: @@ -53,7 +54,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Install libraries uses: ./.github/actions/setup/ubuntu diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 2188bfad331801..f39d3090cc22bc 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -31,7 +31,7 @@ concurrency: # environment variables (plus the "echo $GITHUB_ENV" hack) is to reroute that # restriction. env: - default_cc: clang-16 + default_cc: clang-17 append_cc: '' # -O1 is faster than -O3 in our tests... Majority of time are consumed trying @@ -85,6 +85,7 @@ jobs: optflags: '-O2' shared: disable # check: true + - { name: clang-18, env: { default_cc: clang-18 } } - { name: clang-17, env: { default_cc: clang-17 } } - { name: clang-16, env: { default_cc: clang-16 } } - { name: clang-15, env: { default_cc: clang-15 } } @@ -123,15 +124,17 @@ jobs: # warning generates a lot of noise from use of ANYARGS in # rb_define_method() and friends. # See: https://github.com/llvm/llvm-project/commit/11da1b53d8cd3507959022cd790d5a7ad4573d94 - - { name: c99, env: { append_cc: '-std=c99 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } -# - { name: c11, env: { append_cc: '-std=c11 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } -# - { name: c17, env: { append_cc: '-std=c17 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - - { name: c2x, env: { append_cc: '-std=c2x -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } + - { name: c99, env: { CFLAGS: '-std=c99 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } +# - { name: c11, env: { CFLAGS: '-std=c11 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } +# - { name: c17, env: { CFLAGS: '-std=c17 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } + - { name: c23, env: { CFLAGS: '-std=c2x -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - { name: c++98, env: { CXXFLAGS: '-std=c++98 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } # - { name: c++11, env: { CXXFLAGS: '-std=c++11 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } # - { name: c++14, env: { CXXFLAGS: '-std=c++14 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } # - { name: c++17, env: { CXXFLAGS: '-std=c++17 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { name: c++2a, env: { CXXFLAGS: '-std=c++2a -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } +# - { name: c++20, env: { CXXFLAGS: '-std=c++20 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } +# - { name: c++23, env: { CXXFLAGS: '-std=c++23 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } + - { name: c++26, env: { CXXFLAGS: '-std=c++26 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - { name: '-O0', env: { optflags: '-O0 -march=x86-64 -mtune=generic' } } # - { name: '-O3', env: { optflags: '-O3 -march=x86-64 -mtune=generic' }, check: true } @@ -207,13 +210,14 @@ jobs: - { name: enable-rjit, env: { append_configure: '--enable-rjit --disable-yjit' } } - { name: YJIT_FORCE_ENABLE, env: { cppflags: '-DYJIT_FORCE_ENABLE' } } # - { name: RJIT_FORCE_ENABLE, env: { cppflags: '-DRJIT_FORCE_ENABLE' } } + - { name: UNIVERSAL_PARSER, env: { cppflags: '-DUNIVERSAL_PARSER' } } name: ${{ matrix.entry.name }} runs-on: ubuntu-latest container: - image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-16' }} + image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-17' }} options: --user root if: >- @@ -221,6 +225,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} env: ${{ matrix.entry.env || matrix.env }} @@ -229,7 +234,7 @@ jobs: - run: id working-directory: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d5835183072694..30c30e46265c4e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -52,10 +52,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 0136c7bd2d6836..df24ff394e0c51 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -63,6 +63,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: @@ -98,7 +99,7 @@ jobs: $result working-directory: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/rjit-bindgen.yml b/.github/workflows/rjit-bindgen.yml index cb777c069755ea..6a6769b92afc8d 100644 --- a/.github/workflows/rjit-bindgen.yml +++ b/.github/workflows/rjit-bindgen.yml @@ -44,6 +44,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: @@ -52,7 +53,7 @@ jobs: with: ruby-version: '3.1' - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index 4d9062f1bd79ad..eea20d6be95aa0 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -54,10 +54,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 5c480eed46781d..c6a4110af1b994 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -32,7 +32,7 @@ jobs: steps: - name: 'Checkout code' - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: persist-credentials: false diff --git a/.github/workflows/spec_guards.yml b/.github/workflows/spec_guards.yml index 4a5ff9a90fb087..77677c98596807 100644 --- a/.github/workflows/spec_guards.yml +++ b/.github/workflows/spec_guards.yml @@ -32,6 +32,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} strategy: @@ -44,7 +45,7 @@ jobs: - ruby-3.2 steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: ruby/setup-ruby@250fcd6a742febb1123a77a841497ccaa8b9e939 # v1.152.0 with: diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2e3f228aec6805..66b0228102c023 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -60,10 +60,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 496d5fc8925c4f..3be9bf8d1ed380 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -59,10 +59,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 49e2ce5738439e..71209a5c9c48a3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -45,6 +45,7 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} name: VisualStudio ${{ matrix.vs }} @@ -107,7 +108,7 @@ jobs: Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH shell: pwsh - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 6912e775ee7ef5..45c7a224441356 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 # For now we can't run cargo test --offline because it complains about the # capstone dependency, even though the dependency is optional @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 # Check that we don't have linting errors in release mode, too - run: cargo clippy --all-targets --all-features @@ -114,10 +114,11 @@ jobs: || contains(github.event.head_commit.message, '[DOC]') || contains(github.event.pull_request.title, '[DOC]') || contains(github.event.pull_request.labels.*.name, 'Documentation') + || (github.event_name == 'push' && github.actor == 'dependabot[bot]') )}} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: sparse-checkout-cone-mode: false sparse-checkout: /.github diff --git a/NEWS.md b/NEWS.md index dd40577821fa26..edad5de51e6ca1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -54,7 +54,7 @@ Note: We're only listing outstanding class updates. * Notify the Ruby virtual machine that the boot sequence is finished, and that now is a good time to optimize the application. This is useful for long running applications. The actual optimizations performed are entirely - implementation specific and may change in the future without notice. [[Feature #18885] + implementation specific and may change in the future without notice. [[Feature #18885]] * Refinement @@ -69,20 +69,21 @@ The following default gems are updated. * bigdecimal 3.1.4 * bundler 2.5.0.dev * csv 3.2.8 +* erb 4.0.3 * fiddle 1.1.2 * fileutils 1.7.1 * irb 1.7.4 * nkf 0.1.3 * optparse 0.4.0.pre.1 * psych 5.1.0 -* reline 0.3.7 +* reline 0.3.8 * stringio 3.0.8 * strscan 3.0.7 * syntax_suggest 1.1.0 * time 0.2.2 * timeout 0.4.0 * uri 0.12.2 -* yarp 0.7.0 +* yarp 0.8.0 The following bundled gems are updated. @@ -122,23 +123,29 @@ changelog for details of the default gems or bundled gems. ### YJIT -* Significant performance improvements over 3.2 - * Splat and rest arguments support has been improved. +* Major performance improvements over 3.2 + * Support for splat and rest arguments has been improved. * Registers are allocated for stack operations of the virtual machine. * More calls with optional arguments are compiled. - * `Integer#!=`, `String#!=`, `Kernel#block_given?`, `Kernel#is_a?`, - `Kernel#instance_of?`, `Module#===` are specially optimized. + * Exception handlers are also compiled. * Instance variables no longer exit to the interpreter with megamorphic Object Shapes. + * Unsupported call types no longer exit to the interpreter. + * `Integer#!=`, `String#!=`, `Kernel#block_given?`, `Kernel#is_a?`, + `Kernel#instance_of?`, `Module#===` are specially optimized. + * Now more than 3x faster than the interpreter on optcarrot! * Metadata for compiled code uses a lot less memory. -* Improved code generation on ARM64 +* Generate more compact code on ARM64 * Option to start YJIT in paused mode and then later enable it manually * `--yjit-pause` and `RubyVM::YJIT.resume` * This can be used to enable YJIT only once your application is done booting +* `ratio_in_yjit` stat produced by `--yjit-stats` is now avaiable in release builds, + a special stats or dev build is no longer required. * Exit tracing option now supports sampling * `--trace-exits-sample-rate=N` +* `--yjit-stats=quiet` is added to avoid printing stats on exit. * The default value for `--yjit-exec-mem-size` is changed from 64 to 128. -* Multiple bug fixes +* More thorough testing and multiple bug fixes ### RJIT @@ -149,6 +156,7 @@ changelog for details of the default gems or bundled gems. * You should keep using YJIT in production. [Feature #18498]: https://bugs.ruby-lang.org/issues/18498 +[Feature #18885]: https://bugs.ruby-lang.org/issues/18885 [Bug #19150]: https://bugs.ruby-lang.org/issues/19150 [Feature #19314]: https://bugs.ruby-lang.org/issues/19314 [Feature #19347]: https://bugs.ruby-lang.org/issues/19347 diff --git a/addr2line.c b/addr2line.c index 287a69d5e42739..358eccc5fc841f 100644 --- a/addr2line.c +++ b/addr2line.c @@ -1474,7 +1474,7 @@ static bool di_skip_records(DebugInfoReader *reader) { for (;;) { - DebugInfoValue v = {{}}; + DebugInfoValue v = {{0}}; uint64_t at = uleb128(&reader->q); uint64_t form = uleb128(&reader->q); if (!at || !form) return true; @@ -1791,10 +1791,10 @@ di_read_cu(DebugInfoReader *reader) reader->current_addr_base = 0; reader->current_rnglists_base = 0; - DebugInfoValue low_pc = {{}}; + DebugInfoValue low_pc = {{0}}; /* enumerate abbrev */ for (;;) { - DebugInfoValue v = {{}}; + DebugInfoValue v = {{0}}; if (!di_read_record(reader, &v)) break; switch (v.at) { case DW_AT_low_pc: @@ -1820,7 +1820,7 @@ di_read_cu(DebugInfoReader *reader) break; case VAL_addr: { - addr_header_t header = {}; + addr_header_t header = {0}; if (!addr_header_init(reader->obj, &header)) return -1; reader->current_low_pc = read_addr(&header, reader->current_addr_base, low_pc.as.addr_idx); } @@ -1861,7 +1861,7 @@ read_abstract_origin(DebugInfoReader *reader, uint64_t form, uint64_t abstract_o /* enumerate abbrev */ for (;;) { - DebugInfoValue v = {{}}; + DebugInfoValue v = {{0}}; if (!di_read_record(reader, &v)) break; switch (v.at) { case DW_AT_name: @@ -1880,16 +1880,16 @@ static bool debug_info_read(DebugInfoReader *reader, int num_traces, void **traces, line_info_t *lines, int offset) { - addr_header_t addr_header = {}; + addr_header_t addr_header = {0}; if (!addr_header_init(reader->obj, &addr_header)) return false; - rnglists_header_t rnglists_header = {}; + rnglists_header_t rnglists_header = {0}; if (!rnglists_header_init(reader->obj, &rnglists_header)) return false; while (reader->p < reader->cu_end) { DIE die; - ranges_t ranges = {}; - line_info_t line = {}; + ranges_t ranges = {0}; + line_info_t line = {0}; if (!di_read_die(reader, &die)) continue; /* kprintf("%d:%tx: <%d>\n",__LINE__,die.pos,reader->level,die.tag); */ @@ -1902,7 +1902,7 @@ debug_info_read(DebugInfoReader *reader, int num_traces, void **traces, /* enumerate abbrev */ for (;;) { - DebugInfoValue v = {{}}; + DebugInfoValue v = {{0}}; /* ptrdiff_t pos = reader->p - reader->p0; */ if (!di_read_record(reader, &v)) break; /* kprintf("\n%d:%tx: AT:%lx FORM:%lx\n",__LINE__,pos,v.at,v.form); */ @@ -1993,7 +1993,7 @@ parse_ver5_debug_line_header(const char *p, int idx, uint8_t format, obj_info_t for (j = 0; j < entry_count; j++) { const char *format = entry_format; for (i = 0; i < entry_format_count; i++) { - DebugInfoValue v = {{}}; + DebugInfoValue v = {{0}}; unsigned long dw_lnct = uleb128(&format); unsigned long dw_form = uleb128(&format); if (!debug_info_reader_read_value(&reader, dw_form, &v)) return 0; diff --git a/ast.c b/ast.c index d3e217dafcca2a..4bd3784fbae9ae 100644 --- a/ast.c +++ b/ast.c @@ -97,7 +97,7 @@ rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE StringValue(str); VALUE vparser = ast_parse_new(); - if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser); + if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); ast = rb_parser_compile_string_path(vparser, Qnil, str, 1); @@ -121,7 +121,7 @@ rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VAL f = rb_file_open_str(path, "r"); rb_funcall(f, rb_intern("set_encoding"), 2, rb_enc_from_encoding(enc), rb_str_new_cstr("-")); VALUE vparser = ast_parse_new(); - if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser); + if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); ast = rb_parser_compile_file_path(vparser, Qnil, f, 1); @@ -149,7 +149,7 @@ rb_ast_parse_array(VALUE array, VALUE keep_script_lines, VALUE error_tolerant, V array = rb_check_array_type(array); VALUE vparser = ast_parse_new(); - if (RTEST(keep_script_lines)) rb_parser_keep_script_lines(vparser); + if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue); if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser); if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser); ast = rb_parser_compile_generic(vparser, lex_array, Qnil, array, 1); @@ -184,8 +184,8 @@ node_find(VALUE self, const int node_id) extern VALUE rb_e_script; -static VALUE -script_lines(VALUE path) +VALUE +rb_script_lines_for(VALUE path, bool add) { VALUE hash, lines; ID script_lines; @@ -193,10 +193,19 @@ script_lines(VALUE path) if (!rb_const_defined_at(rb_cObject, script_lines)) return Qnil; hash = rb_const_get_at(rb_cObject, script_lines); if (!RB_TYPE_P(hash, T_HASH)) return Qnil; - lines = rb_hash_lookup(hash, path); - if (!RB_TYPE_P(lines, T_ARRAY)) return Qnil; + if (add) { + rb_hash_aset(hash, path, lines = rb_ary_new()); + } + else if (!RB_TYPE_P((lines = rb_hash_lookup(hash, path)), T_ARRAY)) { + return Qnil; + } return lines; } +static VALUE +script_lines(VALUE path) +{ + return rb_script_lines_for(path, false); +} static VALUE node_id_for_backtrace_location(rb_execution_context_t *ec, VALUE module, VALUE location) diff --git a/ast.rb b/ast.rb index ec870d8c7a725a..fa9b69507c3f43 100644 --- a/ast.rb +++ b/ast.rb @@ -20,7 +20,7 @@ module RubyVM::AbstractSyntaxTree # call-seq: - # RubyVM::AbstractSyntaxTree.parse(string, keep_script_lines: false, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node + # RubyVM::AbstractSyntaxTree.parse(string, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node # # Parses the given _string_ into an abstract syntax tree, # returning the root node of that tree. @@ -55,12 +55,12 @@ module RubyVM::AbstractSyntaxTree # # Note that parsing continues even after the errored expression. # - def self.parse string, keep_script_lines: false, error_tolerant: false, keep_tokens: false + def self.parse string, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false Primitive.ast_s_parse string, keep_script_lines, error_tolerant, keep_tokens end # call-seq: - # RubyVM::AbstractSyntaxTree.parse_file(pathname, keep_script_lines: false, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node + # RubyVM::AbstractSyntaxTree.parse_file(pathname, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node # # Reads the file from _pathname_, then parses it like ::parse, # returning the root node of the abstract syntax tree. @@ -72,13 +72,13 @@ def self.parse string, keep_script_lines: false, error_tolerant: false, keep_tok # # => # # # See ::parse for explanation of keyword argument meaning and usage. - def self.parse_file pathname, keep_script_lines: false, error_tolerant: false, keep_tokens: false + def self.parse_file pathname, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false Primitive.ast_s_parse_file pathname, keep_script_lines, error_tolerant, keep_tokens end # call-seq: - # RubyVM::AbstractSyntaxTree.of(proc, keep_script_lines: false, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node - # RubyVM::AbstractSyntaxTree.of(method, keep_script_lines: false, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node + # RubyVM::AbstractSyntaxTree.of(proc, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node + # RubyVM::AbstractSyntaxTree.of(method, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false) -> RubyVM::AbstractSyntaxTree::Node # # Returns AST nodes of the given _proc_ or _method_. # @@ -93,7 +93,7 @@ def self.parse_file pathname, keep_script_lines: false, error_tolerant: false, k # # => # # # See ::parse for explanation of keyword argument meaning and usage. - def self.of body, keep_script_lines: false, error_tolerant: false, keep_tokens: false + def self.of body, keep_script_lines: RubyVM.keep_script_lines, error_tolerant: false, keep_tokens: false Primitive.ast_s_of body, keep_script_lines, error_tolerant, keep_tokens end diff --git a/common.mk b/common.mk index ea200298c05afb..b0f9369fbc8682 100644 --- a/common.mk +++ b/common.mk @@ -45,7 +45,7 @@ RUN_OPTS = --disable-gems # GITPULLOPTIONS = --no-tags -INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(UNICODE_HDR_DIR) +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(srcdir)/yarp -I$(UNICODE_HDR_DIR) GEM_HOME = GEM_PATH = @@ -202,39 +202,39 @@ $(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time $(Q) $(MAKEDIRS) $(@D) @$(NULLCMD) > $@ -main: $(top_srcdir)/lib/yarp/node.rb -srcs: $(top_srcdir)/lib/yarp/node.rb -$(top_srcdir)/lib/yarp/node.rb: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/lib/yarp/node.rb.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb lib/yarp/node.rb $(top_srcdir)/lib/yarp/node.rb +main: $(srcdir)/lib/yarp/node.rb +srcs: $(srcdir)/lib/yarp/node.rb +$(srcdir)/lib/yarp/node.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/node.rb.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb -main: $(top_srcdir)/lib/yarp/serialize.rb -srcs: $(top_srcdir)/lib/yarp/serialize.rb -$(top_srcdir)/lib/yarp/serialize.rb: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/lib/yarp/serialize.rb.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb lib/yarp/serialize.rb $(top_srcdir)/lib/yarp/serialize.rb +main: $(srcdir)/lib/yarp/serialize.rb +srcs: $(srcdir)/lib/yarp/serialize.rb +$(srcdir)/lib/yarp/serialize.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/serialize.rb.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/serialize.rb $(srcdir)/lib/yarp/serialize.rb -srcs: $(top_srcdir)/yarp/api_node.c -$(top_srcdir)/yarp/api_node.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/ext/yarp/api_node.c.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb ext/yarp/api_node.c $(top_srcdir)/yarp/api_node.c +srcs: yarp/api_node.c +yarp/api_node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/ext/yarp/api_node.c.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb ext/yarp/api_node.c $@ -srcs: $(top_srcdir)/yarp/ast.h -$(top_srcdir)/yarp/ast.h: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/include/yarp/ast.h.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb include/yarp/ast.h $(top_srcdir)/yarp/ast.h +srcs: yarp/ast.h +yarp/ast.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/ast.h.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/ast.h $@ -srcs: $(top_srcdir)/yarp/node.c -$(top_srcdir)/yarp/node.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/node.c.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/node.c $(top_srcdir)/yarp/node.c +srcs: yarp/node.c +yarp/node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/node.c.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/node.c $@ -srcs: $(top_srcdir)/yarp/prettyprint.c -$(top_srcdir)/yarp/prettyprint.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/prettyprint.c.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/prettyprint.c $(top_srcdir)/yarp/prettyprint.c +srcs: yarp/prettyprint.c +yarp/prettyprint.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/prettyprint.c.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/prettyprint.c $@ -srcs: $(top_srcdir)/yarp/serialize.c -$(top_srcdir)/yarp/serialize.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/serialize.c.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/serialize.c $(top_srcdir)/yarp/serialize.c +srcs: yarp/serialize.c +yarp/serialize.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/serialize.c.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/serialize.c $@ -srcs: $(top_srcdir)/yarp/token_type.c -$(top_srcdir)/yarp/token_type.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/token_type.c.erb - $(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/token_type.c $(top_srcdir)/yarp/token_type.c +srcs: yarp/token_type.c +yarp/token_type.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/token_type.c.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/token_type.c $@ EXPORTOBJS = $(DLNOBJ) \ localeinit.$(OBJEXT) \ @@ -6980,6 +6980,7 @@ gc.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h gc.$(OBJEXT): {$(VPATH)}builtin.h gc.$(OBJEXT): {$(VPATH)}config.h gc.$(OBJEXT): {$(VPATH)}constant.h +gc.$(OBJEXT): {$(VPATH)}darray.h gc.$(OBJEXT): {$(VPATH)}debug.h gc.$(OBJEXT): {$(VPATH)}debug_counter.h gc.$(OBJEXT): {$(VPATH)}defines.h @@ -8401,6 +8402,7 @@ load.$(OBJEXT): $(CCAN_DIR)/str/str.h load.$(OBJEXT): $(hdrdir)/ruby/ruby.h load.$(OBJEXT): $(top_srcdir)/internal/array.h load.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h +load.$(OBJEXT): $(top_srcdir)/internal/bits.h load.$(OBJEXT): $(top_srcdir)/internal/compilers.h load.$(OBJEXT): $(top_srcdir)/internal/dir.h load.$(OBJEXT): $(top_srcdir)/internal/error.h @@ -18849,8 +18851,8 @@ weakmap.$(OBJEXT): {$(VPATH)}vm_opts.h weakmap.$(OBJEXT): {$(VPATH)}weakmap.c yarp/api_node.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_node.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/api_node.c -yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19041,7 +19043,7 @@ yarp/api_node.$(OBJEXT): {$(VPATH)}subst.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/api_pack.c -yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19372,7 +19374,7 @@ yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_31j.c yarp/enc/yp_windows_31j.$(OBJEXT): {$(VPATH)}config.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19561,12 +19563,12 @@ yarp/extension.$(OBJEXT): {$(VPATH)}onigmo.h yarp/extension.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/extension.$(OBJEXT): {$(VPATH)}st.h yarp/extension.$(OBJEXT): {$(VPATH)}subst.h -yarp/node.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h -yarp/node.$(OBJEXT): $(top_srcdir)/yarp/node.c +yarp/node.$(OBJEXT): {$(VPATH)}yarp/node.c yarp/node.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19588,12 +19590,12 @@ yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/pack.$(OBJEXT): {$(VPATH)}config.h -yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/parser.h -yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/prettyprint.c +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/prettyprint.c yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h @@ -19601,7 +19603,7 @@ yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/prettyprint.$(OBJEXT): {$(VPATH)}config.h -yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19616,7 +19618,7 @@ yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/regexp.$(OBJEXT): {$(VPATH)}config.h -yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19625,7 +19627,7 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/parser.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/regexp.h -yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/serialize.c +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/serialize.c yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/unescape.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h @@ -19639,14 +19641,14 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/serialize.$(OBJEXT): {$(VPATH)}config.h -yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/defines.h -yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/token_type.c +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/token_type.$(OBJEXT): {$(VPATH)}config.h -yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19690,7 +19692,7 @@ yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.c yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/util/yp_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19727,7 +19729,7 @@ yarp/util/yp_string_list.$(OBJEXT): {$(VPATH)}config.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strncasecmp.c -yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19740,7 +19742,7 @@ yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.c yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}config.h -yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19766,7 +19768,7 @@ yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/yarp.$(OBJEXT): {$(VPATH)}config.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/ast.h +yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h diff --git a/compile.c b/compile.c index d36175ad7362ce..c84572bfd6bf0e 100644 --- a/compile.c +++ b/compile.c @@ -5348,7 +5348,7 @@ defined_expr0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, case NODE_GVAR: ADD_INSN(ret, line_node, putnil); ADD_INSN3(ret, line_node, defined, INT2FIX(DEFINED_GVAR), - ID2SYM(node->nd_entry), PUSH_VAL(DEFINED_GVAR)); + ID2SYM(node->nd_vid), PUSH_VAL(DEFINED_GVAR)); return; case NODE_CVAR: @@ -9476,7 +9476,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no if (!popped) { ADD_INSN(ret, node, dup); } - ADD_INSN1(ret, node, setglobal, ID2SYM(node->nd_entry)); + ADD_INSN1(ret, node, setglobal, ID2SYM(node->nd_vid)); break; } case NODE_IASGN:{ @@ -9606,7 +9606,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no break; } case NODE_GVAR:{ - ADD_INSN1(ret, node, getglobal, ID2SYM(node->nd_entry)); + ADD_INSN1(ret, node, getglobal, ID2SYM(node->nd_vid)); if (popped) { ADD_INSN(ret, node, pop); } diff --git a/configure.ac b/configure.ac index f71c2c260904a2..05dd41e79c9f69 100644 --- a/configure.ac +++ b/configure.ac @@ -2224,6 +2224,27 @@ RUBY_CHECK_BUILTIN_FUNC(__builtin_types_compatible_p, [__builtin_types_compatibl RUBY_CHECK_BUILTIN_FUNC(__builtin_trap, [__builtin_trap()]) RUBY_CHECK_BUILTIN_FUNC(__builtin_expect, [__builtin_expect(0, 0)]) +AS_IF([test "$rb_cv_builtin___builtin_mul_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_mul_overflow with long long arguments, rb_cv_use___builtin_mul_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_mul_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_mul_overflow_long_long=yes, + rb_cv_use___builtin_mul_overflow_long_long=no)]) +]) +AS_IF([test "$rb_cv_use___builtin_mul_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG, 1) +]) + AS_IF([test "$ac_cv_func_qsort_r" != no], [ AC_CACHE_CHECK(whether qsort_r is GNU version, rb_cv_gnu_qsort_r, [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/cont.c b/cont.c index ac3d77f940680a..db8508e724bebf 100644 --- a/cont.c +++ b/cont.c @@ -2118,7 +2118,7 @@ rb_fiber_storage_get(VALUE self) static int fiber_storage_validate_each(VALUE key, VALUE value, VALUE _argument) { - rb_check_id(&key); + Check_Type(key, T_SYMBOL); return ST_CONTINUE; } @@ -2190,8 +2190,7 @@ rb_fiber_storage_set(VALUE self, VALUE value) static VALUE rb_fiber_storage_aref(VALUE class, VALUE key) { - ID id = rb_check_id(&key); - if (!id) return Qnil; + Check_Type(key, T_SYMBOL); VALUE storage = fiber_storage_get(fiber_current(), FALSE); if (storage == Qnil) return Qnil; @@ -2212,8 +2211,7 @@ rb_fiber_storage_aref(VALUE class, VALUE key) static VALUE rb_fiber_storage_aset(VALUE class, VALUE key, VALUE value) { - ID id = rb_check_id(&key); - if (!id) return Qnil; + Check_Type(key, T_SYMBOL); VALUE storage = fiber_storage_get(fiber_current(), value != Qnil); if (storage == Qnil) return Qnil; diff --git a/coroutine/amd64/Context.S b/coroutine/amd64/Context.S index d50732adbc043c..056c276a317c3c 100644 --- a/coroutine/amd64/Context.S +++ b/coroutine/amd64/Context.S @@ -13,29 +13,35 @@ .globl PREFIXED_SYMBOL(SYMBOL_PREFIX,coroutine_transfer) PREFIXED_SYMBOL(SYMBOL_PREFIX,coroutine_transfer): - # Save caller state - pushq %rbp - pushq %rbx - pushq %r12 - pushq %r13 - pushq %r14 - pushq %r15 - - # Save caller stack pointer + # Make space on the stack for 6 registers: + subq $48, %rsp + + # Save caller state: + movq %rbp, 40(%rsp) + movq %rbx, 32(%rsp) + movq %r12, 24(%rsp) + movq %r13, 16(%rsp) + movq %r14, 8(%rsp) + movq %r15, (%rsp) + + # Save caller stack pointer: movq %rsp, (%rdi) - # Restore callee stack pointer + # Restore callee stack pointer: movq (%rsi), %rsp # Restore callee state - popq %r15 - popq %r14 - popq %r13 - popq %r12 - popq %rbx - popq %rbp - - # Put the first argument into the return value + movq 40(%rsp), %rbp + movq 32(%rsp), %rbx + movq 24(%rsp), %r12 + movq 16(%rsp), %r13 + movq 8(%rsp), %r14 + movq (%rsp), %r15 + + # Adjust stack pointer back: + addq $48, %rsp + + # Put the first argument into the return value: movq %rdi, %rax # We pop the return address and jump to it diff --git a/darray.h b/darray.h index db95c8b60e643b..8e1e5763551605 100644 --- a/darray.h +++ b/darray.h @@ -5,6 +5,9 @@ #include #include +#include "internal/bits.h" +#include "internal/gc.h" + // Type for a dynamic array. Use to declare a dynamic array. // It is a pointer so it fits in st_table nicely. Designed // to be fairly type-safe. @@ -37,13 +40,21 @@ // #define rb_darray_ref(ary, idx) (&((ary)->data[(idx)])) -// Copy a new element into the array. ptr_to_ary is evaluated multiple times. -// -// void rb_darray_append(rb_darray(T) *ptr_to_ary, T element); -// -#define rb_darray_append(ptr_to_ary, element) do { \ - rb_darray_ensure_space((ptr_to_ary), sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0])); \ +/* Copy a new element into the array. ptr_to_ary is evaluated multiple times. + * + * void rb_darray_append(rb_darray(T) *ptr_to_ary, T element); + */ +#define rb_darray_append(ptr_to_ary, element) \ + rb_darray_append_impl(ptr_to_ary, element, rb_xrealloc_mul_add) + +#define rb_darray_append_without_gc(ptr_to_ary, element) \ + rb_darray_append_impl(ptr_to_ary, element, rb_darray_realloc_mul_add_without_gc) + +#define rb_darray_append_impl(ptr_to_ary, element, realloc_func) do { \ + rb_darray_ensure_space((ptr_to_ary), \ + sizeof(**(ptr_to_ary)), \ + sizeof((*(ptr_to_ary))->data[0]), \ + realloc_func); \ rb_darray_set(*(ptr_to_ary), \ (*(ptr_to_ary))->meta.size, \ (element)); \ @@ -55,28 +66,53 @@ #define rb_darray_foreach(ary, idx_name, elem_ptr_var) \ for (size_t idx_name = 0; idx_name < rb_darray_size(ary) && ((elem_ptr_var) = rb_darray_ref(ary, idx_name)); ++idx_name) -// Make a dynamic array of a certain size. All bytes backing the elements are set to zero. -// -// Note that NULL is a valid empty dynamic array. -// -// void rb_darray_make(rb_darray(T) *ptr_to_ary, size_t size); +// Iterate over valid indices in the array in a for loop // +#define rb_darray_for(ary, idx_name) \ + for (size_t idx_name = 0; idx_name < rb_darray_size(ary); ++idx_name) + +/* Make a dynamic array of a certain size. All bytes backing the elements are set to zero. + * Return 1 on success and 0 on failure. + * + * Note that NULL is a valid empty dynamic array. + * + * void rb_darray_make(rb_darray(T) *ptr_to_ary, size_t size); + */ #define rb_darray_make(ptr_to_ary, size) \ rb_darray_make_impl((ptr_to_ary), size, sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0])) + sizeof((*(ptr_to_ary))->data[0]), rb_xcalloc_mul_add) -#define rb_darray_data_ptr(ary) ((ary)->data) +#define rb_darray_make_without_gc(ptr_to_ary, size) \ + rb_darray_make_impl((ptr_to_ary), size, sizeof(**(ptr_to_ary)), \ + sizeof((*(ptr_to_ary))->data[0]), rb_darray_calloc_mul_add_without_gc) -// Set the size of the array to zero without freeing the backing memory. -// Allows reusing the same array. -// -#define rb_darray_clear(ary) (ary->meta.size = 0) +/* Resize the darray to a new capacity. The new capacity must be greater than + * or equal to the size of the darray. + * + * void rb_darray_resize_capa(rb_darray(T) *ptr_to_ary, size_t capa); + */ +#define rb_darray_resize_capa_without_gc(ptr_to_ary, capa) \ + rb_darray_resize_capa_impl((ptr_to_ary), rb_darray_next_power_of_two(capa), sizeof(**(ptr_to_ary)), \ + sizeof((*(ptr_to_ary))->data[0]), rb_darray_realloc_mul_add_without_gc) + +#define rb_darray_data_ptr(ary) ((ary)->data) typedef struct rb_darray_meta { size_t size; size_t capa; } rb_darray_meta_t; +/* Set the size of the array to zero without freeing the backing memory. + * Allows reusing the same array. */ +static inline void +rb_darray_clear(void *ary) +{ + rb_darray_meta_t *meta = ary; + if (meta) { + meta->size = 0; + } +} + // Get the size of the dynamic array. // static inline size_t @@ -95,20 +131,86 @@ rb_darray_capa(const void *ary) return meta ? meta->capa : 0; } -// Free the dynamic array. -// +/* Free the dynamic array. */ static inline void rb_darray_free(void *ary) { rb_darray_meta_t *meta = ary; - ruby_sized_xfree(ary, meta->capa); + if (meta) ruby_sized_xfree(ary, meta->capa); +} + +static inline void +rb_darray_free_without_gc(void *ary) +{ + free(ary); +} + +/* Internal function. Like rb_xcalloc_mul_add but does not trigger GC and does + * not check for overflow in arithmetic. */ +static inline void * +rb_darray_calloc_mul_add_without_gc(size_t x, size_t y, size_t z) +{ + size_t size = (x * y) + z; + + void *ptr = calloc(1, size); + if (ptr == NULL) rb_bug("rb_darray_calloc_mul_add_without_gc: failed"); + + return ptr; +} + +/* Internal function. Like rb_xrealloc_mul_add but does not trigger GC and does + * not check for overflow in arithmetic. */ +static inline void * +rb_darray_realloc_mul_add_without_gc(const void *orig_ptr, size_t x, size_t y, size_t z) +{ + size_t size = (x * y) + z; + + void *ptr = realloc((void *)orig_ptr, size); + if (ptr == NULL) rb_bug("rb_darray_realloc_mul_add_without_gc: failed"); + + return ptr; +} + +/* Internal function. Returns the next power of two that is greater than or + * equal to n. */ +static inline size_t +rb_darray_next_power_of_two(size_t n) +{ + return (size_t)(1 << (64 - nlz_int64(n))); +} + +/* Internal function. Resizes the capacity of a darray. The new capacity must + * be greater than or equal to the size of the darray. */ +static inline void +rb_darray_resize_capa_impl(void *ptr_to_ary, size_t new_capa, size_t header_size, size_t element_size, + void *(*realloc_mul_add_impl)(const void *, size_t, size_t, size_t)) +{ + rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; + rb_darray_meta_t *meta = *ptr_to_ptr_to_meta; + + rb_darray_meta_t *new_ary = realloc_mul_add_impl(meta, new_capa, element_size, header_size); + + if (meta == NULL) { + /* First allocation. Initialize size. On subsequence allocations + * realloc takes care of carrying over the size. */ + new_ary->size = 0; + } + + assert(new_ary->size <= new_capa); + + new_ary->capa = new_capa; + + // We don't have access to the type of the dynamic array in function context. + // Write out result with memcpy to avoid strict aliasing issue. + memcpy(ptr_to_ary, &new_ary, sizeof(new_ary)); } // Internal function // Ensure there is space for one more element. // Note: header_size can be bigger than sizeof(rb_darray_meta_t) when T is __int128_t, for example. static inline void -rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size) +rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size, + void *(*realloc_mul_add_impl)(const void *, size_t, size_t, size_t)) { rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; rb_darray_meta_t *meta = *ptr_to_ptr_to_meta; @@ -118,25 +220,12 @@ rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size // Double the capacity size_t new_capa = current_capa == 0 ? 1 : current_capa * 2; - rb_darray_meta_t *doubled_ary = rb_xrealloc_mul_add(meta, new_capa, element_size, header_size); - // rb_xrealloc functions guarantee that NULL is not returned - assert(doubled_ary != NULL); - - if (meta == NULL) { - // First allocation. Initialize size. On subsequence allocations - // realloc takes care of carrying over the size. - doubled_ary->size = 0; - } - - doubled_ary->capa = new_capa; - - // We don't have access to the type of the dynamic array in function context. - // Write out result with memcpy to avoid strict aliasing issue. - memcpy(ptr_to_ary, &doubled_ary, sizeof(doubled_ary)); + rb_darray_resize_capa_impl(ptr_to_ary, new_capa, header_size, element_size, realloc_mul_add_impl); } static inline void -rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, size_t element_size) +rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, size_t element_size, + void *(*calloc_mul_add_impl)(size_t, size_t, size_t)) { rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; if (array_size == 0) { @@ -144,9 +233,7 @@ rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, siz return; } - rb_darray_meta_t *meta = rb_xcalloc_mul_add(array_size, element_size, header_size); - // rb_xcalloc functions guarantee that NULL is not returned - assert(meta != NULL); + rb_darray_meta_t *meta = calloc_mul_add_impl(array_size, element_size, header_size); meta->size = array_size; meta->capa = array_size; diff --git a/doc/contributing/documentation_guide.md b/doc/contributing/documentation_guide.md index 5d5e9e64613ccc..6cb2b8d7dc434b 100644 --- a/doc/contributing/documentation_guide.md +++ b/doc/contributing/documentation_guide.md @@ -127,7 +127,14 @@ a #=> [2, 3, 1] ### Headings -Organize a long discussion with [headings](rdoc-ref:RDoc::MarkupReference@Headings). +Organize a long discussion for a class or module with [headings](rdoc-ref:RDoc::MarkupReference@Headings). + +Do not use formal headings in the documentation for a method or constant. + +In the rare case where heading-like structures are needed +within the documentation for a method or constant, use +[bold text](rdoc-ref:RDoc::MarkupReference@Bold) +as pseudo-headings. ### Blank Lines @@ -151,6 +158,14 @@ For a method name in text: or a hash mark for an instance method: Foo.bar, Foo#baz. +### Embedded Code and Commands + +Code or commands embedded in running text (i.e., not in a code block) +should marked up as +[monofont](rdoc-ref:RDoc::MarkupReference@Monofont). + +Code that is a simple string should include the quote marks. + ### Auto-Linking In general, \RDoc's auto-linking should not be suppressed. @@ -167,9 +182,28 @@ We might consider whether to suppress when: - The reference is to a class or module that users usually don't deal with, including these: - - \Class. - - \Method. - - \Module. + - \Class. + - \Method. + - \Module. + +Most often, the name of a class, module, or method +will be autolinked: + +- Array. +- Enumerable. +- File.new +- File#read. + +If not, or if you suppress autolinking, consider forcing +[monofont](rdoc-ref:RDoc::MarkupReference@Monofont). + +### Variable Names + +The name of a variable (as specified in its call-seq) should be marked up as +[monofont](rdoc-ref:RDoc::MarkupReference@Monofont). + +Also, use monofont text for the name of a transient variable +(i.e., one defined and used only in the discussion, such as +n+). ### HTML Tags diff --git a/doc/yjit/yjit.md b/doc/yjit/yjit.md index 501a539d2c12b8..8aac4ec555e7a5 100644 --- a/doc/yjit/yjit.md +++ b/doc/yjit/yjit.md @@ -166,7 +166,8 @@ YJIT supports all command-line options supported by upstream CRuby, but also add - `--yjit`: enable YJIT (disabled by default) - `--yjit-call-threshold=N`: number of calls after which YJIT begins to compile a function (default 30) - `--yjit-exec-mem-size=N`: size of the executable memory block to allocate, in MiB (default 64 MiB) -- `--yjit-stats`: produce statistics after the execution of a program (incurs a small run-time cost) +- `--yjit-stats`: print statistics after the execution of a program (incurs a run-time cost) +- `--yjit-stats=quiet`: gather statistics while running a program but don't print them. Stats are accessible through `RubyVM::YJIT.runtime_stats`. (incurs a run-time cost) - `--yjit-trace-exits`: produce a Marshal dump of backtraces from specific exits. Automatically enables `--yjit-stats` (must configure and build with `--enable-yjit=stats` to use this) - `--yjit-max-versions=N`: maximum number of versions to generate per basic block (default 4) - `--yjit-greedy-versioning`: greedy versioning mode (disabled by default, may increase code size) @@ -205,9 +206,7 @@ You can also use the `--yjit-stats` command-line option to see which bytecodes c ### Other Statistics -If you configure with --enable-yjit=dev or --enable-yjit=stats and run with `--yjit --yjit-stats`, YJIT will track and return performance statistics in `RubyVM::YJIT.runtime_stats`. - -You will get a much more limited set of statistics with YJIT enabled but not in dev or stats configuration. +If you run `ruby` with `--yjit --yjit-stats`, YJIT will track and return performance statistics in `RubyVM::YJIT.runtime_stats`. ```rb $ RUBYOPT="--yjit --yjit-stats" irb @@ -225,23 +224,23 @@ irb(main):001:0> RubyVM::YJIT.runtime_stats Some of the counters include: -:yjit_insns_count - how many Ruby bytecode instructions have been executed -:binding_allocations - number of bindings allocated -:binding_set - number of variables set via a binding -:code_gc_count - number of garbage collections of compiled code since process start -:vm_insns_count - number of instructions executed by the Ruby interpreter -:compiled_iseq_count - number of bytecode sequences compiled -:inline_code_size - size in bytes of compiled YJIT blocks -:outline_code_size - size in bytes of YJIT error-handling compiled code -:side_exit_count - number of side exits taken at runtime -:total_exit_count - number of exits, including side exits, taken at runtime -:avg_len_in_yjit - avg. number of instructions in compiled blocks before exiting to interpreter +* :yjit_insns_count - how many Ruby bytecode instructions have been executed +* :binding_allocations - number of bindings allocated +* :binding_set - number of variables set via a binding +* :code_gc_count - number of garbage collections of compiled code since process start +* :vm_insns_count - number of instructions executed by the Ruby interpreter +* :compiled_iseq_count - number of bytecode sequences compiled +* :inline_code_size - size in bytes of compiled YJIT blocks +* :outline_code_size - size in bytes of YJIT error-handling compiled code +* :side_exit_count - number of side exits taken at runtime +* :total_exit_count - number of exits, including side exits, taken at runtime +* :avg_len_in_yjit - avg. number of instructions in compiled blocks before exiting to interpreter Counters starting with "exit_" show reasons for YJIT code taking a side exit (return to the interpreter.) See yjit_hacking.md for more details. Performance counter names are not guaranteed to remain the same between Ruby versions. If you're curious what one does, it's usually best to search the source code for it — but it may change in a later Ruby version. -Some of these counters are available in non-stats configuration, but most counters are only in stats or dev configuration. The printed text after a --yjit-stats run includes other information that may be named differently than the information in runtime_stats. +The printed text after a --yjit-stats run includes other information that may be named differently than the information in runtime_stats. ## Contributing diff --git a/ext/fiddle/fiddle.h b/ext/fiddle/fiddle.h index 5a31a4bdb853e6..53db0588079a55 100644 --- a/ext/fiddle/fiddle.h +++ b/ext/fiddle/fiddle.h @@ -199,7 +199,10 @@ /* GCC releases before GCC 4.9 had a bug in _Alignof. See GCC bug 52023 . clang versions < 8.0.0 have the same bug. */ -#if (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112 \ +#if defined(HAVE__ALIGNOF) +# /* Autoconf detected availability of a sane `_Alignof()`. */ +# define ALIGN_OF(type) RB_GNUC_EXTENSION(_Alignof(type)) +#elif (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112 \ || (defined(__GNUC__) && __GNUC__ < 4 + (__GNUC_MINOR__ < 9) \ && !defined(__clang__)) \ || (defined(__clang__) && __clang_major__ < 8)) diff --git a/ext/openssl/ossl.c b/ext/openssl/ossl.c index d27e56dca7452f..00ed7c0c23cd2f 100644 --- a/ext/openssl/ossl.c +++ b/ext/openssl/ossl.c @@ -1165,10 +1165,10 @@ Init_openssl(void) #if defined(LIBRESSL_VERSION_NUMBER) /* * Version number of LibreSSL the ruby OpenSSL extension was built with - * (base 16). The format is 0xMNNFFPPS (major minor fix patch + * (base 16). The format is 0xMNNFF00f (major minor fix 00 * status). This constant is only defined in LibreSSL cases. * - * See also the man page OPENSSL_VERSION_NUMBER(3). + * See also the man page LIBRESSL_VERSION_NUMBER(3). */ rb_define_const(mOSSL, "LIBRESSL_VERSION_NUMBER", INT2NUM(LIBRESSL_VERSION_NUMBER)); #endif diff --git a/ext/ripper/depend b/ext/ripper/depend index 531672d285f6dc..19a7986f0a3d89 100644 --- a/ext/ripper/depend +++ b/ext/ripper/depend @@ -796,6 +796,8 @@ ripper_init.o: $(hdrdir)/ruby/ruby.h ripper_init.o: $(hdrdir)/ruby/st.h ripper_init.o: $(hdrdir)/ruby/subst.h ripper_init.o: $(top_srcdir)/internal.h +ripper_init.o: $(top_srcdir)/internal/array.h +ripper_init.o: $(top_srcdir)/internal/imemo.h ripper_init.o: $(top_srcdir)/internal/parse.h ripper_init.o: $(top_srcdir)/internal/ruby_parser.h ripper_init.o: $(top_srcdir)/internal/serial.h diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl index 98f260f8b1ab6d..40c583eeaebcb4 100644 --- a/ext/ripper/ripper_init.c.tmpl +++ b/ext/ripper/ripper_init.c.tmpl @@ -1,6 +1,7 @@ #include "ruby/ruby.h" #include "ruby/encoding.h" #include "internal.h" +#include "internal/imemo.h" /* needed by ruby_parser.h */ #include "internal/parse.h" #include "internal/ruby_parser.h" #include "node.h" diff --git a/gc.c b/gc.c index 54d11565b8bd91..61de36bb30efba 100644 --- a/gc.c +++ b/gc.c @@ -95,6 +95,7 @@ #undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */ #include "constant.h" +#include "darray.h" #include "debug_counter.h" #include "eval_intern.h" #include "id_table.h" @@ -854,6 +855,10 @@ typedef struct rb_objspace { struct timespec marking_start_time; uint64_t sweeping_time_ns; struct timespec sweeping_start_time; + + /* Weak references */ + size_t weak_references_count; + size_t retained_weak_references_count; } profile; struct gc_list *global_list; @@ -900,6 +905,8 @@ typedef struct rb_objspace { VALUE stress_to_class; #endif + rb_darray(VALUE *) weak_references; + const struct rb_callcache *global_cc_cache_table[VM_GLOBAL_CC_CACHE_TABLE_SIZE]; // vm_eval.c rb_ractor_t *ractor; @@ -2020,6 +2027,8 @@ rb_objspace_alloc(void) ccan_list_head_init(&SIZE_POOL_TOMB_HEAP(size_pool)->pages); } + rb_darray_make_without_gc(&objspace->weak_references, 0); + dont_gc_on(); return objspace; @@ -2073,6 +2082,8 @@ rb_objspace_free(rb_objspace_t *objspace) free_stack_chunks(&objspace->mark_stack); mark_stack_free_cache(&objspace->mark_stack); + rb_darray_free_without_gc(objspace->weak_references); + ccan_list_del(&objspace->objspace_node); if (objspace->ractor == GET_VM()->ractor.main_ractor) { @@ -7892,6 +7903,25 @@ rb_gc_mark_and_move(VALUE *ptr) } } +void +rb_gc_mark_weak(VALUE *ptr) +{ + rb_objspace_t *objspace = &rb_objspace; + + if (UNLIKELY(!during_gc)) return; + + VALUE obj = *ptr; + if (RB_SPECIAL_CONST_P(obj)) return; + + GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED)); + + rgengc_check_relation(objspace, obj); + + rb_darray_append_without_gc(&objspace->weak_references, ptr); + + objspace->profile.weak_references_count++; +} + /* CAUTION: THIS FUNCTION ENABLE *ONLY BEFORE* SWEEPING. * This function is only for GC_END_MARK timing. */ @@ -9141,6 +9171,30 @@ gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap) gc_mark_stacked_objects_all(objspace); } +static void +gc_update_weak_references(rb_objspace_t *objspace) +{ + size_t retained_weak_references_count = 0; + VALUE **ptr_ptr; + rb_darray_foreach(objspace->weak_references, i, ptr_ptr) { + VALUE obj = **ptr_ptr; + + if (RB_SPECIAL_CONST_P(obj)) continue; + + if (!RVALUE_MARKED(obj)) { + **ptr_ptr = Qundef; + } + else { + retained_weak_references_count++; + } + } + + objspace->profile.retained_weak_references_count = retained_weak_references_count; + + rb_darray_clear(objspace->weak_references); + rb_darray_resize_capa_without_gc(&objspace->weak_references, retained_weak_references_count); +} + static void gc_marks_finish(rb_objspace_t *objspace) { @@ -9167,6 +9221,8 @@ gc_marks_finish(rb_objspace_t *objspace) } } + gc_update_weak_references(objspace); + #if RGENGC_CHECK_MODE >= 2 gc_verify_internal_consistency(objspace); #endif @@ -10505,6 +10561,8 @@ gc_set_flags_finish(rb_objspace_t *objspace, unsigned int reason, unsigned int * objspace->profile.latest_gc_info = reason; objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace); objspace->profile.heap_used_at_gc_start = heap_allocated_pages; + objspace->profile.weak_references_count = 0; + objspace->profile.retained_weak_references_count = 0; gc_prof_setup_new_record(objspace, reason); gc_reset_malloc_info(objspace, *do_full_mark); @@ -12276,6 +12334,7 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned #endif static VALUE sym_newobj, sym_malloc, sym_method, sym_capi; static VALUE sym_none, sym_marking, sym_sweeping; + static VALUE sym_weak_references_count, sym_retained_weak_references_count; VALUE hash = Qnil, key = Qnil; VALUE major_by, need_major_by; unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info; @@ -12315,6 +12374,9 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned S(none); S(marking); S(sweeping); + + S(weak_references_count); + S(retained_weak_references_count); #undef S } @@ -12365,6 +12427,9 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned SET(state, gc_mode(objspace) == gc_mode_none ? sym_none : gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping); } + + SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count)); + SET(retained_weak_references_count, LONG2FIX(objspace->profile.retained_weak_references_count)); #undef SET if (!NIL_P(key)) {/* matched key should return above */ @@ -12432,6 +12497,7 @@ enum gc_stat_sym { gc_stat_sym_oldmalloc_increase_bytes, gc_stat_sym_oldmalloc_increase_bytes_limit, #endif + gc_stat_sym_weak_references_count, #if RGENGC_PROFILE gc_stat_sym_total_generated_normal_object_count, gc_stat_sym_total_generated_shady_object_count, @@ -12487,6 +12553,7 @@ setup_gc_stat_symbols(void) S(oldmalloc_increase_bytes); S(oldmalloc_increase_bytes_limit); #endif + S(weak_references_count); #if RGENGC_PROFILE S(total_generated_normal_object_count); S(total_generated_shady_object_count); @@ -14946,7 +15013,7 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU APPEND_F("%s", RSTRING_PTR(class_path)); } else { - APPEND_S("(annon)"); + APPEND_S("(anon)"); } break; } diff --git a/include/ruby/internal/has/c_attribute.h b/include/ruby/internal/has/c_attribute.h index c5c48867bfd561..69b0f402cdc1f8 100644 --- a/include/ruby/internal/has/c_attribute.h +++ b/include/ruby/internal/has/c_attribute.h @@ -21,11 +21,23 @@ * @brief Defines #RBIMPL_HAS_C_ATTRIBUTE. */ +#include "ruby/internal/has/extension.h" +#include "ruby/internal/has/warning.h" + /** Wraps (or simulates) `__has_c_attribute`. */ #if defined(__cplusplus) # /* Makes no sense. */ # define RBIMPL_HAS_C_ATTRIBUTE(_) 0 +#elif RBIMPL_HAS_EXTENSION(c_attributes) +# /* Hmm. It seems Clang 17 has this macro defined even when -std=c99 mode, +# * _and_ fails to compile complaining that attributes are C2X feature. We +# * need to work around this nonsense. */ +# define RBIMPL_HAS_C_ATTRIBUTE(_) __has_c_attribute(_) + +#elif RBIMPL_HAS_WARNING("-Wc2x-extensions") +# define RBIMPL_HAS_C_ATTRIBUTE(_) 0 + #elif defined(__has_c_attribute) # define RBIMPL_HAS_C_ATTRIBUTE(_) __has_c_attribute(_) diff --git a/internal/bits.h b/internal/bits.h index 6248e4cfa9597c..538e6c11ae8688 100644 --- a/internal/bits.h +++ b/internal/bits.h @@ -118,12 +118,16 @@ MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX) #endif -#ifdef MUL_OVERFLOW_P +#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG) # define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b) +#else +# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) +#endif + +#ifdef MUL_OVERFLOW_P # define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_P(a, b) # define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_P(a, b) #else -# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) # define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX) # define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX) #endif diff --git a/internal/gc.h b/internal/gc.h index 5489849695c7f1..7195b9caf6b8dd 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -260,6 +260,8 @@ bool rb_ractor_safe_gc_state(void); void rb_gc_mark_and_move(VALUE *ptr); +void rb_gc_mark_weak(VALUE *ptr); + #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ rb_gc_mark_and_move(&_obj); \ diff --git a/internal/io.h b/internal/io.h index 3b2495a0d22504..f16d9efc9c4483 100644 --- a/internal/io.h +++ b/internal/io.h @@ -16,7 +16,7 @@ struct rb_io; #include "ruby/io.h" /* for rb_io_t */ /** Ruby's IO, metadata and buffers. */ -typedef struct rb_io { +struct rb_io { /** The IO's Ruby level counterpart. */ VALUE self; @@ -108,7 +108,7 @@ typedef struct rb_io { * The timeout associated with this IO when performing blocking operations. */ VALUE timeout; -} rb_io_t; +}; /* io.c */ void ruby_set_inplace_mode(const char *); diff --git a/internal/parse.h b/internal/parse.h index c66c8df7b61684..ce99556ef95842 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -78,7 +78,7 @@ size_t rb_ruby_parser_memsize(const void *ptr); void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp, int split); rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main); -void rb_ruby_parser_keep_script_lines(rb_parser_t *p); +void rb_ruby_parser_set_script_lines(rb_parser_t *p, VALUE lines_array); void rb_ruby_parser_error_tolerant(rb_parser_t *p); rb_ast_t* rb_ruby_parser_compile_file_path(rb_parser_t *p, VALUE fname, VALUE file, int start); void rb_ruby_parser_keep_tokens(rb_parser_t *p); diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h index 97ac49dd3bb34c..fc23177719ea93 100644 --- a/internal/ruby_parser.h +++ b/internal/ruby_parser.h @@ -1,8 +1,9 @@ #ifndef INTERNAL_RUBY_PARSE_H #define INTERNAL_RUBY_PARSE_H -#include "rubyparser.h" #include "internal.h" +#include "internal/imemo.h" +#include "rubyparser.h" #include "vm.h" RUBY_SYMBOL_EXPORT_BEGIN @@ -18,7 +19,7 @@ VALUE rb_parser_encoding(VALUE); VALUE rb_parser_set_yydebug(VALUE, VALUE); void rb_parser_set_options(VALUE, int, int, int, int); void *rb_parser_load_file(VALUE parser, VALUE name); -void rb_parser_keep_script_lines(VALUE vparser); +void rb_parser_set_script_lines(VALUE vparser, VALUE lines_array); void rb_parser_error_tolerant(VALUE vparser); void rb_parser_keep_tokens(VALUE vparser); diff --git a/iseq.c b/iseq.c index 51d7ef78da60d1..80bd280a14b25e 100644 --- a/iseq.c +++ b/iseq.c @@ -1122,6 +1122,7 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V const rb_iseq_t *outer_scope = rb_iseq_new(NULL, name, name, Qnil, 0, ISEQ_TYPE_TOP); VALUE outer_scope_v = (VALUE)outer_scope; rb_parser_set_context(parser, outer_scope, FALSE); + rb_parser_set_script_lines(parser, RBOOL(ruby_vm_keep_script_lines)); RB_GC_GUARD(outer_scope_v); ast = (*parse)(parser, file, src, ln); } diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index 7b8c907041a02c..cebf72798f1810 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -8,6 +8,7 @@ module Gem::BUNDLED_GEMS "net-imap" => "3.1.0", "net-pop" => "3.1.0", "net-smtp" => "3.1.0", + "prime" => "3.1.0", "abbrev" => "3.4.0", "observer" => "3.4.0", "getoptlong" => "3.4.0", @@ -18,24 +19,27 @@ module Gem::BUNDLED_GEMS "drb" => "3.4.0", "mutex_m" => "3.4.0", "csv" => "3.4.0", - "base64" => "3.4.0" + "base64" => "3.4.0", + "bigdecimal" => "3.4.0", }.freeze EXACT = { - "abbrev"=>true, - "base64"=>true, - "csv"=>true, - "drb"=>true, - "getoptlong"=>true, - "mutex_m"=>true, - "nkf"=>true, "kconv"=>"nkf", - "observer"=>true, - "resolv-replace"=>true, - "rinda"=>true, - "syslog"=>true, + "abbrev" => true, + "base64" => true, + "bigdecimal" => true, + "csv" => true, + "drb" => true, + "getoptlong" => true, + "mutex_m" => true, + "nkf" => true, "kconv" => "nkf", + "observer" => true, + "resolv-replace" => true, + "rinda" => true, + "syslog" => true, }.freeze PREFIXED = { + "bigdecimal" => true, "csv" => true, "drb" => true, "rinda" => true, @@ -47,7 +51,9 @@ module Gem::BUNDLED_GEMS conf = ::RbConfig::CONFIG LIBDIR = (conf["rubylibdir"] + "/").freeze ARCHDIR = (conf["rubyarchdir"] + "/").freeze - DLEXT = /\.#{Regexp.union([conf["DLEXT"], "so"].uniq)}\z/ + dlext = [conf["DLEXT"], "so"].uniq + DLEXT = /\.#{Regexp.union(dlext)}\z/ + LIBEXT = /\.#{Regexp.union("rb", *dlext)}\z/ def self.find_gem(path) if !path @@ -65,12 +71,12 @@ def self.find_gem(path) def self.warning?(name) _t, path = $:.resolve_feature_path(name) return unless gem = find_gem(path) - caller, = caller_locations(3, 1) + caller = caller_locations(3, 3).find {|c| c&.absolute_path} return if find_gem(caller&.absolute_path) return if WARNED[name] WARNED[name] = true if gem == true - gem = name + gem = name.sub(LIBEXT, "") # assume "foo.rb"/"foo.so" belongs to "foo" gem elsif gem return if WARNED[gem] WARNED[gem] = true diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index 9f6094b853b026..6f35ce0c28cd2f 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -98,6 +98,17 @@ ruby file: "\.ruby\-version" . .IP "" 0 . +.P +The version file should conform to any of the following formats: +. +.IP "\(bu" 4 +\fB3\.1\.2\fR (\.ruby\-version) +. +.IP "\(bu" 4 +\fBruby 3\.1\.2\fR (\.tool\-versions, read: https://asdf\-vm\.com/manage/configuration\.html#tool\-versions) +. +.IP "" 0 +. .SS "ENGINE" Each application \fImay\fR specify a Ruby engine\. If an engine is specified, an engine version \fImust\fR also be specified\. . diff --git a/lib/bundler/man/gemfile.5.ronn b/lib/bundler/man/gemfile.5.ronn index 39cd18d551cfbc..e8a1f8b79edb69 100644 --- a/lib/bundler/man/gemfile.5.ronn +++ b/lib/bundler/man/gemfile.5.ronn @@ -74,6 +74,11 @@ you can use the `file` option instead. ruby file: ".ruby-version" +The version file should conform to any of the following formats: + + - `3.1.2` (.ruby-version) + - `ruby 3.1.2` (.tool-versions, read: https://asdf-vm.com/manage/configuration.html#tool-versions) + ### ENGINE Each application _may_ specify a Ruby engine. If an engine is specified, an diff --git a/lib/bundler/ruby_dsl.rb b/lib/bundler/ruby_dsl.rb index d054969e8d22cc..542dbac1e4627a 100644 --- a/lib/bundler/ruby_dsl.rb +++ b/lib/bundler/ruby_dsl.rb @@ -11,7 +11,12 @@ def ruby(*ruby_version) if options[:file] raise GemfileError, "Cannot specify version when using the file option" if ruby_version.any? - ruby_version << Bundler.read_file(options[:file]).strip + file_content = Bundler.read_file(Bundler.root.join(options[:file])) + if /^ruby\s+(.*)$/.match(file_content) # match .tool-versions files + ruby_version << $1.split("#", 2).first.strip # remove trailing comment + else + ruby_version << file_content.strip + end end if options[:engine] == "ruby" && options[:engine_version] && diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 7c281e5137207a..060be66c8c3e15 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -254,8 +254,20 @@ def replace_require(specs) "inline Gemfile" end be = RUBY_VERSION < ::Gem::BUNDLED_GEMS::SINCE[name] ? "will be" : "is" - warn "#{name} #{be} not part of the default gems since Ruby #{::Gem::BUNDLED_GEMS::SINCE[name]}." \ - " Add it to your #{target_file}.", uplevel: 1 + message = "#{name} #{be} not part of the default gems since Ruby #{::Gem::BUNDLED_GEMS::SINCE[name]}." \ + " Add #{name} to your #{target_file}." + location = caller_locations(1,1)[0]&.path + if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) + caller_gem = nil + Gem.path.each do |path| + if location =~ /#{path}\/gems\/([\w\-\.]+)/ + caller_gem = $1 + break + end + end + message += " Also contact author of #{caller_gem} to add #{name} into its gemspec." + end + warn message, uplevel: 1 end end kernel_class.send(:no_warning_require, file) diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 3ba378b9c09c3d..f5999481a8f73b 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -90,14 +90,22 @@ class Settings def initialize(root = nil) @root = root @local_config = load_config(local_config_file) - @env_config = ENV.to_h.select {|key, _value| key =~ /\ABUNDLE_.+/ } + + @env_config = ENV.to_h + @env_config.select! {|key, _value| key.start_with?("BUNDLE_") } + @env_config.delete("BUNDLE_") + @global_config = load_config(global_config_file) @temporary = {} end def [](name) key = key_for(name) - value = configs.values.map {|config| config[key] }.compact.first + + values = configs.values + values.map! {|config| config[key] } + values.compact! + value = values.first converted_value(value, name) end @@ -140,17 +148,22 @@ def set_global(key, value) end def all - keys = @temporary.keys | @global_config.keys | @local_config.keys | @env_config.keys + keys = @temporary.keys.union(@global_config.keys, @local_config.keys, @env_config.keys) - keys.map do |key| - key.sub(/^BUNDLE_/, "").gsub(/___/, "-").gsub(/__/, ".").downcase - end.sort + keys.map! do |key| + key = key.delete_prefix("BUNDLE_") + key.gsub!("___", "-") + key.gsub!("__", ".") + key.downcase! + key + end.sort! + keys end def local_overrides repos = {} all.each do |k| - repos[$'] = self[k] if k =~ /^local\./ + repos[k.delete_prefix("local.")] = self[k] if k.start_with?("local.") end repos end @@ -329,16 +342,20 @@ def split_specific_setting_for(name) end def is_bool(name) - BOOL_KEYS.include?(name.to_s) || BOOL_KEYS.include?(parent_setting_for(name.to_s)) + name = name.to_s + BOOL_KEYS.include?(name) || BOOL_KEYS.include?(parent_setting_for(name)) end def is_string(name) - STRING_KEYS.include?(name.to_s) || name.to_s.start_with?("local.") || name.to_s.start_with?("mirror.") || name.to_s.start_with?("build.") + name = name.to_s + STRING_KEYS.include?(name) || name.start_with?("local.") || name.start_with?("mirror.") || name.start_with?("build.") end def to_bool(value) case value - when nil, /\A(false|f|no|n|0|)\z/i, false + when String + value.match?(/\A(false|f|no|n|0|)\z/i) ? false : true + when nil, false false else true @@ -393,6 +410,8 @@ def set_key(raw_key, value, hash, file) end def converted_value(value, key) + key = key.to_s + if is_array(key) to_array(value) elsif value.nil? @@ -491,8 +510,11 @@ def serializer_class def self.key_for(key) key = normalize_uri(key).to_s if key.is_a?(String) && key.start_with?("http", "mirror.http") - key = key.to_s.gsub(".", "__").gsub("-", "___").upcase - "BUNDLE_#{key}" + key = key.to_s.gsub(".", "__") + key.gsub!("-", "___") + key.upcase! + + key.prepend("BUNDLE_") end # TODO: duplicates Rubygems#normalize_uri diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index af57acbbc2739a..9790753204b91e 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -399,16 +399,11 @@ def remote_specs @remote_specs ||= Index.build do |idx| index_fetchers = fetchers - api_fetchers - # gather lists from non-api sites - fetch_names(index_fetchers, nil, idx, false) - - # legacy multi-remote sources need special logic to figure out - # dependency names and that logic can be very costly if one remote - # uses the dependency API but others don't. So use full indexes - # consistently in that particular case. - allow_api = !multiple_remotes? - - fetch_names(api_fetchers, allow_api && dependency_names, idx, false) + if index_fetchers.empty? + fetch_names(api_fetchers, dependency_names, idx, false) + else + fetch_names(fetchers, nil, idx, false) + end end end diff --git a/lib/erb/compiler.rb b/lib/erb/compiler.rb index 547d2c4c4434f6..7096c8dcea802c 100644 --- a/lib/erb/compiler.rb +++ b/lib/erb/compiler.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::Compiler # diff --git a/lib/erb/def_method.rb b/lib/erb/def_method.rb index 17f9c0f9fa48b5..aee989a926ea94 100644 --- a/lib/erb/def_method.rb +++ b/lib/erb/def_method.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::DefMethod # diff --git a/lib/erb/util.rb b/lib/erb/util.rb index 0c1e7482a83b7e..1d2a36275dbbeb 100644 --- a/lib/erb/util.rb +++ b/lib/erb/util.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::Escape # diff --git a/lib/erb/version.rb b/lib/erb/version.rb index 38e1b76ff492c7..295fc5fa6f7964 100644 --- a/lib/erb/version.rb +++ b/lib/erb/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class ERB - VERSION = '4.0.2' + VERSION = '4.0.3' private_constant :VERSION end diff --git a/lib/irb.rb b/lib/irb.rb index 93ab6370ed68d1..aa412583056b3c 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -12,6 +12,7 @@ require_relative "irb/extend-command" require_relative "irb/ruby-lex" +require_relative "irb/statement" require_relative "irb/input-method" require_relative "irb/locale" require_relative "irb/color" @@ -550,27 +551,9 @@ def eval_input @context.io.prompt end - @scanner.set_input do - signal_status(:IN_INPUT) do - if l = @context.io.gets - print l if @context.verbose? - else - if @context.ignore_eof? and @context.io.readable_after_eof? - l = "\n" - if @context.verbose? - printf "Use \"exit\" to leave %s\n", @context.ap_name - end - else - print "\n" if @context.prompting? - end - end - l - end - end - configure_io - @scanner.each_top_level_statement do |statement, line_no| + each_top_level_statement do |statement, line_no| signal_status(:IN_EVAL) do begin # If the integration with debugger is activated, we need to handle certain input differently @@ -600,6 +583,86 @@ def eval_input end end + def read_input + signal_status(:IN_INPUT) do + if l = @context.io.gets + print l if @context.verbose? + else + if @context.ignore_eof? and @context.io.readable_after_eof? + l = "\n" + if @context.verbose? + printf "Use \"exit\" to leave %s\n", @context.ap_name + end + else + print "\n" if @context.prompting? + end + end + l + end + end + + def readmultiline + @scanner.save_prompt_to_context_io([], false, 0) + + # multiline + return read_input if @context.io.respond_to?(:check_termination) + + # nomultiline + code = '' + line_offset = 0 + loop do + line = read_input + unless line + return code.empty? ? nil : code + end + + code << line + + # Accept any single-line input for symbol aliases or commands that transform args + return code if single_line_command?(code) + + tokens, opens, terminated = @scanner.check_code_state(code) + return code if terminated + + line_offset += 1 + continue = @scanner.should_continue?(tokens) + @scanner.save_prompt_to_context_io(opens, continue, line_offset) + end + end + + def each_top_level_statement + loop do + code = readmultiline + break unless code + + if code != "\n" + yield build_statement(code), @scanner.line_no + end + @scanner.increase_line_no(code.count("\n")) + rescue RubyLex::TerminateLineInput + end + end + + def build_statement(code) + code.force_encoding(@context.io.encoding) + command_or_alias, arg = code.split(/\s/, 2) + # Transform a non-identifier alias (@, $) or keywords (next, break) + command_name = @context.command_aliases[command_or_alias.to_sym] + command = command_name || command_or_alias + command_class = ExtendCommandBundle.load_command(command) + + if command_class + Statement::Command.new(code, command, arg, command_class) + else + Statement::Expression.new(code, @scanner.assignment_expression?(code)) + end + end + + def single_line_command?(code) + command = code.split(/\s/, 2).first + @context.symbol_alias?(command) || @context.transform_args?(command) + end + def configure_io if @context.io.respond_to?(:check_termination) @context.io.check_termination do |code| @@ -616,7 +679,7 @@ def configure_io end else # Accept any single-line input for symbol aliases or commands that transform args - next true if @scanner.single_line_command?(code) + next true if single_line_command?(code) _tokens, _opens, terminated = @scanner.check_code_state(code) terminated diff --git a/lib/irb/cmd/edit.rb b/lib/irb/cmd/edit.rb index 523a1dac5d5765..69606beea0db9a 100644 --- a/lib/irb/cmd/edit.rb +++ b/lib/irb/cmd/edit.rb @@ -8,7 +8,7 @@ module IRB module ExtendCommand class Edit < Nop category "Misc" - description 'Open a file with the editor command defined with `ENV["EDITOR"]`.' + description 'Open a file with the editor command defined with `ENV["VISUAL"]` or `ENV["EDITOR"]`.' class << self def transform_args(args) @@ -45,12 +45,12 @@ def execute(*args) end end - if editor = ENV['EDITOR'] + if editor = (ENV['VISUAL'] || ENV['EDITOR']) puts "command: '#{editor}'" puts " path: #{path}" system(*Shellwords.split(editor), path) else - puts "Can not find editor setting: ENV['EDITOR']" + puts "Can not find editor setting: ENV['VISUAL'] or ENV['EDITOR']" end end end diff --git a/lib/irb/debug.rb b/lib/irb/debug.rb index dab9d1846a0514..f72282299322c0 100644 --- a/lib/irb/debug.rb +++ b/lib/irb/debug.rb @@ -73,7 +73,7 @@ def configure_irb_for_debugger(irb) require 'irb/debug/ui' IRB.instance_variable_set(:@debugger_irb, irb) irb.context.with_debugger = true - irb.context.irb_name = "irb:rdbg" + irb.context.irb_name += ":rdbg" end def binding_irb? diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index 3a0173a6be2ffe..502883bd43ca62 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -7,555 +7,495 @@ require "ripper" require "jruby" if RUBY_ENGINE == "jruby" require_relative "nesting_parser" -require_relative "statement" - -# :stopdoc: -class RubyLex - ASSIGNMENT_NODE_TYPES = [ - # Local, instance, global, class, constant, instance, and index assignment: - # "foo = bar", - # "@foo = bar", - # "$foo = bar", - # "@@foo = bar", - # "::Foo = bar", - # "a::Foo = bar", - # "Foo = bar" - # "foo.bar = 1" - # "foo[1] = bar" - :assign, - - # Operation assignment: - # "foo += bar" - # "foo -= bar" - # "foo ||= bar" - # "foo &&= bar" - :opassign, - - # Multiple assignment: - # "foo, bar = 1, 2 - :massign, - ] - - class TerminateLineInput < StandardError - def initialize - super("Terminate Line Input") - end - end - - def initialize(context) - @context = context - @line_no = 1 - @prompt = nil - end - def self.compile_with_errors_suppressed(code, line_no: 1) - begin - result = yield code, line_no - rescue ArgumentError - # Ruby can issue an error for the code if there is an - # incomplete magic comment for encoding in it. Force an - # expression with a new line before the code in this - # case to prevent magic comment handling. To make sure - # line numbers in the lexed code remain the same, - # decrease the line number by one. - code = ";\n#{code}" - line_no -= 1 - result = yield code, line_no +module IRB + # :stopdoc: + class RubyLex + ASSIGNMENT_NODE_TYPES = [ + # Local, instance, global, class, constant, instance, and index assignment: + # "foo = bar", + # "@foo = bar", + # "$foo = bar", + # "@@foo = bar", + # "::Foo = bar", + # "a::Foo = bar", + # "Foo = bar" + # "foo.bar = 1" + # "foo[1] = bar" + :assign, + + # Operation assignment: + # "foo += bar" + # "foo -= bar" + # "foo ||= bar" + # "foo &&= bar" + :opassign, + + # Multiple assignment: + # "foo, bar = 1, 2 + :massign, + ] + + class TerminateLineInput < StandardError + def initialize + super("Terminate Line Input") + end end - result - end - def single_line_command?(code) - command = code.split(/\s/, 2).first - @context.symbol_alias?(command) || @context.transform_args?(command) - end + attr_reader :line_no - # io functions - def set_input(&block) - @input = block - end + def initialize(context) + @context = context + @line_no = 1 + @prompt = nil + end - def set_prompt(&block) - @prompt = block - end + def self.compile_with_errors_suppressed(code, line_no: 1) + begin + result = yield code, line_no + rescue ArgumentError + # Ruby can issue an error for the code if there is an + # incomplete magic comment for encoding in it. Force an + # expression with a new line before the code in this + # case to prevent magic comment handling. To make sure + # line numbers in the lexed code remain the same, + # decrease the line number by one. + code = ";\n#{code}" + line_no -= 1 + result = yield code, line_no + end + result + end - ERROR_TOKENS = [ - :on_parse_error, - :compile_error, - :on_assign_error, - :on_alias_error, - :on_class_name_error, - :on_param_error - ] - - def self.generate_local_variables_assign_code(local_variables) - "#{local_variables.join('=')}=nil;" unless local_variables.empty? - end + def set_prompt(&block) + @prompt = block + end - # Some part of the code is not included in Ripper's token. - # Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr. - # With interpolated tokens, tokens.map(&:tok).join will be equal to code. - def self.interpolate_ripper_ignored_tokens(code, tokens) - line_positions = [0] - code.lines.each do |line| - line_positions << line_positions.last + line.bytesize + ERROR_TOKENS = [ + :on_parse_error, + :compile_error, + :on_assign_error, + :on_alias_error, + :on_class_name_error, + :on_param_error + ] + + def self.generate_local_variables_assign_code(local_variables) + "#{local_variables.join('=')}=nil;" unless local_variables.empty? end - prev_byte_pos = 0 - interpolated = [] - prev_line = 1 - tokens.each do |t| - line, col = t.pos - byte_pos = line_positions[line - 1] + col - if prev_byte_pos < byte_pos - tok = code.byteslice(prev_byte_pos...byte_pos) + + # Some part of the code is not included in Ripper's token. + # Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr. + # With interpolated tokens, tokens.map(&:tok).join will be equal to code. + def self.interpolate_ripper_ignored_tokens(code, tokens) + line_positions = [0] + code.lines.each do |line| + line_positions << line_positions.last + line.bytesize + end + prev_byte_pos = 0 + interpolated = [] + prev_line = 1 + tokens.each do |t| + line, col = t.pos + byte_pos = line_positions[line - 1] + col + if prev_byte_pos < byte_pos + tok = code.byteslice(prev_byte_pos...byte_pos) + pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]] + interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0) + prev_line += tok.count("\n") + end + interpolated << t + prev_byte_pos = byte_pos + t.tok.bytesize + prev_line += t.tok.count("\n") + end + if prev_byte_pos < code.bytesize + tok = code.byteslice(prev_byte_pos..) pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]] interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0) - prev_line += tok.count("\n") end - interpolated << t - prev_byte_pos = byte_pos + t.tok.bytesize - prev_line += t.tok.count("\n") - end - if prev_byte_pos < code.bytesize - tok = code.byteslice(prev_byte_pos..) - pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]] - interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0) + interpolated end - interpolated - end - def self.ripper_lex_without_warning(code, context: nil) - verbose, $VERBOSE = $VERBOSE, nil - lvars_code = generate_local_variables_assign_code(context&.local_variables || []) - original_code = code - if lvars_code - code = "#{lvars_code}\n#{code}" - line_no = 0 - else - line_no = 1 - end + def self.ripper_lex_without_warning(code, context: nil) + verbose, $VERBOSE = $VERBOSE, nil + lvars_code = generate_local_variables_assign_code(context&.local_variables || []) + original_code = code + if lvars_code + code = "#{lvars_code}\n#{code}" + line_no = 0 + else + line_no = 1 + end - compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no| - lexer = Ripper::Lexer.new(inner_code, '-', line_no) - tokens = [] - lexer.scan.each do |t| - next if t.pos.first == 0 - prev_tk = tokens.last - position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize - if position_overlapped - tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event) - else - tokens << t + compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no| + lexer = Ripper::Lexer.new(inner_code, '-', line_no) + tokens = [] + lexer.scan.each do |t| + next if t.pos.first == 0 + prev_tk = tokens.last + position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize + if position_overlapped + tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event) + else + tokens << t + end end + interpolate_ripper_ignored_tokens(original_code, tokens) end - interpolate_ripper_ignored_tokens(original_code, tokens) + ensure + $VERBOSE = verbose end - ensure - $VERBOSE = verbose - end - - def prompt(opens, continue, line_num_offset) - ltype = ltype_from_open_tokens(opens) - indent_level = calc_indent_level(opens) - @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset) - end - - def check_code_state(code) - tokens = self.class.ripper_lex_without_warning(code, context: @context) - opens = IRB::NestingParser.open_tokens(tokens) - [tokens, opens, code_terminated?(code, tokens, opens)] - end - def code_terminated?(code, tokens, opens) - case check_code_syntax(code) - when :unrecoverable_error - true - when :recoverable_error - false - when :other_error - opens.empty? && !should_continue?(tokens) - when :valid - !should_continue?(tokens) + def prompt(opens, continue, line_num_offset) + ltype = ltype_from_open_tokens(opens) + indent_level = calc_indent_level(opens) + @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset) end - end - def save_prompt_to_context_io(opens, continue, line_num_offset) - # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`. - prompt(opens, continue, line_num_offset) - end - - def increase_line_no(addition) - @line_no += addition - end - - def readmultiline - save_prompt_to_context_io([], false, 0) - - # multiline - return @input.call if @context.io.respond_to?(:check_termination) + def check_code_state(code) + tokens = self.class.ripper_lex_without_warning(code, context: @context) + opens = NestingParser.open_tokens(tokens) + [tokens, opens, code_terminated?(code, tokens, opens)] + end - # nomultiline - code = '' - line_offset = 0 - loop do - line = @input.call - unless line - return code.empty? ? nil : code + def code_terminated?(code, tokens, opens) + case check_code_syntax(code) + when :unrecoverable_error + true + when :recoverable_error + false + when :other_error + opens.empty? && !should_continue?(tokens) + when :valid + !should_continue?(tokens) end - - code << line - # Accept any single-line input for symbol aliases or commands that transform args - return code if single_line_command?(code) - - tokens, opens, terminated = check_code_state(code) - return code if terminated - - line_offset += 1 - continue = should_continue?(tokens) - save_prompt_to_context_io(opens, continue, line_offset) end - end - - def each_top_level_statement - loop do - code = readmultiline - break unless code - if code != "\n" - yield build_statement(code), @line_no - end - increase_line_no(code.count("\n")) - rescue TerminateLineInput + def save_prompt_to_context_io(opens, continue, line_num_offset) + # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`. + prompt(opens, continue, line_num_offset) end - end - def build_statement(code) - code.force_encoding(@context.io.encoding) - command_or_alias, arg = code.split(/\s/, 2) - # Transform a non-identifier alias (@, $) or keywords (next, break) - command_name = @context.command_aliases[command_or_alias.to_sym] - command = command_name || command_or_alias - command_class = IRB::ExtendCommandBundle.load_command(command) - - if command_class - IRB::Statement::Command.new(code, command, arg, command_class) - else - IRB::Statement::Expression.new(code, assignment_expression?(code)) + def increase_line_no(addition) + @line_no += addition end - end - def assignment_expression?(code) - # Try to parse the code and check if the last of possibly multiple - # expressions is an assignment type. - - # If the expression is invalid, Ripper.sexp should return nil which will - # result in false being returned. Any valid expression should return an - # s-expression where the second element of the top level array is an - # array of parsed expressions. The first element of each expression is the - # expression's type. - verbose, $VERBOSE = $VERBOSE, nil - code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{code}" - # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part. - node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0) - ASSIGNMENT_NODE_TYPES.include?(node_type) - ensure - $VERBOSE = verbose - end + def assignment_expression?(code) + # Try to parse the code and check if the last of possibly multiple + # expressions is an assignment type. - def should_continue?(tokens) - # Look at the last token and check if IRB need to continue reading next line. - # Example code that should continue: `a\` `a +` `a.` - # Trailing spaces, newline, comments are skipped - return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n" - - tokens.reverse_each do |token| - case token.event - when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end - # Skip - when :on_regexp_end, :on_heredoc_end, :on_semicolon - # State is EXPR_BEG but should not continue - return false - else - # Endless range should not continue - return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/) + # If the expression is invalid, Ripper.sexp should return nil which will + # result in false being returned. Any valid expression should return an + # s-expression where the second element of the top level array is an + # array of parsed expressions. The first element of each expression is the + # expression's type. + verbose, $VERBOSE = $VERBOSE, nil + code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{code}" + # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part. + node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0) + ASSIGNMENT_NODE_TYPES.include?(node_type) + ensure + $VERBOSE = verbose + end + + def should_continue?(tokens) + # Look at the last token and check if IRB need to continue reading next line. + # Example code that should continue: `a\` `a +` `a.` + # Trailing spaces, newline, comments are skipped + return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n" + + tokens.reverse_each do |token| + case token.event + when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end + # Skip + when :on_regexp_end, :on_heredoc_end, :on_semicolon + # State is EXPR_BEG but should not continue + return false + else + # Endless range should not continue + return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/) - # EXPR_DOT and most of the EXPR_BEG should continue - return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT) + # EXPR_DOT and most of the EXPR_BEG should continue + return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT) + end end + false end - false - end - def check_code_syntax(code) - lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables) - code = "#{lvars_code}\n#{code}" + def check_code_syntax(code) + lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables) + code = "#{lvars_code}\n#{code}" - begin # check if parser error are available - verbose, $VERBOSE = $VERBOSE, nil - case RUBY_ENGINE - when 'ruby' - self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| - RubyVM::InstructionSequence.compile(inner_code, nil, nil, line_no) + begin # check if parser error are available + verbose, $VERBOSE = $VERBOSE, nil + case RUBY_ENGINE + when 'ruby' + self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| + RubyVM::InstructionSequence.compile(inner_code, nil, nil, line_no) + end + when 'jruby' + JRuby.compile_ir(code) + else + catch(:valid) do + eval("BEGIN { throw :valid, true }\n#{code}") + false + end end - when 'jruby' - JRuby.compile_ir(code) - else - catch(:valid) do - eval("BEGIN { throw :valid, true }\n#{code}") - false + rescue EncodingError + # This is for a hash with invalid encoding symbol, {"\xAE": 1} + :unrecoverable_error + rescue SyntaxError => e + case e.message + when /unterminated (?:string|regexp) meets end of file/ + # "unterminated regexp meets end of file" + # + # example: + # / + # + # "unterminated string meets end of file" + # + # example: + # ' + return :recoverable_error + when /syntax error, unexpected end-of-input/ + # "syntax error, unexpected end-of-input, expecting keyword_end" + # + # example: + # if true + # hoge + # if false + # fuga + # end + return :recoverable_error + when /syntax error, unexpected keyword_end/ + # "syntax error, unexpected keyword_end" + # + # example: + # if ( + # end + # + # example: + # end + return :unrecoverable_error + when /syntax error, unexpected '\.'/ + # "syntax error, unexpected '.'" + # + # example: + # . + return :unrecoverable_error + when /unexpected tREGEXP_BEG/ + # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('" + # + # example: + # method / f / + return :unrecoverable_error + else + return :other_error end + ensure + $VERBOSE = verbose end - rescue EncodingError - # This is for a hash with invalid encoding symbol, {"\xAE": 1} - :unrecoverable_error - rescue SyntaxError => e - case e.message - when /unterminated (?:string|regexp) meets end of file/ - # "unterminated regexp meets end of file" - # - # example: - # / - # - # "unterminated string meets end of file" - # - # example: - # ' - return :recoverable_error - when /syntax error, unexpected end-of-input/ - # "syntax error, unexpected end-of-input, expecting keyword_end" - # - # example: - # if true - # hoge - # if false - # fuga - # end - return :recoverable_error - when /syntax error, unexpected keyword_end/ - # "syntax error, unexpected keyword_end" - # - # example: - # if ( - # end - # - # example: - # end - return :unrecoverable_error - when /syntax error, unexpected '\.'/ - # "syntax error, unexpected '.'" - # - # example: - # . - return :unrecoverable_error - when /unexpected tREGEXP_BEG/ - # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('" - # - # example: - # method / f / - return :unrecoverable_error - else - return :other_error - end - ensure - $VERBOSE = verbose + :valid end - :valid - end - def calc_indent_level(opens) - indent_level = 0 - opens.each_with_index do |t, index| - case t.event - when :on_heredoc_beg - if opens[index + 1]&.event != :on_heredoc_beg - if t.tok.match?(/^<<[~-]/) - indent_level += 1 - else - indent_level = 0 + def calc_indent_level(opens) + indent_level = 0 + opens.each_with_index do |t, index| + case t.event + when :on_heredoc_beg + if opens[index + 1]&.event != :on_heredoc_beg + if t.tok.match?(/^<<[~-]/) + indent_level += 1 + else + indent_level = 0 + end end + when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick + # No indent: "", //, :"", `` + # Indent: %(), %r(), %i(), %x() + indent_level += 1 if t.tok.start_with? '%' + when :on_embdoc_beg + indent_level = 0 + else + indent_level += 1 end - when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick - # No indent: "", //, :"", `` - # Indent: %(), %r(), %i(), %x() - indent_level += 1 if t.tok.start_with? '%' - when :on_embdoc_beg - indent_level = 0 - else - indent_level += 1 end + indent_level end - indent_level - end - FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg] - - def free_indent_token?(token) - FREE_INDENT_TOKENS.include?(token&.event) - end + FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg] - # Calculates the difference of pasted code's indent and indent calculated from tokens - def indent_difference(lines, line_results, line_index) - loop do - _tokens, prev_opens, _next_opens, min_depth = line_results[line_index] - open_token = prev_opens.last - if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token)) - # If the leading whitespace is an indent, return the difference - indent_level = calc_indent_level(prev_opens.take(min_depth)) - calculated_indent = 2 * indent_level - actual_indent = lines[line_index][/^ */].size - return actual_indent - calculated_indent - elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/) - return 0 - end - # If the leading whitespace is not an indent but part of a multiline token - # Calculate base_indent of the multiline token's beginning line - line_index = open_token.pos[0] - 1 + def free_indent_token?(token) + FREE_INDENT_TOKENS.include?(token&.event) end - end - def process_indent_level(tokens, lines, line_index, is_newline) - line_results = IRB::NestingParser.parse_by_line(tokens) - result = line_results[line_index] - if result - _tokens, prev_opens, next_opens, min_depth = result - else - # When last line is empty - prev_opens = next_opens = line_results.last[2] - min_depth = next_opens.size + # Calculates the difference of pasted code's indent and indent calculated from tokens + def indent_difference(lines, line_results, line_index) + loop do + _tokens, prev_opens, _next_opens, min_depth = line_results[line_index] + open_token = prev_opens.last + if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token)) + # If the leading whitespace is an indent, return the difference + indent_level = calc_indent_level(prev_opens.take(min_depth)) + calculated_indent = 2 * indent_level + actual_indent = lines[line_index][/^ */].size + return actual_indent - calculated_indent + elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/) + return 0 + end + # If the leading whitespace is not an indent but part of a multiline token + # Calculate base_indent of the multiline token's beginning line + line_index = open_token.pos[0] - 1 + end end - # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation. - # Shortest open tokens can be calculated by `opens.take(min_depth)` - indent = 2 * calc_indent_level(prev_opens.take(min_depth)) + def process_indent_level(tokens, lines, line_index, is_newline) + line_results = NestingParser.parse_by_line(tokens) + result = line_results[line_index] + if result + _tokens, prev_opens, next_opens, min_depth = result + else + # When last line is empty + prev_opens = next_opens = line_results.last[2] + min_depth = next_opens.size + end - preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size + # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation. + # Shortest open tokens can be calculated by `opens.take(min_depth)` + indent = 2 * calc_indent_level(prev_opens.take(min_depth)) - prev_open_token = prev_opens.last - next_open_token = next_opens.last + preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size - # Calculates base indent for pasted code on the line where prev_open_token is located - # irb(main):001:1* if a # base_indent is 2, indent calculated from tokens is 0 - # irb(main):002:1* if b # base_indent is 6, indent calculated from tokens is 2 - # irb(main):003:0> c # base_indent is 6, indent calculated from tokens is 4 - if prev_open_token - base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max - else - base_indent = 0 - end + prev_open_token = prev_opens.last + next_open_token = next_opens.last - if free_indent_token?(prev_open_token) - if is_newline && prev_open_token.pos[0] == line_index - # First newline inside free-indent token - base_indent + indent - else - # Accept any number of indent inside free-indent token - preserve_indent - end - elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg - if prev_open_token&.event == next_open_token&.event - # Accept any number of indent inside embdoc content - preserve_indent + # Calculates base indent for pasted code on the line where prev_open_token is located + # irb(main):001:1* if a # base_indent is 2, indent calculated from tokens is 0 + # irb(main):002:1* if b # base_indent is 6, indent calculated from tokens is 2 + # irb(main):003:0> c # base_indent is 6, indent calculated from tokens is 4 + if prev_open_token + base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max else - # =begin or =end - 0 + base_indent = 0 end - elsif prev_open_token&.event == :on_heredoc_beg - tok = prev_open_token.tok - if prev_opens.size <= next_opens.size - if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token - # First line in heredoc - tok.match?(/^<<[-~]/) ? base_indent + indent : indent - elsif tok.match?(/^<<~/) - # Accept extra indent spaces inside `<<~` heredoc - [base_indent + indent, preserve_indent].max + + if free_indent_token?(prev_open_token) + if is_newline && prev_open_token.pos[0] == line_index + # First newline inside free-indent token + base_indent + indent else - # Accept any number of indent inside other heredoc + # Accept any number of indent inside free-indent token preserve_indent end + elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg + if prev_open_token&.event == next_open_token&.event + # Accept any number of indent inside embdoc content + preserve_indent + else + # =begin or =end + 0 + end + elsif prev_open_token&.event == :on_heredoc_beg + tok = prev_open_token.tok + if prev_opens.size <= next_opens.size + if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token + # First line in heredoc + tok.match?(/^<<[-~]/) ? base_indent + indent : indent + elsif tok.match?(/^<<~/) + # Accept extra indent spaces inside `<<~` heredoc + [base_indent + indent, preserve_indent].max + else + # Accept any number of indent inside other heredoc + preserve_indent + end + else + # Heredoc close + prev_line_indent_level = calc_indent_level(prev_opens) + tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0 + end else - # Heredoc close - prev_line_indent_level = calc_indent_level(prev_opens) - tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0 + base_indent + indent end - else - base_indent + indent end - end - LTYPE_TOKENS = %i[ - on_heredoc_beg on_tstring_beg - on_regexp_beg on_symbeg on_backtick - on_symbols_beg on_qsymbols_beg - on_words_beg on_qwords_beg - ] + LTYPE_TOKENS = %i[ + on_heredoc_beg on_tstring_beg + on_regexp_beg on_symbeg on_backtick + on_symbols_beg on_qsymbols_beg + on_words_beg on_qwords_beg + ] - def ltype_from_open_tokens(opens) - start_token = opens.reverse_each.find do |tok| - LTYPE_TOKENS.include?(tok.event) - end - return nil unless start_token - - case start_token&.event - when :on_tstring_beg - case start_token&.tok - when ?" then ?" - when /^%.$/ then ?" - when /^%Q.$/ then ?" - when ?' then ?' - when /^%q.$/ then ?' + def ltype_from_open_tokens(opens) + start_token = opens.reverse_each.find do |tok| + LTYPE_TOKENS.include?(tok.event) end - when :on_regexp_beg then ?/ - when :on_symbeg then ?: - when :on_backtick then ?` - when :on_qwords_beg then ?] - when :on_words_beg then ?] - when :on_qsymbols_beg then ?] - when :on_symbols_beg then ?] - when :on_heredoc_beg - start_token&.tok =~ /<<[-~]?(['"`])\w+\1/ - $1 || ?" - else - nil - end - end - - def check_termination_in_prev_line(code) - tokens = self.class.ripper_lex_without_warning(code, context: @context) - past_first_newline = false - index = tokens.rindex do |t| - # traverse first token before last line - if past_first_newline - if t.tok.include?("\n") - true + return nil unless start_token + + case start_token&.event + when :on_tstring_beg + case start_token&.tok + when ?" then ?" + when /^%.$/ then ?" + when /^%Q.$/ then ?" + when ?' then ?' + when /^%q.$/ then ?' end - elsif t.tok.include?("\n") - past_first_newline = true - false + when :on_regexp_beg then ?/ + when :on_symbeg then ?: + when :on_backtick then ?` + when :on_qwords_beg then ?] + when :on_words_beg then ?] + when :on_qsymbols_beg then ?] + when :on_symbols_beg then ?] + when :on_heredoc_beg + start_token&.tok =~ /<<[-~]?(['"`])\w+\1/ + $1 || ?" else - false + nil end end - if index - first_token = nil - last_line_tokens = tokens[(index + 1)..(tokens.size - 1)] - last_line_tokens.each do |t| - unless [:on_sp, :on_ignored_sp, :on_comment].include?(t.event) - first_token = t - break + def check_termination_in_prev_line(code) + tokens = self.class.ripper_lex_without_warning(code, context: @context) + past_first_newline = false + index = tokens.rindex do |t| + # traverse first token before last line + if past_first_newline + if t.tok.include?("\n") + true + end + elsif t.tok.include?("\n") + past_first_newline = true + false + else + false end end - if first_token && first_token.state != Ripper::EXPR_DOT - tokens_without_last_line = tokens[0..index] - code_without_last_line = tokens_without_last_line.map(&:tok).join - opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line) - if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line) - return last_line_tokens.map(&:tok).join + if index + first_token = nil + last_line_tokens = tokens[(index + 1)..(tokens.size - 1)] + last_line_tokens.each do |t| + unless [:on_sp, :on_ignored_sp, :on_comment].include?(t.event) + first_token = t + break + end + end + + if first_token && first_token.state != Ripper::EXPR_DOT + tokens_without_last_line = tokens[0..index] + code_without_last_line = tokens_without_last_line.map(&:tok).join + opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line) + if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line) + return last_line_tokens.map(&:tok).join + end end end + false end - false end + # :startdoc: end -# :startdoc: + +RubyLex = IRB::RubyLex +Object.deprecate_constant(:RubyLex) diff --git a/lib/open-uri.rb b/lib/open-uri.rb index 0052bb49fbce6a..11af0f7a121cc4 100644 --- a/lib/open-uri.rb +++ b/lib/open-uri.rb @@ -31,6 +31,7 @@ def self.open(name, *rest, &block) super end end + singleton_class.send(:ruby2_keywords, :open) if respond_to?(:ruby2_keywords, true) end # OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP. diff --git a/lib/reline.rb b/lib/reline.rb index 0a2dba4f0d23f3..3d1716c81bb53f 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -1,5 +1,4 @@ require 'io/console' -require 'timeout' require 'forwardable' require 'reline/version' require 'reline/config' @@ -397,7 +396,7 @@ def readline(prompt = '', add_hist = false) private def read_io(keyseq_timeout, &block) buffer = [] loop do - c = io_gate.getc + c = io_gate.getc(Float::INFINITY) if c == -1 result = :unmatched @bracketed_paste_finished = true @@ -434,15 +433,8 @@ def readline(prompt = '', add_hist = false) end private def read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - begin - succ_c = nil - Timeout.timeout(keyseq_timeout / 1000.0) { - succ_c = io_gate.getc - } - rescue Timeout::Error # cancel matching only when first byte - block.([Reline::Key.new(c, c, false)]) - return :break - else + succ_c = io_gate.getc(keyseq_timeout.fdiv(1000)) + if succ_c case key_stroke.match_status(buffer.dup.push(succ_c)) when :unmatched if c == "\e".ord @@ -462,27 +454,23 @@ def readline(prompt = '', add_hist = false) block.(expanded) return :break end + else + block.([Reline::Key.new(c, c, false)]) + return :break end end private def read_escaped_key(keyseq_timeout, c, block) - begin - escaped_c = nil - Timeout.timeout(keyseq_timeout / 1000.0) { - escaped_c = io_gate.getc - } - rescue Timeout::Error # independent ESC + escaped_c = io_gate.getc(keyseq_timeout.fdiv(1000)) + + if escaped_c.nil? block.([Reline::Key.new(c, c, false)]) + elsif escaped_c >= 128 # maybe, first byte of multi byte + block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) + elsif escaped_c == "\e".ord # escape twice + block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) else - if escaped_c.nil? - block.([Reline::Key.new(c, c, false)]) - elsif escaped_c >= 128 # maybe, first byte of multi byte - block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) - elsif escaped_c == "\e".ord # escape twice - block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) - else - block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) - end + block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) end end diff --git a/lib/reline/ansi.rb b/lib/reline/ansi.rb index d1a0e5b13875a6..c2e5075ea8b5b4 100644 --- a/lib/reline/ansi.rb +++ b/lib/reline/ansi.rb @@ -1,6 +1,5 @@ require 'io/console' require 'io/wait' -require 'timeout' require_relative 'terminfo' class Reline::ANSI @@ -154,11 +153,13 @@ def self.with_raw_input end @@buf = [] - def self.inner_getc + def self.inner_getc(timeout_second) unless @@buf.empty? return @@buf.shift end until c = @@input.raw(intr: true) { @@input.wait_readable(0.1) && @@input.getbyte } + timeout_second -= 0.1 + return nil if timeout_second <= 0 Reline.core.line_editor.resize end (c == 0x16 && @@input.raw(min: 0, time: 0, &:getbyte)) || c @@ -172,40 +173,38 @@ def self.inner_getc @@in_bracketed_paste_mode = false START_BRACKETED_PASTE = String.new("\e[200~,", encoding: Encoding::ASCII_8BIT) END_BRACKETED_PASTE = String.new("\e[200~.", encoding: Encoding::ASCII_8BIT) - def self.getc_with_bracketed_paste + def self.getc_with_bracketed_paste(timeout_second) buffer = String.new(encoding: Encoding::ASCII_8BIT) - buffer << inner_getc + buffer << inner_getc(timeout_second) while START_BRACKETED_PASTE.start_with?(buffer) or END_BRACKETED_PASTE.start_with?(buffer) do if START_BRACKETED_PASTE == buffer @@in_bracketed_paste_mode = true - return inner_getc + return inner_getc(timeout_second) elsif END_BRACKETED_PASTE == buffer @@in_bracketed_paste_mode = false ungetc(-1) - return inner_getc + return inner_getc(timeout_second) end - begin - succ_c = nil - Timeout.timeout(Reline.core.config.keyseq_timeout * 100) { - succ_c = inner_getc - } - rescue Timeout::Error - break - else + succ_c = inner_getc(Reline.core.config.keyseq_timeout) + + if succ_c buffer << succ_c + else + break end end buffer.bytes.reverse_each do |ch| ungetc ch end - inner_getc + inner_getc(timeout_second) end - def self.getc + # if the usage expects to wait indefinitely, use Float::INFINITY for timeout_second + def self.getc(timeout_second) if Reline.core.config.enable_bracketed_paste - getc_with_bracketed_paste + getc_with_bracketed_paste(timeout_second) else - inner_getc + inner_getc(timeout_second) end end diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb index 99298465689d01..dd2e87eaf0a441 100644 --- a/lib/reline/general_io.rb +++ b/lib/reline/general_io.rb @@ -1,4 +1,3 @@ -require 'timeout' require 'io/wait' class Reline::GeneralIO @@ -35,7 +34,7 @@ def self.with_raw_input yield end - def self.getc + def self.getc(_timeout_second) unless @@buf.empty? return @@buf.shift end diff --git a/lib/reline/version.rb b/lib/reline/version.rb index c8bbf615ccd7a1..a2538f6fa36431 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.3.7' + VERSION = '0.3.8' end diff --git a/lib/reline/windows.rb b/lib/reline/windows.rb index 7ea2a00f633bd8..6f635f630fda69 100644 --- a/lib/reline/windows.rb +++ b/lib/reline/windows.rb @@ -295,7 +295,7 @@ def self.with_raw_input yield end - def self.getc + def self.getc(_timeout_second) check_input_event @@output_buf.shift end diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 3f132b6f2bf562..0b9bf143dfb698 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -1302,10 +1302,20 @@ def self._load(str) Gem.load_yaml yaml_set = false + retry_count = 0 array = begin Marshal.load str rescue ArgumentError => e + # Avoid an infinite retry loop when the argument error has nothing to do + # with the classes not being defined. + # 1 retry each allowed in case all 3 of + # - YAML + # - YAML::Syck::DefaultKey + # - YAML::PrivateType + # need to be defined + raise if retry_count >= 3 + # # Some very old marshaled specs included references to `YAML::PrivateType` # and `YAML::Syck::DefaultKey` constants due to bugs in the old emitter @@ -1323,11 +1333,12 @@ def self._load(str) if message.include?("YAML::Syck::") YAML.const_set "Syck", YAML unless YAML.const_defined?(:Syck) - YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") - elsif message.include?("YAML::PrivateType") + YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") && !YAML::Syck.const_defined?(:DefaultKey) + elsif message.include?("YAML::PrivateType") && !YAML.const_defined?(:PrivateType) YAML.const_set "PrivateType", Class.new end + retry_count += 1 retry ensure Object.__send__(:remove_const, "YAML") if yaml_set diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index ac5fbfc8cfdd66..c96df2a08536bb 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -133,7 +133,7 @@ def uninstall if index == list.size remove_all list - elsif index >= 0 && index < list.size + elsif index && index >= 0 && index < list.size uninstall_gem list[index] else say "Error: must enter a number [1-#{list.size + 1}]" diff --git a/lib/yarp.rb b/lib/yarp.rb index aa92113d2cc6dc..15217e80f490a7 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -313,6 +313,30 @@ def pretty_print(q) end end + class FloatNode < Node + def value + Float(slice) + end + end + + class ImaginaryNode < Node + def value + Complex(0, numeric.value) + end + end + + class IntegerNode < Node + def value + Integer(slice) + end + end + + class RationalNode < Node + def value + Rational(numeric.value) + end + end + # Load the serialized AST using the source as a reference into a tree. def self.load(source, serialized) Serialize.load(source, serialized) @@ -457,6 +481,8 @@ def self.yarp_locals(source) locals << [] when PostExecutionNode locals.push([], []) + when InterpolatedRegularExpressionNode + locals << [] if node.once? end stack.concat(node.child_nodes.compact) @@ -489,5 +515,5 @@ def self.parse_serialize_file(filepath) if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"] require "yarp/yarp" else - require "yarp/ffi" + require_relative "yarp/ffi" end diff --git a/lib/yarp/lex_compat.rb b/lib/yarp/lex_compat.rb index 7e1d9f36573c78..37bfecd0bca934 100644 --- a/lib/yarp/lex_compat.rb +++ b/lib/yarp/lex_compat.rb @@ -208,18 +208,9 @@ def ==(other) end end - # It is extremely non obvious which state the parser is in when comments get - # dispatched. Because of this we don't both comparing state when comparing - # against other comment tokens. - class CommentToken < Token - def ==(other) - self[0...-1] == other[0...-1] - end - end - - # Heredoc end tokens are emitted in an odd order, so we don't compare the - # state on them. - class HeredocEndToken < Token + # Tokens where state should be ignored + # used for :on_comment, :on_heredoc_end, :on_embexpr_end + class IgnoreStateToken < Token def ==(other) self[0...-1] == other[0...-1] end @@ -252,6 +243,23 @@ def ==(other) end end + # If we have an identifier that follows a method name like: + # + # def foo bar + # + # then Ripper will mark bar as END|LABEL if there is a local in a parent + # scope named bar because it hasn't pushed the local table yet. We do this + # more accurately, so we need to allow comparing against both END and + # END|LABEL. + class ParamToken < Token + def ==(other) + (self[0...-1] == other[0...-1]) && ( + (other[3] == Ripper::EXPR_END) || + (other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL) + ) + end + end + # A heredoc in this case is a list of tokens that belong to the body of the # heredoc that should be appended onto the list of tokens when the heredoc # closes. @@ -558,18 +566,45 @@ def result result_value = result.value previous_state = nil - # If there's a UTF-8 byte-order mark as the start of the file, then ripper - # sets every token's on the first line back by 6 bytes. It also keeps the - # byte order mark in the first token's value. This is weird, and I don't - # want to mirror that in our parser. So instead, we'll match up the values - # here, and then match up the locations as we process the tokens. - bom = source.bytes[0..2] == [0xEF, 0xBB, 0xBF] - result_value[0][0].value.prepend("\xEF\xBB\xBF") if bom + # In previous versions of Ruby, Ripper wouldn't flush the bom before the + # first token, so we had to have a hack in place to account for that. This + # checks for that behavior. + bom_flushed = Ripper.lex("\xEF\xBB\xBF# test")[0][0][1] == 0 + bom = source.byteslice(0..2) == "\xEF\xBB\xBF" result_value.each_with_index do |(token, lex_state), index| lineno = token.location.start_line column = token.location.start_column - column -= index == 0 ? 6 : 3 if bom && lineno == 1 + + # If there's a UTF-8 byte-order mark as the start of the file, then for + # certain tokens ripper sets the first token back by 3 bytes. It also + # keeps the byte order mark in the first token's value. This is weird, + # and I don't want to mirror that in our parser. So instead, we'll match + # up the columns and values here. + if bom && lineno == 1 + column -= 3 + + if index == 0 && column == 0 && !bom_flushed + flushed = + case token.type + when :BACK_REFERENCE, :INSTANCE_VARIABLE, :CLASS_VARIABLE, + :GLOBAL_VARIABLE, :NUMBERED_REFERENCE, :PERCENT_LOWER_I, + :PERCENT_LOWER_X, :PERCENT_LOWER_W, :PERCENT_UPPER_I, + :PERCENT_UPPER_W, :STRING_BEGIN + true + when :REGEXP_BEGIN, :SYMBOL_BEGIN + token.value.start_with?("%") + else + false + end + + unless flushed + column -= 3 + value = token.value + value.prepend(String.new("\xEF\xBB\xBF", encoding: value.encoding)) + end + end + end event = RIPPER.fetch(token.type) value = token.value @@ -580,13 +615,23 @@ def result when :on___end__ EndContentToken.new([[lineno, column], event, value, lex_state]) when :on_comment - CommentToken.new([[lineno, column], event, value, lex_state]) + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_heredoc_end # Heredoc end tokens can be emitted in an odd order, so we don't # want to bother comparing the state on them. - HeredocEndToken.new([[lineno, column], event, value, lex_state]) - when :on_embexpr_end, :on_ident - if lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_ident + if lex_state == Ripper::EXPR_END + # If we have an identifier that follows a method name like: + # + # def foo bar + # + # then Ripper will mark bar as END|LABEL if there is a local in a + # parent scope named bar because it hasn't pushed the local table + # yet. We do this more accurately, so we need to allow comparing + # against both END and END|LABEL. + ParamToken.new([[lineno, column], event, value, lex_state]) + elsif lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL # In the event that we're comparing identifiers, we're going to # allow a little divergence. Ripper doesn't account for local # variables introduced through named captures in regexes, and we @@ -595,6 +640,8 @@ def result else Token.new([[lineno, column], event, value, lex_state]) end + when :on_embexpr_end + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_ignored_nl # Ignored newlines can occasionally have a LABEL state attached to # them which doesn't actually impact anything. We don't mirror that @@ -629,6 +676,26 @@ def result previous_state end + Token.new([[lineno, column], event, value, lex_state]) + when :on_eof + previous_token = result_value[index - 1][0] + + # If we're at the end of the file and the previous token was a + # comment and there is still whitespace after the comment, then + # Ripper will append a on_nl token (even though there isn't + # necessarily a newline). We mirror that here. + start_offset = previous_token.location.end_offset + end_offset = token.location.start_offset + + if previous_token.type == :COMMENT && start_offset < end_offset + if bom + start_offset += 3 + end_offset += 3 + end + + tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state]) + end + Token.new([[lineno, column], event, value, lex_state]) else Token.new([[lineno, column], event, value, lex_state]) @@ -713,7 +780,8 @@ def result end end - tokens.reject! { |t| t.event == :on_eof } + # Drop the EOF token from the list + tokens = tokens[0...-1] # We sort by location to compare against Ripper's output tokens.sort_by!(&:location) diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index f32cd4b6abbac7..db69bcb451b8d7 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "yarp" - spec.version = "0.7.0" + spec.version = "0.8.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] diff --git a/load.c b/load.c index 0d4fbae5a6c786..f9111ee6d81dd3 100644 --- a/load.c +++ b/load.c @@ -24,6 +24,11 @@ static VALUE ruby_dln_librefs; #define IS_SOEXT(e) (strcmp((e), ".so") == 0 || strcmp((e), ".o") == 0) #define IS_DLEXT(e) (strcmp((e), DLEXT) == 0) +enum { + loadable_ext_rb = (0+ /* .rb extension is the first in both tables */ + 1) /* offset by rb_find_file_ext() */ +}; + static const char *const loadable_ext[] = { ".rb", DLEXT, 0 @@ -475,6 +480,12 @@ loaded_feature_path_i(st_data_t v, st_data_t b, st_data_t f) return ST_STOP; } +/* + * Returns the type of already provided feature. + * 'r': ruby script (".rb") + * 's': shared object (".so"/"."DLEXT) + * 'u': unsuffixed + */ static int rb_feature_p(rb_vm_t *vm, const char *feature, const char *ext, int rb, int expanded, const char **fn) { @@ -996,7 +1007,7 @@ search_required(rb_vm_t *vm, VALUE fname, volatile VALUE *path, feature_func rb_ { VALUE tmp; char *ext, *ftptr; - int type, ft = 0; + int ft = 0; const char *loading; *path = 0; @@ -1048,11 +1059,11 @@ search_required(rb_vm_t *vm, VALUE fname, volatile VALUE *path, feature_func rb_ return 'r'; } tmp = fname; - type = rb_find_file_ext(&tmp, ft == 's' ? ruby_ext : loadable_ext); + const unsigned int type = rb_find_file_ext(&tmp, ft == 's' ? ruby_ext : loadable_ext); // Check if it's a statically linked extension when // not already a feature and not found as a dynamic library. - if (!ft && type != 1 && vm->static_ext_inits) { + if (!ft && type != loadable_ext_rb && vm->static_ext_inits) { VALUE lookup_name = tmp; // Append ".so" if not already present so for example "etc" can find "etc.so". // We always register statically linked extensions with a ".so" extension. @@ -1080,13 +1091,13 @@ search_required(rb_vm_t *vm, VALUE fname, volatile VALUE *path, feature_func rb_ goto feature_present; } /* fall through */ - case 1: + case loadable_ext_rb: ext = strrchr(ftptr = RSTRING_PTR(tmp), '.'); - if (rb_feature_p(vm, ftptr, ext, !--type, TRUE, &loading) && !loading) + if (rb_feature_p(vm, ftptr, ext, type == loadable_ext_rb, TRUE, &loading) && !loading) break; *path = tmp; } - return type ? 's' : 'r'; + return type > loadable_ext_rb ? 's' : 'r'; feature_present: if (loading) *path = rb_filesystem_str_new_cstr(loading); diff --git a/marshal.c b/marshal.c index d1218af9ad6375..ec38c2188a7285 100644 --- a/marshal.c +++ b/marshal.c @@ -173,7 +173,7 @@ struct dump_arg { st_table *data; st_table *compat_tbl; st_table *encodings; - unsigned long num_entries; + st_index_t num_entries; }; struct dump_call_arg { diff --git a/misc/gdb.py b/misc/gdb.py index 8f22bc91e1b5a0..6034a389bbf34e 100644 --- a/misc/gdb.py +++ b/misc/gdb.py @@ -92,8 +92,9 @@ def invoke(self, args, from_tty): self.print_env(cfp, -1, self.frame_types(cfp, -1)) print() - # We can't calculate BP for the first frame - if cfp_index > 0: + # We can't calculate BP for the first frame. + # vm_base_ptr doesn't work for C frames either. + if cfp_index > 0 and self.get_int(f'{cfp}->iseq'): if args.stack_size is not None: stack_size = args.stack_size else: diff --git a/node.h b/node.h index 69f73fc7984377..1f365960fcbcc2 100644 --- a/node.h +++ b/node.h @@ -29,10 +29,6 @@ typedef struct { node_buffer_elem_t *last; } node_buffer_list_t; -#ifdef UNIVERSAL_PARSER -typedef struct rb_parser_config_struct rb_parser_config_t; -#endif - struct node_buffer_struct { node_buffer_list_t unmarkable; node_buffer_list_t markable; diff --git a/node_dump.c b/node_dump.c index 52c49df5fae850..c13403d77ffa25 100644 --- a/node_dump.c +++ b/node_dump.c @@ -61,7 +61,6 @@ #define SIMPLE_FIELD1(name, ann) SIMPLE_FIELD(FIELD_NAME_LEN(name, ann), FIELD_NAME_DESC(name, ann)) #define F_CUSTOM1(name, ann) SIMPLE_FIELD1(#name, ann) #define F_ID(name, ann) SIMPLE_FIELD1(#name, ann) A_ID(node->name) -#define F_GENTRY(name, ann) SIMPLE_FIELD1(#name, ann) A_ID(node->name) #define F_INT(name, ann) SIMPLE_FIELD1(#name, ann) A_INT(node->name) #define F_LONG(name, ann) SIMPLE_FIELD1(#name, ann) A_LONG(node->name) #define F_LIT(name, ann) SIMPLE_FIELD1(#name, ann) A_LIT(node->name) @@ -436,9 +435,9 @@ dump_node(VALUE buf, VALUE indent, int comment, const NODE * node) return; case NODE_GASGN: ANN("global variable assignment"); - ANN("format: [nd_entry](gvar) = [nd_value]"); + ANN("format: [nd_vid](gvar) = [nd_value]"); ANN("example: $x = foo"); - F_GENTRY(nd_entry, "global variable"); + F_ID(nd_vid, "global variable"); LAST_NODE; F_NODE(nd_value, "rvalue"); return; @@ -592,7 +591,7 @@ dump_node(VALUE buf, VALUE indent, int comment, const NODE * node) case NODE_HASH: if (!node->nd_brace) { ANN("keyword arguments"); - ANN("format: nd_head"); + ANN("format: [nd_head]"); ANN("example: a: 1, b: 2"); } else { @@ -651,9 +650,9 @@ dump_node(VALUE buf, VALUE indent, int comment, const NODE * node) case NODE_GVAR: ANN("global variable reference"); - ANN("format: [nd_entry](gvar)"); + ANN("format: [nd_vid](gvar)"); ANN("example: $x"); - F_GENTRY(nd_entry, "global variable"); + F_ID(nd_vid, "global variable"); return; case NODE_NTH_REF: diff --git a/parse.y b/parse.y index d5942983eacd14..ba146b7439ece5 100644 --- a/parse.y +++ b/parse.y @@ -29,6 +29,8 @@ # include RUBY_EXTCONF_H #endif +#include "ruby/internal/config.h" + #include #include "internal/ruby_parser.h" #include "parser_node.h" @@ -149,26 +151,6 @@ literal_hash(VALUE a) return rb_iseq_cdhash_hash(a); } -static ID -script_lines(void) -{ - ID script_lines; - CONST_ID(script_lines, "SCRIPT_LINES__"); - return script_lines; -} - -static int -script_lines_defined(void) -{ - return rb_const_defined_at(rb_cObject, script_lines()); -} - -static VALUE -script_lines_get(void) -{ - return rb_const_get_at(rb_cObject, script_lines()); -} - static VALUE syntax_error_new(void) { @@ -548,7 +530,6 @@ struct parser_params { unsigned int do_loop: 1; unsigned int do_chomp: 1; unsigned int do_split: 1; - unsigned int keep_script_lines: 1; unsigned int error_tolerant: 1; unsigned int keep_tokens: 1; @@ -1164,6 +1145,11 @@ static void numparam_pop(struct parser_params *p, NODE *prev_inner); #define idFWD_KWREST idPow /* Use simple "**", as tDSTAR is "**arg" */ #define idFWD_BLOCK '&' #define idFWD_ALL idDot3 +#ifdef RIPPER +#define arg_FWD_BLOCK Qnone +#else +#define arg_FWD_BLOCK idFWD_BLOCK +#endif #define FORWARD_ARGS_WITH_RUBY2_KEYWORDS #define RE_OPTION_ONCE (1<<16) @@ -5647,7 +5633,7 @@ args_tail : f_kwarg ',' f_kwrest opt_f_block_arg | args_forward { add_forwarding_args(p); - $$ = new_args_tail(p, Qnone, $1, ID2VAL(idFWD_BLOCK), &@1); + $$ = new_args_tail(p, Qnone, $1, arg_FWD_BLOCK, &@1); /*%%%*/ ($$->nd_ainfo)->forwarding = 1; /*% %*/ @@ -6842,20 +6828,6 @@ static void parser_prepare(struct parser_params *p); #ifndef RIPPER static NODE *parser_append_options(struct parser_params *p, NODE *node); -static VALUE -debug_lines(struct parser_params *p, VALUE fname) -{ - if (script_lines_defined()) { - VALUE hash = script_lines_get(); - if (RB_TYPE_P(hash, T_HASH)) { - VALUE lines = rb_ary_new(); - rb_hash_aset(hash, fname, lines); - return lines; - } - } - return 0; -} - static int e_option_supplied(struct parser_params *p) { @@ -6871,7 +6843,6 @@ yycompile0(VALUE arg) int cov = FALSE; if (!compile_for_eval && !NIL_P(p->ruby_sourcefile_string)) { - p->debug_lines = debug_lines(p, p->ruby_sourcefile_string); if (p->debug_lines && p->ruby_sourceline > 0) { VALUE str = rb_default_rs; n = p->ruby_sourceline; @@ -6885,11 +6856,7 @@ yycompile0(VALUE arg) } } - if (p->keep_script_lines || ruby_vm_keep_script_lines) { - if (!p->debug_lines) { - p->debug_lines = rb_ary_new(); - } - + if (p->debug_lines) { RB_OBJ_WRITE(p->ast, &p->ast->body.script_lines, p->debug_lines); } @@ -7748,6 +7715,13 @@ parser_mixed_escape(struct parser_params *p, const char *beg, rb_encoding *enc1, p->lex.pcur = pos; } +static inline char +nibble_char_upper(unsigned int c) +{ + c &= 0xf; + return c + (c < 10 ? '0' : 'A' - 10); +} + static int tokadd_string(struct parser_params *p, int func, int term, int paren, long *nest, @@ -7837,12 +7811,11 @@ tokadd_string(struct parser_params *p, pushback(p, c); c = read_escape(p, 0, enc); - int i; - char escbuf[5]; - snprintf(escbuf, sizeof(escbuf), "\\x%02X", c); - for (i = 0; i < 4; i++) { - tokadd(p, escbuf[i]); - } + char *t = tokspace(p, rb_strlen_lit("\\x00")); + *t++ = '\\'; + *t++ = 'x'; + *t++ = nibble_char_upper(c >> 4); + *t++ = nibble_char_upper(c); continue; } } @@ -9059,6 +9032,7 @@ parser_prepare(struct parser_params *p) } #endif p->lex.pbeg = p->lex.pcur; + token_flush(p); return; } break; @@ -11261,7 +11235,7 @@ static NODE * new_kw_arg(struct parser_params *p, NODE *k, const YYLTYPE *loc) { if (!k) return 0; - return NEW_KW_ARG(0, (k), loc); + return NEW_KW_ARG((k), loc); } static NODE * @@ -13981,9 +13955,19 @@ rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, in } void -rb_ruby_parser_keep_script_lines(rb_parser_t *p) +rb_ruby_parser_set_script_lines(rb_parser_t *p, VALUE lines) { - p->keep_script_lines = 1; + if (!RTEST(lines)) { + lines = Qfalse; + } + else if (lines == Qtrue) { + lines = rb_ary_new(); + } + else { + Check_Type(lines, T_ARRAY); + rb_ary_modify(lines); + } + p->debug_lines = lines; } void @@ -14071,12 +14055,12 @@ rb_parser_error_tolerant(VALUE vparser) } void -rb_parser_keep_script_lines(VALUE vparser) +rb_parser_set_script_lines(VALUE vparser, VALUE lines) { struct parser_params *p; TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, p); - rb_ruby_parser_keep_script_lines(p); + rb_ruby_parser_set_script_lines(p, lines); } void diff --git a/parser_node.h b/parser_node.h index b776486fce1d01..886f9af7d28051 100644 --- a/parser_node.h +++ b/parser_node.h @@ -59,7 +59,7 @@ code_loc_gen(const rb_code_location_t *loc1, const rb_code_location_t *loc2) #define NEW_ZLIST(loc) NEW_NODE(NODE_ZLIST,0,0,0,loc) #define NEW_HASH(a,loc) NEW_NODE(NODE_HASH,a,0,0,loc) #define NEW_MASGN(l,r,loc) NEW_NODE(NODE_MASGN,l,0,r,loc) -#define NEW_GASGN(v,val,loc) NEW_NODE(NODE_GASGN,v,val,v,loc) +#define NEW_GASGN(v,val,loc) NEW_NODE(NODE_GASGN,v,val,0,loc) #define NEW_LASGN(v,val,loc) NEW_NODE(NODE_LASGN,v,val,0,loc) #define NEW_DASGN(v,val,loc) NEW_NODE(NODE_DASGN,v,val,0,loc) #define NEW_IASGN(v,val,loc) NEW_NODE(NODE_IASGN,v,val,0,loc) @@ -71,7 +71,7 @@ code_loc_gen(const rb_code_location_t *loc1, const rb_code_location_t *loc2) #define NEW_OP_ASGN_OR(i,val,loc) NEW_NODE(NODE_OP_ASGN_OR,i,val,0,loc) #define NEW_OP_ASGN_AND(i,val,loc) NEW_NODE(NODE_OP_ASGN_AND,i,val,0,loc) #define NEW_OP_CDECL(v,op,val,loc) NEW_NODE(NODE_OP_CDECL,v,val,op,loc) -#define NEW_GVAR(v,loc) NEW_NODE(NODE_GVAR,v,0,v,loc) +#define NEW_GVAR(v,loc) NEW_NODE(NODE_GVAR,v,0,0,loc) #define NEW_LVAR(v,loc) NEW_NODE(NODE_LVAR,v,0,0,loc) #define NEW_DVAR(v,loc) NEW_NODE(NODE_DVAR,v,0,0,loc) #define NEW_IVAR(v,loc) NEW_NODE(NODE_IVAR,v,0,0,loc) @@ -97,7 +97,7 @@ code_loc_gen(const rb_code_location_t *loc1, const rb_code_location_t *loc2) #define NEW_ZSUPER(loc) NEW_NODE(NODE_ZSUPER,0,0,0,loc) #define NEW_ARGS_AUX(r,b,loc) NEW_NODE(NODE_ARGS_AUX,r,b,0,loc) #define NEW_OPT_ARG(i,v,loc) NEW_NODE(NODE_OPT_ARG,i,v,0,loc) -#define NEW_KW_ARG(i,v,loc) NEW_NODE(NODE_KW_ARG,i,v,0,loc) +#define NEW_KW_ARG(v,loc) NEW_NODE(NODE_KW_ARG,0,v,0,loc) #define NEW_POSTARG(i,v,loc) NEW_NODE(NODE_POSTARG,i,v,0,loc) #define NEW_ARGSCAT(a,b,loc) NEW_NODE(NODE_ARGSCAT,a,b,0,loc) #define NEW_ARGSPUSH(a,b,loc) NEW_NODE(NODE_ARGSPUSH,a,b,0,loc) diff --git a/process.c b/process.c index 467efa181b8145..fa2ae7344ab20c 100644 --- a/process.c +++ b/process.c @@ -1419,21 +1419,18 @@ proc_m_wait(int c, VALUE *v, VALUE _) return proc_wait(c, v); } - /* * call-seq: - * Process.wait2(pid=-1, flags=0) -> [pid, status] - * Process.waitpid2(pid=-1, flags=0) -> [pid, status] - * - * Waits for a child process to exit (see Process::waitpid for exact - * semantics) and returns an array containing the process ID and the - * exit status (a Process::Status object) of that - * child. Raises a SystemCallError if there are no child processes. - * - * Process.fork { exit 99 } #=> 27437 - * pid, status = Process.wait2 - * pid #=> 27437 - * status.exitstatus #=> 99 + * Process.wait2(pid = -1, flags = 0) -> [pid, status] + * + * Like Process.waitpid, but returns an array + * containing the child process +pid+ and Process::Status +status+: + * + * pid = Process.spawn('ruby', '-e', 'exit 13') # => 309581 + * Process.wait2(pid) + * # => [309581, #] + * + * Process.waitpid2 is an alias for Process.waitpid. */ static VALUE @@ -1447,22 +1444,17 @@ proc_wait2(int argc, VALUE *argv, VALUE _) /* * call-seq: - * Process.waitall -> [ [pid1,status1], ...] + * Process.waitall -> array * - * Waits for all children, returning an array of - * _pid_/_status_ pairs (where _status_ is a - * Process::Status object). + * Waits for all children, returns an array of 2-element arrays; + * each subarray contains the integer pid and Process::Status status + * for one of the reaped child processes: * - * fork { sleep 0.2; exit 2 } #=> 27432 - * fork { sleep 0.1; exit 1 } #=> 27433 - * fork { exit 0 } #=> 27434 - * p Process.waitall + * pid0 = Process.spawn('ruby', '-e', 'exit 13') # => 325470 + * pid1 = Process.spawn('ruby', '-e', 'exit 14') # => 325495 + * Process.waitall + * # => [[325470, #], [325495, #]] * - * produces: - * - * [[30982, #], - * [30979, #], - * [30976, #]] */ static VALUE @@ -1520,48 +1512,41 @@ rb_detach_process(rb_pid_t pid) /* * call-seq: - * Process.detach(pid) -> thread + * Process.detach(pid) -> thread * - * Some operating systems retain the status of terminated child - * processes until the parent collects that status (normally using - * some variant of wait()). If the parent never collects - * this status, the child stays around as a zombie process. - * Process::detach prevents this by setting up a separate Ruby thread - * whose sole job is to reap the status of the process _pid_ when it - * terminates. Use #detach only when you do not intend to explicitly - * wait for the child to terminate. + * Avoids the potential for a child process to become a + * {zombie process}[https://en.wikipedia.org/wiki/Zombie_process]. + * Process.detach prevents this by setting up a separate Ruby thread + * whose sole job is to reap the status of the process _pid_ when it terminates. * - * The waiting thread returns the exit status of the detached process - * when it terminates, so you can use Thread#join to - * know the result. If specified _pid_ is not a valid child process - * ID, the thread returns +nil+ immediately. + * This method is needed only when the parent process will never wait + * for the child process. * - * The waiting thread has #pid method which returns the pid. + * This example does not reap the second child process; + * that process appears as a zombie in the process status (+ps+) output: * - * In this first example, we don't reap the first child process, so - * it appears as a zombie in the process status display. + * pid = Process.spawn('ruby', '-e', 'exit 13') # => 312691 + * sleep(1) + * # Find zombies. + * system("ps -ho pid,state -p #{pid}") * - * p1 = fork { sleep 0.1 } - * p2 = fork { sleep 0.2 } - * Process.waitpid(p2) - * sleep 2 - * system("ps -ho pid,state -p #{p1}") + * Output: * - * produces: + * 312716 Z * - * 27389 Z + * This example also does not reap the second child process, + * but it does detach the process so that it does not become a zombie: * - * In the next example, Process::detach is used to reap - * the child automatically. + * pid = Process.spawn('ruby', '-e', 'exit 13') # => 313213 + * thread = Process.detach(pid) + * sleep(1) + * # => # + * system("ps -ho pid,state -p #{pid}") # Finds no zombies. * - * p1 = fork { sleep 0.1 } - * p2 = fork { sleep 0.2 } - * Process.detach(p1) - * Process.waitpid(p2) - * sleep 2 - * system("ps -ho pid,state -p #{p1}") + * The waiting thread can return the pid of the detached child process: + * + * thread.join.pid # => 313262 * - * (produces no output) */ static VALUE @@ -3020,77 +3005,65 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); /* * call-seq: - * exec([env,] command... [,options]) - * - * Replaces the current process by running the given external _command_, which - * can take one of the following forms: + * exec([env, ] command_line, options = {}) + * exec([env, ] exe_path, *args, options = {}) * - * [exec(commandline)] - * command line string which is passed to the standard shell - * [exec(cmdname, arg1, ...)] - * command name and one or more arguments (no shell) - * [exec([cmdname, argv0], arg1, ...)] - * command name, +argv[0]+ and zero or more arguments (no shell) + * Replaces the current process by doing one of the following: * - * In the first form, the string is taken as a command line that is subject to - * shell expansion before being executed. + * - Passing string +command_line+ to the shell. + * - Invoking the executable at +exe_path+. * - * The standard shell always means "/bin/sh" on Unix-like systems, - * otherwise, ENV["RUBYSHELL"] or ENV["COMSPEC"] on - * Windows and similar. The command is passed as an argument to the - * "-c" switch to the shell, except in the case of +COMSPEC+. + * The new process is created using the + * {exec system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/execve.html]; + * it may inherit some of its environment from the calling program + * (possibly including open file descriptors). * - * If the string from the first form (exec("command")) follows - * these simple rules: + * Argument +env+, if given, is a hash that affects +ENV+ for the new process; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. * - * * no meta characters, - * * not starting with shell reserved word or special built-in, + * Argument +options+ is a hash of options for the new process; + * see {Execution Options}[rdoc-ref:Process@Execution+Options]. * - * Ruby invokes the command directly without shell. + * The first required argument is one of the following: * - * You can force shell invocation by adding ";" to the string (because ";" is - * a meta character). + * - +command_line+ if it is a string, + * and if it begins with a shell reserved word or special built-in, + * or if it contains one or more metacharacters. + * - +exe_path+ otherwise. * - * Note that this behavior is observable by pid obtained - * (return value of spawn() and IO#pid for IO.popen) is the pid of the invoked - * command, not shell. + * Argument +command_line+ * - * In the second form (exec("command1", "arg1", ...)), the first - * is taken as a command name and the rest are passed as parameters to command - * with no shell expansion. + * \String argument +command_line+ is a command line to be passed to a shell; + * it must begin with a shell reserved word, begin with a special built-in, + * or contain meta characters. + * It may also contain arguments and options for that command. * - * In the third form (exec(["command", "argv0"], "arg1", ...)), - * starting a two-element array at the beginning of the command, the first - * element is the command to be executed, and the second argument is used as - * the argv[0] value, which may show up in process listings. + * On a Unix-like system, the shell is /bin/sh; + * otherwise the shell is determined by environment variable + * ENV['RUBYSHELL'], if defined, or ENV['COMSPEC'] otherwise. * - * In order to execute the command, one of the exec(2) system - * calls are used, so the running command may inherit some of the environment - * of the original program (including open file descriptors). + * Except for the +COMSPEC+ case, + * the entire string +command_line+ is passed as an argument + * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * This behavior is modified by the given +env+ and +options+ parameters. See - * ::spawn for details. + * The shell performs normal shell expansion on the command line. * - * If the command fails to execute (typically Errno::ENOENT when - * it was not found) a SystemCallError exception is raised. + * Raises an exception if the new process fails to execute. * - * This method modifies process attributes according to given +options+ before - * exec(2) system call. See ::spawn for more details about the - * given +options+. + * Argument +exe_path+ * - * The modified attributes may be retained when exec(2) system - * call fails. + * Argument +exe_path+ is one of the following: * - * For example, hard resource limits are not restorable. + * - The string path to an executable to be called. + * - A 2-element array containing the path to an executable + * and the string to be used as the name of the executing process. * - * Consider to create a child process using ::spawn or Kernel#system if this - * is not acceptable. + * Ruby invokes the executable directly, with no shell and no shell expansion. * - * exec "echo *" # echoes list of files in current directory - * # never get here + * If one or more +args+ is given, each is an argument or option + * to be passed to the executable. * - * exec "echo", "*" # echoes an asterisk - * # never get here + * Raises an exception if the new process fails to execute. */ static VALUE @@ -4265,27 +4238,62 @@ rb_proc__fork(VALUE _obj) /* * call-seq: - * Kernel.fork [{ block }] -> integer or nil - * Process.fork [{ block }] -> integer or nil - * - * Creates a subprocess. If a block is specified, that block is run - * in the subprocess, and the subprocess terminates with a status of - * zero. Otherwise, the +fork+ call returns twice, once in the - * parent, returning the process ID of the child, and once in the - * child, returning _nil_. The child process can exit using - * Kernel.exit! to avoid running any at_exit - * functions. The parent process should use Process.wait to collect - * the termination statuses of its children or use Process.detach to - * register disinterest in their status; otherwise, the operating - * system may accumulate zombie processes. - * - * The thread calling fork is the only thread in the created child process. - * fork doesn't copy other threads. - * - * If fork is not usable, Process.respond_to?(:fork) returns false. - * - * Note that fork(2) is not available on some platforms like Windows and NetBSD 4. - * Therefore you should use spawn() instead of fork(). + * Process.fork { ... } -> integer or nil + * Process.fork -> integer or nil + * + * Creates a child process. + * + * With a block given, runs the block in the child process; + * on block exit, the child terminates with a status of zero: + * + * puts "Before the fork: #{Process.pid}" + * fork do + * puts "In the child process: #{Process.pid}" + * end # => 382141 + * puts "After the fork: #{Process.pid}" + * + * Output: + * + * Before the fork: 420496 + * After the fork: 420496 + * In the child process: 420520 + * + * With no block given, the +fork+ call returns twice: + * + * - Once in the parent process, returning the pid of the child process. + * - Once in the child process, returning +nil+. + * + * Example: + * + * puts "This is the first line before the fork (pid #{Process.pid})" + * puts fork + * puts "This is the second line after the fork (pid #{Process.pid})" + * + * Output: + * + * This is the first line before the fork (pid 420199) + * 420223 + * This is the second line after the fork (pid 420199) + * + * This is the second line after the fork (pid 420223) + * + * In either case, the child process may exit using + * Kernel.exit! to avoid the call to Kernel#at_exit. + * + * To avoid zombie processes, the parent process should call either: + * + * - Process.wait, to collect the termination statuses of its children. + * - Process.detach, to register disinterest in their status. + * + * The thread calling +fork+ is the only thread in the created child process; + * +fork+ doesn't copy other threads. + * + * Note that method +fork+ is available on some platforms, + * but not on others: + * + * Process.respond_to?(:fork) # => true # Would be false on some. + * + * If not, you may use ::spawn instead of +fork+. */ static VALUE @@ -4337,13 +4345,18 @@ exit_status_code(VALUE status) NORETURN(static VALUE rb_f_exit_bang(int argc, VALUE *argv, VALUE obj)); /* * call-seq: - * Process.exit!(status=false) + * exit!(status = false) + * Process.exit!(status = false) * - * Exits the process immediately. No exit handlers are - * run. status is returned to the underlying system as the - * exit status. + * Exits the process immediately; no exit handlers are called. + * Returns exit status +status+ to the underlying operating system. * * Process.exit!(true) + * + * Values +true+ and +false+ for argument +status+ + * indicate, respectively, success and failure; + * The meanings of integer values are system-dependent. + * */ static VALUE @@ -4394,43 +4407,47 @@ rb_f_exit(int argc, const VALUE *argv) NORETURN(static VALUE f_exit(int c, const VALUE *a, VALUE _)); /* * call-seq: - * exit(status=true) - * Kernel::exit(status=true) - * Process::exit(status=true) - * - * Initiates the termination of the Ruby script by raising the - * SystemExit exception. This exception may be caught. The - * optional parameter is used to return a status code to the invoking - * environment. - * +true+ and +FALSE+ of _status_ means success and failure - * respectively. The interpretation of other integer values are - * system dependent. - * - * begin - * exit - * puts "never get here" - * rescue SystemExit - * puts "rescued a SystemExit exception" - * end - * puts "after begin block" + * exit(status = true) + * Process.exit(status = true) * - * produces: + * Initiates termination of the Ruby script by raising SystemExit; + * the exception may be caught. + * Returns exit status +status+ to the underlying operating system. * - * rescued a SystemExit exception - * after begin block + * Values +true+ and +false+ for argument +status+ + * indicate, respectively, success and failure; + * The meanings of integer values are system-dependent. * - * Just prior to termination, Ruby executes any at_exit - * functions (see Kernel::at_exit) and runs any object finalizers - * (see ObjectSpace::define_finalizer). + * Example: * - * at_exit { puts "at_exit function" } - * ObjectSpace.define_finalizer("string", proc { puts "in finalizer" }) - * exit + * begin + * exit + * puts 'Never get here.' + * rescue SystemExit + * puts 'Rescued a SystemExit exception.' + * end + * puts 'After begin block.' * - * produces: + * Output: + * + * Rescued a SystemExit exception. + * After begin block. + * + * Just prior to final termination, + * Ruby executes any at-exit procedures (see Kernel::at_exit) + * and any object finalizers (see ObjectSpace::define_finalizer). + * + * Example: + * + * at_exit { puts 'In at_exit function.' } + * ObjectSpace.define_finalizer('string', proc { puts 'In finalizer.' }) + * exit + * + * Output: + * + * In at_exit function. + * In finalizer. * - * at_exit function - * in finalizer */ static VALUE @@ -4469,14 +4486,16 @@ NORETURN(static VALUE f_abort(int c, const VALUE *a, VALUE _)); /* * call-seq: - * abort - * Kernel::abort([msg]) - * Process.abort([msg]) - * - * Terminate execution immediately, effectively by calling - * Kernel.exit(false). If _msg_ is given, it is written - * to STDERR prior to terminating. Otherwise, if an exception was raised, - * print its message and backtrace. + * abort + * Process.abort(msg = nil) + * + * Terminates execution immediately, effectively by calling + * Kernel.exit(false). + * + * If string argument +msg+ is given, + * it is written to STDERR prior to termination; + * otherwise, if an exception was raised, + * prints its message and backtrace. */ static VALUE @@ -8216,7 +8235,9 @@ rb_clock_gettime(int argc, VALUE *argv, VALUE _) VALUE unit = (rb_check_arity(argc, 1, 2) == 2) ? argv[1] : Qnil; VALUE clk_id = argv[0]; +#ifdef HAVE_CLOCK_GETTIME clockid_t c; +#endif if (SYMBOL_P(clk_id)) { #ifdef CLOCK_REALTIME @@ -8444,7 +8465,9 @@ rb_clock_getres(int argc, VALUE *argv, VALUE _) timetick_int_t denominators[2]; int num_numerators = 0; int num_denominators = 0; +#ifdef HAVE_CLOCK_GETRES clockid_t c; +#endif VALUE unit = (rb_check_arity(argc, 1, 2) == 2) ? argv[1] : Qnil; VALUE clk_id = argv[0]; @@ -8669,6 +8692,172 @@ proc_warmup(VALUE _) * \Module +Process+ represents a process in the underlying operating system. * Its methods support management of the current process and its child processes. * + * == \Process Creation + * + * Each of these methods creates a process: + * + * - Process.exec: Replaces the current process by running a given external command. + * - Process.spawn, Kernel#spawn: Executes the given command and returns its pid without waiting for completion. + * - Kernel#system: Executes the given command in a subshell. + * + * Each of these methods accepts: + * + * - An optional hash of environment variable names and values. + * - An optional hash of execution options. + * + * === Execution Environment + * + * Optional leading argument +env+ is a hash of name/value pairs, + * where each name is a string and each value is a string or +nil+; + * each name/value pair is added to ENV in the new process. + * + * Process.spawn( 'ruby -e "p ENV[\"Foo\"]"') + * Process.spawn({'Foo' => '0'}, 'ruby -e "p ENV[\"Foo\"]"') + * + * Output: + * + * nil + * "0" + * + * The effect is usually similar to that of calling ENV#update with argument +env+, + * where each named environment variable is created or updated + * (if the value is non-+nil+), + * or deleted (if the value is +nil+). + * + * However, some modifications to the calling process may remain + * if the new process fails. + * For example, hard resource limits are not restored. + * + * === Execution Options + * + * Optional trailing argument +options+ is a hash of execution options. + * + * ==== Working Directory (+:chdir+) + * + * By default, the working directory for the new process is the same as + * that of the current process: + * + * Dir.chdir('/var') + * Process.spawn('ruby -e "puts Dir.pwd"') + * + * Output: + * + * /var + * + * Use option +:chdir+ to set the working directory for the new process: + * + * Process.spawn('ruby -e "puts Dir.pwd"', {chdir: '/tmp'}) + * + * Output: + * + * /tmp + * + * The working directory of the current process is not changed: + * + * Dir.pwd # => "/var" + * + * ==== \File Redirection (\File Descriptor) + * + * Use execution options for file redirection in the new process. + * + * The key for such an option may be an integer file descriptor (fd), + * specifying a source, + * or an array of fds, specifying multiple sources. + + * An integer source fd may be specified as: + * + * - _n_: Specifies file descriptor _n_. + * + * There are these shorthand symbols for fds: + * + * - +:in+: Specifies file descriptor 0 (STDIN). + * - +:out+: Specifies file descriptor 1 (STDOUT). + * - +:err+: Specifies file descriptor 2 (STDERR). + * + * The value given with a source is one of: + * + * - _n_: + * Redirects to fd _n_ in the parent process. + * - +filepath+: + * Redirects from or to the file at +filepath+ via open(filepath, mode, 0644), + * where +mode+ is 'r' for source +:in+, + * or 'w' for source +:out+ or +:err+. + * - [filepath]: + * Redirects from the file at +filepath+ via open(filepath, 'r', 0644). + * - [filepath, mode]: + * Redirects from or to the file at +filepath+ via open(filepath, mode, 0644). + * - [filepath, mode, perm]: + * Redirects from or to the file at +filepath+ via open(filepath, mode, perm). + * - [:child, fd]: + * Redirects to the redirected +fd+. + * - +:close+: Closes the file descriptor in child process. + * + * See {Access Modes}[rdoc-ref:File@Access+Modes] + * and {File Permissions}[rdoc-ref:File@File+Permissions]. + * + * ==== Environment Variables (+:unsetenv_others+) + * + * By default, the new process inherits environment variables + * from the parent process; + * use execution option key +:unsetenv_others+ with value +true+ + * to clear environment variables in the new process. + * + * Any changes specified by execution option +env+ are made after the new process + * inherits or clears its environment variables; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + * + * ==== \File-Creation Access (+:umask+) + * + * Use execution option +:umask+ to set the file-creation access + * for the new process; + * see {Access Modes}[rdoc-ref:File@Access+Modes]: + * + * command = 'ruby -e "puts sprintf(\"0%o\", File.umask)"' + * options = {:umask => 0644} + * Process.spawn(command, options) + * + * Output: + * + * 0644 + * + * ==== \Process Groups (+:pgroup+ and +:new_pgroup+) + * + * By default, the new process belongs to the same + * {process group}[https://en.wikipedia.org/wiki/Process_group] + * as the parent process. + * + * To specify a different process group. + * use execution option +:pgroup+ with one of the following values: + * + * - +true+: Create a new process group for the new process. + * - _pgid_: Create the new process in the process group + * whose id is _pgid_. + * + * On Windows only, use execution option +:new_pgroup+ with value +true+ + * to create a new process group for the new process. + * + * ==== Resource Limits + * + * Use execution options to set resource limits. + * + * The keys for these options are symbols of the form + * :rlimit_resource_name, + * where _resource_name_ is the downcased form of one of the string + * resource names described at method Process.setrlimit. + * For example, key +:rlimit_cpu+ corresponds to resource limit 'CPU'. + * + * The value for such as key is one of: + * + * - An integer, specifying both the current and maximum limits. + * - A 2-element array of integers, specifying the current and maximum limits. + * + * ==== \File Descriptor Inheritance + * + * By default, the new process inherits file descriptors from the parent process. + * + * Use execution option :close_others => true to modify that inheritance + * by closing non-standard fds (3 and greater) that are not otherwise redirected. + * * == What's Here * * === Current-Process Getters diff --git a/ruby.c b/ruby.c index 5a339fac0d16a4..2b4397c0011617 100644 --- a/ruby.c +++ b/ruby.c @@ -2397,6 +2397,8 @@ struct load_file_arg { VALUE f; }; +VALUE rb_script_lines_for(VALUE path, bool add); + static VALUE load_file_internal(VALUE argp_v) { @@ -2499,6 +2501,12 @@ load_file_internal(VALUE argp_v) } rb_parser_set_options(parser, opt->do_print, opt->do_loop, opt->do_line, opt->do_split); + + VALUE lines = rb_script_lines_for(orig_fname, true); + if (!NIL_P(lines)) { + rb_parser_set_script_lines(parser, lines); + } + if (NIL_P(f)) { f = rb_str_new(0, 0); rb_enc_associate(f, enc); @@ -2888,12 +2896,13 @@ ruby_process_options(int argc, char **argv) set_progname(external_str_new_cstr(script_name)); /* for the time being */ rb_argv0 = rb_str_new4(rb_progname); rb_gc_register_mark_object(rb_argv0); - iseq = process_options(argc, argv, cmdline_options_init(&opt)); #ifndef HAVE_SETPROCTITLE ruby_init_setproctitle(argc, argv); #endif + iseq = process_options(argc, argv, cmdline_options_init(&opt)); + return (void*)(struct RData*)iseq; } diff --git a/ruby_parser.c b/ruby_parser.c index 9ab78be2a9689a..6aaa867c16a55f 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -117,12 +117,6 @@ syntax_error_append(VALUE exc, VALUE file, int line, int column, return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args); } -static int -vm_keep_script_lines(void) -{ - return ruby_vm_keep_script_lines; -} - static int local_defined(ID id, const void *p) { @@ -401,24 +395,6 @@ int2fix(long i) return INT2FIX(i); } -static int -script_lines_defined(void) -{ - ID script_lines; - CONST_ID(script_lines, "SCRIPT_LINES__"); - - return rb_const_defined_at(rb_cObject, script_lines); -} - -static VALUE -script_lines_get(void) -{ - ID script_lines; - CONST_ID(script_lines, "SCRIPT_LINES__"); - - return rb_const_get_at(rb_cObject, script_lines); -} - static VALUE syntax_error_new(void) { @@ -597,8 +573,6 @@ rb_parser_config_initialize(rb_parser_config_t *config) config->compile_callback = rb_suppress_tracing; config->reg_named_capture_assign = reg_named_capture_assign; - config->script_lines_defined = script_lines_defined; - config->script_lines_get = script_lines_get; config->obj_freeze = rb_obj_freeze; config->obj_hide = rb_obj_hide; @@ -743,7 +717,6 @@ rb_parser_config_initialize(rb_parser_config_t *config) config->ractor_make_shareable = rb_ractor_make_shareable; - config->vm_keep_script_lines = vm_keep_script_lines; config->local_defined = local_defined; config->dvar_defined = dvar_defined; @@ -801,7 +774,6 @@ rb_parser_config_initialize(rb_parser_config_t *config) config->long2int = rb_long2int; config->special_const_p = special_const_p; config->builtin_type = builtin_type; - config->snprintf = snprintf; config->node_case_when_optimizable_literal = rb_node_case_when_optimizable_literal; @@ -853,12 +825,12 @@ rb_parser_set_context(VALUE vparser, const struct rb_iseq_struct *base, int main } void -rb_parser_keep_script_lines(VALUE vparser) +rb_parser_set_script_lines(VALUE vparser, VALUE lines) { struct ruby_parser *parser; TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser); - rb_ruby_parser_keep_script_lines(parser->parser_params); + rb_ruby_parser_set_script_lines(parser->parser_params, lines); } void diff --git a/rubyparser.h b/rubyparser.h index 92c64870c06f50..e95b71f049f681 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -4,12 +4,16 @@ * This is a header file for librubyparser interface */ +#include /* for va_list */ + #ifdef UNIVERSAL_PARSER #define rb_encoding void #define OnigCodePoint unsigned int #include "parser_st.h" +#ifndef RUBY_RUBY_H #include "parser_value.h" +#endif #else @@ -146,7 +150,6 @@ enum node_type { #define nd_stts u1.node -#define nd_entry u3.id #define nd_vid u1.id #define nd_var u1.node @@ -328,8 +331,11 @@ typedef struct rb_ast_struct { * Parser Interface */ + typedef struct parser_params rb_parser_t; +#ifndef INTERNAL_IMEMO_H typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t; +#endif #ifdef UNIVERSAL_PARSER typedef struct rb_parser_config_struct { @@ -366,8 +372,6 @@ typedef struct rb_parser_config_struct { // VALUE rb_suppress_tracing(VALUE (*func)(VALUE), VALUE arg); VALUE (*compile_callback)(VALUE (*func)(VALUE), VALUE arg); NODE *(*reg_named_capture_assign)(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc); - int (*script_lines_defined)(void); - VALUE (*script_lines_get)(void); /* Object */ VALUE (*obj_freeze)(VALUE obj); @@ -529,7 +533,6 @@ typedef struct rb_parser_config_struct { VALUE (*ractor_make_shareable)(VALUE obj); /* Compile */ - int (*vm_keep_script_lines)(void); // int rb_local_defined(ID id, const rb_iseq_t *iseq); int (*local_defined)(ID, const void*); // int rb_dvar_defined(ID id, const rb_iseq_t *iseq); @@ -598,7 +601,6 @@ typedef struct rb_parser_config_struct { int (*long2int)(long); int (*special_const_p)(VALUE); int (*builtin_type)(VALUE); - int (*snprintf)(char *str, size_t n, char const *fmt, ...); VALUE (*node_case_when_optimizable_literal)(const NODE *const node); diff --git a/spec/bundler/bundler/ruby_dsl_spec.rb b/spec/bundler/bundler/ruby_dsl_spec.rb index 0ba55e949f7c3d..4af271ad59fff1 100644 --- a/spec/bundler/bundler/ruby_dsl_spec.rb +++ b/spec/bundler/bundler/ruby_dsl_spec.rb @@ -104,7 +104,38 @@ class MockDSL let(:engine_version) { version } let(:patchlevel) { nil } let(:engine) { "ruby" } - before { allow(Bundler).to receive(:read_file).with("foo").and_return("#{version}\n") } + let(:project_root) { Pathname.new("/path/to/project") } + + before do + allow(Bundler).to receive(:read_file).with(project_root.join("foo")).and_return("#{version}\n") + allow(Bundler).to receive(:root).and_return(Pathname.new("/path/to/project")) + end + + it_behaves_like "it stores the ruby version" + + context "and a version" do + let(:ruby_version_arg) { "2.0.0" } + + it "raises an error" do + expect { subject }.to raise_error(Bundler::GemfileError, "Cannot specify version when using the file option") + end + end + end + + context "with a (.tool-versions) file option" do + let(:options) { { :file => "foo" } } + let(:version) { "3.2.2" } + let(:ruby_version) { "3.2.2" } + let(:ruby_version_arg) { nil } + let(:engine_version) { version } + let(:patchlevel) { nil } + let(:engine) { "ruby" } + let(:project_root) { Pathname.new("/path/to/project") } + + before do + allow(Bundler).to receive(:read_file).with(project_root.join("foo")).and_return("nodejs 18.16.0\nruby #{version} # This is a comment\npnpm 8.6.12\n") + allow(Bundler).to receive(:root).and_return(Pathname.new("/path/to/project")) + end it_behaves_like "it stores the ruby version" diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb index 9095eca03bb538..1f89f9f0f9b933 100644 --- a/spec/bundler/install/gemfile/sources_spec.rb +++ b/spec/bundler/install/gemfile/sources_spec.rb @@ -35,6 +35,15 @@ expect(the_bundle).to include_gems("rack-obama 1.0.0", "rack 1.0.0", :source => "remote1") end + it "does not use the full index unnecessarily", :bundler => "< 3" do + bundle :install, :artifice => "compact_index", :verbose => true + + expect(out).to include("https://gem.repo1/versions") + expect(out).to include("https://gem.repo3/versions") + expect(out).not_to include("https://gem.repo1/quick/Marshal.4.8/") + expect(out).not_to include("https://gem.repo3/quick/Marshal.4.8/") + end + it "fails", :bundler => "3" do bundle :install, :artifice => "compact_index", :raise_on_error => false expect(err).to include("Each source after the first must include a block") diff --git a/spec/ruby/core/fiber/storage_spec.rb b/spec/ruby/core/fiber/storage_spec.rb index e99fe6e4df7eb4..e5232c8894a9c6 100644 --- a/spec/ruby/core/fiber/storage_spec.rb +++ b/spec/ruby/core/fiber/storage_spec.rb @@ -74,6 +74,20 @@ Fiber.new { Fiber[:life] }.resume.should be_nil end end + + ruby_version_is "3.2.3" do + it "can use dynamically defined keys" do + key = :"#{self.class.name}#.#{self.object_id}" + Fiber.new { Fiber[key] = 42; Fiber[key] }.resume.should == 42 + end + + it "can't use invalid keys" do + invalid_keys = [Object.new, "Foo", 12] + invalid_keys.each do |key| + -> { Fiber[key] }.should raise_error(TypeError) + end + end + end end describe "Fiber.[]=" do diff --git a/spec/ruby/core/io/write_spec.rb b/spec/ruby/core/io/write_spec.rb index 2e39b4fadd2920..4a26f8dbaf9fec 100644 --- a/spec/ruby/core/io/write_spec.rb +++ b/spec/ruby/core/io/write_spec.rb @@ -203,15 +203,19 @@ rm_r @fifo end - it "writes correctly" do - thr = Thread.new do - IO.read(@fifo) - end - begin - string = "hi" - IO.write(@fifo, string).should == string.length - ensure - thr.join + # rb_cloexec_open() is currently missing a retry on EINTR. + # @ioquatix is looking into fixing it. Quarantined until it's done. + quarantine! do + it "writes correctly" do + thr = Thread.new do + IO.read(@fifo) + end + begin + string = "hi" + IO.write(@fifo, string).should == string.length + ensure + thr.join + end end end end diff --git a/string.c b/string.c index 3979af641db798..9c5022012c273b 100644 --- a/string.c +++ b/string.c @@ -1746,11 +1746,11 @@ static inline VALUE ec_str_duplicate(struct rb_execution_context_struct *ec, VALUE klass, VALUE str) { VALUE dup; - if (FL_TEST(str, STR_NOEMBED)) { - dup = ec_str_alloc_heap(ec, klass); + if (STR_EMBED_P(str)) { + dup = ec_str_alloc_embed(ec, klass, RSTRING_LEN(str) + TERM_LEN(str)); } else { - dup = ec_str_alloc_embed(ec, klass, RSTRING_LEN(str) + TERM_LEN(str)); + dup = ec_str_alloc_heap(ec, klass); } return str_duplicate_setup(klass, str, dup); @@ -1760,11 +1760,11 @@ static inline VALUE str_duplicate(VALUE klass, VALUE str) { VALUE dup; - if (FL_TEST(str, STR_NOEMBED)) { - dup = str_alloc_heap(klass); + if (STR_EMBED_P(str)) { + dup = str_alloc_embed(klass, RSTRING_LEN(str) + TERM_LEN(str)); } else { - dup = str_alloc_embed(klass, RSTRING_LEN(str) + TERM_LEN(str)); + dup = str_alloc_heap(klass); } return str_duplicate_setup(klass, str, dup); @@ -3249,6 +3249,7 @@ rb_str_buf_append(VALUE str, VALUE str2) case ENC_CODERANGE_7BIT: // If RHS is 7bit we can do simple concatenation str_buf_cat4(str, RSTRING_PTR(str2), RSTRING_LEN(str2), true); + RB_GC_GUARD(str2); return str; case ENC_CODERANGE_VALID: // If RHS is valid, we can do simple concatenation if encodings are the same @@ -3258,6 +3259,7 @@ rb_str_buf_append(VALUE str, VALUE str2) if (UNLIKELY(str_cr != ENC_CODERANGE_VALID)) { ENC_CODERANGE_SET(str, RB_ENC_CODERANGE_AND(str_cr, str2_cr)); } + RB_GC_GUARD(str2); return str; } } @@ -10716,11 +10718,11 @@ static VALUE rb_str_b(VALUE str) { VALUE str2; - if (FL_TEST(str, STR_NOEMBED)) { - str2 = str_alloc_heap(rb_cString); + if (STR_EMBED_P(str)) { + str2 = str_alloc_embed(rb_cString, RSTRING_LEN(str) + TERM_LEN(str)); } else { - str2 = str_alloc_embed(rb_cString, RSTRING_LEN(str) + TERM_LEN(str)); + str2 = str_alloc_heap(rb_cString); } str_replace_shared_without_enc(str2, str); diff --git a/template/prelude.c.tmpl b/template/prelude.c.tmpl index aa8b0c1fefa355..74f6c08da7973e 100644 --- a/template/prelude.c.tmpl +++ b/template/prelude.c.tmpl @@ -43,7 +43,10 @@ class Prelude lineno = 0 result = [@preludes.size, @vpath.strip(filename), lines, sub] @vpath.foreach(filename) do |line| - line.force_encoding("ASCII-8BIT") if line.respond_to?(:force_encoding) + if line.respond_to?(:force_encoding) + enc = line.encoding + line.force_encoding("ASCII-8BIT") + end line.rstrip! lineno += 1 @preludes[filename] ||= result @@ -71,6 +74,7 @@ class Prelude orig end end + comment.force_encoding(enc) if enc and comment lines << [line, comment] end result << (start_line || 1) diff --git a/test/irb/helper.rb b/test/irb/helper.rb index e281b8e2f64981..122eb607f1409a 100644 --- a/test/irb/helper.rb +++ b/test/irb/helper.rb @@ -83,6 +83,9 @@ class IntegrationTestCase < TestCase TIMEOUT_SEC = 3 def setup + @envs = {} + @tmpfiles = [] + unless defined?(PTY) omit "Integration tests require PTY." end @@ -90,8 +93,12 @@ def setup if ruby_core? omit "This test works only under ruby/irb" end + end - @envs = {} + def teardown + @tmpfiles.each do |tmpfile| + File.unlink(tmpfile) + end end def run_ruby_file(&block) @@ -133,7 +140,6 @@ def run_ruby_file(&block) MSG assert_block(message) { false } ensure - File.unlink(@ruby_file) if @ruby_file FileUtils.remove_entry tmp_dir end @@ -180,8 +186,17 @@ def type(command) def write_ruby(program) @ruby_file = Tempfile.create(%w{irb- .rb}) + @tmpfiles << @ruby_file @ruby_file.write(program) @ruby_file.close end + + def write_rc(content) + @irbrc = Tempfile.new('irbrc') + @tmpfiles << @irbrc + @irbrc.write(content) + @irbrc.close + @envs['IRBRC'] = @irbrc.path + end end end diff --git a/test/irb/test_cmd.rb b/test/irb/test_cmd.rb index dcea020b1ef9ed..670078679f1152 100644 --- a/test/irb/test_cmd.rb +++ b/test/irb/test_cmd.rb @@ -62,23 +62,6 @@ def test_calling_command_on_a_frozen_main end end - class CommnadAliasTest < CommandTestCase - def test_vars_with_aliases - @foo = "foo" - $bar = "bar" - out, err = execute_lines( - "@foo\n", - "$bar\n", - ) - assert_empty err - assert_match(/"foo"/, out) - assert_match(/"bar"/, out) - ensure - remove_instance_variable(:@foo) - $bar = nil - end - end - class InfoTest < CommandTestCase def setup super @@ -920,12 +903,15 @@ def test_show_doc_without_rdoc class EditTest < CommandTestCase def setup + @original_visual = ENV["VISUAL"] @original_editor = ENV["EDITOR"] # noop the command so nothing gets executed - ENV["EDITOR"] = ": code" + ENV["VISUAL"] = ": code" + ENV["EDITOR"] = ": code2" end def teardown + ENV["VISUAL"] = @original_visual ENV["EDITOR"] = @original_editor end @@ -988,5 +974,18 @@ def test_edit_with_instance_method assert_match(/path: .*\/lib\/irb\.rb/, out) assert_match("command: ': code'", out) end + + def test_edit_with_editor_env_var + ENV.delete("VISUAL") + + out, err = execute_lines( + "edit", + irb_path: __FILE__ + ) + + assert_empty err + assert_match("path: #{__FILE__}", out) + assert_match("command: ': code2'", out) + end end end diff --git a/test/irb/test_debug_cmd.rb b/test/irb/test_debug_cmd.rb index c4e4a04fdd64d4..a99f7a943f62cd 100644 --- a/test/irb/test_debug_cmd.rb +++ b/test/irb/test_debug_cmd.rb @@ -289,6 +289,25 @@ def test_prompt_line_number_continues assert_match(/irb:rdbg\(main\):005> next/, output) end + def test_prompt_irb_name_is_kept + write_rc <<~RUBY + IRB.conf[:IRB_NAME] = "foo" + RUBY + + write_ruby <<~'ruby' + binding.irb + puts "Hello" + ruby + + output = run_ruby_file do + type "next" + type "continue" + end + + assert_match(/foo\(main\):001> next/, output) + assert_match(/foo:rdbg\(main\):002> continue/, output) + end + def test_irb_commands_are_available_after_moving_around_with_the_debugger write_ruby <<~'ruby' class Foo diff --git a/test/irb/test_history.rb b/test/irb/test_history.rb index 9bf146609c2ab2..15628e66e2d46e 100644 --- a/test/irb/test_history.rb +++ b/test/irb/test_history.rb @@ -326,12 +326,9 @@ def write_history(history) @history_file = Tempfile.new('irb_history') @history_file.write(history) @history_file.close - @irbrc = Tempfile.new('irbrc') - @irbrc.write <<~RUBY + write_rc <<~RUBY IRB.conf[:HISTORY_FILE] = "#{@history_file.path}" RUBY - @irbrc.close - @envs['IRBRC'] = @irbrc.path end end end diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index b613cc8a9ad5a3..c685912093dffb 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -4,6 +4,67 @@ require_relative "helper" module TestIRB + class InputTest < IntegrationTestCase + def test_symbol_aliases_are_handled_correctly + write_ruby <<~'RUBY' + class Foo + end + binding.irb + RUBY + + output = run_ruby_file do + type "$ Foo" + type "exit!" + end + + assert_include output, "From: #{@ruby_file.path}:1" + end + + def test_symbol_aliases_are_handled_correctly_with_singleline_mode + @irbrc = Tempfile.new('irbrc') + @irbrc.write <<~RUBY + IRB.conf[:USE_SINGLELINE] = true + RUBY + @irbrc.close + @envs['IRBRC'] = @irbrc.path + + write_ruby <<~'RUBY' + class Foo + end + binding.irb + RUBY + + output = run_ruby_file do + type "irb_info" + type "$ Foo" + type "exit!" + end + + # Make sure it's tested in singleline mode + assert_include output, "InputMethod: ReadlineInputMethod" + assert_include output, "From: #{@ruby_file.path}:1" + ensure + @irbrc.unlink if @irbrc + end + + def test_symbol_aliases_dont_affect_ruby_syntax + write_ruby <<~'RUBY' + $foo = "It's a foo" + @bar = "It's a bar" + binding.irb + RUBY + + output = run_ruby_file do + type "$foo" + type "@bar" + type "exit!" + end + + assert_include output, "=> \"It's a foo\"" + assert_include output, "=> \"It's a bar\"" + end + end + class IrbIOConfigurationTest < TestCase Row = Struct.new(:content, :current_line_spaces, :new_line_spaces, :indent_level) diff --git a/test/irb/test_nesting_parser.rb b/test/irb/test_nesting_parser.rb index 83c7fb08a63eef..ea3a23aaf5bb60 100644 --- a/test/irb/test_nesting_parser.rb +++ b/test/irb/test_nesting_parser.rb @@ -14,7 +14,7 @@ def teardown end def parse_by_line(code) - IRB::NestingParser.parse_by_line(RubyLex.ripper_lex_without_warning(code)) + IRB::NestingParser.parse_by_line(IRB::RubyLex.ripper_lex_without_warning(code)) end def test_open_tokens @@ -27,7 +27,7 @@ def f x: " #{p(1, 2, 3 EOS - opens = IRB::NestingParser.open_tokens(RubyLex.ripper_lex_without_warning(code)) + opens = IRB::NestingParser.open_tokens(IRB::RubyLex.ripper_lex_without_warning(code)) assert_equal(%w[class def if do { " #{ (], opens.map(&:tok)) end diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index 338ff387512c73..336f4fd3660ddf 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -20,7 +20,7 @@ def test_interpolate_token_with_heredoc_and_unclosed_embexpr #{⑤&< + ^C + irb(main):001> + EOC + end + def test_show_cmds_with_pager_can_quit_with_ctrl_c write_irbrc <<~'LINES' puts 'start IRB' diff --git a/test/open-uri/test_open-uri.rb b/test/open-uri/test_open-uri.rb index fdd1969666fe02..86aefc52c8b255 100644 --- a/test/open-uri/test_open-uri.rb +++ b/test/open-uri/test_open-uri.rb @@ -172,6 +172,19 @@ def test_invalid_option assert_raise(ArgumentError) { URI.open("http://127.0.0.1/", :invalid_option=>true) {} } end + def test_pass_keywords + begin + f = URI.open(File::NULL, mode: 0666) + assert_kind_of File, f + ensure + f&.close + end + + o = Object.new + def o.open(foo: ) foo end + assert_equal 1, URI.open(o, foo: 1) + end + def test_mode with_http {|srv, dr, url| srv.mount_proc("/mode", lambda { |req, res| res.body = "mode" } ) diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index c72e47fd38ae8c..8ac0c0c096ebe0 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -3,7 +3,7 @@ begin require 'yamatanooroti' - class Reline::TestRendering < Yamatanooroti::TestCase + class Reline::RenderingTest < Yamatanooroti::TestCase def setup @pwd = Dir.pwd suffix = '%010d' % Random.rand(0..65535) diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb index c4d9a627a68ebc..39dfac6da00f8a 100644 --- a/test/ripper/test_parser_events.rb +++ b/test/ripper/test_parser_events.rb @@ -152,6 +152,7 @@ def test_args_forward thru_args_forward = false parse(code, :on_args_forward) {thru_args_forward = true} assert_equal true, thru_args_forward, "no args_forward for: #{code}" + parse(code, :on_params) {|*, block| assert_nil(block)} end end diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb index 8b565f531d548a..b4bcc03cfe4db2 100644 --- a/test/ruby/test_ast.rb +++ b/test/ruby/test_ast.rb @@ -1082,6 +1082,12 @@ def test_error_tolerant_unexpected_backslash assert_equal([[0, :backslash, "\\", [1, 0, 1, 1]]], node.children.last.tokens) end + def test_with_bom + assert_error_tolerant("\u{feff}nil", <<~EXP) + (SCOPE@1:0-1:3 tbl: [] args: nil body: (NIL@1:0-1:3)) + EXP + end + def assert_error_tolerant(src, expected, keep_tokens: false) begin verbose_bak, $VERBOSE = $VERBOSE, false diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 8d77e65bbe781b..bfcfbbb72f6d3c 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -286,6 +286,54 @@ def test_latest_gc_info_need_major_by assert_not_nil(major_by) end + def test_latest_gc_info_weak_references_count + assert_separately([], __FILE__, __LINE__, <<~RUBY) + count = 10_000 + # Some weak references may be created, so allow some margin of error + error_tolerance = 100 + + # Run full GC to clear out weak references + GC.start + # Run full GC again to collect stats about weak references + GC.start + + before_weak_references_count = GC.latest_gc_info(:weak_references_count) + before_retained_weak_references_count = GC.latest_gc_info(:retained_weak_references_count) + + # Create some objects and place it in a WeakMap + wmap = ObjectSpace::WeakMap.new + ary = Array.new(count) + count.times do |i| + obj = Object.new + ary[i] = obj + wmap[obj] = nil + end + + # Run full GC to collect stats about weak references + GC.start + + assert_operator(GC.latest_gc_info(:weak_references_count), :>=, before_weak_references_count + count - error_tolerance) + assert_operator(GC.latest_gc_info(:retained_weak_references_count), :>=, before_retained_weak_references_count + count - error_tolerance) + assert_operator(GC.latest_gc_info(:retained_weak_references_count), :<=, GC.latest_gc_info(:weak_references_count)) + + before_weak_references_count = GC.latest_gc_info(:weak_references_count) + before_retained_weak_references_count = GC.latest_gc_info(:retained_weak_references_count) + + ary = nil + + # Free ary, which should empty out the wmap + GC.start + # Run full GC again to collect stats about weak references + GC.start + + assert_equal(0, wmap.size) + + assert_operator(GC.latest_gc_info(:weak_references_count), :<=, before_weak_references_count - count + error_tolerance) + assert_operator(GC.latest_gc_info(:retained_weak_references_count), :<=, before_retained_weak_references_count - count + error_tolerance) + assert_operator(GC.latest_gc_info(:retained_weak_references_count), :<=, GC.latest_gc_info(:weak_references_count)) + RUBY + end + def test_stress_compile_send assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "") GC.stress = true diff --git a/test/ruby/test_marshal.rb b/test/ruby/test_marshal.rb index 6cd0b9acc3aacf..13645e3aa805f8 100644 --- a/test/ruby/test_marshal.rb +++ b/test/ruby/test_marshal.rb @@ -1,6 +1,5 @@ # frozen_string_literal: false require 'test/unit' -require 'tempfile' require_relative 'marshaltestlib' class TestMarshal < Test::Unit::TestCase @@ -314,11 +313,10 @@ def test_regexp2 assert_equal(c, Marshal.load(Marshal.dump(c)), bug2109) assert_nothing_raised(ArgumentError, '[ruby-dev:40386]') do - re = Tempfile.create("marshal_regexp") do |f| - f.binmode.write("\x04\bI/\x00\x00\x06:\rencoding\"\rUS-ASCII") - f.rewind - re2 = Marshal.load(f) - re2 + re = IO.pipe do |r, w| + w.write("\x04\bI/\x00\x00\x06:\rencoding\"\rUS-ASCII") + # Marshal.load would not overread and block + Marshal.load(r) end assert_equal(//, re) end diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 12e47464856b26..38dcb8054fc1ca 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -1799,7 +1799,11 @@ def assert_fail_too_long_path((cmd, sep), mesg) loop do Process.spawn(cmds.join(sep), opts) min = [cmds.size, min].max - cmds *= 100 + begin + cmds *= 100 + rescue ArgumentError + raise NoMemoryError + end end rescue NoMemoryError size = cmds.size diff --git a/test/ruby/test_weakkeymap.rb b/test/ruby/test_weakkeymap.rb index 5df53749ca04ae..be3e80cec46932 100644 --- a/test/ruby/test_weakkeymap.rb +++ b/test/ruby/test_weakkeymap.rb @@ -108,6 +108,21 @@ def hash RUBY end + def test_compaction + omit "compaction is not supported on this platform" unless GC.respond_to?(:compact) + + assert_separately(%w(-robjspace), <<-'end;') + wm = ObjectSpace::WeakKeyMap.new + key = Object.new + val = Object.new + wm[key] = val + + GC.verify_compaction_references(expand_heap: true, toward: :empty) + + assert_equal(val, wm[key]) + end; + end + private def assert_weak_include(m, k, n = 100) diff --git a/test/ruby/test_weakmap.rb b/test/ruby/test_weakmap.rb index 6b3dc678ad60a9..b42787ad5e2a3d 100644 --- a/test/ruby/test_weakmap.rb +++ b/test/ruby/test_weakmap.rb @@ -193,9 +193,10 @@ def test_no_memory_leak end; end - def test_compaction_bug_19529 + def test_compaction omit "compaction is not supported on this platform" unless GC.respond_to?(:compact) + # [Bug #19529] obj = Object.new 100.times do |i| GC.compact @@ -211,6 +212,17 @@ def test_compaction_bug_19529 end GC.compact end; + + assert_separately(%w(-robjspace), <<-'end;') + wm = ObjectSpace::WeakMap.new + key = Object.new + val = Object.new + wm[key] = val + + GC.verify_compaction_references(expand_heap: true, toward: :empty) + + assert_equal(val, wm[key]) + end; end def test_replaced_values_bug_19531 diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index 0207f4aaf16577..5347028550dfca 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -1320,6 +1320,16 @@ def test_opt_aref_with RUBY end + def test_proc_block_arg + assert_compiles(<<~RUBY, result: [:proc, :no_block]) + def yield_if_given = block_given? ? yield : :no_block + + def call(block_arg = nil) = yield_if_given(&block_arg) + + [call(-> { :proc }), call] + RUBY + end + private def code_gc_helpers diff --git a/test/rubygems/test_gem_package.rb b/test/rubygems/test_gem_package.rb index 4ca806efc323f3..6161e81f62df75 100644 --- a/test/rubygems/test_gem_package.rb +++ b/test/rubygems/test_gem_package.rb @@ -955,11 +955,15 @@ def test_verify_corrupt_tar_metadata_entry package = Gem::Package.new "corrupt.gem" - e = assert_raise Gem::Package::FormatError do - package.verify + e = nil + out_err = capture_output do + e = assert_raise Gem::Package::FormatError do + package.verify + end end assert_match(/(EOFError|end of file reached) in corrupt.gem/i, e.message) + assert_equal(["", "Exception while verifying corrupt.gem\n"], out_err) end def test_verify_corrupt_tar_checksums_entry @@ -987,11 +991,15 @@ def test_verify_corrupt_tar_data_entry package = Gem::Package.new "corrupt.gem" - e = assert_raise Gem::Package::FormatError do - package.verify + e = nil + out_err = capture_output do + e = assert_raise Gem::Package::FormatError do + package.verify + end end assert_match(/(EOFError|end of file reached) in corrupt.gem/i, e.message) + assert_equal(["", "Exception while verifying corrupt.gem\n"], out_err) end def test_corrupt_data_tar_gz diff --git a/test/rubygems/test_gem_package_tar_reader_entry.rb b/test/rubygems/test_gem_package_tar_reader_entry.rb index 7510e1d199427b..67ab7922b5e00b 100644 --- a/test/rubygems/test_gem_package_tar_reader_entry.rb +++ b/test/rubygems/test_gem_package_tar_reader_entry.rb @@ -216,6 +216,8 @@ def test_read_corrupted_tar corrupt_entry.rewind assert_nil corrupt_entry.read(100), "IO.read with len should return nil as per IO.read docs" + ensure + close_util_entry(corrupt_entry) if corrupt_entry end def test_readpartial_corrupted_tar @@ -228,6 +230,8 @@ def test_readpartial_corrupted_tar assert_raise EOFError do corrupt_entry.readpartial(100) end + ensure + close_util_entry(corrupt_entry) if corrupt_entry end def test_rewind diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 9e68c40b7cb87d..25b72c587f05f8 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -1094,6 +1094,25 @@ def test_handles_dependencies_with_syck_requirements_bug assert_equal(yaml_defined, Object.const_defined?("YAML")) end + def test_handles_dependencies_with_other_syck_requirements_argument_error + yaml_defined = Object.const_defined?("YAML") + + data = Marshal.dump(Gem::Specification.new do |s| + v = Gem::Version.allocate + v.instance_variable_set :@version, "YAML::Syck::DefaultKey" + s.instance_variable_set :@version, v + end) + + assert_raise(ArgumentError) { Marshal.load(data) } + out, err = capture_output do + assert_raise(ArgumentError) { Marshal.load(data) } + end + assert_empty out + assert_empty err + + assert_equal(yaml_defined, Object.const_defined?("YAML")) + end + def test_initialize spec = Gem::Specification.new do |s| s.name = "blah" diff --git a/test/yarp/bom_test.rb b/test/yarp/bom_test.rb new file mode 100644 index 00000000000000..7dc7eabe92409d --- /dev/null +++ b/test/yarp/bom_test.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +# Don't bother checking this on these engines, this is such a specific Ripper +# test. +return if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby" + +require "yarp_test_helper" + +class BOMTest < Test::Unit::TestCase + def test_ident + assert_bom("foo") + end + + def test_back_reference + assert_bom("$+") + end + + def test_instance_variable + assert_bom("@foo") + end + + def test_class_variable + assert_bom("@@foo") + end + + def test_global_variable + assert_bom("$foo") + end + + def test_numbered_reference + assert_bom("$1") + end + + def test_percents + assert_bom("%i[]") + assert_bom("%r[]") + assert_bom("%s[]") + assert_bom("%q{}") + assert_bom("%w[]") + assert_bom("%x[]") + assert_bom("%I[]") + assert_bom("%W[]") + assert_bom("%Q{}") + end + + def test_string + assert_bom("\"\"") + assert_bom("''") + end + + private + + def assert_bom(source) + bommed = "\xEF\xBB\xBF#{source}" + assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value + end +end diff --git a/test/yarp/comments_test.rb b/test/yarp/comments_test.rb index 5c49f7b86e7b5b..fdb70045ca9462 100644 --- a/test/yarp/comments_test.rb +++ b/test/yarp/comments_test.rb @@ -6,7 +6,10 @@ class CommentsTest < Test::Unit::TestCase include ::YARP::DSL def test_comment_inline - assert_comment "# comment", :inline, 0..9 + source = "# comment" + + assert_comment source, :inline, 0..9 + assert_equal [0], YARP.const_get(:Debug).newlines(source) end def test_comment_inline_def @@ -28,6 +31,12 @@ def test_comment___END__ assert_comment source, :__END__, 0..16 end + def test_comment___END__crlf + source = "__END__\r\ncomment\r\n" + + assert_comment source, :__END__, 0..18 + end + def test_comment_embedded_document source = <<~RUBY =begin diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index 95d7bd3223b302..c96a08e60e6c23 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -50,6 +50,13 @@ def test_coding assert_equal Encoding.find("utf-8"), actual end + def test_coding_with_whitespace + result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("ascii-8bit"), actual + end + + def test_emacs_style result = YARP.parse("# -*- coding: utf-8 -*-\nident") actual = result.value.statements.body.first.name.encoding diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index 355e7a8aff298f..d58fd27448a1ad 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -18,9 +18,10 @@ def test_module_name_recoverable Location(), ConstantReadNode(), StatementsNode( - [ModuleNode([], Location(), MissingNode(), nil, Location())] + [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] ), - Location() + Location(), + "Parent" ) assert_errors expected, "module Parent module end", [ @@ -130,6 +131,14 @@ def test_unterminated_regular_expression ] end + def test_unterminated_regular_expression_with_heredoc + source = "<<-END + /b\nEND\n" + + assert_errors expression(source), source, [ + ["Expected a closing delimiter for a regular expression.", 10..10] + ] + end + def test_unterminated_xstring assert_errors expression("`hello"), "`hello", [ ["Expected a closing delimiter for an xstring.", 1..1] @@ -162,6 +171,12 @@ def test_unterminated_argument_expression ] end + def test_cr_without_lf_in_percent_expression + assert_errors expression("%\r"), "%\r", [ + ["Invalid %% token", 0..2], + ] + end + def test_1_2_3 assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ ["Expected to be able to parse an expression.", 2..2], @@ -377,7 +392,7 @@ def test_module_definition_in_method_body Location(), nil, nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location())]), + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), [], Location(), nil, @@ -408,7 +423,7 @@ def test_module_definition_in_method_body_within_block BlockNode( [], nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location())]), + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), Location(), Location() ), @@ -449,7 +464,8 @@ def test_class_definition_in_method_body nil, nil, nil, - Location() + Location(), + "A" )] ), [], @@ -949,7 +965,8 @@ def test_dont_allow_return_inside_class_body nil, nil, StatementsNode([ReturnNode(Location(), nil)]), - Location() + Location(), + "A" ) assert_errors expected, "class A; return; end", [ @@ -963,7 +980,8 @@ def test_dont_allow_return_inside_module_body Location(), ConstantReadNode(), StatementsNode([ReturnNode(Location(), nil)]), - Location() + Location(), + "A" ) assert_errors expected, "module A; return; end", [ diff --git a/test/yarp/fixtures/constants.txt b/test/yarp/fixtures/constants.txt index d3e7140630d83b..d86f8d34020661 100644 --- a/test/yarp/fixtures/constants.txt +++ b/test/yarp/fixtures/constants.txt @@ -12,6 +12,18 @@ ABC Foo 1 +Foo *bar + +Foo **bar + +Foo &bar + +Foo::Bar *baz + +Foo::Bar **baz + +Foo::Bar &baz + ::A::foo ::A = 1 diff --git a/test/yarp/fixtures/dash_heredocs.txt b/test/yarp/fixtures/dash_heredocs.txt index aba782c99ad163..3e663fae632116 100644 --- a/test/yarp/fixtures/dash_heredocs.txt +++ b/test/yarp/fixtures/dash_heredocs.txt @@ -45,3 +45,19 @@ EOF <<-'EOF' a #{1} EOF + +<<-A + <<-B + a +A + b + #{2 + } +B + +<<-A + <<-B + a +A + b + #{ + 2} +B diff --git a/test/yarp/fixtures/heredoc_with_escaped_newline_at_start.txt b/test/yarp/fixtures/heredoc_with_escaped_newline_at_start.txt new file mode 100644 index 00000000000000..81f314e0b3cf67 --- /dev/null +++ b/test/yarp/fixtures/heredoc_with_escaped_newline_at_start.txt @@ -0,0 +1,7 @@ +<<-TARGET.gsub /^\s{/, ''\ +TARGET + + +<<-TARGET.gsub /^\s{/, ''\ +TARGET + diff --git a/test/yarp/fixtures/indented_file_end.txt b/test/yarp/fixtures/indented_file_end.txt new file mode 100644 index 00000000000000..5a96dcacf7826f --- /dev/null +++ b/test/yarp/fixtures/indented_file_end.txt @@ -0,0 +1,4 @@ + def hi + + end # hi there + \ No newline at end of file diff --git a/test/yarp/fixtures/lambda.txt b/test/yarp/fixtures/lambda.txt index 1071810aa18ac2..5c21eafb24f993 100644 --- a/test/yarp/fixtures/lambda.txt +++ b/test/yarp/fixtures/lambda.txt @@ -5,3 +5,7 @@ ->(x: "b#{a}") { } ->(a: b * 3) {} + +-> foo = bar do end + +-> foo: bar do end diff --git a/test/yarp/fixtures/newline_terminated.txt b/test/yarp/fixtures/newline_terminated.txt new file mode 100644 index 00000000000000..12f3bda22972a8 Binary files /dev/null and b/test/yarp/fixtures/newline_terminated.txt differ diff --git a/test/yarp/fixtures/regex.txt b/test/yarp/fixtures/regex.txt index fe7b848b1be4b7..1e917e7de7eb2b 100644 --- a/test/yarp/fixtures/regex.txt +++ b/test/yarp/fixtures/regex.txt @@ -26,3 +26,5 @@ foo /bar/ /(?#\))/ =~ "hi" %r#pound# + +/aaa #{bbb}/o diff --git a/test/yarp/fixtures/seattlerb/TestRubyParserShared.txt b/test/yarp/fixtures/seattlerb/TestRubyParserShared.txt new file mode 100644 index 00000000000000..c55b3e1f70ede2 --- /dev/null +++ b/test/yarp/fixtures/seattlerb/TestRubyParserShared.txt @@ -0,0 +1,92 @@ +%I[ + + +] + +%I[ +line2 +line3 +] + +%W[ + + +] + +%W[ +line2 +line3 +] + +%i[ + + +] + +%i[ +line2 +line3 +] + +%r[ + + +] + +%w[ + + +] + +%w[ +line2 +line3 +] + +[ +:line2, +:line3 +] + +class X # line 1 + def self.y(a, # line 2 + b) # line 3 + a + b # line 4 + end # line 5 +end # line 6 + + +class X # line 1 + class Y # line 2 + Z = 42 # line 3 + end # line 4 +end # line 5 + + +class X # line 1 + def y(a, # line 2 + b) # line 3 + a + b # line 4 + end # line 5 +end # line 6 + + +module X + X = [ + :line3, + :line4, + ] +end + + +module X # line 1 + module Y # line 2 + Z = 42 # line 3 + end # line 4 +end # line 5 + + +x( +:line2, +:line3 +) diff --git a/test/yarp/fixtures/seattlerb/case_in.txt b/test/yarp/fixtures/seattlerb/case_in.txt index f28c28634862e9..0835da095670e8 100644 --- a/test/yarp/fixtures/seattlerb/case_in.txt +++ b/test/yarp/fixtures/seattlerb/case_in.txt @@ -1,5 +1,5 @@ case :a -in "b": +in "b": end case :a diff --git a/test/yarp/fixtures/seattlerb/defn_forward_args__no_parens.txt b/test/yarp/fixtures/seattlerb/defn_forward_args__no_parens.txt new file mode 100644 index 00000000000000..2d34077c936866 --- /dev/null +++ b/test/yarp/fixtures/seattlerb/defn_forward_args__no_parens.txt @@ -0,0 +1,3 @@ +def f ... + m(...) +end diff --git a/test/yarp/fixtures/seattlerb/heredoc_wtf_I_hate_you.txt b/test/yarp/fixtures/seattlerb/difficult0_.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/heredoc_wtf_I_hate_you.txt rename to test/yarp/fixtures/seattlerb/difficult0_.txt diff --git a/test/yarp/fixtures/seattlerb/i_fucking_hate_line_numbers.txt b/test/yarp/fixtures/seattlerb/difficult1_line_numbers.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/i_fucking_hate_line_numbers.txt rename to test/yarp/fixtures/seattlerb/difficult1_line_numbers.txt diff --git a/test/yarp/fixtures/seattlerb/i_fucking_hate_line_numbers2.txt b/test/yarp/fixtures/seattlerb/difficult1_line_numbers2.txt similarity index 82% rename from test/yarp/fixtures/seattlerb/i_fucking_hate_line_numbers2.txt rename to test/yarp/fixtures/seattlerb/difficult1_line_numbers2.txt index 898a70fb756332..1964562416b577 100644 --- a/test/yarp/fixtures/seattlerb/i_fucking_hate_line_numbers2.txt +++ b/test/yarp/fixtures/seattlerb/difficult1_line_numbers2.txt @@ -1,5 +1,5 @@ if true then - p('a') + p("a") b = 1 p b c =1 diff --git a/test/yarp/fixtures/seattlerb/i_have_no_freakin_clue.txt b/test/yarp/fixtures/seattlerb/difficult2_.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/i_have_no_freakin_clue.txt rename to test/yarp/fixtures/seattlerb/difficult2_.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me.txt b/test/yarp/fixtures/seattlerb/difficult3_.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me.txt rename to test/yarp/fixtures/seattlerb/difficult3_.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me2.txt b/test/yarp/fixtures/seattlerb/difficult3_2.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me2.txt rename to test/yarp/fixtures/seattlerb/difficult3_2.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me3.txt b/test/yarp/fixtures/seattlerb/difficult3_3.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me3.txt rename to test/yarp/fixtures/seattlerb/difficult3_3.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me4.txt b/test/yarp/fixtures/seattlerb/difficult3_4.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me4.txt rename to test/yarp/fixtures/seattlerb/difficult3_4.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me5.txt b/test/yarp/fixtures/seattlerb/difficult3_5.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me5.txt rename to test/yarp/fixtures/seattlerb/difficult3_5.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_10.txt b/test/yarp/fixtures/seattlerb/difficult3__10.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_10.txt rename to test/yarp/fixtures/seattlerb/difficult3__10.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_11.txt b/test/yarp/fixtures/seattlerb/difficult3__11.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_11.txt rename to test/yarp/fixtures/seattlerb/difficult3__11.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_12.txt b/test/yarp/fixtures/seattlerb/difficult3__12.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_12.txt rename to test/yarp/fixtures/seattlerb/difficult3__12.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_6.txt b/test/yarp/fixtures/seattlerb/difficult3__6.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_6.txt rename to test/yarp/fixtures/seattlerb/difficult3__6.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_7.txt b/test/yarp/fixtures/seattlerb/difficult3__7.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_7.txt rename to test/yarp/fixtures/seattlerb/difficult3__7.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_8.txt b/test/yarp/fixtures/seattlerb/difficult3__8.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_8.txt rename to test/yarp/fixtures/seattlerb/difficult3__8.txt diff --git a/test/yarp/fixtures/seattlerb/kill_me_9.txt b/test/yarp/fixtures/seattlerb/difficult3__9.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/kill_me_9.txt rename to test/yarp/fixtures/seattlerb/difficult3__9.txt diff --git a/test/yarp/fixtures/seattlerb/motherfuckin_leading_dots.txt b/test/yarp/fixtures/seattlerb/difficult4__leading_dots.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/motherfuckin_leading_dots.txt rename to test/yarp/fixtures/seattlerb/difficult4__leading_dots.txt diff --git a/test/yarp/fixtures/seattlerb/motherfuckin_leading_dots2.txt b/test/yarp/fixtures/seattlerb/difficult4__leading_dots2.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/motherfuckin_leading_dots2.txt rename to test/yarp/fixtures/seattlerb/difficult4__leading_dots2.txt diff --git a/test/yarp/fixtures/seattlerb/wtf.txt b/test/yarp/fixtures/seattlerb/difficult6_.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/wtf.txt rename to test/yarp/fixtures/seattlerb/difficult6_.txt diff --git a/test/yarp/fixtures/seattlerb/wtf_7.txt b/test/yarp/fixtures/seattlerb/difficult6__7.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/wtf_7.txt rename to test/yarp/fixtures/seattlerb/difficult6__7.txt diff --git a/test/yarp/fixtures/seattlerb/wtf_8.txt b/test/yarp/fixtures/seattlerb/difficult6__8.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/wtf_8.txt rename to test/yarp/fixtures/seattlerb/difficult6__8.txt diff --git a/test/yarp/fixtures/seattlerb/zomg_sometimes_i_hate_this_project.txt b/test/yarp/fixtures/seattlerb/difficult7_.txt similarity index 100% rename from test/yarp/fixtures/seattlerb/zomg_sometimes_i_hate_this_project.txt rename to test/yarp/fixtures/seattlerb/difficult7_.txt diff --git a/test/yarp/fixtures/seattlerb/heredoc_squiggly_visually_blank_lines.txt b/test/yarp/fixtures/seattlerb/heredoc_squiggly_visually_blank_lines.txt index 0d89134c87f879..3f9198296d5cfc 100644 --- a/test/yarp/fixtures/seattlerb/heredoc_squiggly_visually_blank_lines.txt +++ b/test/yarp/fixtures/seattlerb/heredoc_squiggly_visually_blank_lines.txt @@ -1,6 +1,6 @@ a = <<~EOF x - + z EOF diff --git a/test/yarp/fixtures/seattlerb/module_comments.txt b/test/yarp/fixtures/seattlerb/module_comments.txt index f8d5aae571ccdf..cecb717c5b75de 100644 --- a/test/yarp/fixtures/seattlerb/module_comments.txt +++ b/test/yarp/fixtures/seattlerb/module_comments.txt @@ -1,5 +1,5 @@ # blah 1 - + # blah 2 module X diff --git a/test/yarp/fixtures/seattlerb/parse_line_trailing_newlines.txt b/test/yarp/fixtures/seattlerb/parse_line_trailing_newlines.txt index 422c2b7ab3b3c6..afa826fb509e62 100644 --- a/test/yarp/fixtures/seattlerb/parse_line_trailing_newlines.txt +++ b/test/yarp/fixtures/seattlerb/parse_line_trailing_newlines.txt @@ -1,2 +1,2 @@ -a +a b diff --git a/test/yarp/fixtures/seattlerb/pct_Q_backslash_nl.txt b/test/yarp/fixtures/seattlerb/pct_Q_backslash_nl.txt index d88e1fd21cf51e..4420560d2bee83 100644 --- a/test/yarp/fixtures/seattlerb/pct_Q_backslash_nl.txt +++ b/test/yarp/fixtures/seattlerb/pct_Q_backslash_nl.txt @@ -1,2 +1,2 @@ -%Q{ \ +%q{ \ } diff --git a/test/yarp/fixtures/seattlerb/slashy_newlines_within_string.txt b/test/yarp/fixtures/seattlerb/slashy_newlines_within_string.txt index cdf7a70cb72c3a..421989c76fa923 100644 --- a/test/yarp/fixtures/seattlerb/slashy_newlines_within_string.txt +++ b/test/yarp/fixtures/seattlerb/slashy_newlines_within_string.txt @@ -4,4 +4,4 @@ puts "hello\ friend" a + b - + diff --git a/test/yarp/fixtures/unparser/corpus/literal/send.txt b/test/yarp/fixtures/unparser/corpus/literal/send.txt index 4361cf73cbbf3a..1e9c2a94be09be 100644 --- a/test/yarp/fixtures/unparser/corpus/literal/send.txt +++ b/test/yarp/fixtures/unparser/corpus/literal/send.txt @@ -81,3 +81,4 @@ self.foo=:bar x(**foo) foo&.! foo.~(b) +a&.+(b) diff --git a/test/yarp/fixtures/whitequark/dedenting_heredoc.txt b/test/yarp/fixtures/whitequark/dedenting_heredoc.txt index 2840ec3e1c0512..84937d84abf4f5 100644 --- a/test/yarp/fixtures/whitequark/dedenting_heredoc.txt +++ b/test/yarp/fixtures/whitequark/dedenting_heredoc.txt @@ -49,7 +49,7 @@ E p <<~E x - + y E diff --git a/test/yarp/fixtures/whitequark/interp_digit_var.txt b/test/yarp/fixtures/whitequark/interp_digit_var.txt index 37487ddbaff5d2..1ae5a2e3e00235 100644 --- a/test/yarp/fixtures/whitequark/interp_digit_var.txt +++ b/test/yarp/fixtures/whitequark/interp_digit_var.txt @@ -1,66 +1,66 @@ - "#@1" + "#@1" - "#@@1" + "#@@1" - %I[#@1] + %I[#@1] - %I[#@@1] + %I[#@@1] - %Q{#@1} + %Q{#@1} - %Q{#@@1} + %Q{#@@1} - %W[#@1] + %W[#@1] - %W[#@@1] + %W[#@@1] - %i[ #@1 ] + %i[ #@1 ] - %i[ #@@1 ] + %i[ #@@1 ] - %q{#@1} + %q{#@1} - %q{#@@1} + %q{#@@1} - %r{#@1} + %r{#@1} - %r{#@@1} + %r{#@@1} - %s{#@1} + %s{#@1} - %s{#@@1} + %s{#@@1} - %w[ #@1 ] + %w[ #@1 ] - %w[ #@@1 ] + %w[ #@@1 ] - %x{#@1} + %x{#@1} - %x{#@@1} + %x{#@@1} - %{#@1} + %{#@1} - %{#@@1} + %{#@@1} - '#@1' + '#@1' - '#@@1' + '#@@1' - /#@1/ + /#@1/ - /#@@1/ + /#@@1/ - :"#@1" + :"#@1" - :"#@@1" + :"#@@1" - :'#@1' + :'#@1' - :'#@@1' + :'#@@1' - `#@1` + `#@1` - `#@@1` + `#@@1` <<-"HERE" #@1 diff --git a/test/yarp/fixtures/whitequark/procarg0.txt b/test/yarp/fixtures/whitequark/procarg0.txt index 70b368e7d5275e..74cae2c2eb4bb8 100644 --- a/test/yarp/fixtures/whitequark/procarg0.txt +++ b/test/yarp/fixtures/whitequark/procarg0.txt @@ -1,3 +1,3 @@ -m { |(foo, bar)| } +m { |(foo, bar)| } -m { |foo| } +m { |foo| } diff --git a/test/yarp/fixtures/wrapping_heredoc.txt b/test/yarp/fixtures/wrapping_heredoc.txt new file mode 100644 index 00000000000000..d5fc7101780c21 --- /dev/null +++ b/test/yarp/fixtures/wrapping_heredoc.txt @@ -0,0 +1,13 @@ +# test regex, string, and lists that wrap a heredoc thanks to an escaped newline + +# ripper incorrectly creates a "b\nc" string instead of two separate string tokens +pp <<-A.gsub(/b\ +a +A +c/, "") + +# ripper incorrectly creates a "e\nf" string instead of two separate string tokens +pp <<-A + "e\ +d +A +f" diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index 789bd49ec1a1dc..703c8e24f087ce 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -17,6 +17,16 @@ def test_AndNode assert_location(AndNode, "foo && bar") end + def test_AndWriteNode + assert_location(AndWriteNode, "foo &&= bar") + assert_location(AndWriteNode, "foo = 1; foo &&= bar", 9...20) + assert_location(AndWriteNode, "@@foo &&= bar") + assert_location(AndWriteNode, "Parent::Child &&= bar") + assert_location(AndWriteNode, "Foo &&= bar") + assert_location(AndWriteNode, "$foo &&= bar") + assert_location(AndWriteNode, "@foo &&= bar") + end + def test_ArgumentsNode assert_location(ArgumentsNode, "foo(bar, baz, qux)", 4...17, &:arguments) end @@ -59,8 +69,8 @@ def test_BeginNode assert_location(BeginNode, "begin foo; rescue bar\nelse baz end") assert_location(BeginNode, "begin foo; rescue bar\nelse baz\nensure qux end") - assert_location(BeginNode, "class Foo\nrescue then end", 10..25, &:statements) - assert_location(BeginNode, "module Foo\nrescue then end", 11..26, &:statements) + assert_location(BeginNode, "class Foo\nrescue then end", 10..25, &:body) + assert_location(BeginNode, "module Foo\nrescue then end", 11..26, &:body) end def test_BlockArgumentNode @@ -199,18 +209,6 @@ def test_ClassNode assert_location(ClassNode, "class Foo < Bar end") end - def test_ClassVariableOperatorAndWriteNode - assert_location(ClassVariableOperatorAndWriteNode, "@@foo &&= bar") - end - - def test_ClassVariableOperatorWriteNode - assert_location(ClassVariableOperatorWriteNode, "@@foo += bar") - end - - def test_ClassVariableOperatorOrWriteNode - assert_location(ClassVariableOperatorOrWriteNode, "@@foo ||= bar") - end - def test_ClassVariableReadNode assert_location(ClassVariableReadNode, "@@foo") end @@ -231,30 +229,6 @@ def test_ConstantPathWriteNode assert_location(ConstantPathWriteNode, "::Foo::Bar = baz") end - def test_ConstantPathOperatorAndWriteNode - assert_location(ConstantPathOperatorAndWriteNode, "Parent::Child &&= bar") - end - - def test_ConstantPathOperatorWriteNode - assert_location(ConstantPathOperatorWriteNode, "Parent::Child += bar") - end - - def test_ConstantPathOperatorOrWriteNode - assert_location(ConstantPathOperatorOrWriteNode, "Parent::Child ||= bar") - end - - def test_ConstantOperatorAndWriteNode - assert_location(ConstantOperatorAndWriteNode, "Foo &&= bar") - end - - def test_ConstantOperatorWriteNode - assert_location(ConstantOperatorWriteNode, "Foo += bar") - end - - def test_ConstantOperatorOrWriteNode - assert_location(ConstantOperatorOrWriteNode, "Foo ||= bar") - end - def test_ConstantReadNode assert_location(ConstantReadNode, "Foo") assert_location(ConstantReadNode, "Foo::Bar", 5...8, &:child) @@ -313,7 +287,7 @@ def test_ForNode def test_ForwardingArgumentsNode assert_location(ForwardingArgumentsNode, "def foo(...); bar(...); end", 18...21) do |node| - node.statements.body.first.arguments.arguments.first + node.body.body.first.arguments.arguments.first end end @@ -328,18 +302,6 @@ def test_ForwardingSuperNode assert_location(ForwardingSuperNode, "super {}") end - def test_GlobalVariableOperatorAndWriteNode - assert_location(GlobalVariableOperatorAndWriteNode, "$foo &&= bar") - end - - def test_GlobalVariableOperatorWriteNode - assert_location(GlobalVariableOperatorWriteNode, "$foo += bar") - end - - def test_GlobalVariableOperatorOrWriteNode - assert_location(GlobalVariableOperatorOrWriteNode, "$foo ||= bar") - end - def test_GlobalVariableReadNode assert_location(GlobalVariableReadNode, "$foo") end @@ -374,18 +336,6 @@ def test_InNode end end - def test_InstanceVariableOperatorAndWriteNode - assert_location(InstanceVariableOperatorAndWriteNode, "@foo &&= bar") - end - - def test_InstanceVariableOperatorWriteNode - assert_location(InstanceVariableOperatorWriteNode, "@foo += bar") - end - - def test_InstanceVariableOperatorOrWriteNode - assert_location(InstanceVariableOperatorOrWriteNode, "@foo ||= bar") - end - def test_InstanceVariableReadNode assert_location(InstanceVariableReadNode, "@foo") end @@ -452,21 +402,6 @@ def test_LambdaNode assert_location(LambdaNode, "-> do foo end") end - def test_LocalVariableOperatorAndWriteNode - assert_location(LocalVariableOperatorAndWriteNode, "foo &&= bar") - assert_location(LocalVariableOperatorAndWriteNode, "foo = 1; foo &&= bar", 9...20) - end - - def test_LocalVariableOperatorWriteNode - assert_location(LocalVariableOperatorWriteNode, "foo += bar") - assert_location(LocalVariableOperatorWriteNode, "foo = 1; foo += bar", 9...19) - end - - def test_LocalVariableOperatorOrWriteNode - assert_location(LocalVariableOperatorOrWriteNode, "foo ||= bar") - assert_location(LocalVariableOperatorOrWriteNode, "foo = 1; foo ||= bar", 9...20) - end - def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) end @@ -510,6 +445,16 @@ def test_NumberedReferenceReadNode assert_location(NumberedReferenceReadNode, "$1") end + def test_OperatorWriteNode + assert_location(OperatorWriteNode, "@@foo += bar") + assert_location(OperatorWriteNode, "Parent::Child += bar") + assert_location(OperatorWriteNode, "Foo += bar") + assert_location(OperatorWriteNode, "$foo += bar") + assert_location(OperatorWriteNode, "@foo += bar") + assert_location(OperatorWriteNode, "foo += bar") + assert_location(OperatorWriteNode, "foo = 1; foo += bar", 9...19) + end + def test_OptionalParameterNode assert_location(OptionalParameterNode, "def foo(bar = nil); end", 8...17) do |node| node.parameters.optionals.first @@ -521,6 +466,16 @@ def test_OrNode assert_location(OrNode, "foo or bar") end + def test_OrWriteNode + assert_location(OrWriteNode, "@@foo ||= bar") + assert_location(OrWriteNode, "Parent::Child ||= bar") + assert_location(OrWriteNode, "Foo ||= bar") + assert_location(OrWriteNode, "$foo ||= bar") + assert_location(OrWriteNode, "@foo ||= bar") + assert_location(OrWriteNode, "foo ||= bar") + assert_location(OrWriteNode, "foo = 1; foo ||= bar", 9...20) + end + def test_ParametersNode assert_location(ParametersNode, "def foo(bar, baz); end", 8...16, &:parameters) end @@ -644,13 +599,13 @@ def test_SplatNode end def test_StatementsNode - assert_location(StatementsNode, "foo { 1 }", 6...7) { |node| node.block.statements } + assert_location(StatementsNode, "foo { 1 }", 6...7) { |node| node.block.body } - assert_location(StatementsNode, "(1)", 1...2, &:statements) + assert_location(StatementsNode, "(1)", 1...2, &:body) - assert_location(StatementsNode, "def foo; 1; end", 9...10, &:statements) - assert_location(StatementsNode, "def foo = 1", 10...11, &:statements) - assert_location(StatementsNode, "def foo; 1\n2; end", 9...12, &:statements) + assert_location(StatementsNode, "def foo; 1; end", 9...10, &:body) + assert_location(StatementsNode, "def foo = 1", 10...11, &:body) + assert_location(StatementsNode, "def foo; 1\n2; end", 9...12, &:body) assert_location(StatementsNode, "if foo; bar; end", 8...11, &:statements) assert_location(StatementsNode, "foo if bar", 0...3, &:statements) @@ -676,11 +631,11 @@ def test_StatementsNode assert_location(StatementsNode, "begin; ensure; foo; end", 15...18) { |node| node.ensure_clause.statements } assert_location(StatementsNode, "begin; rescue; else; foo; end", 21...24) { |node| node.else_clause.statements } - assert_location(StatementsNode, "class Foo; foo; end", 11...14, &:statements) - assert_location(StatementsNode, "module Foo; foo; end", 12...15, &:statements) - assert_location(StatementsNode, "class << self; foo; end", 15...18, &:statements) + assert_location(StatementsNode, "class Foo; foo; end", 11...14, &:body) + assert_location(StatementsNode, "module Foo; foo; end", 12...15, &:body) + assert_location(StatementsNode, "class << self; foo; end", 15...18, &:body) - assert_location(StatementsNode, "-> { foo }", 5...8, &:statements) + assert_location(StatementsNode, "-> { foo }", 5...8, &:body) assert_location(StatementsNode, "BEGIN { foo }", 8...11, &:statements) assert_location(StatementsNode, "END { foo }", 6...9, &:statements) diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index 7a910f7db2aacd..f8c1fe12d13443 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -34,8 +34,12 @@ def test_parse_takes_file_path # running on Ruby 3.2+. check_ripper = RUBY_VERSION >= "3.2.0" + # The FOCUS environment variable allows you to specify one particular fixture + # to test, instead of all of them. base = File.join(__dir__, "fixtures") - Dir["**/*.txt", base: base].each do |relative| + relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] + + relatives.each do |relative| # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) @@ -91,6 +95,13 @@ def test_parse_takes_file_path # Next, assert that the newlines are in the expected places. expected_newlines = [0] source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } + + # If there's a __END__, then we should trip out those newlines because we + # don't actually scan them during parsing (because we don't need to). + if found = result.comments.find { |comment| comment.type == :__END__ } + expected_newlines = expected_newlines[...found.location.start_line] + end + assert_equal expected_newlines, YARP.const_get(:Debug).newlines(source) # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if @@ -101,6 +112,10 @@ def test_parse_takes_file_path # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. return if relative == "seattlerb/heredoc_nested.txt" + # Ripper seems to have a bug that the regex portions before and after the heredoc are combined + # into a single token. + return if relative == "wrapping_heredoc.txt" + # Finally, assert that we can lex the source and get the same tokens as # Ripper. lex_result = YARP.lex_compat(source) @@ -120,6 +135,8 @@ def test_parse_takes_file_path end Dir["*.txt", base: base].each do |relative| + next if relative == "newline_terminated.txt" + # We test every snippet (separated by \n\n) in isolation # to ensure the parser does not try to read bytes further than the end of each snippet define_method "test_individual_snippets_#{relative}" do diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index ea18fbb35a49b4..a26b971b824732 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -19,4 +19,15 @@ def test_ruby_api assert_equal_nodes ast1, ast2 assert_equal_nodes ast2, ast3 end + + def test_literal_value_method + assert_equal 123, YARP.parse("123").value.statements.body.first.value + assert_equal 3.14, YARP.parse("3.14").value.statements.body.first.value + assert_equal 42i, YARP.parse("42i").value.statements.body.first.value + assert_equal 3.14i, YARP.parse("3.14i").value.statements.body.first.value + assert_equal 42r, YARP.parse("42r").value.statements.body.first.value + assert_equal 0.5r, YARP.parse("0.5r").value.statements.body.first.value + assert_equal 42ri, YARP.parse("42ri").value.statements.body.first.value + assert_equal 0.5ri, YARP.parse("0.5ri").value.statements.body.first.value + end end diff --git a/test/yarp/snapshots/blocks.txt b/test/yarp/snapshots/blocks.txt index 3579d99bd75dfb..089b63b89ca539 100644 --- a/test/yarp/snapshots/blocks.txt +++ b/test/yarp/snapshots/blocks.txt @@ -88,12 +88,11 @@ ProgramNode(0...402)( (61...62) ), StatementsNode(63...72)( - [LocalVariableOperatorWriteNode(63...72)( - (63...67), + [OperatorWriteNode(63...72)( + LocalVariableWriteNode(63...67)(:memo, 0, nil, (63...67), nil), (68...70), - LocalVariableReadNode(71...72)(:x, 0), - :memo, - :+ + :+, + LocalVariableReadNode(71...72)(:x, 0) )] ), (51...52), diff --git a/test/yarp/snapshots/boolean_operators.txt b/test/yarp/snapshots/boolean_operators.txt index 1b8a2aaed91214..c6cfc311e560a8 100644 --- a/test/yarp/snapshots/boolean_operators.txt +++ b/test/yarp/snapshots/boolean_operators.txt @@ -1,24 +1,21 @@ ProgramNode(0...24)( [:a], StatementsNode(0...24)( - [LocalVariableOperatorAndWriteNode(0...7)( - (0...1), - (2...5), + [AndWriteNode(0...7)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), - :a + (2...5) ), - LocalVariableOperatorWriteNode(9...15)( - (9...10), + OperatorWriteNode(9...15)( + LocalVariableWriteNode(9...10)(:a, 0, nil, (9...10), nil), (11...13), - CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b"), - :a, - :+ + :+, + CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b") ), - LocalVariableOperatorOrWriteNode(17...24)( - (17...18), - (19...22), + OrWriteNode(17...24)( + LocalVariableWriteNode(17...18)(:a, 0, nil, (17...18), nil), CallNode(23...24)(nil, nil, (23...24), nil, nil, nil, nil, 2, "b"), - :a + (19...22) )] ) ) diff --git a/test/yarp/snapshots/classes.txt b/test/yarp/snapshots/classes.txt index 60ab0eadc5e075..33b3d042148f9b 100644 --- a/test/yarp/snapshots/classes.txt +++ b/test/yarp/snapshots/classes.txt @@ -16,7 +16,8 @@ ProgramNode(0...370)( (10...11) )] ), - (14...17) + (14...17), + "A" ), ClassNode(19...39)( [], @@ -32,7 +33,8 @@ ProgramNode(0...370)( EnsureNode(28...39)((28...34), nil, (36...39)), (36...39) ), - (36...39) + (36...39), + "A" ), ClassNode(41...75)( [], @@ -48,7 +50,8 @@ ProgramNode(0...370)( EnsureNode(64...75)((64...70), nil, (72...75)), (72...75) ), - (72...75) + (72...75), + "A" ), ClassNode(77...98)( [:a], @@ -65,7 +68,8 @@ ProgramNode(0...370)( (91...92) )] ), - (95...98) + (95...98), + "A" ), SingletonClassNode(100...120)( [], @@ -118,7 +122,8 @@ ProgramNode(0...370)( (154...157) )] ), - (159...162) + (159...162), + "A" ), ClassNode(164...218)( [], @@ -143,7 +148,8 @@ ProgramNode(0...370)( (210...213) )] ), - (215...218) + (215...218), + "A" ), SingletonClassNode(220...240)( [], @@ -274,7 +280,8 @@ ProgramNode(0...370)( "[]" ), nil, - (367...370) + (367...370), + "A" )] ) ) diff --git a/test/yarp/snapshots/constants.txt b/test/yarp/snapshots/constants.txt index 7e82efd5b662cc..05529a2bc84c62 100644 --- a/test/yarp/snapshots/constants.txt +++ b/test/yarp/snapshots/constants.txt @@ -1,6 +1,6 @@ -ProgramNode(0...715)( +ProgramNode(0...792)( [], - StatementsNode(0...715)( + StatementsNode(0...792)( [ConstantPathNode(0...4)( ConstantReadNode(0...1)(), ConstantReadNode(3...4)(), @@ -43,9 +43,173 @@ ProgramNode(0...715)( "Foo" ), CallNode(50...58)( - ConstantPathNode(50...53)(nil, ConstantReadNode(52...53)(), (50...52)), - (53...55), - (55...58), + nil, + nil, + (50...53), + nil, + ArgumentsNode(54...58)( + [SplatNode(54...58)( + (54...55), + CallNode(55...58)( + nil, + nil, + (55...58), + nil, + nil, + nil, + nil, + 2, + "bar" + ) + )] + ), + nil, + nil, + 0, + "Foo" + ), + CallNode(60...69)( + nil, + nil, + (60...63), + nil, + ArgumentsNode(64...69)( + [KeywordHashNode(64...69)( + [AssocSplatNode(64...69)( + CallNode(66...69)( + nil, + nil, + (66...69), + nil, + nil, + nil, + nil, + 2, + "bar" + ), + (64...66) + )] + )] + ), + nil, + nil, + 0, + "Foo" + ), + CallNode(71...79)( + nil, + nil, + (71...74), + nil, + ArgumentsNode(75...79)( + [BlockArgumentNode(75...79)( + CallNode(76...79)( + nil, + nil, + (76...79), + nil, + nil, + nil, + nil, + 2, + "bar" + ), + (75...76) + )] + ), + nil, + nil, + 0, + "Foo" + ), + CallNode(81...94)( + ConstantReadNode(81...84)(), + (84...86), + (86...89), + nil, + ArgumentsNode(90...94)( + [SplatNode(90...94)( + (90...91), + CallNode(91...94)( + nil, + nil, + (91...94), + nil, + nil, + nil, + nil, + 2, + "baz" + ) + )] + ), + nil, + nil, + 0, + "Bar" + ), + CallNode(96...110)( + ConstantReadNode(96...99)(), + (99...101), + (101...104), + nil, + ArgumentsNode(105...110)( + [KeywordHashNode(105...110)( + [AssocSplatNode(105...110)( + CallNode(107...110)( + nil, + nil, + (107...110), + nil, + nil, + nil, + nil, + 2, + "baz" + ), + (105...107) + )] + )] + ), + nil, + nil, + 0, + "Bar" + ), + CallNode(112...125)( + ConstantReadNode(112...115)(), + (115...117), + (117...120), + nil, + ArgumentsNode(121...125)( + [BlockArgumentNode(121...125)( + CallNode(122...125)( + nil, + nil, + (122...125), + nil, + nil, + nil, + nil, + 2, + "baz" + ), + (121...122) + )] + ), + nil, + nil, + 0, + "Bar" + ), + CallNode(127...135)( + ConstantPathNode(127...130)( + nil, + ConstantReadNode(129...130)(), + (127...129) + ), + (130...132), + (132...135), nil, nil, nil, @@ -53,34 +217,46 @@ ProgramNode(0...715)( 0, "foo" ), - ConstantPathWriteNode(60...67)( - ConstantPathNode(60...63)(nil, ConstantReadNode(62...63)(), (60...62)), - (64...65), - IntegerNode(66...67)() + ConstantPathWriteNode(137...144)( + ConstantPathNode(137...140)( + nil, + ConstantReadNode(139...140)(), + (137...139) + ), + (141...142), + IntegerNode(143...144)() ), - ConstantPathWriteNode(69...79)( - ConstantPathNode(69...75)( - ConstantPathNode(69...72)( + ConstantPathWriteNode(146...156)( + ConstantPathNode(146...152)( + ConstantPathNode(146...149)( nil, - ConstantReadNode(71...72)(), - (69...71) + ConstantReadNode(148...149)(), + (146...148) ), - ConstantReadNode(74...75)(), - (72...74) + ConstantReadNode(151...152)(), + (149...151) ), - (76...77), - IntegerNode(78...79)() + (153...154), + IntegerNode(155...156)() ), - ConstantPathNode(81...87)( - ConstantPathNode(81...84)(nil, ConstantReadNode(83...84)(), (81...83)), - ConstantReadNode(86...87)(), - (84...86) + ConstantPathNode(158...164)( + ConstantPathNode(158...161)( + nil, + ConstantReadNode(160...161)(), + (158...160) + ), + ConstantReadNode(163...164)(), + (161...163) + ), + ConstantPathNode(166...169)( + nil, + ConstantReadNode(168...169)(), + (166...168) ), - ConstantPathNode(89...92)(nil, ConstantReadNode(91...92)(), (89...91)), - CallNode(94...102)( - ConstantReadNode(94...95)(), - (95...97), - (97...102), + CallNode(171...179)( + ConstantReadNode(171...172)(), + (172...174), + (174...179), nil, nil, nil, @@ -88,14 +264,14 @@ ProgramNode(0...715)( 0, "false" ), - CallNode(104...114)( - ConstantPathNode(104...108)( - ConstantReadNode(104...105)(), - ConstantReadNode(107...108)(), - (105...107) + CallNode(181...191)( + ConstantPathNode(181...185)( + ConstantReadNode(181...182)(), + ConstantReadNode(184...185)(), + (182...184) ), - (108...110), - (110...114), + (185...187), + (187...191), nil, nil, nil, @@ -103,10 +279,10 @@ ProgramNode(0...715)( 0, "true" ), - CallNode(116...120)( - ConstantReadNode(116...117)(), - (117...119), - (119...120), + CallNode(193...197)( + ConstantReadNode(193...194)(), + (194...196), + (196...197), nil, nil, nil, @@ -114,10 +290,10 @@ ProgramNode(0...715)( 0, "&" ), - CallNode(122...126)( - ConstantReadNode(122...123)(), - (123...125), - (125...126), + CallNode(199...203)( + ConstantReadNode(199...200)(), + (200...202), + (202...203), nil, nil, nil, @@ -125,10 +301,10 @@ ProgramNode(0...715)( 0, "`" ), - CallNode(128...132)( - ConstantReadNode(128...129)(), - (129...131), - (131...132), + CallNode(205...209)( + ConstantReadNode(205...206)(), + (206...208), + (208...209), nil, nil, nil, @@ -136,10 +312,10 @@ ProgramNode(0...715)( 0, "!" ), - CallNode(134...139)( - ConstantReadNode(134...135)(), - (135...137), - (137...139), + CallNode(211...216)( + ConstantReadNode(211...212)(), + (212...214), + (214...216), nil, nil, nil, @@ -147,10 +323,10 @@ ProgramNode(0...715)( 0, "!=" ), - CallNode(141...145)( - ConstantReadNode(141...142)(), - (142...144), - (144...145), + CallNode(218...222)( + ConstantReadNode(218...219)(), + (219...221), + (221...222), nil, nil, nil, @@ -158,10 +334,10 @@ ProgramNode(0...715)( 0, "^" ), - CallNode(147...152)( - ConstantReadNode(147...148)(), - (148...150), - (150...152), + CallNode(224...229)( + ConstantReadNode(224...225)(), + (225...227), + (227...229), nil, nil, nil, @@ -169,10 +345,10 @@ ProgramNode(0...715)( 0, "==" ), - CallNode(154...160)( - ConstantReadNode(154...155)(), - (155...157), - (157...160), + CallNode(231...237)( + ConstantReadNode(231...232)(), + (232...234), + (234...237), nil, nil, nil, @@ -180,10 +356,10 @@ ProgramNode(0...715)( 0, "===" ), - CallNode(162...167)( - ConstantReadNode(162...163)(), - (163...165), - (165...167), + CallNode(239...244)( + ConstantReadNode(239...240)(), + (240...242), + (242...244), nil, nil, nil, @@ -191,10 +367,10 @@ ProgramNode(0...715)( 0, "=~" ), - CallNode(169...173)( - ConstantReadNode(169...170)(), - (170...172), - (172...173), + CallNode(246...250)( + ConstantReadNode(246...247)(), + (247...249), + (249...250), nil, nil, nil, @@ -202,10 +378,10 @@ ProgramNode(0...715)( 0, ">" ), - CallNode(175...180)( - ConstantReadNode(175...176)(), - (176...178), - (178...180), + CallNode(252...257)( + ConstantReadNode(252...253)(), + (253...255), + (255...257), nil, nil, nil, @@ -213,10 +389,10 @@ ProgramNode(0...715)( 0, ">=" ), - CallNode(182...187)( - ConstantReadNode(182...183)(), - (183...185), - (185...187), + CallNode(259...264)( + ConstantReadNode(259...260)(), + (260...262), + (262...264), nil, nil, nil, @@ -224,10 +400,10 @@ ProgramNode(0...715)( 0, ">>" ), - CallNode(189...194)( - ConstantReadNode(189...190)(), - (190...192), - (192...194), + CallNode(266...271)( + ConstantReadNode(266...267)(), + (267...269), + (269...271), nil, nil, nil, @@ -235,15 +411,15 @@ ProgramNode(0...715)( 0, "<<" ), - ConstantPathNode(196...204)( - ConstantReadNode(196...197)(), - ConstantReadNode(203...204)(), - (197...199) + ConstantPathNode(273...281)( + ConstantReadNode(273...274)(), + ConstantReadNode(280...281)(), + (274...276) ), - CallNode(206...214)( - ConstantReadNode(206...207)(), - (207...209), - (209...214), + CallNode(283...291)( + ConstantReadNode(283...284)(), + (284...286), + (286...291), nil, nil, nil, @@ -251,10 +427,10 @@ ProgramNode(0...715)( 0, "alias" ), - CallNode(216...222)( - ConstantReadNode(216...217)(), - (217...219), - (219...222), + CallNode(293...299)( + ConstantReadNode(293...294)(), + (294...296), + (296...299), nil, nil, nil, @@ -262,10 +438,10 @@ ProgramNode(0...715)( 0, "and" ), - CallNode(224...232)( - ConstantReadNode(224...225)(), - (225...227), - (227...232), + CallNode(301...309)( + ConstantReadNode(301...302)(), + (302...304), + (304...309), nil, nil, nil, @@ -273,15 +449,15 @@ ProgramNode(0...715)( 0, "begin" ), - ConstantPathNode(234...242)( - ConstantReadNode(234...235)(), - ConstantReadNode(237...242)(), - (235...237) + ConstantPathNode(311...319)( + ConstantReadNode(311...312)(), + ConstantReadNode(314...319)(), + (312...314) ), - CallNode(244...252)( - ConstantReadNode(244...245)(), - (245...247), - (247...252), + CallNode(321...329)( + ConstantReadNode(321...322)(), + (322...324), + (324...329), nil, nil, nil, @@ -289,10 +465,10 @@ ProgramNode(0...715)( 0, "break" ), - CallNode(254...262)( - ConstantReadNode(254...255)(), - (255...257), - (257...262), + CallNode(331...339)( + ConstantReadNode(331...332)(), + (332...334), + (334...339), nil, nil, nil, @@ -300,10 +476,10 @@ ProgramNode(0...715)( 0, "class" ), - CallNode(264...270)( - ConstantReadNode(264...265)(), - (265...267), - (267...270), + CallNode(341...347)( + ConstantReadNode(341...342)(), + (342...344), + (344...347), nil, nil, nil, @@ -311,10 +487,10 @@ ProgramNode(0...715)( 0, "def" ), - CallNode(272...282)( - ConstantReadNode(272...273)(), - (273...275), - (275...282), + CallNode(349...359)( + ConstantReadNode(349...350)(), + (350...352), + (352...359), nil, nil, nil, @@ -322,10 +498,10 @@ ProgramNode(0...715)( 0, "defined" ), - CallNode(284...289)( - ConstantReadNode(284...285)(), - (285...287), - (287...289), + CallNode(361...366)( + ConstantReadNode(361...362)(), + (362...364), + (364...366), nil, nil, nil, @@ -333,10 +509,10 @@ ProgramNode(0...715)( 0, "do" ), - CallNode(291...298)( - ConstantReadNode(291...292)(), - (292...294), - (294...298), + CallNode(368...375)( + ConstantReadNode(368...369)(), + (369...371), + (371...375), nil, nil, nil, @@ -344,10 +520,10 @@ ProgramNode(0...715)( 0, "else" ), - CallNode(300...308)( - ConstantReadNode(300...301)(), - (301...303), - (303...308), + CallNode(377...385)( + ConstantReadNode(377...378)(), + (378...380), + (380...385), nil, nil, nil, @@ -355,10 +531,10 @@ ProgramNode(0...715)( 0, "elsif" ), - CallNode(310...316)( - ConstantReadNode(310...311)(), - (311...313), - (313...316), + CallNode(387...393)( + ConstantReadNode(387...388)(), + (388...390), + (390...393), nil, nil, nil, @@ -366,15 +542,15 @@ ProgramNode(0...715)( 0, "end" ), - ConstantPathNode(318...324)( - ConstantReadNode(318...319)(), - ConstantReadNode(321...324)(), - (319...321) + ConstantPathNode(395...401)( + ConstantReadNode(395...396)(), + ConstantReadNode(398...401)(), + (396...398) ), - CallNode(326...335)( - ConstantReadNode(326...327)(), - (327...329), - (329...335), + CallNode(403...412)( + ConstantReadNode(403...404)(), + (404...406), + (406...412), nil, nil, nil, @@ -382,10 +558,10 @@ ProgramNode(0...715)( 0, "ensure" ), - CallNode(337...345)( - ConstantReadNode(337...338)(), - (338...340), - (340...345), + CallNode(414...422)( + ConstantReadNode(414...415)(), + (415...417), + (417...422), nil, nil, nil, @@ -393,10 +569,10 @@ ProgramNode(0...715)( 0, "false" ), - CallNode(347...353)( - ConstantReadNode(347...348)(), - (348...350), - (350...353), + CallNode(424...430)( + ConstantReadNode(424...425)(), + (425...427), + (427...430), nil, nil, nil, @@ -404,10 +580,10 @@ ProgramNode(0...715)( 0, "for" ), - CallNode(355...360)( - ConstantReadNode(355...356)(), - (356...358), - (358...360), + CallNode(432...437)( + ConstantReadNode(432...433)(), + (433...435), + (435...437), nil, nil, nil, @@ -415,10 +591,10 @@ ProgramNode(0...715)( 0, "if" ), - CallNode(362...367)( - ConstantReadNode(362...363)(), - (363...365), - (365...367), + CallNode(439...444)( + ConstantReadNode(439...440)(), + (440...442), + (442...444), nil, nil, nil, @@ -426,10 +602,10 @@ ProgramNode(0...715)( 0, "in" ), - CallNode(369...376)( - ConstantReadNode(369...370)(), - (370...372), - (372...376), + CallNode(446...453)( + ConstantReadNode(446...447)(), + (447...449), + (449...453), nil, nil, nil, @@ -437,10 +613,10 @@ ProgramNode(0...715)( 0, "next" ), - CallNode(378...384)( - ConstantReadNode(378...379)(), - (379...381), - (381...384), + CallNode(455...461)( + ConstantReadNode(455...456)(), + (456...458), + (458...461), nil, nil, nil, @@ -448,10 +624,10 @@ ProgramNode(0...715)( 0, "nil" ), - CallNode(386...392)( - ConstantReadNode(386...387)(), - (387...389), - (389...392), + CallNode(463...469)( + ConstantReadNode(463...464)(), + (464...466), + (466...469), nil, nil, nil, @@ -459,10 +635,10 @@ ProgramNode(0...715)( 0, "not" ), - CallNode(394...399)( - ConstantReadNode(394...395)(), - (395...397), - (397...399), + CallNode(471...476)( + ConstantReadNode(471...472)(), + (472...474), + (474...476), nil, nil, nil, @@ -470,10 +646,10 @@ ProgramNode(0...715)( 0, "or" ), - CallNode(401...408)( - ConstantReadNode(401...402)(), - (402...404), - (404...408), + CallNode(478...485)( + ConstantReadNode(478...479)(), + (479...481), + (481...485), nil, nil, nil, @@ -481,10 +657,10 @@ ProgramNode(0...715)( 0, "redo" ), - CallNode(410...419)( - ConstantReadNode(410...411)(), - (411...413), - (413...419), + CallNode(487...496)( + ConstantReadNode(487...488)(), + (488...490), + (490...496), nil, nil, nil, @@ -492,10 +668,10 @@ ProgramNode(0...715)( 0, "rescue" ), - CallNode(421...429)( - ConstantReadNode(421...422)(), - (422...424), - (424...429), + CallNode(498...506)( + ConstantReadNode(498...499)(), + (499...501), + (501...506), nil, nil, nil, @@ -503,10 +679,10 @@ ProgramNode(0...715)( 0, "retry" ), - CallNode(431...440)( - ConstantReadNode(431...432)(), - (432...434), - (434...440), + CallNode(508...517)( + ConstantReadNode(508...509)(), + (509...511), + (511...517), nil, nil, nil, @@ -514,10 +690,10 @@ ProgramNode(0...715)( 0, "return" ), - CallNode(442...449)( - ConstantReadNode(442...443)(), - (443...445), - (445...449), + CallNode(519...526)( + ConstantReadNode(519...520)(), + (520...522), + (522...526), nil, nil, nil, @@ -525,10 +701,10 @@ ProgramNode(0...715)( 0, "self" ), - CallNode(451...459)( - ConstantReadNode(451...452)(), - (452...454), - (454...459), + CallNode(528...536)( + ConstantReadNode(528...529)(), + (529...531), + (531...536), nil, nil, nil, @@ -536,10 +712,10 @@ ProgramNode(0...715)( 0, "super" ), - CallNode(461...468)( - ConstantReadNode(461...462)(), - (462...464), - (464...468), + CallNode(538...545)( + ConstantReadNode(538...539)(), + (539...541), + (541...545), nil, nil, nil, @@ -547,10 +723,10 @@ ProgramNode(0...715)( 0, "then" ), - CallNode(470...477)( - ConstantReadNode(470...471)(), - (471...473), - (473...477), + CallNode(547...554)( + ConstantReadNode(547...548)(), + (548...550), + (550...554), nil, nil, nil, @@ -558,10 +734,10 @@ ProgramNode(0...715)( 0, "true" ), - CallNode(479...487)( - ConstantReadNode(479...480)(), - (480...482), - (482...487), + CallNode(556...564)( + ConstantReadNode(556...557)(), + (557...559), + (559...564), nil, nil, nil, @@ -569,10 +745,10 @@ ProgramNode(0...715)( 0, "undef" ), - CallNode(489...498)( - ConstantReadNode(489...490)(), - (490...492), - (492...498), + CallNode(566...575)( + ConstantReadNode(566...567)(), + (567...569), + (569...575), nil, nil, nil, @@ -580,10 +756,10 @@ ProgramNode(0...715)( 0, "unless" ), - CallNode(500...508)( - ConstantReadNode(500...501)(), - (501...503), - (503...508), + CallNode(577...585)( + ConstantReadNode(577...578)(), + (578...580), + (580...585), nil, nil, nil, @@ -591,10 +767,10 @@ ProgramNode(0...715)( 0, "until" ), - CallNode(510...517)( - ConstantReadNode(510...511)(), - (511...513), - (513...517), + CallNode(587...594)( + ConstantReadNode(587...588)(), + (588...590), + (590...594), nil, nil, nil, @@ -602,10 +778,10 @@ ProgramNode(0...715)( 0, "when" ), - CallNode(519...527)( - ConstantReadNode(519...520)(), - (520...522), - (522...527), + CallNode(596...604)( + ConstantReadNode(596...597)(), + (597...599), + (599...604), nil, nil, nil, @@ -613,10 +789,10 @@ ProgramNode(0...715)( 0, "while" ), - CallNode(529...537)( - ConstantReadNode(529...530)(), - (530...532), - (532...537), + CallNode(606...614)( + ConstantReadNode(606...607)(), + (607...609), + (609...614), nil, nil, nil, @@ -624,10 +800,10 @@ ProgramNode(0...715)( 0, "yield" ), - CallNode(539...554)( - ConstantReadNode(539...540)(), - (540...542), - (542...554), + CallNode(616...631)( + ConstantReadNode(616...617)(), + (617...619), + (619...631), nil, nil, nil, @@ -635,10 +811,10 @@ ProgramNode(0...715)( 0, "__ENCODING__" ), - CallNode(556...567)( - ConstantReadNode(556...557)(), - (557...559), - (559...567), + CallNode(633...644)( + ConstantReadNode(633...634)(), + (634...636), + (636...644), nil, nil, nil, @@ -646,10 +822,10 @@ ProgramNode(0...715)( 0, "__FILE__" ), - CallNode(569...580)( - ConstantReadNode(569...570)(), - (570...572), - (572...580), + CallNode(646...657)( + ConstantReadNode(646...647)(), + (647...649), + (649...657), nil, nil, nil, @@ -657,10 +833,10 @@ ProgramNode(0...715)( 0, "__LINE__" ), - CallNode(582...586)( - ConstantReadNode(582...583)(), - (583...585), - (585...586), + CallNode(659...663)( + ConstantReadNode(659...660)(), + (660...662), + (662...663), nil, nil, nil, @@ -668,10 +844,10 @@ ProgramNode(0...715)( 0, "<" ), - CallNode(588...594)( - ConstantReadNode(588...589)(), - (589...591), - (591...594), + CallNode(665...671)( + ConstantReadNode(665...666)(), + (666...668), + (668...671), nil, nil, nil, @@ -679,10 +855,10 @@ ProgramNode(0...715)( 0, "<=>" ), - CallNode(596...601)( - ConstantReadNode(596...597)(), - (597...599), - (599...601), + CallNode(673...678)( + ConstantReadNode(673...674)(), + (674...676), + (676...678), nil, nil, nil, @@ -690,10 +866,10 @@ ProgramNode(0...715)( 0, "<<" ), - CallNode(603...607)( - ConstantReadNode(603...604)(), - (604...606), - (606...607), + CallNode(680...684)( + ConstantReadNode(680...681)(), + (681...683), + (683...684), nil, nil, nil, @@ -701,10 +877,10 @@ ProgramNode(0...715)( 0, "-" ), - CallNode(609...613)( - ConstantReadNode(609...610)(), - (610...612), - (612...613), + CallNode(686...690)( + ConstantReadNode(686...687)(), + (687...689), + (689...690), nil, nil, nil, @@ -712,16 +888,16 @@ ProgramNode(0...715)( 0, "%" ), - CallNode(615...620)( - ConstantReadNode(615...616)(), - (616...618), - (618...619), + CallNode(692...697)( + ConstantReadNode(692...693)(), + (693...695), + (695...696), nil, - ArgumentsNode(619...620)( - [CallNode(619...620)( + ArgumentsNode(696...697)( + [CallNode(696...697)( nil, nil, - (619...620), + (696...697), nil, nil, nil, @@ -735,16 +911,16 @@ ProgramNode(0...715)( 0, "%" ), - CallNode(622...627)( - ConstantReadNode(622...623)(), - (623...625), - (625...626), + CallNode(699...704)( + ConstantReadNode(699...700)(), + (700...702), + (702...703), nil, - ArgumentsNode(626...627)( - [CallNode(626...627)( + ArgumentsNode(703...704)( + [CallNode(703...704)( nil, nil, - (626...627), + (703...704), nil, nil, nil, @@ -758,16 +934,16 @@ ProgramNode(0...715)( 0, "%" ), - CallNode(629...634)( - ConstantReadNode(629...630)(), - (630...632), - (632...633), + CallNode(706...711)( + ConstantReadNode(706...707)(), + (707...709), + (709...710), nil, - ArgumentsNode(633...634)( - [CallNode(633...634)( + ArgumentsNode(710...711)( + [CallNode(710...711)( nil, nil, - (633...634), + (710...711), nil, nil, nil, @@ -781,32 +957,32 @@ ProgramNode(0...715)( 0, "%" ), - CallNode(636...641)( - ConstantReadNode(636...637)(), - (637...639), - (639...640), + CallNode(713...718)( + ConstantReadNode(713...714)(), + (714...716), + (716...717), nil, - ArgumentsNode(640...641)([ConstantReadNode(640...641)()]), + ArgumentsNode(717...718)([ConstantReadNode(717...718)()]), nil, nil, 0, "%" ), - CallNode(643...648)( - ConstantReadNode(643...644)(), - (644...646), - (646...647), + CallNode(720...725)( + ConstantReadNode(720...721)(), + (721...723), + (723...724), nil, - ArgumentsNode(647...648)([ConstantReadNode(647...648)()]), + ArgumentsNode(724...725)([ConstantReadNode(724...725)()]), nil, nil, 0, "%" ), - CallNode(650...654)( - ConstantReadNode(650...651)(), - (651...653), - (653...654), + CallNode(727...731)( + ConstantReadNode(727...728)(), + (728...730), + (730...731), nil, nil, nil, @@ -814,10 +990,10 @@ ProgramNode(0...715)( 0, "|" ), - CallNode(656...660)( - ConstantReadNode(656...657)(), - (657...659), - (659...660), + CallNode(733...737)( + ConstantReadNode(733...734)(), + (734...736), + (736...737), nil, nil, nil, @@ -825,10 +1001,10 @@ ProgramNode(0...715)( 0, "+" ), - CallNode(662...666)( - ConstantReadNode(662...663)(), - (663...665), - (665...666), + CallNode(739...743)( + ConstantReadNode(739...740)(), + (740...742), + (742...743), nil, nil, nil, @@ -836,10 +1012,10 @@ ProgramNode(0...715)( 0, "/" ), - CallNode(668...672)( - ConstantReadNode(668...669)(), - (669...671), - (671...672), + CallNode(745...749)( + ConstantReadNode(745...746)(), + (746...748), + (748...749), nil, nil, nil, @@ -847,10 +1023,10 @@ ProgramNode(0...715)( 0, "*" ), - CallNode(674...679)( - ConstantReadNode(674...675)(), - (675...677), - (677...679), + CallNode(751...756)( + ConstantReadNode(751...752)(), + (752...754), + (754...756), nil, nil, nil, @@ -858,10 +1034,10 @@ ProgramNode(0...715)( 0, "**" ), - CallNode(681...685)( - ConstantReadNode(681...682)(), - (682...684), - (684...685), + CallNode(758...762)( + ConstantReadNode(758...759)(), + (759...761), + (761...762), nil, nil, nil, @@ -869,11 +1045,11 @@ ProgramNode(0...715)( 0, "~" ), - ConstantPathNode(687...695)( - CallNode(687...691)( - ConstantReadNode(687...688)(), - (688...690), - (690...691), + ConstantPathNode(764...772)( + CallNode(764...768)( + ConstantReadNode(764...765)(), + (765...767), + (767...768), nil, nil, nil, @@ -881,14 +1057,14 @@ ProgramNode(0...715)( 0, "_" ), - ConstantReadNode(694...695)(), - (691...693) - ), - RangeNode(697...715)( - CallNode(697...701)( - ConstantReadNode(697...698)(), - (698...700), - (700...701), + ConstantReadNode(771...772)(), + (768...770) + ), + RangeNode(774...792)( + CallNode(774...778)( + ConstantReadNode(774...775)(), + (775...777), + (777...778), nil, nil, nil, @@ -896,10 +1072,10 @@ ProgramNode(0...715)( 0, "_" ), - CallNode(705...715)( - ConstantReadNode(705...706)(), - (706...708), - (708...715), + CallNode(782...792)( + ConstantReadNode(782...783)(), + (783...785), + (785...792), nil, nil, nil, @@ -907,7 +1083,7 @@ ProgramNode(0...715)( 0, "__END__" ), - (701...703), + (778...780), 0 )] ) diff --git a/test/yarp/snapshots/dash_heredocs.txt b/test/yarp/snapshots/dash_heredocs.txt index 24871b27358a04..e2eea3c2a53666 100644 --- a/test/yarp/snapshots/dash_heredocs.txt +++ b/test/yarp/snapshots/dash_heredocs.txt @@ -1,6 +1,6 @@ -ProgramNode(0...217)( +ProgramNode(0...278)( [], - StatementsNode(0...217)( + StatementsNode(0...278)( [InterpolatedStringNode(0...6)( (0...6), [StringNode(7...11)(nil, (7...11), nil, " a\n")], @@ -117,6 +117,60 @@ ProgramNode(0...217)( (209...217), [StringNode(218...227)(nil, (218...227), nil, " a \#{1}\n")], (227...231) + ), + CallNode(232...243)( + InterpolatedStringNode(232...236)( + (232...236), + [StringNode(244...248)(nil, (244...248), nil, " a\n")], + (248...250) + ), + nil, + (237...238), + nil, + ArgumentsNode(239...243)( + [InterpolatedStringNode(239...243)( + (239...243), + [StringNode(250...256)(nil, (250...256), nil, " b\n" + " "), + EmbeddedStatementsNode(256...263)( + (256...258), + StatementsNode(258...259)([IntegerNode(258...259)()]), + (262...263) + ), + StringNode(263...264)(nil, (263...264), nil, "\n")], + (264...266) + )] + ), + nil, + nil, + 0, + "+" + ), + CallNode(267...278)( + InterpolatedStringNode(267...271)( + (267...271), + [StringNode(279...283)(nil, (279...283), nil, " a\n")], + (283...285) + ), + nil, + (272...273), + nil, + ArgumentsNode(274...278)( + [InterpolatedStringNode(274...278)( + (274...278), + [StringNode(285...291)(nil, (285...291), nil, " b\n" + " "), + EmbeddedStatementsNode(291...298)( + (291...293), + StatementsNode(296...297)([IntegerNode(296...297)()]), + (297...298) + ), + StringNode(298...299)(nil, (298...299), nil, "\n")], + (299...301) + )] + ), + nil, + nil, + 0, + "+" )] ) ) diff --git a/test/yarp/snapshots/defined.txt b/test/yarp/snapshots/defined.txt index 4d0b377a8a7ac1..6c2cad656e0feb 100644 --- a/test/yarp/snapshots/defined.txt +++ b/test/yarp/snapshots/defined.txt @@ -8,12 +8,11 @@ ProgramNode(0...78)( ), DefinedNode(27...43)( (35...36), - LocalVariableOperatorWriteNode(36...42)( - (36...37), + OperatorWriteNode(36...42)( + LocalVariableWriteNode(36...37)(:x, 0, nil, (36...37), nil), (38...40), - IntegerNode(41...42)(), - :x, - :% + :%, + IntegerNode(41...42)() ), (42...43), (27...35) diff --git a/test/snapshots/heredoc_with_escaped_newline_at_start.txt b/test/yarp/snapshots/heredoc_with_escaped_newline_at_start.txt similarity index 100% rename from test/snapshots/heredoc_with_escaped_newline_at_start.txt rename to test/yarp/snapshots/heredoc_with_escaped_newline_at_start.txt diff --git a/test/yarp/snapshots/if.txt b/test/yarp/snapshots/if.txt index c12691abfe5c2d..d151d6b5471d97 100644 --- a/test/yarp/snapshots/if.txt +++ b/test/yarp/snapshots/if.txt @@ -203,7 +203,7 @@ ProgramNode(0...382)( (269...271) ), nil, - IfNode(274...289)( + IfNode(274...293)( (274...279), MatchPredicateNode(280...289)( CallNode(280...284)( @@ -222,7 +222,7 @@ ProgramNode(0...382)( ), nil, nil, - nil + (290...293) ), (290...293) ), diff --git a/test/yarp/snapshots/indented_file_end.txt b/test/yarp/snapshots/indented_file_end.txt new file mode 100644 index 00000000000000..513a7ab0f5ba48 --- /dev/null +++ b/test/yarp/snapshots/indented_file_end.txt @@ -0,0 +1,18 @@ +ProgramNode(4...23)( + [], + StatementsNode(4...23)( + [DefNode(4...23)( + (8...10), + nil, + nil, + nil, + [], + (4...7), + nil, + nil, + nil, + nil, + (20...23) + )] + ) +) diff --git a/test/yarp/snapshots/lambda.txt b/test/yarp/snapshots/lambda.txt index bf59e6aa6848b2..23c4ebc072e31b 100644 --- a/test/yarp/snapshots/lambda.txt +++ b/test/yarp/snapshots/lambda.txt @@ -1,6 +1,6 @@ -ProgramNode(0...51)( +ProgramNode(0...92)( [], - StatementsNode(0...51)( + StatementsNode(0...92)( [LambdaNode(0...14)( [:foo], (0...2), @@ -104,6 +104,72 @@ ProgramNode(0...51)( (47...48) ), nil + ), + LambdaNode(53...72)( + [:foo], + (53...55), + BlockParametersNode(56...65)( + ParametersNode(56...65)( + [], + [OptionalParameterNode(56...65)( + :foo, + (56...59), + (60...61), + CallNode(62...65)( + nil, + nil, + (62...65), + nil, + nil, + nil, + nil, + 2, + "bar" + ) + )], + [], + nil, + [], + nil, + nil + ), + [], + nil, + nil + ), + nil + ), + LambdaNode(74...92)( + [:foo], + (74...76), + BlockParametersNode(77...85)( + ParametersNode(77...85)( + [], + [], + [], + nil, + [KeywordParameterNode(77...85)( + (77...81), + CallNode(82...85)( + nil, + nil, + (82...85), + nil, + nil, + nil, + nil, + 2, + "bar" + ) + )], + nil, + nil + ), + [], + nil, + nil + ), + nil )] ) ) diff --git a/test/yarp/snapshots/method_calls.txt b/test/yarp/snapshots/method_calls.txt index f59fb7e46d0bf9..b2c84d0d3573c2 100644 --- a/test/yarp/snapshots/method_calls.txt +++ b/test/yarp/snapshots/method_calls.txt @@ -1191,7 +1191,8 @@ ProgramNode(0...1237)( "baz" )] ), - (926...929) + (926...929), + "Bar" )] ), nil, @@ -1222,7 +1223,8 @@ ProgramNode(0...1237)( "baz" )] ), - (957...960) + (957...960), + "Bar" )] ), nil, diff --git a/test/yarp/snapshots/modules.txt b/test/yarp/snapshots/modules.txt index 705563a9f04cdf..a3cf30b9ff8786 100644 --- a/test/yarp/snapshots/modules.txt +++ b/test/yarp/snapshots/modules.txt @@ -14,7 +14,8 @@ ProgramNode(0...140)( (11...12) )] ), - (15...18) + (15...18), + "A" ), InterpolatedStringNode(20...38)( (20...23), @@ -48,7 +49,8 @@ ProgramNode(0...140)( (48...50) ), nil, - (52...55) + (52...55), + "M" ), ModuleNode(57...85)( [:x], @@ -70,14 +72,16 @@ ProgramNode(0...140)( nil, (82...85) ), - (82...85) + (82...85), + "A" ), ModuleNode(87...101)( [], (87...93), ConstantPathNode(94...97)(nil, ConstantReadNode(96...97)(), (94...96)), nil, - (98...101) + (98...101), + "A" ), ModuleNode(103...120)( [], @@ -98,7 +102,8 @@ ProgramNode(0...140)( (113...115) ), nil, - (117...120) + (117...120), + "B" ), ModuleNode(122...140)( [], @@ -119,7 +124,8 @@ ProgramNode(0...140)( (133...135) ), nil, - (137...140) + (137...140), + "B" )] ) ) diff --git a/test/yarp/snapshots/newline_terminated.txt b/test/yarp/snapshots/newline_terminated.txt new file mode 100644 index 00000000000000..06ad6a989eead8 --- /dev/null +++ b/test/yarp/snapshots/newline_terminated.txt @@ -0,0 +1,28 @@ +ProgramNode(76...212)( + [], + StatementsNode(76...212)( + [StringNode(76...82)((76...78), (78...81), (81...82), "abc"), + StringNode(84...90)((84...86), (86...89), (89...90), "abc"), + StringNode(92...98)((92...94), (94...97), (97...98), "abc"), + StringNode(100...106)((100...102), (102...105), (105...106), "abc"), + StringNode(108...114)((108...110), (110...113), (113...114), "abc"), + StringNode(116...122)((116...118), (118...121), (121...122), "abc"), + StringNode(124...130)((124...126), (126...129), (129...130), "abc"), + StringNode(132...139)((132...134), (134...138), (138...139), "\rabc"), + StringNode(142...149)((142...144), (144...148), (148...149), "\rabc"), + StringNode(151...157)((151...153), (153...156), (156...157), "abc"), + StringNode(159...165)((159...161), (161...164), (164...165), "abc"), + StringNode(167...173)((167...169), (169...172), (172...173), "abc"), + StringNode(175...181)((175...177), (177...180), (180...181), "abc"), + StringNode(182...188)((182...184), (184...187), (187...188), "foo"), + StringNode(189...196)((189...192), (192...195), (195...196), "foo"), + StringNode(197...204)((197...200), (200...203), (203...204), "foo"), + RegularExpressionNode(205...212)( + (205...208), + (208...211), + (211...212), + "foo", + 0 + )] + ) +) diff --git a/test/yarp/snapshots/regex.txt b/test/yarp/snapshots/regex.txt index 67d7ab0480cf02..6ba394271aafce 100644 --- a/test/yarp/snapshots/regex.txt +++ b/test/yarp/snapshots/regex.txt @@ -1,6 +1,6 @@ -ProgramNode(0...278)( +ProgramNode(0...293)( [:foo], - StatementsNode(0...278)( + StatementsNode(0...293)( [CallNode(0...9)( nil, nil, @@ -138,6 +138,29 @@ ProgramNode(0...278)( (277...278), "pound", 0 + ), + InterpolatedRegularExpressionNode(280...293)( + (280...281), + [StringNode(281...285)(nil, (281...285), nil, "aaa "), + EmbeddedStatementsNode(285...291)( + (285...287), + StatementsNode(287...290)( + [CallNode(287...290)( + nil, + nil, + (287...290), + nil, + nil, + nil, + nil, + 2, + "bbb" + )] + ), + (290...291) + )], + (291...293), + 128 )] ) ) diff --git a/test/yarp/snapshots/seattlerb/TestRubyParserShared.txt b/test/yarp/snapshots/seattlerb/TestRubyParserShared.txt new file mode 100644 index 00000000000000..0eee2a39d7d165 --- /dev/null +++ b/test/yarp/snapshots/seattlerb/TestRubyParserShared.txt @@ -0,0 +1,223 @@ +ProgramNode(0...689)( + [], + StatementsNode(0...689)( + [ArrayNode(0...7)([], (0...3), (6...7)), + ArrayNode(9...26)( + [SymbolNode(13...18)(nil, (13...18), nil, "line2"), + SymbolNode(19...24)(nil, (19...24), nil, "line3")], + (9...12), + (25...26) + ), + ArrayNode(28...35)([], (28...31), (34...35)), + ArrayNode(37...54)( + [StringNode(41...46)(nil, (41...46), nil, "line2"), + StringNode(47...52)(nil, (47...52), nil, "line3")], + (37...40), + (53...54) + ), + ArrayNode(56...63)([], (56...59), (62...63)), + ArrayNode(65...82)( + [SymbolNode(69...74)(nil, (69...74), nil, "line2"), + SymbolNode(75...80)(nil, (75...80), nil, "line3")], + (65...68), + (81...82) + ), + RegularExpressionNode(84...91)( + (84...87), + (87...90), + (90...91), + "\n" + "\n" + "\n", + 0 + ), + ArrayNode(93...100)([], (93...96), (99...100)), + ArrayNode(102...119)( + [StringNode(106...111)(nil, (106...111), nil, "line2"), + StringNode(112...117)(nil, (112...117), nil, "line3")], + (102...105), + (118...119) + ), + ArrayNode(121...139)( + [SymbolNode(123...129)((123...124), (124...129), nil, "line2"), + SymbolNode(131...137)((131...132), (132...137), nil, "line3")], + (121...122), + (138...139) + ), + ClassNode(141...269)( + [], + (141...146), + ConstantReadNode(147...148)(), + nil, + nil, + StatementsNode(168...246)( + [DefNode(168...246)( + (177...178), + SelfNode(172...176)(), + ParametersNode(179...200)( + [RequiredParameterNode(179...180)(:a), + RequiredParameterNode(199...200)(:b)], + [], + [], + nil, + [], + nil, + nil + ), + StatementsNode(220...225)( + [CallNode(220...225)( + LocalVariableReadNode(220...221)(:a, 0), + nil, + (222...223), + nil, + ArgumentsNode(224...225)( + [LocalVariableReadNode(224...225)(:b, 0)] + ), + nil, + nil, + 0, + "+" + )] + ), + [:a, :b], + (168...171), + (176...177), + (178...179), + (200...201), + nil, + (243...246) + )] + ), + (266...269), + "X" + ), + ClassNode(293...376)( + [], + (293...298), + ConstantReadNode(299...300)(), + nil, + nil, + StatementsNode(315...358)( + [ClassNode(315...358)( + [], + (315...320), + ConstantReadNode(321...322)(), + nil, + nil, + StatementsNode(337...343)( + [ConstantWriteNode(337...343)( + (337...338), + IntegerNode(341...343)(), + (339...340) + )] + ), + (355...358), + "Y" + )] + ), + (373...376), + "X" + ), + ClassNode(395...498)( + [], + (395...400), + ConstantReadNode(401...402)(), + nil, + nil, + StatementsNode(417...480)( + [DefNode(417...480)( + (421...422), + nil, + ParametersNode(423...444)( + [RequiredParameterNode(423...424)(:a), + RequiredParameterNode(443...444)(:b)], + [], + [], + nil, + [], + nil, + nil + ), + StatementsNode(459...464)( + [CallNode(459...464)( + LocalVariableReadNode(459...460)(:a, 0), + nil, + (461...462), + nil, + ArgumentsNode(463...464)( + [LocalVariableReadNode(463...464)(:b, 0)] + ), + nil, + nil, + 0, + "+" + )] + ), + [:a, :b], + (417...420), + nil, + (422...423), + (444...445), + nil, + (477...480) + )] + ), + (495...498), + "X" + ), + ModuleNode(517...565)( + [], + (517...523), + ConstantReadNode(524...525)(), + StatementsNode(528...561)( + [ConstantWriteNode(528...561)( + (528...529), + ArrayNode(532...561)( + [SymbolNode(538...544)((538...539), (539...544), nil, "line3"), + SymbolNode(550...556)((550...551), (551...556), nil, "line4")], + (532...533), + (560...561) + ), + (530...531) + )] + ), + (562...565), + "X" + ), + ModuleNode(568...651)( + [], + (568...574), + ConstantReadNode(575...576)(), + StatementsNode(590...633)( + [ModuleNode(590...633)( + [], + (590...596), + ConstantReadNode(597...598)(), + StatementsNode(612...618)( + [ConstantWriteNode(612...618)( + (612...613), + IntegerNode(616...618)(), + (614...615) + )] + ), + (630...633), + "Y" + )] + ), + (648...651), + "X" + ), + CallNode(670...689)( + nil, + nil, + (670...671), + (671...672), + ArgumentsNode(673...687)( + [SymbolNode(673...679)((673...674), (674...679), nil, "line2"), + SymbolNode(681...687)((681...682), (682...687), nil, "line3")] + ), + (688...689), + nil, + 0, + "x" + )] + ) +) diff --git a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt index e54cde69f5737f..c672ef6b79bcae 100644 --- a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt +++ b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt @@ -1,15 +1,14 @@ ProgramNode(0...18)( [:a], StatementsNode(0...18)( - [LocalVariableOperatorOrWriteNode(0...18)( - (0...1), - (2...5), + [OrWriteNode(0...18)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), RescueModifierNode(6...18)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (8...14), NilNode(15...18)() ), - :a + (2...5) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index e339f35c8c8b61..8277b654c117ea 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -1,7 +1,7 @@ -ProgramNode(0...746)( +ProgramNode(0...747)( [:b, :_, :lhs, :x, :rhs, :c, :e], - StatementsNode(0...746)( - [CaseNode(0...20)( + StatementsNode(0...747)( + [CaseNode(0...21)( SymbolNode(5...7)((5...6), (6...7), nil, "a"), [InNode(8...16)( HashPatternNode(12...16)( @@ -21,207 +21,207 @@ ProgramNode(0...746)( )], nil, (0...4), - (17...20) + (18...21) ), - CaseNode(22...44)( - SymbolNode(27...29)((27...28), (28...29), nil, "a"), - [InNode(30...40)( - ArrayNode(33...40)( - [SymbolNode(36...37)(nil, (36...37), nil, "a"), - SymbolNode(38...39)(nil, (38...39), nil, "b")], - (33...36), - (39...40) + CaseNode(23...45)( + SymbolNode(28...30)((28...29), (29...30), nil, "a"), + [InNode(31...41)( + ArrayNode(34...41)( + [SymbolNode(37...38)(nil, (37...38), nil, "a"), + SymbolNode(39...40)(nil, (39...40), nil, "b")], + (34...37), + (40...41) ), nil, - (30...32), + (31...33), nil )], nil, - (22...26), - (41...44) + (23...27), + (42...45) ), - CaseNode(46...68)( - SymbolNode(51...53)((51...52), (52...53), nil, "a"), - [InNode(54...64)( - ArrayNode(57...64)( - [StringNode(60...61)(nil, (60...61), nil, "a"), - StringNode(62...63)(nil, (62...63), nil, "b")], - (57...60), - (63...64) + CaseNode(47...69)( + SymbolNode(52...54)((52...53), (53...54), nil, "a"), + [InNode(55...65)( + ArrayNode(58...65)( + [StringNode(61...62)(nil, (61...62), nil, "a"), + StringNode(63...64)(nil, (63...64), nil, "b")], + (58...61), + (64...65) ), nil, - (54...56), + (55...57), nil )], nil, - (46...50), - (65...68) + (47...51), + (66...69) ), - CaseNode(70...92)( - SymbolNode(75...77)((75...76), (76...77), nil, "a"), - [InNode(78...88)( - ArrayNode(81...88)( - [SymbolNode(84...85)(nil, (84...85), nil, "a"), - SymbolNode(86...87)(nil, (86...87), nil, "b")], - (81...84), - (87...88) + CaseNode(71...93)( + SymbolNode(76...78)((76...77), (77...78), nil, "a"), + [InNode(79...89)( + ArrayNode(82...89)( + [SymbolNode(85...86)(nil, (85...86), nil, "a"), + SymbolNode(87...88)(nil, (87...88), nil, "b")], + (82...85), + (88...89) ), nil, - (78...80), + (79...81), nil )], nil, - (70...74), - (89...92) + (71...75), + (90...93) ), - CaseNode(94...116)( - SymbolNode(99...101)((99...100), (100...101), nil, "a"), - [InNode(102...112)( - ArrayNode(105...112)( - [StringNode(108...109)(nil, (108...109), nil, "a"), - StringNode(110...111)(nil, (110...111), nil, "b")], - (105...108), - (111...112) + CaseNode(95...117)( + SymbolNode(100...102)((100...101), (101...102), nil, "a"), + [InNode(103...113)( + ArrayNode(106...113)( + [StringNode(109...110)(nil, (109...110), nil, "a"), + StringNode(111...112)(nil, (111...112), nil, "b")], + (106...109), + (112...113) ), nil, - (102...104), + (103...105), nil )], nil, - (94...98), - (113...116) + (95...99), + (114...117) ), - CaseNode(118...140)( - SymbolNode(123...125)((123...124), (124...125), nil, "a"), - [InNode(126...135)( - RangeNode(130...135)(nil, IntegerNode(133...135)(), (130...133), 1), + CaseNode(119...141)( + SymbolNode(124...126)((124...125), (125...126), nil, "a"), + [InNode(127...136)( + RangeNode(131...136)(nil, IntegerNode(134...136)(), (131...134), 1), nil, - (126...128), + (127...129), nil )], nil, - (118...122), - (137...140) + (119...123), + (138...141) ), - CaseNode(142...163)( - SymbolNode(147...149)((147...148), (148...149), nil, "a"), - [InNode(150...158)( - RangeNode(154...158)(nil, IntegerNode(156...158)(), (154...156), 0), + CaseNode(143...164)( + SymbolNode(148...150)((148...149), (149...150), nil, "a"), + [InNode(151...159)( + RangeNode(155...159)(nil, IntegerNode(157...159)(), (155...157), 0), nil, - (150...152), + (151...153), nil )], nil, - (142...146), - (160...163) + (143...147), + (161...164) ), - CaseNode(165...186)( - SymbolNode(170...172)((170...171), (171...172), nil, "a"), - [InNode(173...181)( - RangeNode(177...181)(IntegerNode(177...178)(), nil, (178...181), 1), + CaseNode(166...187)( + SymbolNode(171...173)((171...172), (172...173), nil, "a"), + [InNode(174...182)( + RangeNode(178...182)(IntegerNode(178...179)(), nil, (179...182), 1), nil, - (173...175), + (174...176), nil )], nil, - (165...169), - (183...186) + (166...170), + (184...187) ), - CaseNode(188...210)( - SymbolNode(193...195)((193...194), (194...195), nil, "a"), - [InNode(196...205)( - RangeNode(200...205)( - IntegerNode(200...201)(), - IntegerNode(204...205)(), - (201...204), + CaseNode(189...211)( + SymbolNode(194...196)((194...195), (195...196), nil, "a"), + [InNode(197...206)( + RangeNode(201...206)( + IntegerNode(201...202)(), + IntegerNode(205...206)(), + (202...205), 1 ), nil, - (196...198), + (197...199), nil )], nil, - (188...192), - (207...210) + (189...193), + (208...211) ), - CaseNode(212...231)( - SymbolNode(217...219)((217...218), (218...219), nil, "a"), - [InNode(220...226)(IntegerNode(224...226)(), nil, (220...222), nil)], + CaseNode(213...232)( + SymbolNode(218...220)((218...219), (219...220), nil, "a"), + [InNode(221...227)(IntegerNode(225...227)(), nil, (221...223), nil)], nil, - (212...216), - (228...231) + (213...217), + (229...232) ), - CaseNode(233...253)( - SymbolNode(238...240)((238...239), (239...240), nil, "a"), - [InNode(241...249)( - HashPatternNode(244...249)( + CaseNode(234...254)( + SymbolNode(239...241)((239...240), (240...241), nil, "a"), + [InNode(242...250)( + HashPatternNode(245...250)( nil, - [NoKeywordsParameterNode(244...249)((244...246), (246...249))], + [NoKeywordsParameterNode(245...250)((245...247), (247...250))], nil, nil, nil ), nil, - (241...243), + (242...244), nil )], nil, - (233...237), - (250...253) + (234...238), + (251...254) ), - CaseNode(255...278)( - SymbolNode(260...262)((260...261), (261...262), nil, "a"), - [InNode(263...274)( - RegularExpressionNode(266...274)( - (266...267), - (267...273), - (273...274), + CaseNode(256...279)( + SymbolNode(261...263)((261...262), (262...263), nil, "a"), + [InNode(264...275)( + RegularExpressionNode(267...275)( + (267...268), + (268...274), + (274...275), "regexp", 0 ), nil, - (263...265), + (264...266), nil )], nil, - (255...259), - (275...278) + (256...260), + (276...279) ), - CaseNode(280...305)( - SymbolNode(285...287)((285...286), (286...287), nil, "a"), - [InNode(288...301)( - ArrayPatternNode(291...301)( + CaseNode(281...306)( + SymbolNode(286...288)((286...287), (287...288), nil, "a"), + [InNode(289...302)( + ArrayPatternNode(292...302)( nil, - [SymbolNode(291...293)((291...292), (292...293), nil, "b")], - SplatNode(295...297)( - (295...296), - LocalVariableWriteNode(296...297)(:_, 0, nil, (296...297), nil) + [SymbolNode(292...294)((292...293), (293...294), nil, "b")], + SplatNode(296...298)( + (296...297), + LocalVariableWriteNode(297...298)(:_, 0, nil, (297...298), nil) ), - [SymbolNode(299...301)((299...300), (300...301), nil, "c")], + [SymbolNode(300...302)((300...301), (301...302), nil, "c")], nil, nil ), nil, - (288...290), + (289...291), nil )], nil, - (280...284), - (302...305) + (281...285), + (303...306) ), - CaseNode(307...330)( - SymbolNode(312...314)((312...313), (313...314), nil, "a"), - [InNode(315...326)( - ArrayPatternNode(318...326)( + CaseNode(308...331)( + SymbolNode(313...315)((313...314), (314...315), nil, "a"), + [InNode(316...327)( + ArrayPatternNode(319...327)( nil, - [SymbolNode(318...320)((318...319), (319...320), nil, "b"), - ArrayPatternNode(322...326)( + [SymbolNode(319...321)((319...320), (320...321), nil, "b"), + ArrayPatternNode(323...327)( nil, - [SymbolNode(323...325)((323...324), (324...325), nil, "c")], + [SymbolNode(324...326)((324...325), (325...326), nil, "c")], nil, [], - (322...323), - (325...326) + (323...324), + (326...327) )], nil, [], @@ -229,93 +229,93 @@ ProgramNode(0...746)( nil ), nil, - (315...317), + (316...318), nil )], nil, - (307...311), - (327...330) + (308...312), + (328...331) ), - CaseNode(332...355)( - SymbolNode(337...339)((337...338), (338...339), nil, "a"), - [InNode(340...351)( - ArrayPatternNode(343...351)( - ConstantReadNode(343...349)(), + CaseNode(333...356)( + SymbolNode(338...340)((338...339), (339...340), nil, "a"), + [InNode(341...352)( + ArrayPatternNode(344...352)( + ConstantReadNode(344...350)(), [], nil, [], - (349...350), - (350...351) + (350...351), + (351...352) ), nil, - (340...342), + (341...343), nil )], nil, - (332...336), - (352...355) + (333...337), + (353...356) ), - CaseNode(357...393)( - SymbolNode(362...364)((362...363), (363...364), nil, "a"), - [InNode(365...389)( - FindPatternNode(368...389)( - ConstantReadNode(368...374)(), - SplatNode(375...379)( - (375...376), - LocalVariableWriteNode(376...379)(:lhs, 0, nil, (376...379), nil) + CaseNode(358...394)( + SymbolNode(363...365)((363...364), (364...365), nil, "a"), + [InNode(366...390)( + FindPatternNode(369...390)( + ConstantReadNode(369...375)(), + SplatNode(376...380)( + (376...377), + LocalVariableWriteNode(377...380)(:lhs, 0, nil, (377...380), nil) ), - [LocalVariableWriteNode(381...382)(:x, 0, nil, (381...382), nil)], - SplatNode(384...388)( - (384...385), - LocalVariableWriteNode(385...388)(:rhs, 0, nil, (385...388), nil) + [LocalVariableWriteNode(382...383)(:x, 0, nil, (382...383), nil)], + SplatNode(385...389)( + (385...386), + LocalVariableWriteNode(386...389)(:rhs, 0, nil, (386...389), nil) ), - (374...375), - (388...389) + (375...376), + (389...390) ), nil, - (365...367), + (366...368), nil )], nil, - (357...361), - (390...393) + (358...362), + (391...394) ), - CaseNode(395...431)( - SymbolNode(400...402)((400...401), (401...402), nil, "a"), - [InNode(403...427)( - FindPatternNode(406...427)( - ConstantReadNode(406...412)(), - SplatNode(413...417)( - (413...414), - LocalVariableWriteNode(414...417)(:lhs, 0, nil, (414...417), nil) + CaseNode(396...432)( + SymbolNode(401...403)((401...402), (402...403), nil, "a"), + [InNode(404...428)( + FindPatternNode(407...428)( + ConstantReadNode(407...413)(), + SplatNode(414...418)( + (414...415), + LocalVariableWriteNode(415...418)(:lhs, 0, nil, (415...418), nil) ), - [LocalVariableWriteNode(419...420)(:x, 0, nil, (419...420), nil)], - SplatNode(422...426)( - (422...423), - LocalVariableWriteNode(423...426)(:rhs, 0, nil, (423...426), nil) + [LocalVariableWriteNode(420...421)(:x, 0, nil, (420...421), nil)], + SplatNode(423...427)( + (423...424), + LocalVariableWriteNode(424...427)(:rhs, 0, nil, (424...427), nil) ), - (412...413), - (426...427) + (413...414), + (427...428) ), nil, - (403...405), + (404...406), nil )], nil, - (395...399), - (428...431) + (396...400), + (429...432) ), - CaseNode(433...467)( - SymbolNode(438...440)((438...439), (439...440), nil, "a"), - [InNode(441...463)( - ArrayPatternNode(444...463)( + CaseNode(434...468)( + SymbolNode(439...441)((439...440), (440...441), nil, "a"), + [InNode(442...464)( + ArrayPatternNode(445...464)( nil, - [LambdaNode(445...459)( + [LambdaNode(446...460)( [:b], - (445...447), - BlockParametersNode(447...450)( - ParametersNode(448...449)( - [RequiredParameterNode(448...449)(:b)], + (446...448), + BlockParametersNode(448...451)( + ParametersNode(449...450)( + [RequiredParameterNode(449...450)(:b)], [], [], nil, @@ -324,156 +324,156 @@ ProgramNode(0...746)( nil ), [], - (447...448), - (449...450) + (448...449), + (450...451) ), - StatementsNode(453...457)([TrueNode(453...457)()]) + StatementsNode(454...458)([TrueNode(454...458)()]) ), - LocalVariableWriteNode(461...462)(:c, 0, nil, (461...462), nil)], + LocalVariableWriteNode(462...463)(:c, 0, nil, (462...463), nil)], nil, [], - (444...445), - (462...463) + (445...446), + (463...464) ), nil, - (441...443), + (442...444), nil )], nil, - (433...437), - (464...467) + (434...438), + (465...468) ), - CaseNode(469...509)( - SymbolNode(474...476)((474...475), (475...476), nil, "a"), - [InNode(477...505)( - ArrayPatternNode(480...505)( + CaseNode(470...510)( + SymbolNode(475...477)((475...476), (476...477), nil, "a"), + [InNode(478...506)( + ArrayPatternNode(481...506)( nil, - [SymbolNode(481...483)((481...482), (482...483), nil, "a"), - LocalVariableWriteNode(485...486)(:b, 0, nil, (485...486), nil), - LocalVariableWriteNode(488...489)(:c, 0, nil, (488...489), nil), - ArrayPatternNode(491...504)( + [SymbolNode(482...484)((482...483), (483...484), nil, "a"), + LocalVariableWriteNode(486...487)(:b, 0, nil, (486...487), nil), + LocalVariableWriteNode(489...490)(:c, 0, nil, (489...490), nil), + ArrayPatternNode(492...505)( nil, - [SymbolNode(492...494)((492...493), (493...494), nil, "d")], - SplatNode(496...498)( - (496...497), - LocalVariableWriteNode(497...498)( + [SymbolNode(493...495)((493...494), (494...495), nil, "d")], + SplatNode(497...499)( + (497...498), + LocalVariableWriteNode(498...499)( :e, 0, nil, - (497...498), + (498...499), nil ) ), - [NilNode(500...503)()], - (491...492), - (503...504) + [NilNode(501...504)()], + (492...493), + (504...505) )], nil, [], - (480...481), - (504...505) + (481...482), + (505...506) ), nil, - (477...479), + (478...480), nil )], nil, - (469...473), - (506...509) + (470...474), + (507...510) ), - CaseNode(511...535)( - SymbolNode(516...518)((516...517), (517...518), nil, "a"), - [InNode(519...531)( - ArrayPatternNode(522...531)( + CaseNode(512...536)( + SymbolNode(517...519)((517...518), (518...519), nil, "a"), + [InNode(520...532)( + ArrayPatternNode(523...532)( nil, - [ConstantReadNode(523...524)()], - SplatNode(526...527)((526...527), nil), - [ConstantReadNode(529...530)()], - (522...523), - (530...531) + [ConstantReadNode(524...525)()], + SplatNode(527...528)((527...528), nil), + [ConstantReadNode(530...531)()], + (523...524), + (531...532) ), nil, - (519...521), + (520...522), nil )], nil, - (511...515), - (532...535) + (512...516), + (533...536) ), - CaseNode(537...571)( - SymbolNode(542...544)((542...543), (543...544), nil, "a"), - [InNode(545...567)( - ArrayPatternNode(548...567)( + CaseNode(538...572)( + SymbolNode(543...545)((543...544), (544...545), nil, "a"), + [InNode(546...568)( + ArrayPatternNode(549...568)( nil, - [ArrayPatternNode(549...556)( + [ArrayPatternNode(550...557)( nil, - [SymbolNode(550...552)((550...551), (551...552), nil, "b"), - LocalVariableWriteNode(554...555)( + [SymbolNode(551...553)((551...552), (552...553), nil, "b"), + LocalVariableWriteNode(555...556)( :c, 0, nil, - (554...555), + (555...556), nil )], nil, [], - (549...550), - (555...556) + (550...551), + (556...557) ), - ArrayPatternNode(558...566)( + ArrayPatternNode(559...567)( nil, - [SymbolNode(559...561)((559...560), (560...561), nil, "d"), - PinnedVariableNode(563...565)( - LocalVariableReadNode(564...565)(:e, 0), - (563...564) + [SymbolNode(560...562)((560...561), (561...562), nil, "d"), + PinnedVariableNode(564...566)( + LocalVariableReadNode(565...566)(:e, 0), + (564...565) )], nil, [], - (558...559), - (565...566) + (559...560), + (566...567) )], nil, [], - (548...549), - (566...567) + (549...550), + (567...568) ), nil, - (545...547), + (546...548), nil )], nil, - (537...541), - (568...571) + (538...542), + (569...572) ), - CaseNode(573...590)( - SymbolNode(578...580)((578...579), (579...580), nil, "a"), - [InNode(581...586)( - ArrayPatternNode(584...586)( + CaseNode(574...591)( + SymbolNode(579...581)((579...580), (580...581), nil, "a"), + [InNode(582...587)( + ArrayPatternNode(585...587)( nil, [], nil, [], - (584...585), - (585...586) + (585...586), + (586...587) ), nil, - (581...583), + (582...584), nil )], nil, - (573...577), - (587...590) + (574...578), + (588...591) ), - CaseNode(592...613)( - SymbolNode(597...599)((597...598), (598...599), nil, "a"), - [InNode(600...609)( - ArrayPatternNode(603...609)( + CaseNode(593...614)( + SymbolNode(598...600)((598...599), (599...600), nil, "a"), + [InNode(601...610)( + ArrayPatternNode(604...610)( nil, - [PinnedExpressionNode(604...608)( - CallNode(606...607)( + [PinnedExpressionNode(605...609)( + CallNode(607...608)( nil, nil, - (606...607), + (607...608), nil, nil, nil, @@ -481,127 +481,127 @@ ProgramNode(0...746)( 2, "a" ), - (604...605), (605...606), - (607...608) + (606...607), + (608...609) )], nil, [], - (603...604), - (608...609) + (604...605), + (609...610) ), nil, - (600...602), + (601...603), nil )], nil, - (592...596), - (610...613) + (593...597), + (611...614) ), - CaseNode(615...646)( - SymbolNode(620...622)((620...621), (621...622), nil, "a"), - [InNode(623...642)( - ArrayPatternNode(626...642)( + CaseNode(616...647)( + SymbolNode(621...623)((621...622), (622...623), nil, "a"), + [InNode(624...643)( + ArrayPatternNode(627...643)( nil, - [PinnedVariableNode(627...630)( - InstanceVariableReadNode(628...630)(), - (627...628) + [PinnedVariableNode(628...631)( + InstanceVariableReadNode(629...631)(), + (628...629) ), - PinnedVariableNode(632...635)( - GlobalVariableReadNode(633...635)(), - (632...633) + PinnedVariableNode(633...636)( + GlobalVariableReadNode(634...636)(), + (633...634) ), - PinnedVariableNode(637...641)( - ClassVariableReadNode(638...641)(), - (637...638) + PinnedVariableNode(638...642)( + ClassVariableReadNode(639...642)(), + (638...639) )], nil, [], - (626...627), - (641...642) + (627...628), + (642...643) ), nil, - (623...625), + (624...626), nil )], nil, - (615...619), - (643...646) + (616...620), + (644...647) ), - CaseNode(648...672)( - SymbolNode(653...655)((653...654), (654...655), nil, "a"), - [InNode(656...668)( - XStringNode(659...668)( - (659...660), - (660...667), - (667...668), + CaseNode(649...673)( + SymbolNode(654...656)((654...655), (655...656), nil, "a"), + [InNode(657...669)( + XStringNode(660...669)( + (660...661), + (661...668), + (668...669), "echo hi" ), nil, - (656...658), + (657...659), nil )], nil, - (648...652), - (669...672) + (649...653), + (670...673) ), - CaseNode(674...702)( - SymbolNode(679...681)((679...680), (680...681), nil, "a"), - [InNode(682...698)( - ArrayPatternNode(685...698)( + CaseNode(675...703)( + SymbolNode(680...682)((680...681), (681...682), nil, "a"), + [InNode(683...699)( + ArrayPatternNode(686...699)( nil, - [NilNode(685...688)(), NilNode(690...693)(), NilNode(695...698)()], + [NilNode(686...689)(), NilNode(691...694)(), NilNode(696...699)()], nil, [], nil, nil ), nil, - (682...684), + (683...685), nil )], nil, - (674...678), - (699...702) + (675...679), + (700...703) ), - CaseNode(704...727)( - SymbolNode(709...711)((709...710), (710...711), nil, "a"), - [InNode(712...723)( - HashPatternNode(715...723)( + CaseNode(705...728)( + SymbolNode(710...712)((710...711), (711...712), nil, "a"), + [InNode(713...724)( + HashPatternNode(716...724)( nil, - [AssocNode(717...721)( - SymbolNode(717...721)( - (717...718), + [AssocNode(718...722)( + SymbolNode(718...722)( (718...719), - (719...721), + (719...720), + (720...722), "b" ), nil, nil )], nil, - (715...716), - (722...723) + (716...717), + (723...724) ), nil, - (712...714), + (713...715), nil )], nil, - (704...708), - (724...727) + (705...709), + (725...728) ), - CaseNode(729...746)( - SymbolNode(734...736)((734...735), (735...736), nil, "a"), - [InNode(737...742)( - HashPatternNode(740...742)(nil, [], nil, (740...741), (741...742)), + CaseNode(730...747)( + SymbolNode(735...737)((735...736), (736...737), nil, "a"), + [InNode(738...743)( + HashPatternNode(741...743)(nil, [], nil, (741...742), (742...743)), nil, - (737...739), + (738...740), nil )], nil, - (729...733), - (743...746) + (730...734), + (744...747) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/class_comments.txt b/test/yarp/snapshots/seattlerb/class_comments.txt index 12b347ee8ab4ee..b96c5d6367ea0e 100644 --- a/test/yarp/snapshots/seattlerb/class_comments.txt +++ b/test/yarp/snapshots/seattlerb/class_comments.txt @@ -22,7 +22,8 @@ ProgramNode(19...71)( (64...67) )] ), - (68...71) + (68...71), + "X" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt index dfaac5447cfc80..7405ad8e5d9d22 100644 --- a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt +++ b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt @@ -1,14 +1,18 @@ ProgramNode(0...12)( [], StatementsNode(0...12)( - [ConstantPathOperatorOrWriteNode(0...12)( - ConstantPathNode(0...6)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - ConstantReadNode(5...6)(), - (3...5) + [OrWriteNode(0...12)( + ConstantPathWriteNode(0...6)( + ConstantPathNode(0...6)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + ConstantReadNode(5...6)(), + (3...5) + ), + nil, + nil ), - (7...10), - IntegerNode(11...12)() + IntegerNode(11...12)(), + (7...10) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt index c9d4dbfe3132ca..838287c59fa9f7 100644 --- a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt @@ -1,10 +1,14 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [ConstantPathOperatorOrWriteNode(0...9)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - (4...7), - IntegerNode(8...9)() + [OrWriteNode(0...9)( + ConstantPathWriteNode(0...3)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + nil, + nil + ), + IntegerNode(8...9)(), + (4...7) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt index 1e7dce022689a5..2fdcd97205fd82 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt @@ -1,11 +1,15 @@ ProgramNode(0...8)( [], StatementsNode(0...8)( - [ConstantPathOperatorWriteNode(0...8)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + [OperatorWriteNode(0...8)( + ConstantPathWriteNode(0...3)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + nil, + nil + ), (4...6), - IntegerNode(7...8)(), - :& + :&, + IntegerNode(7...8)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt index 85739dac2d6b62..275aa8238d4735 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt @@ -1,10 +1,14 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [ConstantPathOperatorAndWriteNode(0...9)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - (4...7), - IntegerNode(8...9)() + [AndWriteNode(0...9)( + ConstantPathWriteNode(0...3)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + nil, + nil + ), + IntegerNode(8...9)(), + (4...7) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt index d1bfe6cd4dd2e4..c14175c4e3a8a2 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt @@ -1,14 +1,18 @@ ProgramNode(0...10)( [], StatementsNode(0...10)( - [ConstantPathOperatorOrWriteNode(0...10)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) + [OrWriteNode(0...10)( + ConstantPathWriteNode(0...4)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) + ), + nil, + nil ), - (5...8), - IntegerNode(9...10)() + IntegerNode(9...10)(), + (5...8) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/defn_forward_args__no_parens.txt b/test/yarp/snapshots/seattlerb/defn_forward_args__no_parens.txt new file mode 100644 index 00000000000000..14d94bc244a9cc --- /dev/null +++ b/test/yarp/snapshots/seattlerb/defn_forward_args__no_parens.txt @@ -0,0 +1,38 @@ +ProgramNode(0...22)( + [], + StatementsNode(0...22)( + [DefNode(0...22)( + (4...5), + nil, + ParametersNode(6...9)( + [], + [], + [], + nil, + [], + ForwardingParameterNode(6...9)(), + nil + ), + StatementsNode(12...18)( + [CallNode(12...18)( + nil, + nil, + (12...13), + (13...14), + ArgumentsNode(14...17)([ForwardingArgumentsNode(14...17)()]), + (17...18), + nil, + 0, + "m" + )] + ), + [:"..."], + (0...3), + nil, + nil, + nil, + nil, + (19...22) + )] + ) +) diff --git a/test/yarp/snapshots/seattlerb/defn_oneliner_eq2.txt b/test/yarp/snapshots/seattlerb/defn_oneliner_eq2.txt index 30aaf7190e8da5..d5798209244334 100644 --- a/test/yarp/snapshots/seattlerb/defn_oneliner_eq2.txt +++ b/test/yarp/snapshots/seattlerb/defn_oneliner_eq2.txt @@ -30,7 +30,8 @@ ProgramNode(0...28)( nil )] ), - (25...28) + (25...28), + "X" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/defs_oneliner_eq2.txt b/test/yarp/snapshots/seattlerb/defs_oneliner_eq2.txt index 8a4f9e9086092c..2ec8dc867263f3 100644 --- a/test/yarp/snapshots/seattlerb/defs_oneliner_eq2.txt +++ b/test/yarp/snapshots/seattlerb/defs_oneliner_eq2.txt @@ -30,7 +30,8 @@ ProgramNode(0...33)( nil )] ), - (30...33) + (30...33), + "X" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_wtf_I_hate_you.txt b/test/yarp/snapshots/seattlerb/difficult0_.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/heredoc_wtf_I_hate_you.txt rename to test/yarp/snapshots/seattlerb/difficult0_.txt diff --git a/test/yarp/snapshots/seattlerb/i_fucking_hate_line_numbers.txt b/test/yarp/snapshots/seattlerb/difficult1_line_numbers.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/i_fucking_hate_line_numbers.txt rename to test/yarp/snapshots/seattlerb/difficult1_line_numbers.txt diff --git a/test/yarp/snapshots/seattlerb/i_fucking_hate_line_numbers2.txt b/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/i_fucking_hate_line_numbers2.txt rename to test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt diff --git a/test/yarp/snapshots/seattlerb/i_have_no_freakin_clue.txt b/test/yarp/snapshots/seattlerb/difficult2_.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/i_have_no_freakin_clue.txt rename to test/yarp/snapshots/seattlerb/difficult2_.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me.txt b/test/yarp/snapshots/seattlerb/difficult3_.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me.txt rename to test/yarp/snapshots/seattlerb/difficult3_.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me2.txt b/test/yarp/snapshots/seattlerb/difficult3_2.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me2.txt rename to test/yarp/snapshots/seattlerb/difficult3_2.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me3.txt b/test/yarp/snapshots/seattlerb/difficult3_3.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me3.txt rename to test/yarp/snapshots/seattlerb/difficult3_3.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me4.txt b/test/yarp/snapshots/seattlerb/difficult3_4.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me4.txt rename to test/yarp/snapshots/seattlerb/difficult3_4.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me5.txt b/test/yarp/snapshots/seattlerb/difficult3_5.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me5.txt rename to test/yarp/snapshots/seattlerb/difficult3_5.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_10.txt b/test/yarp/snapshots/seattlerb/difficult3__10.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_10.txt rename to test/yarp/snapshots/seattlerb/difficult3__10.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_11.txt b/test/yarp/snapshots/seattlerb/difficult3__11.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_11.txt rename to test/yarp/snapshots/seattlerb/difficult3__11.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_12.txt b/test/yarp/snapshots/seattlerb/difficult3__12.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_12.txt rename to test/yarp/snapshots/seattlerb/difficult3__12.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_6.txt b/test/yarp/snapshots/seattlerb/difficult3__6.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_6.txt rename to test/yarp/snapshots/seattlerb/difficult3__6.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_7.txt b/test/yarp/snapshots/seattlerb/difficult3__7.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_7.txt rename to test/yarp/snapshots/seattlerb/difficult3__7.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_8.txt b/test/yarp/snapshots/seattlerb/difficult3__8.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_8.txt rename to test/yarp/snapshots/seattlerb/difficult3__8.txt diff --git a/test/yarp/snapshots/seattlerb/kill_me_9.txt b/test/yarp/snapshots/seattlerb/difficult3__9.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/kill_me_9.txt rename to test/yarp/snapshots/seattlerb/difficult3__9.txt diff --git a/test/yarp/snapshots/seattlerb/motherfuckin_leading_dots.txt b/test/yarp/snapshots/seattlerb/difficult4__leading_dots.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/motherfuckin_leading_dots.txt rename to test/yarp/snapshots/seattlerb/difficult4__leading_dots.txt diff --git a/test/yarp/snapshots/seattlerb/motherfuckin_leading_dots2.txt b/test/yarp/snapshots/seattlerb/difficult4__leading_dots2.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/motherfuckin_leading_dots2.txt rename to test/yarp/snapshots/seattlerb/difficult4__leading_dots2.txt diff --git a/test/yarp/snapshots/seattlerb/wtf.txt b/test/yarp/snapshots/seattlerb/difficult6_.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/wtf.txt rename to test/yarp/snapshots/seattlerb/difficult6_.txt diff --git a/test/yarp/snapshots/seattlerb/wtf_7.txt b/test/yarp/snapshots/seattlerb/difficult6__7.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/wtf_7.txt rename to test/yarp/snapshots/seattlerb/difficult6__7.txt diff --git a/test/yarp/snapshots/seattlerb/wtf_8.txt b/test/yarp/snapshots/seattlerb/difficult6__8.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/wtf_8.txt rename to test/yarp/snapshots/seattlerb/difficult6__8.txt diff --git a/test/yarp/snapshots/seattlerb/zomg_sometimes_i_hate_this_project.txt b/test/yarp/snapshots/seattlerb/difficult7_.txt similarity index 100% rename from test/yarp/snapshots/seattlerb/zomg_sometimes_i_hate_this_project.txt rename to test/yarp/snapshots/seattlerb/difficult7_.txt diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt index 3bb7807c5c6d6a..680ffbe0a68dd0 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt @@ -6,8 +6,8 @@ ProgramNode(0...10)( 0, InterpolatedStringNode(4...10)( (4...10), - [StringNode(11...20)(nil, (11...20), nil, "x\n" + "\n" + "z\n")], - (20...24) + [StringNode(11...21)(nil, (11...21), nil, "x\n" + "\n" + "z\n")], + (21...25) ), (0...1), (2...3) diff --git a/test/yarp/snapshots/seattlerb/if_elsif.txt b/test/yarp/snapshots/seattlerb/if_elsif.txt index 6f1edfb62ef4ab..51c89a9a32b540 100644 --- a/test/yarp/snapshots/seattlerb/if_elsif.txt +++ b/test/yarp/snapshots/seattlerb/if_elsif.txt @@ -5,7 +5,7 @@ ProgramNode(0...18)( (0...2), IntegerNode(3...4)(), nil, - IfNode(6...13)((6...11), IntegerNode(12...13)(), nil, nil, nil), + IfNode(6...18)((6...11), IntegerNode(12...13)(), nil, nil, (15...18)), (15...18) )] ) diff --git a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt index 2ab320eda88614..08c8d9ea3fca24 100644 --- a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt +++ b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt @@ -30,7 +30,8 @@ ProgramNode(18...90)( (83...86) )] ), - (87...90) + (87...90), + "ExampleUTF8ClassNameVarietà" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt index 71a28870bd2240..8c678e890d92d8 100644 --- a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt +++ b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt @@ -9,13 +9,18 @@ ProgramNode(0...15)( ArgumentsNode(2...15)( [ParenthesesNode(2...15)( StatementsNode(3...14)( - [ConstantPathOperatorWriteNode(3...14)( - ConstantPathNode(3...7)( - ConstantReadNode(3...4)(), - ConstantReadNode(6...7)(), - (4...6) + [OperatorWriteNode(3...14)( + ConstantPathWriteNode(3...7)( + ConstantPathNode(3...7)( + ConstantReadNode(3...4)(), + ConstantReadNode(6...7)(), + (4...6) + ), + nil, + nil ), (8...10), + :*, CallNode(11...14)( nil, nil, @@ -38,8 +43,7 @@ ProgramNode(0...15)( nil, 0, "d" - ), - :* + ) )] ), (2...3), diff --git a/test/yarp/snapshots/seattlerb/module_comments.txt b/test/yarp/snapshots/seattlerb/module_comments.txt index 5d11f23a4d014a..4f399904111e29 100644 --- a/test/yarp/snapshots/seattlerb/module_comments.txt +++ b/test/yarp/snapshots/seattlerb/module_comments.txt @@ -1,26 +1,27 @@ -ProgramNode(22...75)( +ProgramNode(24...77)( [], - StatementsNode(22...75)( - [ModuleNode(22...75)( + StatementsNode(24...77)( + [ModuleNode(24...77)( [], - (22...28), - ConstantReadNode(29...30)(), - StatementsNode(44...71)( - [DefNode(44...71)( - (48...52), + (24...30), + ConstantReadNode(31...32)(), + StatementsNode(46...73)( + [DefNode(46...73)( + (50...54), nil, nil, nil, [], - (44...47), + (46...49), nil, nil, nil, nil, - (68...71) + (70...73) )] ), - (72...75) + (74...77), + "X" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt index 1daadaab67948b..deb7e17066b389 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt @@ -1,9 +1,8 @@ ProgramNode(0...11)( [:a], StatementsNode(0...11)( - [LocalVariableOperatorOrWriteNode(0...11)( - (0...1), - (2...5), + [OrWriteNode(0...11)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), CallNode(6...11)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (7...8), @@ -15,7 +14,7 @@ ProgramNode(0...11)( 0, "c" ), - :a + (2...5) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt index 0cd3775202dff6..d071a45442fe3c 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt @@ -1,13 +1,18 @@ ProgramNode(0...11)( [], StatementsNode(0...11)( - [ConstantPathOperatorWriteNode(0...11)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) + [OperatorWriteNode(0...11)( + ConstantPathWriteNode(0...4)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) + ), + nil, + nil ), (5...7), + :*, CallNode(8...11)( nil, nil, @@ -20,8 +25,7 @@ ProgramNode(0...11)( nil, 0, "c" - ), - :* + ) )] ) ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt index adf81b25d11eb6..5d4820bef3cf7f 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt @@ -25,12 +25,11 @@ ProgramNode(0...40)( 0, "p" ), - LocalVariableOperatorWriteNode(18...24)( - (18...19), + OperatorWriteNode(18...24)( + LocalVariableWriteNode(18...19)(:y, 0, nil, (18...19), nil), (20...22), - IntegerNode(23...24)(), - :y, - :* + :*, + IntegerNode(23...24)() ), ReturnNode(27...35)( (27...33), diff --git a/test/yarp/snapshots/seattlerb/parse_line_heredoc_hardnewline.txt b/test/yarp/snapshots/seattlerb/parse_line_heredoc_hardnewline.txt index 4f7ae811cfdf51..4445e447d015e9 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_heredoc_hardnewline.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_heredoc_hardnewline.txt @@ -18,7 +18,8 @@ ProgramNode(0...48)( nil, nil, nil, - (45...48) + (45...48), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt index 011858c31e5bf9..d8bd9891848f16 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt @@ -1,12 +1,11 @@ ProgramNode(6...34)( [:foo], StatementsNode(6...34)( - [LocalVariableOperatorWriteNode(6...24)( - (6...9), + [OperatorWriteNode(6...24)( + LocalVariableWriteNode(6...9)(:foo, 0, nil, (6...9), nil), (10...12), - CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar"), - :foo, - :+ + :+, + CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar") ), CallNode(31...34)(nil, nil, (31...34), nil, nil, nil, nil, 2, "baz")] ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_trailing_newlines.txt b/test/yarp/snapshots/seattlerb/parse_line_trailing_newlines.txt index e48c600649bbf7..7617e15479c07b 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_trailing_newlines.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_trailing_newlines.txt @@ -1,7 +1,7 @@ -ProgramNode(0...3)( +ProgramNode(0...4)( [], - StatementsNode(0...3)( + StatementsNode(0...4)( [CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), - CallNode(2...3)(nil, nil, (2...3), nil, nil, nil, nil, 2, "b")] + CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b")] ) ) diff --git a/test/yarp/snapshots/seattlerb/pct_Q_backslash_nl.txt b/test/yarp/snapshots/seattlerb/pct_Q_backslash_nl.txt index 69721af37214f0..0725aaee499427 100644 --- a/test/yarp/snapshots/seattlerb/pct_Q_backslash_nl.txt +++ b/test/yarp/snapshots/seattlerb/pct_Q_backslash_nl.txt @@ -1,4 +1,6 @@ ProgramNode(0...7)( [], - StatementsNode(0...7)([StringNode(0...7)((0...3), (3...6), (6...7), " ")]) + StatementsNode(0...7)( + [StringNode(0...7)((0...3), (3...6), (6...7), " \\\n")] + ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 1423b7e4d071a7..ffaedf4f1834b5 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -610,10 +610,10 @@ ProgramNode(0...704)( ), (543...546) ), - InstanceVariableOperatorOrWriteNode(551...561)( - (551...553), - (554...557), - StringNode(558...561)((558...560), (560...560), (560...561), "") + OrWriteNode(551...561)( + InstanceVariableWriteNode(551...553)((551...553), nil, nil), + StringNode(558...561)((558...560), (560...560), (560...561), ""), + (554...557) ), LocalVariableWriteNode(562...576)( :x, @@ -703,16 +703,16 @@ ProgramNode(0...704)( ), (665...668) ), - InstanceVariableOperatorOrWriteNode(687...704)( - (687...689), - (690...693), + OrWriteNode(687...704)( + InstanceVariableWriteNode(687...689)((687...689), nil, nil), InterpolatedStringNode(694...704)( (694...704), [StringNode(705...707)(nil, (705...707), nil, " "), EmbeddedStatementsNode(707...710)((707...709), nil, (709...710)), StringNode(710...711)(nil, (710...711), nil, "\n")], (711...719) - ) + ), + (690...693) )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/class.txt b/test/yarp/snapshots/unparser/corpus/literal/class.txt index 96b50c7adf3634..aa457618026144 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/class.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/class.txt @@ -8,7 +8,8 @@ ProgramNode(0...213)( nil, nil, nil, - (8...11) + (8...11), + "A" ), SingletonClassNode(13...27)( [], @@ -39,7 +40,8 @@ ProgramNode(0...213)( nil, nil, nil, - (60...63) + (60...63), + "B" ), ClassNode(65...82)( [], @@ -56,7 +58,8 @@ ProgramNode(0...213)( nil, nil, nil, - (79...82) + (79...82), + "C" ), ClassNode(84...99)( [], @@ -65,7 +68,8 @@ ProgramNode(0...213)( (92...93), ConstantReadNode(94...95)(), nil, - (96...99) + (96...99), + "A" ), ClassNode(101...119)( [], @@ -78,7 +82,8 @@ ProgramNode(0...213)( (112...114) ), nil, - (116...119) + (116...119), + "A" ), ClassNode(121...142)( [], @@ -95,7 +100,8 @@ ProgramNode(0...213)( (135...137) ), nil, - (139...142) + (139...142), + "B" ), ClassNode(144...198)( [], @@ -143,7 +149,8 @@ ProgramNode(0...213)( (191...194) )] ), - (195...198) + (195...198), + "A" ), ClassNode(200...213)( [], @@ -156,7 +163,8 @@ ProgramNode(0...213)( nil, nil, nil, - (210...213) + (210...213), + "A" )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/if.txt b/test/yarp/snapshots/unparser/corpus/literal/if.txt index 8375e9acb36e52..1bfbe7e26ff5c5 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/if.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/if.txt @@ -80,7 +80,8 @@ ProgramNode(0...246)( nil )] ), - (130...133) + (130...133), + "A" ), ModuleNode(135...170)( [:foo], @@ -113,7 +114,8 @@ ProgramNode(0...246)( nil )] ), - (167...170) + (167...170), + "B" ), UnlessNode(171...197)( (171...177), diff --git a/test/yarp/snapshots/unparser/corpus/literal/module.txt b/test/yarp/snapshots/unparser/corpus/literal/module.txt index c917fedd23b291..4c805d3d67f0be 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/module.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/module.txt @@ -1,7 +1,14 @@ ProgramNode(0...106)( [], StatementsNode(0...106)( - [ModuleNode(0...12)([], (0...6), ConstantReadNode(7...8)(), nil, (9...12)), + [ModuleNode(0...12)( + [], + (0...6), + ConstantReadNode(7...8)(), + nil, + (9...12), + "A" + ), ModuleNode(14...29)( [], (14...20), @@ -11,7 +18,8 @@ ProgramNode(0...106)( (22...24) ), nil, - (26...29) + (26...29), + "B" ), ModuleNode(31...49)( [], @@ -26,7 +34,8 @@ ProgramNode(0...106)( (42...44) ), nil, - (46...49) + (46...49), + "C" ), ModuleNode(51...106)( [], @@ -72,7 +81,8 @@ ProgramNode(0...106)( (99...102) )] ), - (103...106) + (103...106), + "A" )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt index 04f547fc9ef616..980459766311ca 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt @@ -1,61 +1,53 @@ ProgramNode(0...233)( [:a, :h], StatementsNode(0...233)( - [LocalVariableOperatorWriteNode(0...6)( - (0...1), + [OperatorWriteNode(0...6)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), (2...4), - IntegerNode(5...6)(), - :a, - :+ + :+, + IntegerNode(5...6)() ), - LocalVariableOperatorWriteNode(7...13)( - (7...8), + OperatorWriteNode(7...13)( + LocalVariableWriteNode(7...8)(:a, 0, nil, (7...8), nil), (9...11), - IntegerNode(12...13)(), - :a, - :- + :-, + IntegerNode(12...13)() ), - LocalVariableOperatorWriteNode(14...21)( - (14...15), + OperatorWriteNode(14...21)( + LocalVariableWriteNode(14...15)(:a, 0, nil, (14...15), nil), (16...19), - IntegerNode(20...21)(), - :a, - :** + :**, + IntegerNode(20...21)() ), - LocalVariableOperatorWriteNode(22...28)( - (22...23), + OperatorWriteNode(22...28)( + LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), (24...26), - IntegerNode(27...28)(), - :a, - :* + :*, + IntegerNode(27...28)() ), - LocalVariableOperatorWriteNode(29...35)( - (29...30), + OperatorWriteNode(29...35)( + LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), (31...33), - IntegerNode(34...35)(), - :a, - :/ + :/, + IntegerNode(34...35)() ), - LocalVariableOperatorAndWriteNode(36...43)( - (36...37), - (38...41), + AndWriteNode(36...43)( + LocalVariableWriteNode(36...37)(:a, 0, nil, (36...37), nil), CallNode(42...43)(nil, nil, (42...43), nil, nil, nil, nil, 2, "b"), - :a + (38...41) ), - LocalVariableOperatorOrWriteNode(44...51)( - (44...45), - (46...49), + OrWriteNode(44...51)( + LocalVariableWriteNode(44...45)(:a, 0, nil, (44...45), nil), IntegerNode(50...51)(), - :a + (46...49) ), CallNode(52...65)( ParenthesesNode(52...61)( StatementsNode(53...60)( - [LocalVariableOperatorOrWriteNode(53...60)( - (53...54), - (55...58), + [OrWriteNode(53...60)( + LocalVariableWriteNode(53...54)(:a, 0, nil, (53...54), nil), IntegerNode(59...60)(), - :a + (55...58) )] ), (52...53), @@ -73,11 +65,10 @@ ProgramNode(0...233)( CallNode(66...83)( ParenthesesNode(66...76)( StatementsNode(67...75)( - [LocalVariableOperatorOrWriteNode(67...75)( - (67...68), - (69...72), + [OrWriteNode(67...75)( + LocalVariableWriteNode(67...68)(:h, 0, nil, (67...68), nil), HashNode(73...75)((73...74), [], (74...75)), - :h + (69...72) )] ), (66...67), diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index e6ba96546f22ff..083a0140d9b974 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -1,14 +1,13 @@ -ProgramNode(0...991)( +ProgramNode(0...999)( [], - StatementsNode(0...991)( + StatementsNode(0...999)( [ModuleNode(0...35)( [:foo, :a, :_], (0...6), ConstantReadNode(7...8)(), StatementsNode(11...31)( - [LocalVariableOperatorOrWriteNode(11...31)( - (11...14), - (15...18), + [OrWriteNode(11...31)( + LocalVariableWriteNode(11...14)(:foo, 0, nil, (11...14), nil), ParenthesesNode(19...31)( StatementsNode(21...30)( [MultiWriteNode(21...30)( @@ -45,10 +44,11 @@ ProgramNode(0...991)( (19...20), (30...31) ), - :foo + (15...18) )] ), - (32...35) + (32...35), + "A" ), ModuleNode(37...73)( [:local], @@ -74,7 +74,8 @@ ProgramNode(0...991)( "bar" )] ), - (70...73) + (70...73), + "A" ), CallNode(74...89)( ClassNode(74...85)( @@ -84,7 +85,8 @@ ProgramNode(0...991)( nil, nil, nil, - (82...85) + (82...85), + "A" ), (85...86), (86...89), @@ -101,7 +103,8 @@ ProgramNode(0...991)( (90...96), ConstantReadNode(97...98)(), nil, - (99...102) + (99...102), + "A" ), (102...103), (103...106), @@ -2026,6 +2029,29 @@ ProgramNode(0...991)( nil, 0, "~" + ), + CallNode(992...999)( + CallNode(992...993)(nil, nil, (992...993), nil, nil, nil, nil, 2, "a"), + (993...995), + (995...996), + (996...997), + ArgumentsNode(997...998)( + [CallNode(997...998)( + nil, + nil, + (997...998), + nil, + nil, + nil, + nil, + 2, + "b" + )] + ), + (998...999), + nil, + 1, + "+" )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/while.txt b/test/yarp/snapshots/unparser/corpus/literal/while.txt index 9808846085f0c9..dae7e0f0f31215 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/while.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/while.txt @@ -62,7 +62,8 @@ ProgramNode(0...620)( "foo" )] ), - (65...68) + (65...68), + "A" ), DefNode(70...110)( (74...77), @@ -154,7 +155,8 @@ ProgramNode(0...620)( 0 )] ), - (143...146) + (143...146), + "A" ), ModuleNode(148...182)( [:foo], @@ -186,7 +188,8 @@ ProgramNode(0...620)( 0 )] ), - (179...182) + (179...182), + "A" ), ModuleNode(184...228)( [:foo], @@ -228,7 +231,8 @@ ProgramNode(0...620)( 0 )] ), - (225...228) + (225...228), + "A" ), ModuleNode(230...299)( [], @@ -301,7 +305,8 @@ ProgramNode(0...620)( "each" )] ), - (296...299) + (296...299), + "A" ), ModuleNode(301...370)( [], @@ -364,7 +369,8 @@ ProgramNode(0...620)( "each" )] ), - (367...370) + (367...370), + "A" ), LocalVariableWriteNode(371...402)( :x, diff --git a/test/yarp/snapshots/unparser/corpus/semantic/while.txt b/test/yarp/snapshots/unparser/corpus/semantic/while.txt index fd495fbd587488..148db33da6523f 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/while.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/while.txt @@ -178,7 +178,8 @@ ProgramNode(0...188)( 0 )] ), - (185...188) + (185...188), + "A" )] ) ) diff --git a/test/yarp/snapshots/while.txt b/test/yarp/snapshots/while.txt index cc0d2fbf6ebe76..484eb68b2c9c02 100644 --- a/test/yarp/snapshots/while.txt +++ b/test/yarp/snapshots/while.txt @@ -130,7 +130,8 @@ ProgramNode(0...314)( (181...182) )] ), - (195...198) + (195...198), + "Foo" ), StatementsNode(200...205)([BreakNode(200...205)(nil, (200...205))]), 0 diff --git a/test/yarp/snapshots/whitequark/class.txt b/test/yarp/snapshots/whitequark/class.txt index 0061b34a8a52d8..c176350d059daa 100644 --- a/test/yarp/snapshots/whitequark/class.txt +++ b/test/yarp/snapshots/whitequark/class.txt @@ -8,7 +8,8 @@ ProgramNode(0...29)( nil, nil, nil, - (10...13) + (10...13), + "Foo" ), ClassNode(15...29)( [], @@ -17,7 +18,8 @@ ProgramNode(0...29)( nil, nil, nil, - (26...29) + (26...29), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt index a7bee91d90074b..0af9f40e8a7ec7 100644 --- a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt @@ -84,7 +84,8 @@ ProgramNode(0...197)( (122...123) )] ), - (136...139) + (136...139), + "Foo" ), StatementsNode(141...146)([BreakNode(141...146)(nil, (141...146))]), 0 @@ -110,7 +111,8 @@ ProgramNode(0...197)( "tap" )] ), - (182...185) + (182...185), + "Foo" ), StatementsNode(187...192)([BreakNode(187...192)(nil, (187...192))]), 0 diff --git a/test/yarp/snapshots/whitequark/class_super.txt b/test/yarp/snapshots/whitequark/class_super.txt index aa2845a13edc57..3dda8280a1628e 100644 --- a/test/yarp/snapshots/whitequark/class_super.txt +++ b/test/yarp/snapshots/whitequark/class_super.txt @@ -8,7 +8,8 @@ ProgramNode(0...20)( (10...11), ConstantReadNode(12...15)(), nil, - (17...20) + (17...20), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/whitequark/class_super_label.txt b/test/yarp/snapshots/whitequark/class_super_label.txt index ae6759c1e42d37..17e92e1693d97c 100644 --- a/test/yarp/snapshots/whitequark/class_super_label.txt +++ b/test/yarp/snapshots/whitequark/class_super_label.txt @@ -20,7 +20,8 @@ ProgramNode(0...20)( "a" ), nil, - (17...20) + (17...20), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/whitequark/const_op_asgn.txt b/test/yarp/snapshots/whitequark/const_op_asgn.txt index ef5e755c8ea1c7..e212170a1d799d 100644 --- a/test/yarp/snapshots/whitequark/const_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/const_op_asgn.txt @@ -1,41 +1,53 @@ ProgramNode(0...77)( [], StatementsNode(0...77)( - [ConstantPathOperatorWriteNode(0...8)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + [OperatorWriteNode(0...8)( + ConstantPathWriteNode(0...3)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + nil, + nil + ), (4...6), - IntegerNode(7...8)(), - :+ + :+, + IntegerNode(7...8)() ), - ConstantOperatorWriteNode(10...16)( - (10...11), + OperatorWriteNode(10...16)( + ConstantWriteNode(10...11)((10...11), nil, nil), (12...14), - IntegerNode(15...16)(), - :+ + :+, + IntegerNode(15...16)() ), - ConstantPathOperatorWriteNode(18...27)( - ConstantPathNode(18...22)( - ConstantReadNode(18...19)(), - ConstantReadNode(21...22)(), - (19...21) + OperatorWriteNode(18...27)( + ConstantPathWriteNode(18...22)( + ConstantPathNode(18...22)( + ConstantReadNode(18...19)(), + ConstantReadNode(21...22)(), + (19...21) + ), + nil, + nil ), (23...25), - IntegerNode(26...27)(), - :+ + :+, + IntegerNode(26...27)() ), DefNode(29...50)( (33...34), nil, nil, StatementsNode(36...45)( - [ConstantPathOperatorOrWriteNode(36...45)( - ConstantPathNode(36...39)( + [OrWriteNode(36...45)( + ConstantPathWriteNode(36...39)( + ConstantPathNode(36...39)( + nil, + ConstantReadNode(38...39)(), + (36...38) + ), nil, - ConstantReadNode(38...39)(), - (36...38) + nil ), - (40...43), - IntegerNode(44...45)() + IntegerNode(44...45)(), + (40...43) )] ), [], @@ -51,14 +63,18 @@ ProgramNode(0...77)( nil, nil, StatementsNode(59...72)( - [ConstantPathOperatorOrWriteNode(59...72)( - ConstantPathNode(59...66)( - SelfNode(59...63)(), - ConstantReadNode(65...66)(), - (63...65) + [OrWriteNode(59...72)( + ConstantPathWriteNode(59...66)( + ConstantPathNode(59...66)( + SelfNode(59...63)(), + ConstantReadNode(65...66)(), + (63...65) + ), + nil, + nil ), - (67...70), - IntegerNode(71...72)() + IntegerNode(71...72)(), + (67...70) )] ), [], diff --git a/test/yarp/snapshots/whitequark/cpath.txt b/test/yarp/snapshots/whitequark/cpath.txt index 94714a1ac6dce0..2f8387115c62fd 100644 --- a/test/yarp/snapshots/whitequark/cpath.txt +++ b/test/yarp/snapshots/whitequark/cpath.txt @@ -6,7 +6,8 @@ ProgramNode(0...39)( (0...6), ConstantPathNode(7...12)(nil, ConstantReadNode(9...12)(), (7...9)), nil, - (14...17) + (14...17), + "Foo" ), ModuleNode(19...39)( [], @@ -17,7 +18,8 @@ ProgramNode(0...39)( (29...31) ), nil, - (36...39) + (36...39), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/whitequark/dedenting_heredoc.txt b/test/yarp/snapshots/whitequark/dedenting_heredoc.txt index 10692b50cfebd3..d74deb09b08f56 100644 --- a/test/yarp/snapshots/whitequark/dedenting_heredoc.txt +++ b/test/yarp/snapshots/whitequark/dedenting_heredoc.txt @@ -1,6 +1,6 @@ -ProgramNode(0...309)( +ProgramNode(0...313)( [], - StatementsNode(0...309)( + StatementsNode(0...313)( [CallNode(0...8)( nil, nil, @@ -206,13 +206,13 @@ ProgramNode(0...309)( ArgumentsNode(225...229)( [InterpolatedStringNode(225...229)( (225...229), - [StringNode(230...239)( + [StringNode(230...243)( nil, - (230...239), + (230...243), nil, - "x\n" + "\n" + "y\n" + "x\n" + " \n" + "y\n" )], - (239...241) + (243...245) )] ), nil, @@ -220,16 +220,16 @@ ProgramNode(0...309)( 0, "p" ), - CallNode(242...248)( + CallNode(246...252)( nil, nil, - (242...243), + (246...247), nil, - ArgumentsNode(244...248)( - [InterpolatedStringNode(244...248)( - (244...248), - [StringNode(249...259)(nil, (249...259), nil, "x\n" + " y\n")], - (259...261) + ArgumentsNode(248...252)( + [InterpolatedStringNode(248...252)( + (248...252), + [StringNode(253...263)(nil, (253...263), nil, "x\n" + " y\n")], + (263...265) )] ), nil, @@ -237,16 +237,16 @@ ProgramNode(0...309)( 0, "p" ), - CallNode(262...268)( + CallNode(266...272)( nil, nil, - (262...263), + (266...267), nil, - ArgumentsNode(264...268)( - [InterpolatedStringNode(264...268)( - (264...268), - [StringNode(269...273)(nil, (269...273), nil, "x\n")], - (273...275) + ArgumentsNode(268...272)( + [InterpolatedStringNode(268...272)( + (268...272), + [StringNode(273...277)(nil, (273...277), nil, "x\n")], + (277...279) )] ), nil, @@ -254,16 +254,16 @@ ProgramNode(0...309)( 0, "p" ), - CallNode(276...282)( + CallNode(280...286)( nil, nil, - (276...277), + (280...281), nil, - ArgumentsNode(278...282)( - [InterpolatedStringNode(278...282)( - (278...282), - [StringNode(283...288)(nil, (283...288), nil, "ð\n")], - (288...290) + ArgumentsNode(282...286)( + [InterpolatedStringNode(282...286)( + (282...286), + [StringNode(287...292)(nil, (287...292), nil, "ð\n")], + (292...294) )] ), nil, @@ -271,35 +271,35 @@ ProgramNode(0...309)( 0, "p" ), - CallNode(291...297)( + CallNode(295...301)( nil, nil, - (291...292), + (295...296), nil, - ArgumentsNode(293...297)( - [InterpolatedStringNode(293...297)((293...297), [], (298...300))] + ArgumentsNode(297...301)( + [InterpolatedStringNode(297...301)((297...301), [], (302...304))] ), nil, nil, 0, "p" ), - CallNode(301...309)( + CallNode(305...313)( nil, nil, - (301...302), + (305...306), nil, - ArgumentsNode(303...309)( - [InterpolatedXStringNode(303...309)( - (303...309), - [StringNode(310...318)(nil, (310...318), nil, " x\n"), - EmbeddedStatementsNode(318...324)( - (318...320), - StatementsNode(320...323)( - [CallNode(320...323)( + ArgumentsNode(307...313)( + [InterpolatedXStringNode(307...313)( + (307...313), + [StringNode(314...322)(nil, (314...322), nil, " x\n"), + EmbeddedStatementsNode(322...328)( + (322...324), + StatementsNode(324...327)( + [CallNode(324...327)( nil, nil, - (320...323), + (324...327), nil, nil, nil, @@ -308,10 +308,10 @@ ProgramNode(0...309)( "foo" )] ), - (323...324) + (327...328) ), - StringNode(324...325)(nil, (324...325), nil, "\n")], - (325...327) + StringNode(328...329)(nil, (328...329), nil, "\n")], + (329...331) )] ), nil, diff --git a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt index 77f264878d5c08..2096097f5c6ee9 100644 --- a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt +++ b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt @@ -18,7 +18,8 @@ ProgramNode(0...178)( nil, nil, nil, - (35...38) + (35...38), + "Kernel" ), ClassNode(40...87)( [], @@ -41,7 +42,8 @@ ProgramNode(0...178)( nil, nil, nil, - (84...87) + (84...87), + "Kernel" ), ModuleNode(89...128)( [], @@ -58,7 +60,8 @@ ProgramNode(0...178)( (115...117) ), nil, - (125...128) + (125...128), + "Kernel" ), ModuleNode(130...178)( [], @@ -79,7 +82,8 @@ ProgramNode(0...178)( (165...167) ), nil, - (175...178) + (175...178), + "Kernel" )] ) ) diff --git a/test/yarp/snapshots/whitequark/interp_digit_var.txt b/test/yarp/snapshots/whitequark/interp_digit_var.txt index 45d18f9ef22e15..4af55a97a8bb00 100644 --- a/test/yarp/snapshots/whitequark/interp_digit_var.txt +++ b/test/yarp/snapshots/whitequark/interp_digit_var.txt @@ -1,123 +1,123 @@ -ProgramNode(1...433)( +ProgramNode(1...465)( [], - StatementsNode(1...433)( + StatementsNode(1...465)( [StringNode(1...6)((1...2), (2...5), (5...6), "\#@1"), - StringNode(9...15)((9...10), (10...14), (14...15), "\#@@1"), - ArrayNode(18...25)( - [SymbolNode(21...24)(nil, (21...24), nil, "\#@1")], - (18...21), - (24...25) - ), - ArrayNode(28...36)( - [SymbolNode(31...35)(nil, (31...35), nil, "\#@@1")], - (28...31), - (35...36) - ), - StringNode(39...46)((39...42), (42...45), (45...46), "\#@1"), - StringNode(49...57)((49...52), (52...56), (56...57), "\#@@1"), - ArrayNode(60...67)( - [StringNode(63...66)(nil, (63...66), nil, "\#@1")], - (60...63), - (66...67) - ), - ArrayNode(70...78)( - [StringNode(73...77)(nil, (73...77), nil, "\#@@1")], - (70...73), - (77...78) - ), - ArrayNode(81...90)( - [SymbolNode(85...88)(nil, (85...88), nil, "\#@1")], - (81...84), - (89...90) - ), - ArrayNode(93...103)( - [SymbolNode(97...101)(nil, (97...101), nil, "\#@@1")], - (93...96), - (102...103) - ), - StringNode(106...113)((106...109), (109...112), (112...113), "\#@1"), - StringNode(116...124)((116...119), (119...123), (123...124), "\#@@1"), - RegularExpressionNode(127...134)( - (127...130), - (130...133), - (133...134), + StringNode(10...16)((10...11), (11...15), (15...16), "\#@@1"), + ArrayNode(20...27)( + [SymbolNode(23...26)(nil, (23...26), nil, "\#@1")], + (20...23), + (26...27) + ), + ArrayNode(31...39)( + [SymbolNode(34...38)(nil, (34...38), nil, "\#@@1")], + (31...34), + (38...39) + ), + StringNode(43...50)((43...46), (46...49), (49...50), "\#@1"), + StringNode(54...62)((54...57), (57...61), (61...62), "\#@@1"), + ArrayNode(66...73)( + [StringNode(69...72)(nil, (69...72), nil, "\#@1")], + (66...69), + (72...73) + ), + ArrayNode(77...85)( + [StringNode(80...84)(nil, (80...84), nil, "\#@@1")], + (77...80), + (84...85) + ), + ArrayNode(89...98)( + [SymbolNode(93...96)(nil, (93...96), nil, "\#@1")], + (89...92), + (97...98) + ), + ArrayNode(102...112)( + [SymbolNode(106...110)(nil, (106...110), nil, "\#@@1")], + (102...105), + (111...112) + ), + StringNode(116...123)((116...119), (119...122), (122...123), "\#@1"), + StringNode(127...135)((127...130), (130...134), (134...135), "\#@@1"), + RegularExpressionNode(139...146)( + (139...142), + (142...145), + (145...146), "\#@1", 0 ), - RegularExpressionNode(137...145)( - (137...140), - (140...144), - (144...145), + RegularExpressionNode(150...158)( + (150...153), + (153...157), + (157...158), "\#@@1", 0 ), - SymbolNode(148...155)((148...151), (151...154), (154...155), "\#@1"), - SymbolNode(158...166)((158...161), (161...165), (165...166), "\#@@1"), - ArrayNode(169...178)( - [StringNode(173...176)(nil, (173...176), nil, "\#@1")], - (169...172), - (177...178) - ), - ArrayNode(181...191)( - [StringNode(185...189)(nil, (185...189), nil, "\#@@1")], - (181...184), - (190...191) - ), - XStringNode(194...201)((194...197), (197...200), (200...201), "\#@1"), - XStringNode(204...212)((204...207), (207...211), (211...212), "\#@@1"), - StringNode(215...221)((215...217), (217...220), (220...221), "\#@1"), - StringNode(224...231)((224...226), (226...230), (230...231), "\#@@1"), - StringNode(234...239)((234...235), (235...238), (238...239), "\#@1"), - StringNode(242...248)((242...243), (243...247), (247...248), "\#@@1"), - RegularExpressionNode(251...256)( - (251...252), - (252...255), - (255...256), + SymbolNode(162...169)((162...165), (165...168), (168...169), "\#@1"), + SymbolNode(173...181)((173...176), (176...180), (180...181), "\#@@1"), + ArrayNode(185...194)( + [StringNode(189...192)(nil, (189...192), nil, "\#@1")], + (185...188), + (193...194) + ), + ArrayNode(198...208)( + [StringNode(202...206)(nil, (202...206), nil, "\#@@1")], + (198...201), + (207...208) + ), + XStringNode(212...219)((212...215), (215...218), (218...219), "\#@1"), + XStringNode(223...231)((223...226), (226...230), (230...231), "\#@@1"), + StringNode(235...241)((235...237), (237...240), (240...241), "\#@1"), + StringNode(245...252)((245...247), (247...251), (251...252), "\#@@1"), + StringNode(256...261)((256...257), (257...260), (260...261), "\#@1"), + StringNode(265...271)((265...266), (266...270), (270...271), "\#@@1"), + RegularExpressionNode(275...280)( + (275...276), + (276...279), + (279...280), "\#@1", 0 ), - RegularExpressionNode(259...265)( - (259...260), - (260...264), - (264...265), + RegularExpressionNode(284...290)( + (284...285), + (285...289), + (289...290), "\#@@1", 0 ), - SymbolNode(268...274)(nil, (270...273), nil, "\#@1"), - SymbolNode(277...284)(nil, (279...283), nil, "\#@@1"), - SymbolNode(287...293)((287...289), (289...292), (292...293), "\#@1"), - SymbolNode(296...303)((296...298), (298...302), (302...303), "\#@@1"), - XStringNode(306...311)((306...307), (307...310), (310...311), "\#@1"), - XStringNode(314...320)((314...315), (315...319), (319...320), "\#@@1"), - InterpolatedStringNode(322...331)( - (322...331), - [StringNode(332...336)(nil, (332...336), nil, "\#@1\n")], - (336...341) - ), - InterpolatedStringNode(342...351)( - (342...351), - [StringNode(352...357)(nil, (352...357), nil, "\#@@1\n")], - (357...362) - ), - InterpolatedStringNode(363...372)( - (363...372), - [StringNode(373...377)(nil, (373...377), nil, "\#@1\n")], - (377...382) - ), - InterpolatedStringNode(383...392)( - (383...392), - [StringNode(393...398)(nil, (393...398), nil, "\#@@1\n")], - (398...403) - ), - InterpolatedXStringNode(404...413)( - (404...413), - [StringNode(414...418)(nil, (414...418), nil, "\#@1\n")], - (418...423) - ), - InterpolatedXStringNode(424...433)( - (424...433), - [StringNode(434...439)(nil, (434...439), nil, "\#@@1\n")], - (439...444) + SymbolNode(294...300)(nil, (296...299), nil, "\#@1"), + SymbolNode(304...311)(nil, (306...310), nil, "\#@@1"), + SymbolNode(315...321)((315...317), (317...320), (320...321), "\#@1"), + SymbolNode(325...332)((325...327), (327...331), (331...332), "\#@@1"), + XStringNode(336...341)((336...337), (337...340), (340...341), "\#@1"), + XStringNode(345...351)((345...346), (346...350), (350...351), "\#@@1"), + InterpolatedStringNode(354...363)( + (354...363), + [StringNode(364...368)(nil, (364...368), nil, "\#@1\n")], + (368...373) + ), + InterpolatedStringNode(374...383)( + (374...383), + [StringNode(384...389)(nil, (384...389), nil, "\#@@1\n")], + (389...394) + ), + InterpolatedStringNode(395...404)( + (395...404), + [StringNode(405...409)(nil, (405...409), nil, "\#@1\n")], + (409...414) + ), + InterpolatedStringNode(415...424)( + (415...424), + [StringNode(425...430)(nil, (425...430), nil, "\#@@1\n")], + (430...435) + ), + InterpolatedXStringNode(436...445)( + (436...445), + [StringNode(446...450)(nil, (446...450), nil, "\#@1\n")], + (450...455) + ), + InterpolatedXStringNode(456...465)( + (456...465), + [StringNode(466...471)(nil, (466...471), nil, "\#@@1\n")], + (471...476) )] ) ) diff --git a/test/yarp/snapshots/whitequark/module.txt b/test/yarp/snapshots/whitequark/module.txt index 54fccb1fa32830..1695fb8fa92608 100644 --- a/test/yarp/snapshots/whitequark/module.txt +++ b/test/yarp/snapshots/whitequark/module.txt @@ -6,7 +6,8 @@ ProgramNode(0...15)( (0...6), ConstantReadNode(7...10)(), nil, - (12...15) + (12...15), + "Foo" )] ) ) diff --git a/test/yarp/snapshots/whitequark/numparam_outside_block.txt b/test/yarp/snapshots/whitequark/numparam_outside_block.txt index 99cb79b9e2980b..ac120e8188d330 100644 --- a/test/yarp/snapshots/whitequark/numparam_outside_block.txt +++ b/test/yarp/snapshots/whitequark/numparam_outside_block.txt @@ -21,7 +21,8 @@ ProgramNode(0...83)( StatementsNode(36...38)( [CallNode(36...38)(nil, nil, (36...38), nil, nil, nil, nil, 2, "_1")] ), - (40...43) + (40...43), + "A" ), DefNode(45...64)( (54...55), @@ -45,7 +46,8 @@ ProgramNode(0...83)( StatementsNode(76...78)( [CallNode(76...78)(nil, nil, (76...78), nil, nil, nil, nil, 2, "_1")] ), - (80...83) + (80...83), + "A" )] ) ) diff --git a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt index 409aa26ee25c84..ccc8730d85e7c9 100644 --- a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt @@ -77,13 +77,28 @@ ProgramNode(0...64)( ), :+ ), - ConstantPathOperatorWriteNode(32...47)( - ConstantPathNode(32...38)( - CallNode(32...35)(nil, nil, (32...35), nil, nil, nil, nil, 2, "foo"), - ConstantReadNode(37...38)(), - (35...37) + OperatorWriteNode(32...47)( + ConstantPathWriteNode(32...38)( + ConstantPathNode(32...38)( + CallNode(32...35)( + nil, + nil, + (32...35), + nil, + nil, + nil, + nil, + 2, + "foo" + ), + ConstantReadNode(37...38)(), + (35...37) + ), + nil, + nil ), (39...41), + :+, CallNode(42...47)( nil, nil, @@ -106,8 +121,7 @@ ProgramNode(0...64)( nil, 0, "m" - ), - :+ + ) ), CallOperatorWriteNode(49...64)( CallNode(49...55)( diff --git a/test/yarp/snapshots/whitequark/parser_bug_490.txt b/test/yarp/snapshots/whitequark/parser_bug_490.txt index 6f4c3a8442cfc7..8ea556d927713e 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_490.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_490.txt @@ -47,7 +47,8 @@ ProgramNode(0...132)( nil, nil, nil, - (72...75) + (72...75), + "C" )] ), (77...80) @@ -77,7 +78,8 @@ ProgramNode(0...132)( (109...115), ConstantReadNode(116...117)(), nil, - (119...122) + (119...122), + "M" )] ), (124...127) diff --git a/test/yarp/snapshots/whitequark/parser_bug_518.txt b/test/yarp/snapshots/whitequark/parser_bug_518.txt index 4de48dad9f265d..bee652b20a9402 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_518.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_518.txt @@ -8,7 +8,8 @@ ProgramNode(0...15)( (8...9), ConstantReadNode(10...11)(), nil, - (12...15) + (12...15), + "A" )] ) ) diff --git a/test/yarp/snapshots/whitequark/procarg0.txt b/test/yarp/snapshots/whitequark/procarg0.txt index 8153bfdd9d8dac..fb43e8721080b1 100644 --- a/test/yarp/snapshots/whitequark/procarg0.txt +++ b/test/yarp/snapshots/whitequark/procarg0.txt @@ -1,6 +1,6 @@ -ProgramNode(0...31)( +ProgramNode(0...32)( [], - StatementsNode(0...31)( + StatementsNode(0...32)( [CallNode(0...18)( nil, nil, @@ -36,18 +36,18 @@ ProgramNode(0...31)( 0, "m" ), - CallNode(20...31)( + CallNode(21...32)( nil, nil, - (20...21), + (21...22), nil, nil, nil, - BlockNode(22...31)( + BlockNode(23...32)( [:foo], - BlockParametersNode(24...29)( - ParametersNode(25...28)( - [RequiredParameterNode(25...28)(:foo)], + BlockParametersNode(25...30)( + ParametersNode(26...29)( + [RequiredParameterNode(26...29)(:foo)], [], [], nil, @@ -56,12 +56,12 @@ ProgramNode(0...31)( nil ), [], - (24...25), - (28...29) + (25...26), + (29...30) ), nil, - (22...23), - (30...31) + (23...24), + (31...32) ), 0, "m" diff --git a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt index 051ead127c090d..8ae81caa8f5b1b 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt @@ -1,16 +1,15 @@ ProgramNode(0...22)( [:foo], StatementsNode(0...22)( - [LocalVariableOperatorWriteNode(0...22)( - (0...3), + [OperatorWriteNode(0...22)( + LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), (4...6), + :+, RescueModifierNode(7...22)( CallNode(7...11)(nil, nil, (7...11), nil, nil, nil, nil, 2, "meth"), (12...18), CallNode(19...22)(nil, nil, (19...22), nil, nil, nil, nil, 2, "bar") - ), - :foo, - :+ + ) )] ) ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt index fb00d826a21a57..4ca03bab0109ad 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt @@ -1,9 +1,10 @@ ProgramNode(0...437)( [:foo], StatementsNode(0...437)( - [LocalVariableOperatorWriteNode(0...27)( - (0...3), + [OperatorWriteNode(0...27)( + LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), (4...6), + :+, CallNode(7...27)( nil, nil, @@ -30,13 +31,12 @@ ProgramNode(0...437)( nil, 0, "raise" - ), - :foo, - :+ + ) ), - LocalVariableOperatorWriteNode(29...57)( - (29...32), + OperatorWriteNode(29...57)( + LocalVariableWriteNode(29...32)(:foo, 0, nil, (29...32), nil), (33...35), + :+, RescueModifierNode(36...57)( CallNode(36...46)( nil, @@ -63,9 +63,7 @@ ProgramNode(0...437)( ), (47...53), NilNode(54...57)() - ), - :foo, - :+ + ) ), LocalVariableWriteNode(59...85)( :foo, @@ -301,13 +299,16 @@ ProgramNode(0...437)( ), :+ ), - ConstantPathOperatorOrWriteNode(242...273)( - ConstantPathNode(242...248)( - LocalVariableReadNode(242...245)(:foo, 0), - ConstantReadNode(247...248)(), - (245...247) + OrWriteNode(242...273)( + ConstantPathWriteNode(242...248)( + ConstantPathNode(242...248)( + LocalVariableReadNode(242...245)(:foo, 0), + ConstantReadNode(247...248)(), + (245...247) + ), + nil, + nil ), - (249...252), CallNode(253...273)( nil, nil, @@ -334,15 +335,19 @@ ProgramNode(0...437)( nil, 0, "raise" - ) + ), + (249...252) ), - ConstantPathOperatorOrWriteNode(275...307)( - ConstantPathNode(275...281)( - LocalVariableReadNode(275...278)(:foo, 0), - ConstantReadNode(280...281)(), - (278...280) + OrWriteNode(275...307)( + ConstantPathWriteNode(275...281)( + ConstantPathNode(275...281)( + LocalVariableReadNode(275...278)(:foo, 0), + ConstantReadNode(280...281)(), + (278...280) + ), + nil, + nil ), - (282...285), RescueModifierNode(286...307)( CallNode(286...296)( nil, @@ -369,7 +374,8 @@ ProgramNode(0...437)( ), (297...303), NilNode(304...307)() - ) + ), + (282...285) ), CallOperatorWriteNode(309...339)( CallNode(309...315)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt index fc6b673c50bd94..6eb9e9a0c81ba0 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt @@ -1,12 +1,14 @@ ProgramNode(0...74)( [:a, :b], StatementsNode(0...74)( - [LocalVariableOperatorWriteNode(0...18)( - (0...1), + [OperatorWriteNode(0...18)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), (2...4), - LocalVariableOperatorWriteNode(5...18)( - (5...6), + :+, + OperatorWriteNode(5...18)( + LocalVariableWriteNode(5...6)(:b, 0, nil, (5...6), nil), (7...9), + :+, CallNode(10...18)( nil, nil, @@ -19,16 +21,13 @@ ProgramNode(0...74)( nil, 0, "raise" - ), - :b, - :+ - ), - :a, - :+ + ) + ) ), - LocalVariableOperatorWriteNode(20...37)( - (20...21), + OperatorWriteNode(20...37)( + LocalVariableWriteNode(20...21)(:a, 0, nil, (20...21), nil), (22...24), + :+, LocalVariableWriteNode(25...37)( :b, 0, @@ -47,16 +46,15 @@ ProgramNode(0...74)( ), (25...26), (27...28) - ), - :a, - :+ + ) ), LocalVariableWriteNode(39...56)( :a, 0, - LocalVariableOperatorWriteNode(43...56)( - (43...44), + OperatorWriteNode(43...56)( + LocalVariableWriteNode(43...44)(:b, 0, nil, (43...44), nil), (45...47), + :+, CallNode(48...56)( nil, nil, @@ -69,9 +67,7 @@ ProgramNode(0...74)( nil, 0, "raise" - ), - :b, - :+ + ) ), (39...40), (41...42) diff --git a/test/yarp/snapshots/whitequark/var_and_asgn.txt b/test/yarp/snapshots/whitequark/var_and_asgn.txt index 76469d6b9344f2..a082413a7d05a7 100644 --- a/test/yarp/snapshots/whitequark/var_and_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_and_asgn.txt @@ -1,11 +1,10 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [LocalVariableOperatorAndWriteNode(0...7)( - (0...1), - (2...5), + [AndWriteNode(0...7)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), IntegerNode(6...7)(), - :a + (2...5) )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 2bc8667b6a219d..08b724031b7743 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -1,35 +1,34 @@ ProgramNode(0...53)( [:a], StatementsNode(0...53)( - [ClassVariableOperatorWriteNode(0...11)( - (0...5), + [OperatorWriteNode(0...11)( + ClassVariableWriteNode(0...5)((0...5), nil, nil), (6...8), - IntegerNode(9...11)(), - :| + :|, + IntegerNode(9...11)() ), - InstanceVariableOperatorWriteNode(13...20)( - (13...15), + OperatorWriteNode(13...20)( + InstanceVariableWriteNode(13...15)((13...15), nil, nil), (16...18), - IntegerNode(19...20)(), - :| + :|, + IntegerNode(19...20)() ), - LocalVariableOperatorWriteNode(22...28)( - (22...23), + OperatorWriteNode(22...28)( + LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), (24...26), - IntegerNode(27...28)(), - :a, - :+ + :+, + IntegerNode(27...28)() ), DefNode(30...53)( (34...35), nil, nil, StatementsNode(37...48)( - [ClassVariableOperatorWriteNode(37...48)( - (37...42), + [OperatorWriteNode(37...48)( + ClassVariableWriteNode(37...42)((37...42), nil, nil), (43...45), - IntegerNode(46...48)(), - :| + :|, + IntegerNode(46...48)() )] ), [], diff --git a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt index 4ef661f2fdb7d3..f78329a1eba157 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt @@ -1,9 +1,10 @@ ProgramNode(0...12)( [:foo], StatementsNode(0...12)( - [LocalVariableOperatorWriteNode(0...12)( - (0...3), + [OperatorWriteNode(0...12)( + LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), (4...6), + :+, CallNode(7...12)( nil, nil, @@ -14,9 +15,7 @@ ProgramNode(0...12)( nil, 0, "m" - ), - :foo, - :+ + ) )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_or_asgn.txt b/test/yarp/snapshots/whitequark/var_or_asgn.txt index 39cc8238675f5d..bb2f8c6b39cfaa 100644 --- a/test/yarp/snapshots/whitequark/var_or_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_or_asgn.txt @@ -1,11 +1,10 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [LocalVariableOperatorOrWriteNode(0...7)( - (0...1), - (2...5), + [OrWriteNode(0...7)( + LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), IntegerNode(6...7)(), - :a + (2...5) )] ) ) diff --git a/test/yarp/snapshots/wrapping_heredoc.txt b/test/yarp/snapshots/wrapping_heredoc.txt new file mode 100644 index 00000000000000..674db56ed12546 --- /dev/null +++ b/test/yarp/snapshots/wrapping_heredoc.txt @@ -0,0 +1,80 @@ +ProgramNode(165...298)( + [], + StatementsNode(165...298)( + [CallNode(165...193)( + nil, + nil, + (165...167), + nil, + ArgumentsNode(168...193)( + [CallNode(168...193)( + InterpolatedStringNode(168...172)( + (168...172), + [StringNode(182...184)(nil, (182...184), nil, "a\n")], + (184...186) + ), + (172...173), + (173...177), + (177...178), + ArgumentsNode(178...192)( + [InterpolatedRegularExpressionNode(178...188)( + (178...179), + [StringNode(179...182)(nil, (179...182), nil, "b"), + StringNode(186...187)(nil, (186...187), nil, "c")], + (187...188), + 0 + ), + StringNode(190...192)( + (190...191), + (191...191), + (191...192), + "" + )] + ), + (192...193), + nil, + 0, + "gsub" + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(278...298)( + nil, + nil, + (278...280), + nil, + ArgumentsNode(281...298)( + [CallNode(281...298)( + InterpolatedStringNode(281...285)( + (281...285), + [StringNode(292...294)(nil, (292...294), nil, "d\n")], + (294...296) + ), + nil, + (286...287), + nil, + ArgumentsNode(288...298)( + [InterpolatedStringNode(288...298)( + (288...289), + [StringNode(289...292)(nil, (289...292), nil, "e"), + StringNode(296...297)(nil, (296...297), nil, "f")], + (297...298) + )] + ), + nil, + nil, + 0, + "+" + )] + ), + nil, + nil, + 0, + "pp" + )] + ) +) diff --git a/tool/lib/output.rb b/tool/lib/output.rb index 51e3d32010fd01..5c645daca6c0d2 100644 --- a/tool/lib/output.rb +++ b/tool/lib/output.rb @@ -6,19 +6,27 @@ class Output def initialize @path = @timestamp = @ifchange = @color = nil + @overwrite = @create_only = false @vpath = VPath.new end + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + def def_options(opt) opt.separator(" Output common options:") opt.on('-o', '--output=PATH') {|v| @path = v} opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} - opt.on('--color') {@color = true} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} @vpath.def_options(opt) end - def write(data, overwrite: false, create_only: false) + def write(data, overwrite: @overwrite, create_only: @create_only) unless @path $stdout.print data return true diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index 6adef0059bf8d1..aefdb23a792bef 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -25,6 +25,8 @@ test_subtract(RBS::CliTest) running tests without Bundler test_subtract_several_subtrahends(RBS::CliTest) running tests without Bundler test_subtract_write(RBS::CliTest) running tests without Bundler +test_aref(FiberSingletonTest) the method should not accept String keys + NetSingletonTest depending on external resources NetInstanceTest depending on external resources TestHTTPRequest depending on external resources diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 34fafd1e34010d..2a99e408daeb4b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -17,7 +17,10 @@ module RubyVM::CEscape # generate comment, with escaps. def commentify str - return "/* #{str.b.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" end # Mimic gensym of CL. diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 242bbc07f2e95a..29107fb48f317a 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -432,26 +432,17 @@ def sync_default_gems(gem) |\.git.* |[A-Z]\w+file |COPYING + |Gemfile.lock + |bin\/.* |rakelib\/.* |test\/lib\/.* )\z/mx YARP_IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |CONTRIBUTING\.md - |Gemfile - |Gemfile\.lock - |Makefile\.in - |README\.md - |bin\/.* + /\A(?:Makefile\.in |configure\.ac - |rakelib\/.* + |fuzz\/.* |rust\/.* - |test\/lib\/.* |tasks\/.* |ext\/yarp\/extconf\.rb )\z/mx @@ -502,7 +493,7 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) # Fetch the repository to be synchronized IO.popen(%W"git remote") do |f| unless f.read.split.include?(gem) - `git remote add #{gem} git@github.com:#{repo}.git` + `git remote add #{gem} https://github.com/#{repo}.git` end end system(*%W"git fetch --no-tags #{gem}") @@ -526,11 +517,9 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) end # Ignore Merge commits and already-merged commits. - case gem - when "yarp" - ignore_file_pattern = YARP_IGNORE_FILE_PATTERN - else - ignore_file_pattern = IGNORE_FILE_PATTERN + ignore_file_pattern = IGNORE_FILE_PATTERN + if gem == "yarp" + ignore_file_pattern = Regexp.union(ignore_file_pattern, YARP_IGNORE_FILE_PATTERN) end commits.delete_if do |sha, subject| files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") @@ -558,30 +547,36 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) puts "Pick #{sha} from #{repo}." # Attempt to cherry-pick a commit - skipped = false result = IO.popen(%W"git cherry-pick #{sha}", &:read) if result =~ /nothing\ to\ commit/ `git reset` - skipped = true puts "Skip empty commit #{sha}" + next end - next if skipped # Skip empty commits or deal with conflicts + skipped = false if result.empty? skipped = true elsif /^CONFLICT/ =~ result # Forcibly remove any files that we don't want to copy to this repository. # We also ignore them as new `toplevels` even when they don't conflict. + ignored_paths = [] case gem when "rubygems" # We don't copy any vcr_cassettes to this repository. Because the directory does not # exist, rename detection doesn't work. So it starts with the original path `bundler/`. - %w[bundler/spec/support/artifice/vcr_cassettes].each do |rem| - if File.exist?(rem) - system("git", "reset", rem) - rm_rf(rem) - end + ignored_paths += %w[bundler/spec/support/artifice/vcr_cassettes] + when "yarp" + # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs + # since ruby/ruby/doc is not something owned by YARP. + ignored_paths += %w[docs/] + end + ignored_paths.each do |path| + if File.exist?(path) + puts "Removing: #{path}" + system("git", "reset", path) + rm_rf(path) end end @@ -597,14 +592,28 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) system(*%w[git add yarp]) end + # Skip this commit if everything has been removed as `ignored_paths`. + changes = pipe_readlines(%W"git status --porcelain -z") + if changes.empty? + `git reset` && `git checkout .` && `git clean -fd` + puts "Skip empty commit #{sha}" + next + end + + # For YARP, we want to skip DD: deleted by both. + if gem == "yarp" + deleted = changes.grep(/^DD /) + deleted.map! { |line| line.delete_prefix("DD ") } + system(*%W"git rm -f --", *deleted) unless deleted.empty? + end + # Discover unmerged files # AU: unmerged, added by us # DU: unmerged, deleted by us # UU: unmerged, both modified # UA: unmerged, added by them # AA: unmerged, both added - unmerged = pipe_readlines(%W"git status --porcelain -z") - unmerged.map! {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} + unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} unmerged.compact! ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} # Reset ignored files if they conflict @@ -812,7 +821,7 @@ def update_default_gems(gem, release: false) ruby #$0 rubygems 97e9768612..9e53702832 \e[1mPick all commits since the last picked commit\e[0m - ruby #$0 rubygems -a + ruby #$0 -a rubygems \e[1mList known libraries\e[0m ruby #$0 list diff --git a/tool/update-deps b/tool/update-deps index a3676694354d46..5662188deec402 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -149,6 +149,13 @@ FILES_NEED_VPATH = %w[ enc/trans/single_byte.c enc/trans/utf8_mac.c enc/trans/utf_16_32.c + + yarp/api_node.c + yarp/ast.h + yarp/node.c + yarp/prettyprint.c + yarp/serialize.c + yarp/token_type.c ] # Multiple files with same filename. diff --git a/universal_parser.c b/universal_parser.c index 1e7fc41fa0cbd7..0f1395322e263b 100644 --- a/universal_parser.c +++ b/universal_parser.c @@ -77,6 +77,7 @@ #undef T_RATIONAL #define T_RATIONAL 0x0f +#ifndef INTERNAL_IMEMO_H struct rb_imemo_tmpbuf_struct { VALUE flags; VALUE reserved; @@ -84,6 +85,7 @@ struct rb_imemo_tmpbuf_struct { struct rb_imemo_tmpbuf_struct *next; /* next imemo */ size_t cnt; /* buffer size in VALUE */ }; +#endif #undef xmalloc #define xmalloc p->config->malloc diff --git a/variable.c b/variable.c index 2e0aca0421f941..0cfa23069b9d9e 100644 --- a/variable.c +++ b/variable.c @@ -1437,36 +1437,20 @@ generic_ivar_set(VALUE obj, ID id, VALUE val) } } -static VALUE * -obj_ivar_heap_alloc(VALUE obj, size_t newsize) -{ - return ALLOC_N(VALUE, newsize); -} - -static VALUE * -obj_ivar_heap_realloc(VALUE obj, int32_t len, size_t newsize) -{ - REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, newsize); - VALUE *newptr = ROBJECT(obj)->as.heap.ivptr; - - return newptr; -} - void rb_ensure_iv_list_size(VALUE obj, uint32_t current_capacity, uint32_t new_capacity) { RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); - VALUE *ptr = ROBJECT_IVPTR(obj); - VALUE *newptr; if (RBASIC(obj)->flags & ROBJECT_EMBED) { - newptr = obj_ivar_heap_alloc(obj, new_capacity); + VALUE *ptr = ROBJECT_IVPTR(obj); + VALUE *newptr = ALLOC_N(VALUE, new_capacity); MEMCPY(newptr, ptr, VALUE, current_capacity); RB_FL_UNSET_RAW(obj, ROBJECT_EMBED); ROBJECT(obj)->as.heap.ivptr = newptr; } else { - newptr = obj_ivar_heap_realloc(obj, current_capacity, new_capacity); + REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, new_capacity); } } diff --git a/version.c b/version.c index c4fad96fe881cd..c4756771e1cecc 100644 --- a/version.c +++ b/version.c @@ -86,12 +86,14 @@ void Init_version(void) { enum {ruby_patchlevel = RUBY_PATCHLEVEL}; - VALUE version; - VALUE ruby_engine_name; + VALUE version = MKSTR(version); + VALUE ruby_engine_name = MKSTR(engine); + // MKSTR macro is a marker for fake.rb + /* * The running version of ruby */ - rb_define_global_const("RUBY_VERSION", (version = MKSTR(version))); + rb_define_global_const("RUBY_VERSION", /* MKSTR(version) */ version); /* * The date this ruby was released */ @@ -116,12 +118,12 @@ Init_version(void) /* * The engine or interpreter this ruby uses. */ - rb_define_global_const("RUBY_ENGINE", ruby_engine_name = MKSTR(engine)); + rb_define_global_const("RUBY_ENGINE", /* MKSTR(engine) */ ruby_engine_name); ruby_set_script_name(ruby_engine_name); /* * The version of the engine or interpreter this ruby uses. */ - rb_define_global_const("RUBY_ENGINE_VERSION", (1 ? version : MKSTR(version))); + rb_define_global_const("RUBY_ENGINE_VERSION", /* MKSTR(version) */ version); rb_provide("ruby2_keywords.rb"); } diff --git a/vm.c b/vm.c index 04d0f4b818340e..f838c94022e8cf 100644 --- a/vm.c +++ b/vm.c @@ -381,19 +381,16 @@ static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s static inline rb_jit_func_t jit_compile(rb_execution_context_t *ec) { - // Increment the ISEQ's call counter const rb_iseq_t *iseq = ec->cfp->iseq; struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); bool yjit_enabled = rb_yjit_compile_new_iseqs(); - if (yjit_enabled || rb_rjit_call_p) { - body->jit_entry_calls++; - } - else { + if (!(yjit_enabled || rb_rjit_call_p)) { return NULL; } - // Trigger JIT compilation if not compiled + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled if (body->jit_entry == NULL) { + body->jit_entry_calls++; if (yjit_enabled) { if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) { rb_yjit_compile_iseq(iseq, ec, false); @@ -436,19 +433,18 @@ jit_exec(rb_execution_context_t *ec) static inline rb_jit_func_t jit_compile_exception(rb_execution_context_t *ec) { - // Increment the ISEQ's call counter const rb_iseq_t *iseq = ec->cfp->iseq; struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); - if (rb_yjit_compile_new_iseqs()) { - body->jit_exception_calls++; - } - else { + if (!rb_yjit_compile_new_iseqs()) { return NULL; } - // Trigger JIT compilation if not compiled - if (body->jit_exception == NULL && rb_yjit_threshold_hit(iseq, body->jit_exception_calls)) { - rb_yjit_compile_iseq(iseq, ec, true); + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled + if (body->jit_exception == NULL) { + body->jit_exception_calls++; + if (rb_yjit_threshold_hit(iseq, body->jit_exception_calls)) { + rb_yjit_compile_iseq(iseq, ec, true); + } } return body->jit_exception; } @@ -2191,14 +2187,13 @@ frame_name(const rb_control_frame_t *cfp) // cfp_returning_with_value: // Whether cfp is the last frame in the unwinding process for a non-local return. static void -hook_before_rewind(rb_execution_context_t *ec, const rb_control_frame_t *cfp, - bool cfp_returning_with_value, int state, struct vm_throw_data *err) +hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err) { if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) { return; } else { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = ec->cfp->iseq; rb_hook_list_t *local_hooks = iseq->aux.exec.local_hooks; switch (VM_FRAME_TYPE(ec->cfp)) { @@ -2455,7 +2450,6 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const struct iseq_catch_table *ct; unsigned long epc, cont_pc, cont_sp; const rb_iseq_t *catch_iseq; - rb_control_frame_t *cfp; VALUE type; const rb_control_frame_t *escape_cfp; @@ -2475,7 +2469,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V rb_vm_pop_frame(ec); } - cfp = ec->cfp; + rb_control_frame_t *const cfp = ec->cfp; epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; escape_cfp = NULL; @@ -2505,7 +2499,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V ec->errinfo = Qnil; THROW_DATA_CATCH_FRAME_SET(err, cfp + 1); // cfp == escape_cfp here so calling with cfp_returning_with_value = true - hook_before_rewind(ec, ec->cfp, true, state, err); + hook_before_rewind(ec, true, state, err); rb_vm_pop_frame(ec); return THROW_DATA_VAL(err); } @@ -2514,7 +2508,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } else { /* TAG_BREAK */ - *ec->cfp->sp++ = THROW_DATA_VAL(err); + *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; return Qundef; } @@ -2587,7 +2581,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { - *ec->cfp->sp++ = THROW_DATA_VAL(err); + *cfp->sp++ = THROW_DATA_VAL(err); } ec->errinfo = Qnil; VM_ASSERT(ec->tag->state == TAG_NONE); @@ -2638,7 +2632,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V return Qundef; } else { - hook_before_rewind(ec, ec->cfp, (cfp == escape_cfp), state, err); + hook_before_rewind(ec, (cfp == escape_cfp), state, err); if (VM_FRAME_FINISHED_P(ec->cfp)) { rb_vm_pop_frame(ec); diff --git a/vm_eval.c b/vm_eval.c index 37ed96add22ff4..676c2211cd30dc 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1682,6 +1682,7 @@ eval_make_iseq(VALUE src, VALUE fname, int line, const rb_binding_t *bind, } rb_parser_set_context(parser, parent, FALSE); + rb_parser_set_script_lines(parser, RBOOL(ruby_vm_keep_script_lines)); ast = rb_parser_compile_string_path(parser, fname, src, line); if (ast->body.root) { ast->body.coverage_enabled = coverage_enabled; diff --git a/weakmap.c b/weakmap.c index 9c2737478583a4..e93d2f7af108e1 100644 --- a/weakmap.c +++ b/weakmap.c @@ -6,68 +6,78 @@ #include "ruby/st.h" #include "ruby/st.h" +/* ===== WeakMap ===== + * + * WeakMap contains one ST table which contains a pointer to the object as the + * key and a pointer to the object as the value. This means that the key and + * value of the table are both of the type `VALUE *`. + * + * The objects are not directly stored as keys and values in the table because + * `rb_gc_mark_weak` requires a pointer to the memory location to overwrite + * when the object is reclaimed. Using a pointer into the ST table entry is not + * safe because the pointer can change when the ST table is resized. + * + * WeakMap hashes and compares using the pointer address of the object. + * + * For performance and memory efficiency reasons, the key and value + * are allocated at the same time and adjacent to each other. + * + * During GC and while iterating, reclaimed entries (i.e. either the key or + * value points to `Qundef`) are removed from the ST table. + */ + struct weakmap { - st_table *obj2wmap; /* obj -> [ref,...] */ - st_table *wmap2obj; /* ref -> obj */ - VALUE final; + st_table *table; }; -static int -wmap_replace_ref(st_data_t *key, st_data_t *value, st_data_t _argp, int existing) +static bool +wmap_live_p(VALUE obj) { - *key = rb_gc_location((VALUE)*key); - - VALUE *values = (VALUE *)*value; - VALUE size = values[0]; + return !UNDEF_P(obj); +} - for (VALUE index = 1; index <= size; index++) { - values[index] = rb_gc_location(values[index]); - } +static void +wmap_free_entry(VALUE *key, VALUE *val) +{ + assert(key + 1 == val); - return ST_CONTINUE; + /* We only need to free key because val is allocated beside key on in the + * same malloc call. */ + ruby_sized_xfree(key, sizeof(VALUE) * 2); } static int -wmap_foreach_replace(st_data_t key, st_data_t value, st_data_t _argp, int error) +wmap_mark_weak_table_i(st_data_t key, st_data_t val, st_data_t _) { - if (rb_gc_location((VALUE)key) != (VALUE)key) { - return ST_REPLACE; - } + VALUE key_obj = *(VALUE *)key; + VALUE val_obj = *(VALUE *)val; - VALUE *values = (VALUE *)value; - VALUE size = values[0]; + if (wmap_live_p(key_obj) && wmap_live_p(val_obj)) { + rb_gc_mark_weak((VALUE *)key); + rb_gc_mark_weak((VALUE *)val); - for (VALUE index = 1; index <= size; index++) { - VALUE val = values[index]; - if (rb_gc_location(val) != val) { - return ST_REPLACE; - } + return ST_CONTINUE; } + else { + wmap_free_entry((VALUE *)key, (VALUE *)val); - return ST_CONTINUE; -} - -static void -wmap_compact(void *ptr) -{ - struct weakmap *w = ptr; - if (w->wmap2obj) rb_gc_update_tbl_refs(w->wmap2obj); - if (w->obj2wmap) st_foreach_with_replace(w->obj2wmap, wmap_foreach_replace, wmap_replace_ref, (st_data_t)NULL); - w->final = rb_gc_location(w->final); + return ST_DELETE; + } } static void wmap_mark(void *ptr) { struct weakmap *w = ptr; - rb_gc_mark_movable(w->final); + if (w->table) { + st_foreach(w->table, wmap_mark_weak_table_i, (st_data_t)0); + } } static int -wmap_free_map(st_data_t key, st_data_t val, st_data_t arg) +wmap_free_table_i(st_data_t key, st_data_t val, st_data_t arg) { - VALUE *ptr = (VALUE *)val; - ruby_sized_xfree(ptr, (ptr[0] + 1) * sizeof(VALUE)); + wmap_free_entry((VALUE *)key, (VALUE *)val); return ST_CONTINUE; } @@ -75,30 +85,63 @@ static void wmap_free(void *ptr) { struct weakmap *w = ptr; - st_foreach(w->obj2wmap, wmap_free_map, 0); - st_free_table(w->obj2wmap); - st_free_table(w->wmap2obj); + + st_foreach(w->table, wmap_free_table_i, 0); + st_free_table(w->table); xfree(w); } +static size_t +wmap_memsize(const void *ptr) +{ + const struct weakmap *w = ptr; + + size_t size = sizeof(*w); + size += st_memsize(w->table); + /* The key and value of the table each take sizeof(VALUE) in size. */ + size += st_table_size(w->table) * (2 * sizeof(VALUE)); + + return size; +} + static int -wmap_memsize_map(st_data_t key, st_data_t val, st_data_t arg) +wmap_compact_table_i(st_data_t key, st_data_t val, st_data_t data) { - VALUE *ptr = (VALUE *)val; - *(size_t *)arg += (ptr[0] + 1) * sizeof(VALUE); + st_table *table = (st_table *)data; + + VALUE key_obj = *(VALUE *)key; + VALUE val_obj = *(VALUE *)val; + + if (wmap_live_p(key_obj) && wmap_live_p(val_obj)) { + VALUE new_key_obj = rb_gc_location(key_obj); + + *(VALUE *)val = rb_gc_location(val_obj); + + /* If the key object moves, then we must reinsert because the hash is + * based on the pointer rather than the object itself. */ + if (key_obj != new_key_obj) { + *(VALUE *)key = new_key_obj; + + st_insert(table, key, val); + + return ST_DELETE; + } + } + else { + wmap_free_entry((VALUE *)key, (VALUE *)val); + + return ST_DELETE; + } + return ST_CONTINUE; } -static size_t -wmap_memsize(const void *ptr) +static void +wmap_compact(void *ptr) { - size_t size; - const struct weakmap *w = ptr; - size = sizeof(*w); - size += st_memsize(w->obj2wmap); - size += st_memsize(w->wmap2obj); - st_foreach(w->obj2wmap, wmap_memsize_map, (st_data_t)&size); - return size; + struct weakmap *w = ptr; + + st_foreach(w->table, wmap_compact_table_i, (st_data_t)w->table); } static const rb_data_type_t weakmap_type = { @@ -112,111 +155,68 @@ static const rb_data_type_t weakmap_type = { 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED }; -static VALUE wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self)); +static int +wmap_cmp(st_data_t x, st_data_t y) +{ + return *(VALUE *)x != *(VALUE *)y; +} + +static st_index_t +wmap_hash(st_data_t n) +{ + return st_numhash(*(VALUE *)n); +} + +static const struct st_hash_type wmap_hash_type = { + wmap_cmp, + wmap_hash, +}; static VALUE wmap_allocate(VALUE klass) { struct weakmap *w; VALUE obj = TypedData_Make_Struct(klass, struct weakmap, &weakmap_type, w); - w->obj2wmap = rb_init_identtable(); - w->wmap2obj = rb_init_identtable(); - RB_OBJ_WRITE(obj, &w->final, rb_func_lambda_new(wmap_finalize, obj, 1, 1)); + w->table = st_init_table(&wmap_hash_type); return obj; } -static int -wmap_live_p(VALUE obj) -{ - if (SPECIAL_CONST_P(obj)) return TRUE; - /* If rb_gc_is_ptr_to_obj returns false, the page could be in the tomb heap - * or have already been freed. */ - if (!rb_gc_is_ptr_to_obj((void *)obj)) return FALSE; - - void *poisoned = asan_poisoned_object_p(obj); - asan_unpoison_object(obj, false); - - enum ruby_value_type t = BUILTIN_TYPE(obj); - int ret = (!(t == T_NONE || t >= T_FIXNUM || t == T_ICLASS) && - !rb_objspace_garbage_object_p(obj)); - - if (poisoned) { - asan_poison_object(obj); - } - - return ret; -} +struct wmap_foreach_data { + struct weakmap *w; + void (*func)(VALUE, VALUE, st_data_t); + st_data_t arg; +}; static int -wmap_remove_inverse_ref(st_data_t *key, st_data_t *val, st_data_t arg, int existing) +wmap_foreach_i(st_data_t key, st_data_t val, st_data_t arg) { - if (!existing) return ST_STOP; - - VALUE old_ref = (VALUE)arg; - - VALUE *values = (VALUE *)*val; - VALUE size = values[0]; + struct wmap_foreach_data *data = (struct wmap_foreach_data *)arg; - if (size == 1) { - // fast path, we only had one backref - RUBY_ASSERT(values[1] == old_ref); - ruby_sized_xfree(values, 2 * sizeof(VALUE)); - return ST_DELETE; - } + VALUE key_obj = *(VALUE *)key; + VALUE val_obj = *(VALUE *)val; - bool found = false; - VALUE index = 1; - for (; index <= size; index++) { - if (values[index] == old_ref) { - found = true; - break; - } + if (wmap_live_p(key_obj) && wmap_live_p(val_obj)) { + data->func(key_obj, val_obj, data->arg); } - if (!found) return ST_STOP; + else { + wmap_free_entry((VALUE *)key, (VALUE *)val); - if (size > index) { - MEMMOVE(&values[index], &values[index + 1], VALUE, size - index); + return ST_DELETE; } - size -= 1; - values[0] = size; - SIZED_REALLOC_N(values, VALUE, size + 1, size + 2); - *val = (st_data_t)values; return ST_CONTINUE; } -/* :nodoc: */ -static VALUE -wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self)) +static void +wmap_foreach(struct weakmap *w, void (*func)(VALUE, VALUE, st_data_t), st_data_t arg) { - st_data_t orig, wmap, data; - VALUE obj, *rids, i, size; - struct weakmap *w; - - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - /* Get reference from object id. */ - if (UNDEF_P(obj = rb_gc_id2ref_obj_tbl(objid))) { - rb_bug("wmap_finalize: objid is not found."); - } - - /* obj is original referenced object and/or weak reference. */ - orig = (st_data_t)obj; - if (st_delete(w->obj2wmap, &orig, &data)) { - rids = (VALUE *)data; - size = *rids++; - for (i = 0; i < size; ++i) { - wmap = (st_data_t)rids[i]; - st_delete(w->wmap2obj, &wmap, NULL); - } - ruby_sized_xfree((VALUE *)data, (size + 1) * sizeof(VALUE)); - } + struct wmap_foreach_data foreach_data = { + .w = w, + .func = func, + .arg = arg, + }; - wmap = (st_data_t)obj; - if (st_delete(w->wmap2obj, &wmap, &orig)) { - wmap = (st_data_t)obj; - st_update(w->obj2wmap, orig, wmap_remove_inverse_ref, wmap); - } - return self; + st_foreach(w->table, wmap_foreach_i, (st_data_t)&foreach_data); } static VALUE @@ -225,19 +225,15 @@ wmap_inspect_append(VALUE str, VALUE obj) if (SPECIAL_CONST_P(obj)) { return rb_str_append(str, rb_inspect(obj)); } - else if (wmap_live_p(obj)) { - return rb_str_append(str, rb_any_to_s(obj)); - } else { - return rb_str_catf(str, "#", (void*)obj); + return rb_str_append(str, rb_any_to_s(obj)); } } -static int -wmap_inspect_i(st_data_t key, st_data_t val, st_data_t arg) +static void +wmap_inspect_i(VALUE key, VALUE val, st_data_t data) { - VALUE str = (VALUE)arg; - VALUE k = (VALUE)key, v = (VALUE)val; + VALUE str = (VALUE)data; if (RSTRING_PTR(str)[0] == '#') { rb_str_cat2(str, ", "); @@ -246,11 +242,10 @@ wmap_inspect_i(st_data_t key, st_data_t val, st_data_t arg) rb_str_cat2(str, ": "); RSTRING_PTR(str)[0] = '#'; } - wmap_inspect_append(str, k); - rb_str_cat2(str, " => "); - wmap_inspect_append(str, v); - return ST_CONTINUE; + wmap_inspect_append(str, key); + rb_str_cat2(str, " => "); + wmap_inspect_append(str, val); } static VALUE @@ -261,30 +256,19 @@ wmap_inspect(VALUE self) TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); VALUE str = rb_sprintf("-<%"PRIsVALUE":%p", c, (void *)self); - if (w->wmap2obj) { - st_foreach(w->wmap2obj, wmap_inspect_i, (st_data_t)str); - } + + wmap_foreach(w, wmap_inspect_i, (st_data_t)str); + RSTRING_PTR(str)[0] = '#'; rb_str_cat2(str, ">"); - return str; -} -static inline bool -wmap_live_entry_p(st_data_t key, st_data_t val) -{ - return wmap_live_p((VALUE)key) && wmap_live_p((VALUE)val); + return str; } -static int -wmap_each_i(st_data_t key, st_data_t val, st_data_t _) +static void +wmap_each_i(VALUE key, VALUE val, st_data_t _) { - if (wmap_live_entry_p(key, val)) { - rb_yield_values(2, (VALUE)key, (VALUE)val); - return ST_CONTINUE; - } - else { - return ST_DELETE; - } + rb_yield_values(2, key, val); } /* Iterates over keys and objects in a weakly referenced object */ @@ -292,22 +276,17 @@ static VALUE wmap_each(VALUE self) { struct weakmap *w; - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - st_foreach(w->wmap2obj, wmap_each_i, (st_data_t)0); + + wmap_foreach(w, wmap_each_i, (st_data_t)0); + return self; } -static int -wmap_each_key_i(st_data_t key, st_data_t val, st_data_t arg) +static void +wmap_each_key_i(VALUE key, VALUE _val, st_data_t _data) { - if (wmap_live_entry_p(key, val)) { - rb_yield((VALUE)key); - return ST_CONTINUE; - } - else { - return ST_DELETE; - } + rb_yield(key); } /* Iterates over keys and objects in a weakly referenced object */ @@ -315,22 +294,17 @@ static VALUE wmap_each_key(VALUE self) { struct weakmap *w; - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - st_foreach(w->wmap2obj, wmap_each_key_i, (st_data_t)0); + + wmap_foreach(w, wmap_each_key_i, (st_data_t)0); + return self; } -static int -wmap_each_value_i(st_data_t key, st_data_t val, st_data_t arg) +static void +wmap_each_value_i(VALUE _key, VALUE val, st_data_t _data) { - if (wmap_live_entry_p(key, val)) { - rb_yield((VALUE)val); - return ST_CONTINUE; - } - else { - return ST_DELETE; - } + rb_yield(val); } /* Iterates over keys and objects in a weakly referenced object */ @@ -338,24 +312,19 @@ static VALUE wmap_each_value(VALUE self) { struct weakmap *w; - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - st_foreach(w->wmap2obj, wmap_each_value_i, (st_data_t)0); + + wmap_foreach(w, wmap_each_value_i, (st_data_t)0); + return self; } -static int -wmap_keys_i(st_data_t key, st_data_t val, st_data_t arg) +static void +wmap_keys_i(st_data_t key, st_data_t _, st_data_t arg) { VALUE ary = (VALUE)arg; - if (wmap_live_entry_p(key, val)) { - rb_ary_push(ary, (VALUE)key); - return ST_CONTINUE; - } - else { - return ST_DELETE; - } + rb_ary_push(ary, key); } /* Iterates over keys and objects in a weakly referenced object */ @@ -366,22 +335,17 @@ wmap_keys(VALUE self) TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); VALUE ary = rb_ary_new(); - st_foreach(w->wmap2obj, wmap_keys_i, (st_data_t)ary); + wmap_foreach(w, wmap_keys_i, (st_data_t)ary); + return ary; } -static int +static void wmap_values_i(st_data_t key, st_data_t val, st_data_t arg) { VALUE ary = (VALUE)arg; - if (wmap_live_entry_p(key, val)) { - rb_ary_push(ary, (VALUE)val); - return ST_CONTINUE; - } - else { - return ST_DELETE; - } + rb_ary_push(ary, (VALUE)val); } /* Iterates over values and objects in a weakly referenced object */ @@ -392,37 +356,9 @@ wmap_values(VALUE self) TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); VALUE ary = rb_ary_new(); - st_foreach(w->wmap2obj, wmap_values_i, (st_data_t)ary); - return ary; -} - -static int -wmap_aset_update(st_data_t *key, st_data_t *val, st_data_t arg, int existing) -{ - VALUE size, *ptr, *optr; - if (existing) { - size = (ptr = optr = (VALUE *)*val)[0]; + wmap_foreach(w, wmap_values_i, (st_data_t)ary); - for (VALUE index = 1; index <= size; index++) { - if (ptr[index] == (VALUE)arg) { - // The reference was already registered. - return ST_STOP; - } - } - - ++size; - SIZED_REALLOC_N(ptr, VALUE, size + 1, size); - } - else { - optr = 0; - size = 1; - ptr = ruby_xmalloc(2 * sizeof(VALUE)); - } - ptr[0] = size; - ptr[size] = (VALUE)arg; - if (ptr == optr) return ST_STOP; - *val = (st_data_t)ptr; - return ST_CONTINUE; + return ary; } static VALUE @@ -437,57 +373,39 @@ nonspecial_obj_id(VALUE obj) #endif } -struct wmap_aset_replace_args { - VALUE new_value; - VALUE old_value; -}; - static int -wmap_aset_replace_value(st_data_t *key, st_data_t *val, st_data_t _args, int existing) +wmap_aset_replace(st_data_t *key, st_data_t *val, st_data_t new_key, int existing) { - struct wmap_aset_replace_args *args = (struct wmap_aset_replace_args *)_args; - if (existing) { - args->old_value = *val; + VALUE *orig_pair = ((VALUE *)*key); + assert(orig_pair[0] == *(VALUE *)new_key); + + wmap_free_entry(orig_pair, orig_pair + 1); } - *val = (st_data_t)args->new_value; + + *key = new_key; + *val = (st_data_t)(((VALUE *)new_key) + 1); + return ST_CONTINUE; } /* Creates a weak reference from the given key to the given value */ static VALUE -wmap_aset(VALUE self, VALUE key, VALUE value) +wmap_aset(VALUE self, VALUE key, VALUE val) { struct weakmap *w; - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - if (FL_ABLE(value)) { - rb_define_finalizer_no_check(value, w->final); - } - if (FL_ABLE(key)) { - rb_define_finalizer_no_check(key, w->final); - } - struct wmap_aset_replace_args aset_args = { - .new_value = value, - .old_value = Qundef, - }; - st_update(w->wmap2obj, (st_data_t)key, wmap_aset_replace_value, (st_data_t)&aset_args); + VALUE *pair = xmalloc(sizeof(VALUE) * 2); + pair[0] = key; + pair[1] = val; - // If the value is unchanged, we have nothing to do. - if (value != aset_args.old_value) { - if (!UNDEF_P(aset_args.old_value) && FL_ABLE(aset_args.old_value)) { - // That key existed and had an inverse reference, we need to clear the outdated inverse reference. - st_update(w->obj2wmap, (st_data_t)aset_args.old_value, wmap_remove_inverse_ref, key); - } + st_update(w->table, (st_data_t)pair, wmap_aset_replace, (st_data_t)pair); - if (FL_ABLE(value)) { - // If the value has no finalizer, we don't need to keep the inverse reference - st_update(w->obj2wmap, (st_data_t)value, wmap_aset_update, key); - } - } + RB_OBJ_WRITTEN(self, Qundef, key); + RB_OBJ_WRITTEN(self, Qundef, val); - return nonspecial_obj_id(value); + return nonspecial_obj_id(val); } /* Retrieves a weakly referenced object with the given key */ @@ -500,12 +418,12 @@ wmap_lookup(VALUE self, VALUE key) TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); st_data_t data; - if (!st_lookup(w->wmap2obj, (st_data_t)key, &data)) return Qundef; + if (!st_lookup(w->table, (st_data_t)&key, &data)) return Qundef; - VALUE obj = (VALUE)data; - if (SPECIAL_CONST_P(obj)) return obj; - if (!wmap_live_p(obj)) return Qundef; - return obj; + if (SPECIAL_CONST_P(*(VALUE *)data)) return *(VALUE *)data; + if (!wmap_live_p(*(VALUE *)data)) return Qundef; + + return *(VALUE *)data; } /* Retrieves a weakly referenced object with the given key */ @@ -520,20 +438,23 @@ wmap_aref(VALUE self, VALUE key) static VALUE wmap_delete(VALUE self, VALUE key) { - assert(wmap_live_p(key)); - struct weakmap *w; TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - VALUE old_value = Qnil; - if (st_delete(w->wmap2obj, (st_data_t *)&key, (st_data_t *)&old_value)) { - if (FL_ABLE(old_value)) { - // That key existed and had an inverse reference, we need to clear the outdated inverse reference. - st_update(w->obj2wmap, (st_data_t)old_value, wmap_remove_inverse_ref, key); + VALUE orig_key = key; + st_data_t orig_key_data = (st_data_t)&orig_key; + st_data_t orig_val_data; + if (st_delete(w->table, &orig_key_data, &orig_val_data)) { + VALUE orig_val = *(VALUE *)orig_val_data; + + wmap_free_entry((VALUE *)orig_key_data, (VALUE *)orig_val_data); + + if (wmap_live_p(orig_val)) { + return orig_val; } - return old_value; } - else if (rb_block_given_p()) { + + if (rb_block_given_p()) { return rb_yield(key); } else { @@ -553,10 +474,10 @@ static VALUE wmap_size(VALUE self) { struct weakmap *w; - st_index_t n; - TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w); - n = w->wmap2obj->num_entries; + + st_index_t n = st_table_size(w->table); + #if SIZEOF_ST_INDEX_T <= SIZEOF_LONG return ULONG2NUM(n); #else @@ -564,64 +485,69 @@ wmap_size(VALUE self) #endif } -typedef struct weakkeymap_entry { - VALUE obj; - st_index_t hash; -} weakkeymap_entry_t; +/* ===== WeakKeyMap ===== + * + * WeakKeyMap contains one ST table which contains a pointer to the object as + * the key and the object as the value. This means that the key is of the type + * `VALUE *` while the value is of the type `VALUE`. + * + * The object is not not directly stored as keys in the table because + * `rb_gc_mark_weak` requires a pointer to the memory location to overwrite + * when the object is reclaimed. Using a pointer into the ST table entry is not + * safe because the pointer can change when the ST table is resized. + * + * WeakKeyMap hashes and compares using the `#hash` and `#==` methods of the + * object, respectively. + * + * During GC and while iterating, reclaimed entries (i.e. the key points to + * `Qundef`) are removed from the ST table. + */ struct weakkeymap { - st_table *map; - st_table *obj2hash; - VALUE final; + st_table *table; }; static int -weakkeymap_cmp_entry(st_data_t a, st_data_t b) +wkmap_mark_table_i(st_data_t key, st_data_t val_obj, st_data_t _) { - struct weakkeymap_entry *entry_a = (struct weakkeymap_entry *)a; - struct weakkeymap_entry *entry_b = (struct weakkeymap_entry *)b; - if (entry_a == entry_b) { - return 0; + VALUE key_obj = *(VALUE *)key; + + if (wmap_live_p(key_obj)) { + rb_gc_mark_weak((VALUE *)key); + rb_gc_mark_movable((VALUE)val_obj); + + return ST_CONTINUE; } else { - return rb_any_cmp(entry_a->obj, entry_b->obj); - } -} + ruby_sized_xfree((VALUE *)key, sizeof(VALUE)); -static st_index_t -weakkeymap_hash_entry(st_data_t a) -{ - struct weakkeymap_entry *entry_a = (struct weakkeymap_entry *)a; - return entry_a->hash; + return ST_DELETE; + } } -static const struct st_hash_type weakkeymap_hash = { - weakkeymap_cmp_entry, - weakkeymap_hash_entry, -}; - static void -wkmap_compact(void *ptr) +wkmap_mark(void *ptr) { struct weakkeymap *w = ptr; - if (w->map) rb_gc_update_tbl_refs(w->map); - w->final = rb_gc_location(w->final); + if (w->table) { + st_foreach(w->table, wkmap_mark_table_i, (st_data_t)0); + } } -static void -wkmap_mark(void *ptr) +static int +wkmap_free_table_i(st_data_t key, st_data_t _val, st_data_t _arg) { - struct weakkeymap *w = ptr; - rb_mark_tbl_no_pin(w->map); - rb_gc_mark_movable(w->final); + ruby_sized_xfree((VALUE *)key, sizeof(VALUE)); + return ST_CONTINUE; } static void wkmap_free(void *ptr) { struct weakkeymap *w = ptr; - st_free_table(w->map); - st_free_table(w->obj2hash); + + st_foreach(w->table, wkmap_free_table_i, 0); + st_free_table(w->table); xfree(w); } @@ -629,7 +555,51 @@ static size_t wkmap_memsize(const void *ptr) { const struct weakkeymap *w = ptr; - return sizeof(struct weakkeymap) + st_memsize(w->map) + st_memsize(w->obj2hash); + + size_t size = sizeof(*w); + size += st_memsize(w->table); + /* Each key of the table takes sizeof(VALUE) in size. */ + size += st_table_size(w->table) * sizeof(VALUE); + + return size; +} + +static int +wkmap_compact_table_i(st_data_t key, st_data_t val_obj, st_data_t _data, int _error) +{ + VALUE key_obj = *(VALUE *)key; + + if (wmap_live_p(key_obj)) { + if (key_obj != rb_gc_location(key_obj) || val_obj != rb_gc_location(val_obj)) { + return ST_REPLACE; + } + } + else { + ruby_sized_xfree((VALUE *)key, sizeof(VALUE)); + + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static int +wkmap_compact_table_replace(st_data_t *key_ptr, st_data_t *val_ptr, st_data_t _data, int existing) +{ + assert(existing); + + *(VALUE *)*key_ptr = rb_gc_location(*(VALUE *)*key_ptr); + *val_ptr = (st_data_t)rb_gc_location((VALUE)*val_ptr); + + return ST_CONTINUE; +} + +static void +wkmap_compact(void *ptr) +{ + struct weakkeymap *w = ptr; + + st_foreach_with_replace(w->table, wkmap_compact_table_i, wkmap_compact_table_replace, (st_data_t)0); } static const rb_data_type_t weakkeymap_type = { @@ -643,81 +613,54 @@ static const rb_data_type_t weakkeymap_type = { 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED }; -static VALUE -wkmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self)) +static int +wkmap_cmp(st_data_t x, st_data_t y) { - struct weakkeymap *w; - VALUE key; - - TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); + VALUE x_obj = *(VALUE *)x; + VALUE y_obj = *(VALUE *)y; - /* Get reference from object id. */ - if ((key = rb_gc_id2ref_obj_tbl(objid)) == Qundef) { - rb_bug("wkmap_finalize: objid is not found."); + if (wmap_live_p(x_obj) && wmap_live_p(y_obj)) { + return rb_any_cmp(x_obj, y_obj); } - - st_index_t hash; - if (st_delete(w->obj2hash, (st_data_t *)key, &hash)) { - weakkeymap_entry_t lookup_entry = {key, hash}; - weakkeymap_entry_t *deleted_entry = NULL; - if (st_get_key(w->map, (st_data_t)&lookup_entry, (st_data_t *)deleted_entry)) { - st_data_t deleted_value; - st_delete(w->map, (st_data_t *)deleted_entry, &deleted_value); - xfree(deleted_entry); - } + else { + /* If one of the objects is dead, then they cannot be the same. */ + return 1; } +} - return self; +static st_index_t +wkmap_hash(st_data_t n) +{ + VALUE obj = *(VALUE *)n; + assert(wmap_live_p(obj)); + + return rb_any_hash(obj); } +static const struct st_hash_type wkmap_hash_type = { + wkmap_cmp, + wkmap_hash, +}; + static VALUE wkmap_allocate(VALUE klass) { struct weakkeymap *w; VALUE obj = TypedData_Make_Struct(klass, struct weakkeymap, &weakkeymap_type, w); - w->map = st_init_table(&weakkeymap_hash); - w->obj2hash = rb_init_identtable(); - RB_OBJ_WRITE(obj, &w->final, rb_func_lambda_new(wkmap_finalize, obj, 1, 1)); + w->table = st_init_table(&wkmap_hash_type); return obj; } -static st_index_t -wkmap_lookup_hash(struct weakkeymap *w, VALUE key) -{ - st_index_t hash; - if (!st_lookup(w->obj2hash, (st_data_t)key, &hash)) { - hash = rb_any_hash(key); - } - return hash; -} - -static weakkeymap_entry_t* -wkmap_lookup_entry(struct weakkeymap *w, VALUE key, st_index_t hash) -{ - st_data_t data; - weakkeymap_entry_t lookup_entry = {key, hash}; - - if (st_get_key(w->map, (st_data_t)&lookup_entry, &data)) { - return (weakkeymap_entry_t *)data; - } - - return NULL; -} - static VALUE wkmap_lookup(VALUE self, VALUE key) { - st_data_t data; struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); - st_index_t hash = rb_any_hash(key); - weakkeymap_entry_t lookup_entry = {key, hash}; + st_data_t data; + if (!st_lookup(w->table, (st_data_t)&key, &data)) return Qundef; - if (st_lookup(w->map, (st_data_t)&lookup_entry, &data)) { - return (VALUE)data; - } - return Qundef; + return (VALUE)data; } /* @@ -735,6 +678,28 @@ wkmap_aref(VALUE self, VALUE key) return obj != Qundef ? obj : Qnil; } +struct wkmap_aset_args { + VALUE *new_key; + VALUE new_val; +}; + +static int +wkmap_aset_replace(st_data_t *key, st_data_t *val, st_data_t data_args, int existing) +{ + struct wkmap_aset_args *args = (struct wkmap_aset_args *)data_args; + + if (existing) { + VALUE *orig_key_ptr = ((VALUE *)*key); + + ruby_sized_xfree(orig_key_ptr, sizeof(VALUE)); + } + + *key = (st_data_t)args->new_key; + *val = (st_data_t)args->new_val; + + return ST_CONTINUE; +} + /* * call-seq: * map[key] = value -> value @@ -748,7 +713,7 @@ wkmap_aref(VALUE self, VALUE key) * the ordering is not affected */ static VALUE -wkmap_aset(VALUE self, VALUE key, VALUE value) +wkmap_aset(VALUE self, VALUE key, VALUE val) { struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); @@ -758,23 +723,20 @@ wkmap_aset(VALUE self, VALUE key, VALUE value) UNREACHABLE_RETURN(Qnil); } - st_index_t hash = wkmap_lookup_hash(w, key); - weakkeymap_entry_t *key_entry = wkmap_lookup_entry(w, key, hash); + VALUE *key_ptr = xmalloc(sizeof(VALUE)); + *key_ptr = key; - if (!key_entry) { - key_entry = ALLOC(weakkeymap_entry_t); - key_entry->obj = key; - key_entry->hash = hash; - } + struct wkmap_aset_args args = { + .new_key = key_ptr, + .new_val = val, + }; - if (!st_insert(w->map, (st_data_t)key_entry, (st_data_t)value)) { - st_insert(w->obj2hash, (st_data_t)key, (st_data_t)hash); - rb_define_finalizer_no_check(key, w->final); - } + st_update(w->table, (st_data_t)key_ptr, wkmap_aset_replace, (st_data_t)&args); - RB_OBJ_WRITTEN(self, Qundef, value); + RB_OBJ_WRITTEN(self, Qundef, key); + RB_OBJ_WRITTEN(self, Qundef, val); - return value; + return val; } /* @@ -810,21 +772,18 @@ wkmap_delete(VALUE self, VALUE key) struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); - st_index_t hash = rb_any_hash(key); - weakkeymap_entry_t lookup_entry = {key, hash}; - weakkeymap_entry_t *deleted_entry = NULL; - if (st_get_key(w->map, (st_data_t)&lookup_entry, (st_data_t *)&deleted_entry)) { - st_data_t deleted_value; - if (st_delete(w->map, (st_data_t *)&deleted_entry, &deleted_value)) { - xfree(deleted_entry); - st_delete(w->obj2hash, (st_data_t *)key, &hash); - return (VALUE)deleted_value; - } - else { - rb_bug("WeakKeyMap: miss on delete, corrupted memory?"); - } + VALUE orig_key = key; + st_data_t orig_key_data = (st_data_t)&orig_key; + st_data_t orig_val_data; + if (st_delete(w->table, &orig_key_data, &orig_val_data)) { + VALUE orig_val = (VALUE)orig_val_data; + + ruby_sized_xfree((VALUE *)orig_key_data, sizeof(VALUE)); + + return orig_val; } - else if (rb_block_given_p()) { + + if (rb_block_given_p()) { return rb_yield(key); } else { @@ -844,19 +803,10 @@ wkmap_getkey(VALUE self, VALUE key) struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); - st_index_t hash = rb_any_hash(key); - weakkeymap_entry_t lookup_entry = {key, hash}; - - weakkeymap_entry_t *key_entry = NULL; - if (st_get_key(w->map, (st_data_t)&lookup_entry, (st_data_t *)&key_entry)) { - assert(key_entry != NULL); + st_data_t orig_key; + if (!st_get_key(w->table, (st_data_t)&key, &orig_key)) return Qnil; - VALUE obj = key_entry->obj; - if (wmap_live_p(obj)) { - return obj; - } - } - return Qnil; + return *(VALUE *)orig_key; } /* @@ -882,12 +832,10 @@ wkmap_clear(VALUE self) { struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); - if (w->map) { - st_clear(w->map); - } - if (w->obj2hash) { - st_clear(w->obj2hash); - } + + st_foreach(w->table, wkmap_free_table_i, 0); + st_clear(w->table); + return self; } @@ -908,10 +856,7 @@ wkmap_inspect(VALUE self) struct weakkeymap *w; TypedData_Get_Struct(self, struct weakkeymap, &weakkeymap_type, w); - st_index_t n = 0; - if (w->map) { - n = w->map->num_entries; - } + st_index_t n = st_table_size(w->table); #if SIZEOF_ST_INDEX_T <= SIZEOF_LONG const char * format = "#<%"PRIsVALUE":%p size=%lu>"; diff --git a/yarp/config.yml b/yarp/config.yml index 8b33b6675d4233..3662c176098a50 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -399,6 +399,19 @@ nodes: left and right ^^^^^^^^^^^^^^ + - name: AndWriteNode + child_nodes: + - name: target + type: node + - name: value + type: node + - name: operator_loc + type: location + comment: | + Represents the use of the `&&=` operator. + + target &&= value + ^^^^^^^^^^^^^^^^ - name: ArgumentsNode child_nodes: - name: arguments @@ -527,7 +540,7 @@ nodes: - name: parameters type: node? kind: BlockParametersNode - - name: statements + - name: body type: node? - name: opening_loc type: location @@ -714,56 +727,17 @@ nodes: type: location? - name: superclass type: node? - - name: statements + - name: body type: node? - name: end_keyword_loc type: location + - name: name + type: string comment: | Represents a class declaration involving the `class` keyword. class Foo end ^^^^^^^^^^^^^ - - name: ClassVariableOperatorAndWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `&&=` operator for assignment to a class variable. - - @@target &&= value - ^^^^^^^^^^^^^^^^ - - name: ClassVariableOperatorOrWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `||=` operator for assignment to a class variable. - - @@target ||= value - ^^^^^^^^^^^^^^^^^^ - - name: ClassVariableOperatorWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: operator - type: constant - comment: | - Represents assigning to a class variable using an operator that isn't `=`. - - @@target += value - ^^^^^^^^^^^^^^^^^ - name: ClassVariableReadNode comment: | Represents referencing a class variable. @@ -783,47 +757,6 @@ nodes: @@foo = 1 ^^^^^^^^^ - - name: ConstantOperatorAndWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `&&=` operator for assignment to a constant. - - Target &&= value - ^^^^^^^^^^^^^^^^ - - name: ConstantOperatorOrWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `||=` operator for assignment to a constant. - - Target ||= value - ^^^^^^^^^^^^^^^^ - - name: ConstantOperatorWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: operator - type: constant - comment: | - Represents assigning to a constant using an operator that isn't `=`. - - Target += value - ^^^^^^^^^^^^^^^ - name: ConstantPathNode child_nodes: - name: parent @@ -837,50 +770,6 @@ nodes: Foo::Bar ^^^^^^^^ - - name: ConstantPathOperatorAndWriteNode - child_nodes: - - name: target - type: node - kind: ConstantPathNode - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `&&=` operator for assignment to a constant path. - - Parent::Child &&= value - ^^^^^^^^^^^^^^^^^^^^^^^ - - name: ConstantPathOperatorOrWriteNode - child_nodes: - - name: target - type: node - kind: ConstantPathNode - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `||=` operator for assignment to a constant path. - - Parent::Child ||= value - ^^^^^^^^^^^^^^^^^^^^^^^ - - name: ConstantPathOperatorWriteNode - child_nodes: - - name: target - type: node - kind: ConstantPathNode - - name: operator_loc - type: location - - name: value - type: node - - name: operator - type: constant - comment: | - Represents assigning to a constant path using an operator that isn't `=`. - - Parent::Child += value - ^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathWriteNode child_nodes: - name: target @@ -929,7 +818,7 @@ nodes: - name: parameters type: node? kind: ParametersNode - - name: statements + - name: body type: node? - name: locals type: constant[] @@ -1123,47 +1012,6 @@ nodes: super ^^^^^ - - name: GlobalVariableOperatorAndWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `&&=` operator for assignment to a global variable. - - $target &&= value - ^^^^^^^^^^^^^^^^^ - - name: GlobalVariableOperatorOrWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `||=` operator for assignment to a global variable. - - $target ||= value - ^^^^^^^^^^^^^^^^^ - - name: GlobalVariableOperatorWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: operator - type: constant - comment: | - Represents assigning to a global variable using an operator that isn't `=`. - - $target += value - ^^^^^^^^^^^^^^^^ - name: GlobalVariableReadNode comment: | Represents referencing a global variable. @@ -1263,47 +1111,6 @@ nodes: case a; in b then c end ^^^^^^^^^^^ - - name: InstanceVariableOperatorAndWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `&&=` operator for assignment to an instance variable. - - @target &&= value - ^^^^^^^^^^^^^^^^^ - - name: InstanceVariableOperatorOrWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - comment: | - Represents the use of the `||=` operator for assignment to an instance variable. - - @target ||= value - ^^^^^^^^^^^^^^^^^ - - name: InstanceVariableOperatorWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: operator - type: constant - comment: | - Represents assigning to an instance variable using an operator that isn't `=`. - - @target += value - ^^^^^^^^^^^^^^^^ - name: InstanceVariableReadNode comment: | Represents referencing an instance variable. @@ -1434,60 +1241,13 @@ nodes: - name: parameters type: node? kind: BlockParametersNode - - name: statements + - name: body type: node? comment: | Represents using a lambda literal (not the lambda method call). ->(value) { value * 2 } ^^^^^^^^^^^^^^^^^^^^^^^ - - name: LocalVariableOperatorAndWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: constant_id - type: constant - comment: | - Represents the use of the `&&=` operator for assignment to a local variable. - - target &&= value - ^^^^^^^^^^^^^^^^ - - name: LocalVariableOperatorOrWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: constant_id - type: constant - comment: | - Represents the use of the `||=` operator for assignment to a local variable. - - target ||= value - ^^^^^^^^^^^^^^^^ - - name: LocalVariableOperatorWriteNode - child_nodes: - - name: name_loc - type: location - - name: operator_loc - type: location - - name: value - type: node - - name: constant_id - type: constant - - name: operator_id - type: constant - comment: | - Represents assigning to a local variable using an operator that isn't `=`. - - target += value - ^^^^^^^^^^^^^^^ - name: LocalVariableReadNode child_nodes: - name: constant_id @@ -1556,10 +1316,12 @@ nodes: type: location - name: constant_path type: node - - name: statements + - name: body type: node? - name: end_keyword_loc type: location + - name: name + type: string comment: | Represents a module declaration involving the `module` keyword. @@ -1618,6 +1380,21 @@ nodes: $1 ^^ + - name: OperatorWriteNode + child_nodes: + - name: target + type: node + - name: operator_loc + type: location + - name: operator + type: constant + - name: value + type: node + comment: | + Represents the use of an operator on a write. + + target += value + ^^^^^^^^^^^^^^^ - name: OptionalParameterNode child_nodes: - name: constant_id @@ -1647,6 +1424,19 @@ nodes: left or right ^^^^^^^^^^^^^ + - name: OrWriteNode + child_nodes: + - name: target + type: node + - name: value + type: node + - name: operator_loc + type: location + comment: | + Represents the use of the `||=` operator. + + target ||= value + ^^^^^^^^^^^^^^^^ - name: ParametersNode child_nodes: - name: requireds @@ -1673,7 +1463,7 @@ nodes: end - name: ParenthesesNode child_nodes: - - name: statements + - name: body type: node? - name: opening_loc type: location @@ -1916,7 +1706,7 @@ nodes: type: location - name: expression type: node - - name: statements + - name: body type: node? - name: end_keyword_loc type: location @@ -1932,7 +1722,6 @@ nodes: __ENCODING__ ^^^^^^^^^^^^ - name: SourceFileNode - is_migrated: true child_nodes: - name: filepath type: string diff --git a/yarp/extension.c b/yarp/extension.c index 455cdcadccce5b..8aef456c0010a9 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -221,6 +221,20 @@ static void lex_encoding_changed_callback(yp_parser_t *parser) { lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; lex_data->encoding = rb_enc_find(parser->encoding.name); + + // Since we got a new encoding, we need to go back and change the encoding + // of the tokens that we've already lexed. This should be a tiny amount + // since encoding magic comments need to be the first or second line of the + // file. + VALUE tokens = lex_data->tokens; + for (long index = 0; index < RARRAY_LEN(tokens); index++) { + VALUE yields = rb_ary_entry(tokens, index); + VALUE token = rb_ary_entry(yields, 0); + + VALUE value = rb_ivar_get(token, rb_intern("@value")); + rb_enc_associate(value, lex_data->encoding); + ENC_CODERANGE_CLEAR(value); + } } // Return an array of tokens corresponding to the given source. diff --git a/yarp/extension.h b/yarp/extension.h index 12381317dcdeaf..75d5cc84a7d77b 100644 --- a/yarp/extension.h +++ b/yarp/extension.h @@ -5,7 +5,7 @@ #include #include "yarp.h" -#define EXPECTED_YARP_VERSION "0.7.0" +#define EXPECTED_YARP_VERSION "0.8.0" VALUE yp_source_new(yp_parser_t *parser); VALUE yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALUE source); diff --git a/yarp/node.h b/yarp/node.h index a107baddb8d48e..f2d5be8bf22e91 100644 --- a/yarp/node.h +++ b/yarp/node.h @@ -34,3 +34,13 @@ YP_EXPORTED_FUNCTION const char * yp_node_type_to_str(yp_node_type_t node_type); #define YP_EMPTY_LOCATION_LIST ((yp_location_list_t) { .locations = NULL, .size = 0, .capacity = 0 }) #endif // YARP_NODE_H + +// ScopeNodes are helper nodes, and will never +// be part of the AST. We manually declare them +// here to avoid generating them +typedef struct yp_scope_node { + yp_node_t base; + struct yp_parameters_node *parameters; + yp_node_t *body; + yp_constant_id_list_t locals; +} yp_scope_node_t; diff --git a/yarp/templates/include/yarp/ast.h.erb b/yarp/templates/include/yarp/ast.h.erb index 5b69c0c8b16231..6fe3bc2c2456bb 100644 --- a/yarp/templates/include/yarp/ast.h.erb +++ b/yarp/templates/include/yarp/ast.h.erb @@ -50,6 +50,7 @@ enum yp_node_type { <%- nodes.each_with_index do |node, index| -%> <%= node.type %> = <%= index + 1 %>, <%- end -%> + YP_NODE_SCOPE_NODE }; typedef uint16_t yp_node_type_t; diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb index f78770508afeac..312e232182592e 100644 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ b/yarp/templates/java/org/yarp/Loader.java.erb @@ -49,7 +49,7 @@ public class Loader { private final Nodes.Source source; private byte MAJOR_VERSION = (byte) 0; - private byte MINOR_VERSION = (byte) 7; + private byte MINOR_VERSION = (byte) 8; private byte PATCH_VERSION = (byte) 0; private Loader(byte[] serialized, Nodes.Source source) { @@ -95,13 +95,28 @@ public class Loader { return new ParseResult(node, comments, errors, warnings); } - private byte[] loadString() { + private byte[] loadEmbeddedString() { int length = loadVarInt(); byte[] string = new byte[length]; buffer.get(string); return string; } + private byte[] loadString() { + switch (buffer.get()) { + case 1: + int start = loadVarInt(); + int length = loadVarInt(); + byte[] string = new byte[length]; + System.arraycopy(source.bytes, start, string, 0, length); + return string; + case 2: + return loadEmbeddedString(); + default: + throw new Error("Expected 0 or 1 but was " + buffer.get()); + } + } + private ParseResult.Comment[] loadComments() { int count = loadVarInt(); ParseResult.Comment[] comments = new ParseResult.Comment[count]; @@ -123,7 +138,7 @@ public class Loader { // error messages only contain ASCII characters for (int i = 0; i < count; i++) { - byte[] bytes = loadString(); + byte[] bytes = loadEmbeddedString(); String message = new String(bytes, StandardCharsets.US_ASCII); Nodes.Location location = loadLocation(); @@ -140,7 +155,7 @@ public class Loader { // warning messages only contain ASCII characters for (int i = 0; i < count; i++) { - byte[] bytes = loadString(); + byte[] bytes = loadEmbeddedString(); String message = new String(bytes, StandardCharsets.US_ASCII); Nodes.Location location = loadLocation(); diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 4194025c32b977..8ee072c0b18a19 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -14,7 +14,7 @@ end module YARP module Serialize MAJOR_VERSION = 0 - MINOR_VERSION = 7 + MINOR_VERSION = 8 PATCH_VERSION = 0 def self.load(input, serialized) @@ -54,8 +54,8 @@ module YARP end comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } - errors = load_varint.times.map { ParseError.new(load_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) } + errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } + warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } raise "Expected to consume all bytes while deserializing" unless @io.eof? @@ -70,8 +70,8 @@ module YARP @input = input.force_encoding(@encoding).freeze comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) } - errors = load_varint.times.map { ParseError.new(load_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) } + errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } + warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } @constant_pool_offset = io.read(4).unpack1("L") @constant_pool = Array.new(load_varint, nil) @@ -110,10 +110,21 @@ module YARP end end - def load_string + def load_embedded_string io.read(load_varint).force_encoding(encoding) end + def load_string + case io.getbyte + when 1 + input.byteslice(load_varint, load_varint).force_encoding(encoding) + when 2 + load_embedded_string + else + raise + end + end + def load_location Location.new(source, load_varint, load_varint) end diff --git a/yarp/templates/src/node.c.erb b/yarp/templates/src/node.c.erb index 371655fdb5fc57..d288628fffa666 100644 --- a/yarp/templates/src/node.c.erb +++ b/yarp/templates/src/node.c.erb @@ -115,6 +115,10 @@ yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) { memsize->node_count++; switch (YP_NODE_TYPE(node)) { + // We do not calculate memsize of a ScopeNode + // as it should never be generated + case YP_NODE_SCOPE_NODE: + return; <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { diff --git a/yarp/templates/src/prettyprint.c.erb b/yarp/templates/src/prettyprint.c.erb index 29d0194dca2bfd..cf2f12f2aeff73 100644 --- a/yarp/templates/src/prettyprint.c.erb +++ b/yarp/templates/src/prettyprint.c.erb @@ -16,6 +16,10 @@ prettyprint_location(yp_buffer_t *buffer, yp_parser_t *parser, yp_location_t *lo static void prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) { switch (YP_NODE_TYPE(node)) { + // We do not need to print a ScopeNode as it's not part + // of the AST + case YP_NODE_SCOPE_NODE: + return; <%- nodes.each do |node| -%> case <%= node.type %>: { yp_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>); diff --git a/yarp/templates/src/serialize.c.erb b/yarp/templates/src/serialize.c.erb index 63a43f282a687b..9b49540566c435 100644 --- a/yarp/templates/src/serialize.c.erb +++ b/yarp/templates/src/serialize.c.erb @@ -15,7 +15,7 @@ yp_sizet_to_u32(size_t value) { } static void -serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) { +yp_serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) { assert(location->start); assert(location->end); assert(location->start <= location->end); @@ -24,15 +24,42 @@ serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *bu yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->end - location->start)); } +static void +yp_serialize_string(yp_parser_t *parser, yp_string_t *string, yp_buffer_t *buffer) { + switch (string->type) { + case YP_STRING_SHARED: { + yp_buffer_append_u8(buffer, 1); + yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(yp_string_source(string) - parser->start)); + yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_string_length(string))); + break; + } + case YP_STRING_OWNED: + case YP_STRING_CONSTANT: { + uint32_t length = yp_sizet_to_u32(yp_string_length(string)); + yp_buffer_append_u8(buffer, 2); + yp_buffer_append_u32(buffer, length); + yp_buffer_append_str(buffer, yp_string_source(string), length); + break; + } + case YP_STRING_MAPPED: + assert(false && "Cannot serialize mapped strings."); + break; + } +} + void yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { yp_buffer_append_u8(buffer, (uint8_t) YP_NODE_TYPE(node)); size_t offset = buffer->length; - serialize_location(parser, &node->location, buffer); + yp_serialize_location(parser, &node->location, buffer); switch (YP_NODE_TYPE(node)) { + // We do not need to serialize a ScopeNode ever as + // it is not part of the AST + case YP_NODE_SCOPE_NODE: + return; <%- nodes.each do |node| -%> case <%= node.type %>: { <%- if node.needs_serialized_length? -%> @@ -52,9 +79,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); } <%- when StringParam -%> - uint32_t <%= param.name %>_length = yp_sizet_to_u32(yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>)); - yp_buffer_append_u32(buffer, <%= param.name %>_length); - yp_buffer_append_str(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), <%= param.name %>_length); + yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); <%- when NodeListParam -%> uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); yp_buffer_append_u32(buffer, <%= param.name %>_size); @@ -65,7 +90,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); yp_buffer_append_u32(buffer, <%= param.name %>_size); for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index], buffer); + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index], buffer); } <%- when ConstantParam -%> yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>)); @@ -76,13 +101,13 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index])); } <%- when LocationParam -%> - serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); <%- when OptionalLocationParam -%> if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) { yp_buffer_append_u8(buffer, 0); } else { yp_buffer_append_u8(buffer, 1); - serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); } <%- when UInt32Param -%> yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= param.name %>); @@ -103,7 +128,8 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { } } -void yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) { +static void +yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) { // serialize type yp_buffer_append_u8(buffer, (uint8_t) comment->type); @@ -112,16 +138,18 @@ void yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_ yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start)); } -void yp_serialize_comment_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) { - yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list))); +static void +yp_serialize_comment_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) { + yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list))); yp_comment_t *comment; - for (comment = (yp_comment_t *) list.head; comment != NULL; comment = (yp_comment_t *) comment->node.next) { + for (comment = (yp_comment_t *) list->head; comment != NULL; comment = (yp_comment_t *) comment->node.next) { yp_serialize_comment(parser, comment, buffer); } } -void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) { +static void +yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) { // serialize message size_t message_length = strlen(diagnostic->message); yp_buffer_append_u32(buffer, yp_sizet_to_u32(message_length)); @@ -132,11 +160,12 @@ void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, y yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->end - diagnostic->start)); } -void yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) { - yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list))); +static void +yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) { + yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list))); yp_diagnostic_t *diagnostic; - for (diagnostic = (yp_diagnostic_t *) list.head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) { + for (diagnostic = (yp_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) { yp_serialize_diagnostic(parser, diagnostic, buffer); } } @@ -149,14 +178,9 @@ yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length)); yp_buffer_append_str(buffer, parser->encoding.name, encoding_length); - // Serialize the comments - yp_serialize_comment_list(parser, parser->comment_list, buffer); - - // Serialize the errors - yp_serialize_diagnostic_list(parser, parser->error_list, buffer); - - // Serialize the warnings - yp_serialize_diagnostic_list(parser, parser->warning_list, buffer); + yp_serialize_comment_list(parser, &parser->comment_list, buffer); + yp_serialize_diagnostic_list(parser, &parser->error_list, buffer); + yp_serialize_diagnostic_list(parser, &parser->warning_list, buffer); // Here we're going to leave space for the offset of the constant pool in // the buffer. @@ -220,16 +244,11 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe yp_node_t *node = yp_parse(&parser); // Append 0 to mark end of tokens - yp_buffer_append_u32(buffer, 0); - - // Serialize the comments - yp_serialize_comment_list(&parser, parser.comment_list, buffer); - - // Serialize the errors - yp_serialize_diagnostic_list(&parser, parser.error_list, buffer); + yp_buffer_append_u8(buffer, 0); - // Serialize the warnings - yp_serialize_diagnostic_list(&parser, parser.warning_list, buffer); + yp_serialize_comment_list(&parser, &parser.comment_list, buffer); + yp_serialize_diagnostic_list(&parser, &parser.error_list, buffer); + yp_serialize_diagnostic_list(&parser, &parser.warning_list, buffer); yp_node_destroy(&parser, node); yp_parser_free(&parser); diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 68df4524f74c8b..8b8a175172ffea 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -23,6 +23,10 @@ def c_type end end + def ruby_type + options[:kind] || "Node" + end + def java_type options[:kind] || "Node" end @@ -42,7 +46,7 @@ class NodeParam < Param include KindTypes def rbs_class - "Node" + ruby_type end end @@ -52,7 +56,7 @@ class OptionalNodeParam < Param include KindTypes def rbs_class - "Node?" + "#{ruby_type}?" end end @@ -266,8 +270,8 @@ def template(name, locals, write_to: nil) non_ruby_heading = <<~HEADING /******************************************************************************/ - /* This file is generated by the bin/template script and should not be */ - /* modified manually. See */ + /* This file is generated by the templates/template.rb script and should not */ + /* be modified manually. See */ /* #{filepath + " " * (74 - filepath.size) } */ /* if you are looking to modify the */ /* template */ @@ -277,7 +281,7 @@ def template(name, locals, write_to: nil) ruby_heading = <<~HEADING # frozen_string_literal: true =begin - This file is generated by the bin/template script and should not be + This file is generated by the templates/template.rb script and should not be modified manually. See #{filepath} if you are looking to modify the template =end diff --git a/yarp/unescape.c b/yarp/unescape.c index f1c40347a41e6f..7cf2631b9b943f 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -14,6 +14,20 @@ yp_char_is_hexadecimal_digits(const char *c, size_t length) { return true; } +// We don't call the char_width function unless we have to because it's +// expensive to go through the indirection of the function pointer. Instead we +// provide a fast path that will check if we can just return 1. +static inline size_t +yp_char_width(yp_parser_t *parser, const char *start, const char *end) { + const unsigned char *uc = (const unsigned char *) start; + + if (parser->encoding_changed || (*uc >= 0x80)) { + return parser->encoding.char_width(start, end - start); + } else { + return 1; + } +} + /******************************************************************************/ /* Lookup tables for characters */ /******************************************************************************/ @@ -178,7 +192,7 @@ unescape_char(const unsigned char value, const unsigned char flags) { // Read a specific escape sequence into the given destination. static const char * -unescape(char *dest, size_t *dest_length, const char *backslash, const char *end, yp_list_t *error_list, const unsigned char flags, bool write_to_str) { +unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backslash, const char *end, const unsigned char flags, bool write_to_str) { switch (backslash[1]) { case 'a': case 'b': @@ -218,7 +232,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \unnnn Unicode character, where nnnn is exactly 4 hexadecimal digits ([0-9a-fA-F]) case 'u': { if ((flags & YP_UNESCAPE_FLAG_CONTROL) | (flags & YP_UNESCAPE_FLAG_META)) { - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); return backslash + 2; } @@ -235,11 +249,11 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \u{nnnn} character literal allows only 1-6 hexadecimal digits if (hexadecimal_length > 6) - yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); + yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); // there are not hexadecimal characters if (hexadecimal_length == 0) { - yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); + yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); return unicode_cursor; } @@ -252,7 +266,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end uint32_t value; unescape_unicode(unicode_start, (size_t) (unicode_cursor - unicode_start), &value); if (write_to_str) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, error_list); + *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, &parser->error_list); } unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); @@ -260,7 +274,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // ?\u{nnnn} character literal should contain only one codepoint and cannot be like ?\u{nnnn mmmm} if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) - yp_diagnostic_list_append(error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); + yp_diagnostic_list_append(&parser->error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); return unicode_cursor + 1; } @@ -270,12 +284,12 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end unescape_unicode(backslash + 2, 4, &value); if (write_to_str) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, error_list); + *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, &parser->error_list); } return backslash + 6; } - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); return backslash + 2; } // \c\M-x meta control character, where x is an ASCII printable character @@ -283,18 +297,18 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \cx control character, where x is an ASCII printable character case 'c': if (backslash + 2 >= end) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } switch (backslash[2]) { case '\\': - return unescape(dest, dest_length, backslash + 2, end, error_list, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); case '?': if (write_to_str) { dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); @@ -302,7 +316,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end return backslash + 3; default: { if (!char_is_ascii_printable(backslash[2])) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } @@ -316,23 +330,23 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \C-? delete, ASCII 7Fh (DEL) case 'C': if (backslash + 3 >= end) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } switch (backslash[3]) { case '\\': - return unescape(dest, dest_length, backslash + 3, end, error_list, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); case '?': if (write_to_str) { dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); @@ -340,7 +354,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end return backslash + 4; default: if (!char_is_ascii_printable(backslash[3])) { - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid control escape sequence"); return backslash + 2; } @@ -354,22 +368,22 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \M-x meta character, where x is an ASCII printable character case 'M': { if (backslash + 3 >= end) { - yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_META) { - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 2; } if (backslash[3] == '\\') { - return unescape(dest, dest_length, backslash + 3, end, error_list, flags | YP_UNESCAPE_FLAG_META, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META, write_to_str); } if (char_is_ascii_printable(backslash[3])) { @@ -379,7 +393,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end return backslash + 4; } - yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 3; } // \n @@ -390,14 +404,17 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end if (backslash + 2 < end && backslash[2] == '\n') { return backslash + 3; } - - /* fallthrough */ + /* fallthrough */ // In this case we're escaping something that doesn't need escaping. default: { + size_t width = yp_char_width(parser, backslash + 1, end); + if (write_to_str) { - dest[(*dest_length)++] = backslash[1]; + memcpy(dest + *dest_length, backslash + 1, width); + *dest_length += width; } - return backslash + 2; + + return backslash + 1 + width; } } } @@ -431,7 +448,7 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end // \c? or \C-? delete, ASCII 7Fh (DEL) // YP_EXPORTED_FUNCTION void -yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list) { +yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { if (unescape_type == YP_UNESCAPE_NONE) { // If we're not unescaping then we can reference the source directly. return; @@ -448,7 +465,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // within the string. char *allocated = malloc(string->length); if (allocated == NULL) { - yp_diagnostic_list_append(error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping."); + yp_diagnostic_list_append(&parser->error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping."); return; } @@ -493,7 +510,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // This is the only type of unescaping left. In this case we need to // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - cursor = unescape(dest, &dest_length, backslash, end, error_list, YP_UNESCAPE_FLAG_NONE, true); + cursor = unescape(parser, dest, &dest_length, backslash, end, YP_UNESCAPE_FLAG_NONE, true); break; } @@ -521,29 +538,11 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc yp_string_owned_init(string, allocated, dest_length + ((size_t) (end - cursor))); } -YP_EXPORTED_FUNCTION bool -yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { - bool success; - - yp_parser_t parser; - yp_parser_init(&parser, start, length, ""); - - yp_list_t error_list = YP_LIST_EMPTY; - yp_string_shared_init(result, start, start + length); - yp_unescape_manipulate_string(&parser, result, unescape_type, &error_list); - success = yp_list_empty_p(&error_list); - - yp_list_free(&error_list); - yp_parser_free(&parser); - - return success; -} - // This function is similar to yp_unescape_manipulate_string, except it doesn't // actually perform any string manipulations. Instead, it calculates how long // the unescaped character is, and returns that value -YP_EXPORTED_FUNCTION size_t -yp_unescape_calculate_difference(const char *backslash, const char *end, yp_unescape_type_t unescape_type, bool expect_single_codepoint, yp_list_t *error_list) { +size_t +yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { assert(unescape_type != YP_UNESCAPE_NONE); switch (backslash[1]) { @@ -551,7 +550,9 @@ yp_unescape_calculate_difference(const char *backslash, const char *end, yp_unes case '\'': return 2; default: { - if (unescape_type == YP_UNESCAPE_MINIMAL) return 2; + if (unescape_type == YP_UNESCAPE_MINIMAL) { + return 1 + yp_char_width(parser, backslash + 1, parser->end); + } // This is the only type of unescaping left. In this case we need to // handle all of the different unescapes. @@ -561,10 +562,27 @@ yp_unescape_calculate_difference(const char *backslash, const char *end, yp_unes if (expect_single_codepoint) flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; - const char *cursor = unescape(NULL, 0, backslash, end, error_list, flags, false); + const char *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, false); assert(cursor > backslash); return (size_t) (cursor - backslash); } } } + +// This is one of the main entry points into the extension. It accepts a source +// string, a type of unescaping, and a pointer to a result string. It returns a +// boolean indicating whether or not the unescaping was successful. +YP_EXPORTED_FUNCTION bool +yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { + yp_parser_t parser; + yp_parser_init(&parser, start, length, NULL); + + yp_string_shared_init(result, start, start + length); + yp_unescape_manipulate_string(&parser, result, unescape_type); + + bool success = yp_list_empty_p(&parser.error_list); + yp_parser_free(&parser); + + return success; +} diff --git a/yarp/unescape.h b/yarp/unescape.h index d3c68fb015b0dd..30c433febd525f 100644 --- a/yarp/unescape.h +++ b/yarp/unescape.h @@ -31,12 +31,14 @@ typedef enum { // Unescape the contents of the given token into the given string using the // given unescape mode. -YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list); +YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type); // Accepts a source string and a type of unescaping and returns the unescaped version. // The caller must yp_string_free(result); after calling this function. YP_EXPORTED_FUNCTION bool yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result); -YP_EXPORTED_FUNCTION size_t yp_unescape_calculate_difference(const char *value, const char *end, yp_unescape_type_t unescape_type, bool expect_single_codepoint, yp_list_t *error_list); +// Returns the number of bytes that encompass the first escape sequence in the +// given string. +size_t yp_unescape_calculate_difference(yp_parser_t *parser, const char *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint); #endif diff --git a/yarp/util/yp_newline_list.c b/yarp/util/yp_newline_list.c index 8b24f82a0248ea..de353acf62a869 100644 --- a/yarp/util/yp_newline_list.c +++ b/yarp/util/yp_newline_list.c @@ -30,6 +30,7 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { if (list->offsets == NULL) return false; } + assert(*cursor == '\n'); assert(cursor >= list->start); size_t newline_offset = (size_t) (cursor - list->start + 1); assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]); @@ -38,6 +39,15 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { return true; } +// Conditionally append a new offset to the newline list, if the value passed in is a newline. +bool +yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor) { + if (*cursor != '\n') { + return true; + } + return yp_newline_list_append(list, cursor); +} + // Returns the line and column of the given offset, assuming we don't have any // information about the previous index that we found. static yp_line_column_t diff --git a/yarp/util/yp_newline_list.h b/yarp/util/yp_newline_list.h index 095acd5168e840..b7c8c1f3aace57 100644 --- a/yarp/util/yp_newline_list.h +++ b/yarp/util/yp_newline_list.h @@ -47,6 +47,9 @@ bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t cap // the offsets succeeds (if one was necessary), otherwise returns false. bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor); +// Conditionally append a new offset to the newline list, if the value passed in is a newline. +bool yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor); + // Returns the line and column of the given offset. If the offset is not in the // list, the line and column of the closest offset less than the given offset // are returned. diff --git a/yarp/version.h b/yarp/version.h index a798d76c7ad6b8..179543f54d751b 100644 --- a/yarp/version.h +++ b/yarp/version.h @@ -1,4 +1,4 @@ #define YP_VERSION_MAJOR 0 -#define YP_VERSION_MINOR 7 +#define YP_VERSION_MINOR 8 #define YP_VERSION_PATCH 0 -#define YP_VERSION "0.7.0" +#define YP_VERSION "0.8.0" diff --git a/yarp/yarp.c b/yarp/yarp.c index 715c2819491d62..012f8136e56c05 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -362,7 +362,7 @@ lex_state_ignored_p(yp_parser_t *parser) { if (ignored) { return YP_IGNORED_NEWLINE_ALL; - } else if (parser->lex_state == (YP_LEX_STATE_ARG | YP_LEX_STATE_LABELED)) { + } else if ((parser->lex_state & ~((unsigned int) YP_LEX_STATE_LABEL)) == (YP_LEX_STATE_ARG | YP_LEX_STATE_LABELED)) { return YP_IGNORED_NEWLINE_PATTERN; } else { return YP_IGNORED_NEWLINE_NONE; @@ -450,8 +450,8 @@ yp_flip_flop(yp_node_t *node) { case YP_NODE_PARENTHESES_NODE: { yp_parentheses_node_t *cast = (yp_parentheses_node_t *) node; - if ((cast->statements != NULL) && YP_NODE_TYPE_P(cast->statements, YP_NODE_STATEMENTS_NODE)) { - yp_statements_node_t *statements = (yp_statements_node_t *) cast->statements; + if ((cast->body != NULL) && YP_NODE_TYPE_P(cast->body, YP_NODE_STATEMENTS_NODE)) { + yp_statements_node_t *statements = (yp_statements_node_t *) cast->body; if (statements->body.size == 1) yp_flip_flop(statements->body.nodes[0]); } @@ -536,6 +536,73 @@ yp_arguments_validate(yp_parser_t *parser, yp_arguments_t *arguments) { } } +/******************************************************************************/ +/* Scope node functions */ +/******************************************************************************/ + +// Generate a scope node from the given node. +void +yp_scope_node_init(yp_node_t *node, yp_scope_node_t *scope) { + scope->base.type = YP_NODE_SCOPE_NODE; + scope->base.location.start = node->location.start; + scope->base.location.end = node->location.end; + + scope->parameters = NULL; + scope->body = NULL; + yp_constant_id_list_init(&scope->locals); + + switch (YP_NODE_TYPE(node)) { + case YP_NODE_BLOCK_NODE: { + yp_block_node_t *cast = (yp_block_node_t *) node; + if (cast->parameters) scope->parameters = cast->parameters->parameters; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + case YP_NODE_CLASS_NODE: { + yp_class_node_t *cast = (yp_class_node_t *) node; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + case YP_NODE_DEF_NODE: { + yp_def_node_t *cast = (yp_def_node_t *) node; + scope->parameters = cast->parameters; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + case YP_NODE_LAMBDA_NODE: { + yp_lambda_node_t *cast = (yp_lambda_node_t *) node; + if (cast->parameters) scope->parameters = cast->parameters->parameters; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + case YP_NODE_MODULE_NODE: { + yp_module_node_t *cast = (yp_module_node_t *) node; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + case YP_NODE_PROGRAM_NODE: { + yp_program_node_t *cast = (yp_program_node_t *) node; + scope->body = (yp_node_t *) cast->statements; + scope->locals = cast->locals; + break; + } + case YP_NODE_SINGLETON_CLASS_NODE: { + yp_singleton_class_node_t *cast = (yp_singleton_class_node_t *) node; + scope->body = cast->body; + scope->locals = cast->locals; + break; + } + default: + assert(false && "unreachable"); + break; + } +} + /******************************************************************************/ /* Node creation functions */ /******************************************************************************/ @@ -658,6 +725,27 @@ yp_and_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *opera return node; } +// Allocate and initialize a new AndWriteNode. +static yp_and_write_node_t * +yp_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_and_write_node_t); + + *node = (yp_and_write_node_t) { + { + .type = YP_NODE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + }, + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate an initialize a new arguments node. static yp_arguments_node_t * yp_arguments_node_create(yp_parser_t *parser) { @@ -997,7 +1085,7 @@ yp_block_argument_node_create(yp_parser_t *parser, const yp_token_t *operator, y // Allocate and initialize a new BlockNode node. static yp_block_node_t * -yp_block_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *opening, yp_block_parameters_node_t *parameters, yp_node_t *statements, const yp_token_t *closing) { +yp_block_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *opening, yp_block_parameters_node_t *parameters, yp_node_t *body, const yp_token_t *closing) { yp_block_node_t *node = YP_ALLOC_NODE(parser, yp_block_node_t); *node = (yp_block_node_t) { @@ -1007,7 +1095,7 @@ yp_block_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const y }, .locals = *locals, .parameters = parameters, - .statements = statements, + .body = body, .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), .closing_loc = YP_LOCATION_TOKEN_VALUE(closing) }; @@ -1465,7 +1553,7 @@ yp_case_node_end_keyword_loc_set(yp_case_node_t *node, const yp_token_t *end_key // Allocate a new ClassNode node. static yp_class_node_t * -yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *class_keyword, yp_node_t *constant_path, const yp_token_t *inheritance_operator, yp_node_t *superclass, yp_node_t *statements, const yp_token_t *end_keyword) { +yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *class_keyword, yp_node_t *constant_path, const yp_token_t *name, const yp_token_t *inheritance_operator, yp_node_t *superclass, yp_node_t *body, const yp_token_t *end_keyword) { yp_class_node_t *node = YP_ALLOC_NODE(parser, yp_class_node_t); *node = (yp_class_node_t) { @@ -1478,78 +1566,12 @@ yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const y .constant_path = constant_path, .inheritance_operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(inheritance_operator), .superclass = superclass, - .statements = statements, - .end_keyword_loc = YP_LOCATION_TOKEN_VALUE(end_keyword) - }; - - return node; -} - -// Allocate and initialize a new ClassVariableOperatorAndWriteNode node. -static yp_class_variable_operator_and_write_node_t * -yp_class_variable_operator_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_class_variable_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_and_write_node_t); - - *node = (yp_class_variable_operator_and_write_node_t) { - { - .type = YP_NODE_CLASS_VARIABLE_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - -// Allocate and initialize a new ClassVariableOperatorWriteNode node. -static yp_class_variable_operator_write_node_t * -yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_class_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_write_node_t); - - *node = (yp_class_variable_operator_write_node_t) { - { - .type = YP_NODE_CLASS_VARIABLE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new ClassVariableOperatorOrWriteNode node. -static yp_class_variable_operator_or_write_node_t * -yp_class_variable_operator_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_class_variable_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_or_write_node_t); - - *node = (yp_class_variable_operator_or_write_node_t) { - { - .type = YP_NODE_CLASS_VARIABLE_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value + .body = body, + .end_keyword_loc = YP_LOCATION_TOKEN_VALUE(end_keyword), + .name = YP_EMPTY_STRING }; + yp_string_shared_init(&node->name, name->start, name->end); return node; } @@ -1583,72 +1605,6 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp return node; } -// Allocate and initialize a new ConstantPathOperatorAndWriteNode node. -static yp_constant_path_operator_and_write_node_t * -yp_constant_path_operator_and_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_constant_path_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_operator_and_write_node_t); - - *node = (yp_constant_path_operator_and_write_node_t) { - { - .type = YP_NODE_CONSTANT_PATH_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->base.location.start, - .end = value->location.end - } - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - -// Allocate and initialize a new ConstantPathOperatorWriteNode node. -static yp_constant_path_operator_write_node_t * -yp_constant_path_operator_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_constant_path_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_operator_write_node_t); - - *node = (yp_constant_path_operator_write_node_t) { - { - .type = YP_NODE_CONSTANT_PATH_OPERATOR_WRITE_NODE, - .location = { - .start = target->base.location.start, - .end = value->location.end - } - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new ConstantPathOperatorOrWriteNode node. -static yp_constant_path_operator_or_write_node_t * -yp_constant_path_operator_or_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_constant_path_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_operator_or_write_node_t); - - *node = (yp_constant_path_operator_or_write_node_t) { - { - .type = YP_NODE_CONSTANT_PATH_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->base.location.start, - .end = value->location.end - } - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate and initialize a new ConstantPathNode node. static yp_constant_path_node_t * yp_constant_path_node_create(yp_parser_t *parser, yp_node_t *parent, const yp_token_t *delimiter, yp_node_t *child) { @@ -1691,74 +1647,6 @@ yp_constant_path_write_node_create(yp_parser_t *parser, yp_constant_path_node_t return node; } -// Allocate and initialize a new ConstantOperatorAndWriteNode node. -static yp_constant_operator_and_write_node_t * -yp_constant_operator_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_constant_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_operator_and_write_node_t); - - *node = (yp_constant_operator_and_write_node_t) { - { - .type = YP_NODE_CONSTANT_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - -// Allocate and initialize a new ConstantOperatorWriteNode node. -static yp_constant_operator_write_node_t * -yp_constant_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_constant_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_operator_write_node_t); - - *node = (yp_constant_operator_write_node_t) { - { - .type = YP_NODE_CONSTANT_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new ConstantOperatorOrWriteNode node. -static yp_constant_operator_or_write_node_t * -yp_constant_operator_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_constant_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_operator_or_write_node_t); - - *node = (yp_constant_operator_or_write_node_t) { - { - .type = YP_NODE_CONSTANT_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate and initialize a new ConstantReadNode node. static yp_constant_read_node_t * yp_constant_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -1797,7 +1685,7 @@ yp_def_node_create( const yp_token_t *name, yp_node_t *receiver, yp_parameters_node_t *parameters, - yp_node_t *statements, + yp_node_t *body, yp_constant_id_list_t *locals, const yp_token_t *def_keyword, const yp_token_t *operator, @@ -1810,7 +1698,7 @@ yp_def_node_create( const char *end; if (end_keyword->type == YP_TOKEN_NOT_PROVIDED) { - end = statements->location.end; + end = body->location.end; } else { end = end_keyword->end; } @@ -1823,7 +1711,7 @@ yp_def_node_create( .name_loc = YP_LOCATION_TOKEN_VALUE(name), .receiver = receiver, .parameters = parameters, - .statements = statements, + .body = body, .locals = *locals, .def_keyword_loc = YP_LOCATION_TOKEN_VALUE(def_keyword), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), @@ -2193,74 +2081,6 @@ yp_hash_pattern_node_node_list_create(yp_parser_t *parser, yp_node_list_t *assoc return node; } -// Allocate and initialize a new GlobalVariableOperatorAndWriteNode node. -static yp_global_variable_operator_and_write_node_t * -yp_global_variable_operator_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_global_variable_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_operator_and_write_node_t); - - *node = (yp_global_variable_operator_and_write_node_t) { - { - .type = YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - -// Allocate and initialize a new GlobalVariableOperatorWriteNode node. -static yp_global_variable_operator_write_node_t * -yp_global_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_global_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_operator_write_node_t); - - *node = (yp_global_variable_operator_write_node_t) { - { - .type = YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new GlobalVariableOperatorOrWriteNode node. -static yp_global_variable_operator_or_write_node_t * -yp_global_variable_operator_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_global_variable_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_operator_or_write_node_t); - - *node = (yp_global_variable_operator_or_write_node_t) { - { - .type = YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate a new GlobalVariableReadNode node. static yp_global_variable_read_node_t * yp_global_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -2551,74 +2371,6 @@ yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t return node; } -// Allocate and initialize a new InstanceVariableOperatorAndWriteNode node. -static yp_instance_variable_operator_and_write_node_t * -yp_instance_variable_operator_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_instance_variable_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_and_write_node_t); - - *node = (yp_instance_variable_operator_and_write_node_t) { - { - .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - -// Allocate and initialize a new InstanceVariableOperatorWriteNode node. -static yp_instance_variable_operator_write_node_t * -yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_instance_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_write_node_t); - - *node = (yp_instance_variable_operator_write_node_t) { - { - .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new InstanceVariableOperatorOrWriteNode node. -static yp_instance_variable_operator_or_write_node_t * -yp_instance_variable_operator_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_instance_variable_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_or_write_node_t); - - *node = (yp_instance_variable_operator_or_write_node_t) { - { - .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate and initialize a new InstanceVariableReadNode node. static yp_instance_variable_read_node_t * yp_instance_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { @@ -2870,7 +2622,7 @@ yp_lambda_node_create( yp_constant_id_list_t *locals, const yp_token_t *opening, yp_block_parameters_node_t *parameters, - yp_node_t *statements, + yp_node_t *body, const yp_token_t *closing ) { yp_lambda_node_t *node = YP_ALLOC_NODE(parser, yp_lambda_node_t); @@ -2886,78 +2638,7 @@ yp_lambda_node_create( .locals = *locals, .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), .parameters = parameters, - .statements = statements - }; - - return node; -} - -// Allocate and initialize a new LocalVariableOperatorAndWriteNode node. -static yp_local_variable_operator_and_write_node_t * -yp_local_variable_operator_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id) { - assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); - assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); - yp_local_variable_operator_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_and_write_node_t); - - *node = (yp_local_variable_operator_and_write_node_t) { - { - .type = YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .constant_id = constant_id - }; - - return node; -} - -// Allocate and initialize a new LocalVariableOperatorWriteNode node. -static yp_local_variable_operator_write_node_t * -yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id) { - yp_local_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_write_node_t); - - *node = (yp_local_variable_operator_write_node_t) { - { - .type = YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .constant_id = constant_id, - .operator_id = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) - }; - - return node; -} - -// Allocate and initialize a new LocalVariableOperatorOrWriteNode node. -static yp_local_variable_operator_or_write_node_t * -yp_local_variable_operator_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id) { - assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); - assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); - yp_local_variable_operator_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_or_write_node_t); - - *node = (yp_local_variable_operator_or_write_node_t) { - { - .type = YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - } - }, - .name_loc = target->location, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value, - .constant_id = constant_id + .body = body }; return node; @@ -3067,7 +2748,7 @@ yp_match_required_node_create(yp_parser_t *parser, yp_node_t *value, yp_node_t * // Allocate a new ModuleNode node. static yp_module_node_t * -yp_module_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *module_keyword, yp_node_t *constant_path, yp_node_t *statements, const yp_token_t *end_keyword) { +yp_module_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *module_keyword, yp_node_t *constant_path, const yp_token_t *name, yp_node_t *body, const yp_token_t *end_keyword) { yp_module_node_t *node = YP_ALLOC_NODE(parser, yp_module_node_t); *node = (yp_module_node_t) { @@ -3081,10 +2762,12 @@ yp_module_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const .locals = (locals == NULL ? ((yp_constant_id_list_t) { .ids = NULL, .size = 0, .capacity = 0 }) : *locals), .module_keyword_loc = YP_LOCATION_TOKEN_VALUE(module_keyword), .constant_path = constant_path, - .statements = statements, - .end_keyword_loc = YP_LOCATION_TOKEN_VALUE(end_keyword) + .body = body, + .end_keyword_loc = YP_LOCATION_TOKEN_VALUE(end_keyword), + .name = YP_EMPTY_STRING }; + yp_string_shared_init(&node->name, name->start, name->end); return node; } @@ -3196,6 +2879,28 @@ yp_numbered_reference_read_node_create(yp_parser_t *parser, const yp_token_t *na return node; } +// Allocate and initialize a new OperatorWriteNode. +static yp_operator_write_node_t * +yp_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_operator_write_node_t); + + *node = (yp_operator_write_node_t) { + { + .type = YP_NODE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + }, + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), + .value = value + }; + + return node; +} + // Allocate a new OptionalParameterNode node. static yp_optional_parameter_node_t * yp_optional_parameter_node_create(yp_parser_t *parser, const yp_token_t *name, const yp_token_t *operator, yp_node_t *value) { @@ -3239,6 +2944,27 @@ yp_or_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *operat return node; } +// Allocate and initialize a new OrWriteNode. +static yp_or_write_node_t * +yp_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_or_write_node_t); + + *node = (yp_or_write_node_t) { + { + .type = YP_NODE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + }, + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ParametersNode node. static yp_parameters_node_t * yp_parameters_node_create(yp_parser_t *parser) { @@ -3351,7 +3077,7 @@ yp_program_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, yp_st // Allocate and initialize new ParenthesesNode node. static yp_parentheses_node_t * -yp_parentheses_node_create(yp_parser_t *parser, const yp_token_t *opening, yp_node_t *statements, const yp_token_t *closing) { +yp_parentheses_node_create(yp_parser_t *parser, const yp_token_t *opening, yp_node_t *body, const yp_token_t *closing) { yp_parentheses_node_t *node = YP_ALLOC_NODE(parser, yp_parentheses_node_t); *node = (yp_parentheses_node_t) { @@ -3362,7 +3088,7 @@ yp_parentheses_node_create(yp_parser_t *parser, const yp_token_t *opening, yp_no .end = closing->end } }, - .statements = statements, + .body = body, .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), .closing_loc = YP_LOCATION_TOKEN_VALUE(closing) }; @@ -3708,7 +3434,7 @@ yp_self_node_create(yp_parser_t *parser, const yp_token_t *token) { // Allocate a new SingletonClassNode node. static yp_singleton_class_node_t * -yp_singleton_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *class_keyword, const yp_token_t *operator, yp_node_t *expression, yp_node_t *statements, const yp_token_t *end_keyword) { +yp_singleton_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const yp_token_t *class_keyword, const yp_token_t *operator, yp_node_t *expression, yp_node_t *body, const yp_token_t *end_keyword) { yp_singleton_class_node_t *node = YP_ALLOC_NODE(parser, yp_singleton_class_node_t); *node = (yp_singleton_class_node_t) { @@ -3723,7 +3449,7 @@ yp_singleton_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *local .class_keyword_loc = YP_LOCATION_TOKEN_VALUE(class_keyword), .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .expression = expression, - .statements = statements, + .body = body, .end_keyword_loc = YP_LOCATION_TOKEN_VALUE(end_keyword) }; @@ -3945,7 +3671,7 @@ yp_symbol_node_label_create(yp_parser_t *parser, const yp_token_t *token) { assert((label.end - label.start) >= 0); yp_string_shared_init(&node->unescaped, label.start, label.end); - yp_unescape_manipulate_string(parser, &node->unescaped, YP_UNESCAPE_ALL, &parser->error_list); + yp_unescape_manipulate_string(parser, &node->unescaped, YP_UNESCAPE_ALL); break; } case YP_TOKEN_MISSING: { @@ -4502,27 +4228,30 @@ yp_do_loop_stack_p(yp_parser_t *parser) { /* Lexer check helpers */ /******************************************************************************/ +// Get the next character in the source starting from +cursor+. If that position +// is beyond the end of the source then return '\0'. +static inline char +peek_at(yp_parser_t *parser, const char *cursor) { + if (cursor < parser->end) { + return *cursor; + } else { + return '\0'; + } +} + // Get the next character in the source starting from parser->current.end and // adding the given offset. If that position is beyond the end of the source // then return '\0'. static inline char -peek_at(yp_parser_t *parser, size_t offset) { - if (parser->current.end + offset < parser->end) { - return parser->current.end[offset]; - } else { - return '\0'; - } +peek_offset(yp_parser_t *parser, ptrdiff_t offset) { + return peek_at(parser, parser->current.end + offset); } // Get the next character in the source starting from parser->current.end. If // that position is beyond the end of the source then return '\0'. static inline char peek(yp_parser_t *parser) { - if (parser->current.end < parser->end) { - return *parser->current.end; - } else { - return '\0'; - } + return peek_at(parser, parser->current.end); } // Get the next string of length len in the source starting from parser->current.end. @@ -4547,6 +4276,35 @@ match(yp_parser_t *parser, char value) { return false; } +// Return the length of the line ending string starting at +cursor+, or 0 if it +// is not a line ending. This function is intended to be CRLF/LF agnostic. +static inline size_t +match_eol_at(yp_parser_t *parser, const char *cursor) { + if (peek_at(parser, cursor) == '\n') { + return 1; + } + if (peek_at(parser, cursor) == '\r' && peek_at(parser, cursor + 1) == '\n') { + return 2; + } + return 0; +} + +// Return the length of the line ending string starting at +// parser->current.end + offset, or 0 if it is not a line ending. This function +// is intended to be CRLF/LF agnostic. +static inline size_t +match_eol_offset(yp_parser_t *parser, ptrdiff_t offset) { + return match_eol_at(parser, parser->current.end + offset); +} + +// Return the length of the line ending string starting at parser->current.end, +// or 0 if it is not a line ending. This function is intended to be CRLF/LF +// agnostic. +static inline size_t +match_eol(yp_parser_t *parser) { + return match_eol_at(parser, parser->current.end); +} + // Skip to the next newline character or NUL byte. static inline const char * next_newline(const char *cursor, ptrdiff_t length) { @@ -4570,11 +4328,13 @@ parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdi const char *cursor_limit = cursor + length - key_length + 1; while ((cursor = yp_memchr(cursor, 'c', (size_t) (cursor_limit - cursor), parser->encoding_changed, &parser->encoding)) != NULL) { - if ( - (strncmp(cursor, "coding", key_length - 1) == 0) && - (cursor[key_length - 1] == ':' || cursor[key_length - 1] == '=') - ) { - return cursor + key_length; + if (strncmp(cursor, "coding", key_length - 1) == 0) { + size_t whitespace_after_coding = yp_strspn_inline_whitespace(cursor + key_length - 1, parser->end - (cursor + key_length - 1)); + size_t cur_pos = key_length + whitespace_after_coding; + + if (cursor[cur_pos - 1] == ':' || cursor[cur_pos - 1] == '=') { + return cursor + cur_pos; + } } cursor++; @@ -4830,7 +4590,7 @@ lex_optional_float_suffix(yp_parser_t *parser) { // Here we're going to attempt to parse the optional decimal portion of a // float. If it's not there, then it's okay and we'll just continue on. if (peek(parser) == '.') { - if (yp_char_is_decimal_digit(peek_at(parser, 1))) { + if (yp_char_is_decimal_digit(peek_offset(parser, 1))) { parser->current.end += 2; parser->current.end += yp_strspn_decimal_number(parser->current.end, parser->end - parser->current.end); type = YP_TOKEN_FLOAT; @@ -4863,7 +4623,7 @@ static yp_token_type_t lex_numeric_prefix(yp_parser_t *parser) { yp_token_type_t type = YP_TOKEN_INTEGER; - if (parser->current.end[-1] == '0') { + if (peek_offset(parser, -1) == '0') { switch (*parser->current.end) { // 0d1111 is a decimal number case 'd': @@ -4946,7 +4706,7 @@ lex_numeric_prefix(yp_parser_t *parser) { // If the last character that we consumed was an underscore, then this is // actually an invalid integer value, and we should return an invalid token. - if (parser->current.end[-1] == '_') { + if (peek_offset(parser, -1) == '_') { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Number literal cannot end with a `_`."); } @@ -5127,7 +4887,7 @@ lex_identifier(yp_parser_t *parser, bool previous_command_start) { if ( ((lex_state_p(parser, YP_LEX_STATE_LABEL | YP_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser)) && - (peek(parser) == ':') && (peek_at(parser, 1) != ':') + (peek(parser) == ':') && (peek_offset(parser, 1) != ':') ) { // If we're in a position where we can accept a : at the end of an // identifier, then we'll optionally accept it. @@ -5143,7 +4903,7 @@ lex_identifier(yp_parser_t *parser, bool previous_command_start) { } return YP_TOKEN_IDENTIFIER; - } else if (lex_state_p(parser, YP_LEX_STATE_FNAME) && peek_at(parser, 1) != '~' && peek_at(parser, 1) != '>' && (peek_at(parser, 1) != '=' || peek_at(parser, 2) == '>') && match(parser, '=')) { + } else if (lex_state_p(parser, YP_LEX_STATE_FNAME) && peek_offset(parser, 1) != '~' && peek_offset(parser, 1) != '>' && (peek_offset(parser, 1) != '=' || peek_offset(parser, 2) == '>') && match(parser, '=')) { // If we're in a position where we can accept a = at the end of an // identifier, then we'll optionally accept it. return YP_TOKEN_IDENTIFIER; @@ -5151,7 +4911,7 @@ lex_identifier(yp_parser_t *parser, bool previous_command_start) { if ( ((lex_state_p(parser, YP_LEX_STATE_LABEL | YP_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser)) && - peek(parser) == ':' && peek_at(parser, 1) != ':' + peek(parser) == ':' && peek_offset(parser, 1) != ':' ) { // If we're in a position where we can accept a : at the end of an // identifier, then we'll optionally accept it. @@ -5419,7 +5179,7 @@ lex_question_mark(yp_parser_t *parser) { if (parser->current.start[1] == '\\') { lex_state_set(parser, YP_LEX_STATE_END); - parser->current.end += yp_unescape_calculate_difference(parser->current.start + 1, parser->end, YP_UNESCAPE_ALL, true, &parser->error_list); + parser->current.end += yp_unescape_calculate_difference(parser, parser->current.start + 1, YP_UNESCAPE_ALL, true); return YP_TOKEN_CHARACTER_LITERAL; } else { size_t encoding_width = parser->encoding.char_width(parser->current.end, parser->end - parser->current.end); @@ -5428,7 +5188,7 @@ lex_question_mark(yp_parser_t *parser) { // an underscore. We check for this case if ( !(parser->encoding.alnum_char(parser->current.end, parser->end - parser->current.end) || - *parser->current.end == '_') || + peek(parser) == '_') || ( (parser->current.end + encoding_width >= parser->end) || !char_is_identifier(parser, parser->current.end + encoding_width) @@ -5644,30 +5404,22 @@ parser_lex(yp_parser_t *parser) { space_seen = true; break; case '\r': - if (peek_at(parser, 1) == '\n') { + if (match_eol_offset(parser, 1)) { chomping = false; } else { parser->current.end++; space_seen = true; } break; - case '\\': - if (peek_at(parser, 1) == '\n') { - if (parser->heredoc_end) { - parser->current.end = parser->heredoc_end; - parser->heredoc_end = NULL; - } else { - yp_newline_list_append(&parser->newline_list, parser->current.end + 1); - parser->current.end += 2; - space_seen = true; - } - } else if (peek_at(parser, 1) == '\r' && peek_at(parser, 2) == '\n') { + case '\\': { + size_t eol_length = match_eol_offset(parser, 1); + if (eol_length) { if (parser->heredoc_end) { parser->current.end = parser->heredoc_end; parser->heredoc_end = NULL; } else { - yp_newline_list_append(&parser->newline_list, parser->current.end + 2); - parser->current.end += 3; + parser->current.end += eol_length + 1; + yp_newline_list_append(&parser->newline_list, parser->current.end - 1); space_seen = true; } } else if (yp_char_is_inline_whitespace(*parser->current.end)) { @@ -5675,7 +5427,9 @@ parser_lex(yp_parser_t *parser) { } else { chomping = false; } + break; + } default: chomping = false; break; @@ -5685,13 +5439,14 @@ parser_lex(yp_parser_t *parser) { // Next, we'll set to start of this token to be the current end. parser->current.start = parser->current.end; - // We'll check if we're at the end of the file. If we are, then we need to - // return the EOF token. + // We'll check if we're at the end of the file. If we are, then we + // need to return the EOF token. if (parser->current.end >= parser->end) { LEX(YP_TOKEN_EOF); } - // Finally, we'll check the current character to determine the next token. + // Finally, we'll check the current character to determine the next + // token. switch (*parser->current.end++) { case '\0': // NUL or end of script case '\004': // ^D @@ -5701,16 +5456,14 @@ parser_lex(yp_parser_t *parser) { case '#': { // comments const char *ending = next_newline(parser->current.end, parser->end - parser->current.end); - while (ending && ending < parser->end && *ending != '\n') { - ending = next_newline(ending + 1, parser->end - ending); - } parser->current.end = ending == NULL ? parser->end : ending + 1; parser->current.type = YP_TOKEN_COMMENT; parser_lex_callback(parser); - // If we found a comment while lexing, then we're going to add it to the - // list of comments in the file and keep lexing. + // If we found a comment while lexing, then we're going to + // add it to the list of comments in the file and keep + // lexing. yp_comment_t *comment = parser_comment(parser, YP_COMMENT_INLINE); yp_list_append(&parser->comment_list, (yp_list_node_t *) comment); @@ -5721,21 +5474,29 @@ parser_lex(yp_parser_t *parser) { lexed_comment = true; } /* fallthrough */ - case '\r': { - // The only way you can have carriage returns in this particular loop - // is if you have a carriage return followed by a newline. In that - // case we'll just skip over the carriage return and continue lexing, - // in order to make it so that the newline token encapsulates both the - // carriage return and the newline. Note that we need to check that - // we haven't already lexed a comment here because that falls through - // into here as well. - if (!lexed_comment) parser->current.end++; - } - /* fallthrough */ + case '\r': case '\n': { - if (parser->heredoc_end == NULL) { - yp_newline_list_append(&parser->newline_list, parser->current.end - 1); - } else { + size_t eol_length = match_eol_at(parser, parser->current.end - 1); + if (eol_length) { + // The only way you can have carriage returns in this + // particular loop is if you have a carriage return + // followed by a newline. In that case we'll just skip + // over the carriage return and continue lexing, in + // order to make it so that the newline token + // encapsulates both the carriage return and the + // newline. Note that we need to check that we haven't + // already lexed a comment here because that falls + // through into here as well. + if (!lexed_comment) { + parser->current.end += eol_length - 1; // skip CR + } + + if (parser->heredoc_end == NULL) { + yp_newline_list_append(&parser->newline_list, parser->current.end - 1); + } + } + + if (parser->heredoc_end) { parser_flush_heredoc_end(parser); } @@ -5791,7 +5552,13 @@ parser_lex(yp_parser_t *parser) { // If the lex state was ignored, or we hit a '.' or a '&.', // we will lex the ignored newline - if (lex_state_ignored_p(parser) || (following && ((following[0] == '.') || (following + 1 < parser->end && following[0] == '&' && following[1] == '.')))) { + if ( + lex_state_ignored_p(parser) || + (following && ( + (peek_at(parser, following) == '.') || + (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.') + )) + ) { if (!lexed_comment) parser_lex_ignored_newline(parser); lexed_comment = false; goto lex_next_token; @@ -5804,7 +5571,7 @@ parser_lex(yp_parser_t *parser) { // To match ripper, we need to emit an ignored newline even though // its a real newline in the case that we have a beginless range // on a subsequent line. - if ((next_content + 1 < parser->end) && (next_content[1] == '.')) { + if (peek_at(parser, next_content + 1) == '.') { if (!lexed_comment) parser_lex_ignored_newline(parser); lex_state_set(parser, YP_LEX_STATE_BEG); parser->command_start = true; @@ -5822,7 +5589,7 @@ parser_lex(yp_parser_t *parser) { // If we hit a &. after a newline, then we're in a call chain and // we need to return the call operator. - if (next_content + 1 < parser->end && next_content[0] == '&' && next_content[1] == '.') { + if (peek_at(parser, next_content) == '&' && peek_at(parser, next_content + 1) == '.') { if (!lexed_comment) parser_lex_ignored_newline(parser); lex_state_set(parser, YP_LEX_STATE_DOT); parser->current.start = next_content; @@ -6019,7 +5786,7 @@ parser_lex(yp_parser_t *parser) { // = => =~ == === =begin case '=': - if (current_token_starts_line(parser) && strncmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_at(parser, 5))) { + if (current_token_starts_line(parser) && strncmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_offset(parser, 5))) { yp_token_type_t type = lex_embdoc(parser); if (type == YP_TOKEN_EOF) { @@ -6443,13 +6210,13 @@ parser_lex(yp_parser_t *parser) { LEX(YP_TOKEN_COLON_COLON); } - if (lex_state_end_p(parser) || yp_char_is_whitespace(*parser->current.end) || (*parser->current.end == '#')) { + if (lex_state_end_p(parser) || yp_char_is_whitespace(*parser->current.end) || peek(parser) == '#') { lex_state_set(parser, YP_LEX_STATE_BEG); LEX(YP_TOKEN_COLON); } - if ((*parser->current.end == '"') || (*parser->current.end == '\'')) { - lex_mode_push_string(parser, *parser->current.end == '"', false, '\0', *parser->current.end); + if (peek(parser) == '"' || peek(parser) == '\'') { + lex_mode_push_string(parser, peek(parser) == '"', false, '\0', *parser->current.end); parser->current.end++; } @@ -6518,25 +6285,26 @@ parser_lex(yp_parser_t *parser) { } else if( lex_state_beg_p(parser) || - (lex_state_p(parser, YP_LEX_STATE_FITEM) && (*parser->current.end == 's')) || + (lex_state_p(parser, YP_LEX_STATE_FITEM) && (peek(parser) == 's')) || lex_state_spcarg_p(parser, space_seen) ) { if (!parser->encoding.alnum_char(parser->current.end, parser->end - parser->current.end)) { lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); - if (*parser->current.end == '\r') { + size_t eol_length = match_eol(parser); + if (eol_length) { + parser->current.end += eol_length; + yp_newline_list_append(&parser->newline_list, parser->current.end - 1); + } else { parser->current.end++; } - if (*parser->current.end == '\n') { - yp_newline_list_append(&parser->newline_list, parser->current.end); + if (parser->current.end < parser->end) { + LEX(YP_TOKEN_STRING_BEGIN); } - - parser->current.end++; - LEX(YP_TOKEN_STRING_BEGIN); } - switch (*parser->current.end) { + switch (peek(parser)) { case 'i': { parser->current.end++; @@ -6560,6 +6328,7 @@ parser_lex(yp_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_regexp(parser, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); + yp_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; } @@ -6570,6 +6339,7 @@ parser_lex(yp_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); + yp_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; } @@ -6580,6 +6350,7 @@ parser_lex(yp_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); + yp_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; } @@ -6629,7 +6400,7 @@ parser_lex(yp_parser_t *parser) { // unparseable. In this case we'll just drop it from the parser // and skip past it and hope that the next token is something // that we can parse. - yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "invalid %% token"); + yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid %% token"); goto lex_next_token; } } @@ -6681,8 +6452,9 @@ parser_lex(yp_parser_t *parser) { ((parser->current.end - parser->current.start) == 7) && current_token_starts_line(parser) && (strncmp(parser->current.start, "__END__", 7) == 0) && - (*parser->current.end == '\n' || (*parser->current.end == '\r' && parser->current.end[1] == '\n')) - ) { + (parser->current.end == parser->end || match_eol(parser)) + ) + { parser->current.end = parser->end; parser->current.type = YP_TOKEN___END__; parser_lex_callback(parser); @@ -6739,7 +6511,7 @@ parser_lex(yp_parser_t *parser) { if ((whitespace = yp_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, should_stop)) > 0) { parser->current.end += whitespace; - if (parser->current.end[-1] == '\n') { + if (peek_offset(parser, -1) == '\n') { // mutates next_start parser_flush_heredoc_end(parser); } @@ -6803,13 +6575,11 @@ parser_lex(yp_parser_t *parser) { // and find the next breakpoint. if (*breakpoint == '\\') { yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; - size_t difference = yp_unescape_calculate_difference(breakpoint, parser->end, unescape_type, false, &parser->error_list); + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); // If the result is an escaped newline, then we need to // track that newline. - if (breakpoint[difference - 1] == '\n') { - yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); - } + yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); continue; @@ -6844,7 +6614,13 @@ parser_lex(yp_parser_t *parser) { case YP_LEX_REGEXP: { // First, we'll set to start of this token to be the current end. - parser->current.start = parser->current.end; + if (parser->next_start == NULL) { + parser->current.start = parser->current.end; + } else { + parser->current.start = parser->next_start; + parser->current.end = parser->next_start; + parser->next_start = NULL; + } // We'll check if we're at the end of the file. If we are, then we need to // return the EOF token. @@ -6871,7 +6647,16 @@ parser_lex(yp_parser_t *parser) { // If we've hit a newline, then we need to track that in the // list of newlines. if (*breakpoint == '\n') { - yp_newline_list_append(&parser->newline_list, breakpoint); + // For the special case of a newline-terminated regular expression, we will pass + // through this branch twice -- once with YP_TOKEN_REGEXP_BEGIN and then again + // with YP_TOKEN_STRING_CONTENT. Let's avoid tracking the newline twice, by + // tracking it only in the REGEXP_BEGIN case. + if ( + !(lex_mode->as.regexp.terminator == '\n' && parser->current.type != YP_TOKEN_REGEXP_BEGIN) + && parser->heredoc_end == NULL + ) { + yp_newline_list_append(&parser->newline_list, breakpoint); + } if (lex_mode->as.regexp.terminator != '\n') { // If the terminator is not a newline, then we can set @@ -6912,12 +6697,20 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character // and find the next breakpoint. if (*breakpoint == '\\') { - size_t difference = yp_unescape_calculate_difference(breakpoint, parser->end, YP_UNESCAPE_ALL, false, &parser->error_list); - - // If the result is an escaped newline, then we need to - // track that newline. - if (breakpoint[difference - 1] == '\n') { - yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, YP_UNESCAPE_ALL, false); + + // If the result is an escaped newline ... + if (*(breakpoint + difference - 1) == '\n') { + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, flush the heredoc and + // continue parsing after heredoc_end. + parser->current.end = breakpoint + difference; + parser_flush_heredoc_end(parser); + LEX(YP_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + } } breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); @@ -7005,21 +6798,18 @@ parser_lex(yp_parser_t *parser) { // Otherwise we need to switch back to the parent lex mode and // return the end of the string. - if (*parser->current.end == '\r' && parser->current.end + 1 < parser->end && parser->current.end[1] == '\n') { - parser->current.end = breakpoint + 2; - yp_newline_list_append(&parser->newline_list, breakpoint + 1); + size_t eol_length = match_eol_at(parser, breakpoint); + if (eol_length) { + parser->current.end = breakpoint + eol_length; + yp_newline_list_append(&parser->newline_list, parser->current.end - 1); } else { - if (*parser->current.end == '\n') { - yp_newline_list_append(&parser->newline_list, parser->current.end); - } - parser->current.end = breakpoint + 1; } if ( parser->lex_modes.current->as.string.label_allowed && (peek(parser) == ':') && - (peek_at(parser, 1) != ':') + (peek_offset(parser, 1) != ':') ) { parser->current.end++; lex_state_set(parser, YP_LEX_STATE_ARG | YP_LEX_STATE_LABELED); @@ -7057,12 +6847,20 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character and // find the next breakpoint. yp_unescape_type_t unescape_type = parser->lex_modes.current->as.string.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; - size_t difference = yp_unescape_calculate_difference(breakpoint, parser->end, unescape_type, false, &parser->error_list); + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); - // If the result is an escaped newline, then we need to - // track that newline. - if (breakpoint[difference - 1] == '\n') { - yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + // If the result is an escaped newline ... + if (*(breakpoint + difference - 1) == '\n') { + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, flush the heredoc and + // continue parsing after heredoc_end. + parser->current.end = breakpoint + difference; + parser_flush_heredoc_end(parser); + LEX(YP_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + } } breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); @@ -7098,6 +6896,7 @@ parser_lex(yp_parser_t *parser) { } else { parser->current.start = parser->next_start; parser->current.end = parser->next_start; + parser->heredoc_end = NULL; parser->next_start = NULL; } @@ -7114,7 +6913,7 @@ parser_lex(yp_parser_t *parser) { // If we are immediately following a newline and we have hit the // terminator, then we need to return the ending of the heredoc. - if (parser->current.start[-1] == '\n') { + if (current_token_starts_line(parser)) { const char *start = parser->current.start; if (parser->lex_modes.current->as.heredoc.indent != YP_HEREDOC_INDENT_NONE) { start += yp_strspn_inline_whitespace(start, parser->end - start); @@ -7124,12 +6923,10 @@ parser_lex(yp_parser_t *parser) { bool matched = true; bool at_end = false; - if ((start + ident_length < parser->end) && (start[ident_length] == '\n')) { - parser->current.end = start + ident_length + 1; - yp_newline_list_append(&parser->newline_list, start + ident_length); - } else if ((start + ident_length + 1 < parser->end) && (start[ident_length] == '\r') && (start[ident_length + 1] == '\n')) { - parser->current.end = start + ident_length + 2; - yp_newline_list_append(&parser->newline_list, start + ident_length + 1); + size_t eol_length = match_eol_at(parser, start + ident_length); + if (eol_length) { + parser->current.end = start + ident_length + eol_length; + yp_newline_list_append(&parser->newline_list, parser->current.end - 1); } else if (parser->end == (start + ident_length)) { parser->current.end = start + ident_length; at_end = true; @@ -7194,19 +6991,10 @@ parser_lex(yp_parser_t *parser) { (start + ident_length <= parser->end) && (strncmp(start, ident_start, ident_length) == 0) ) { - // Heredoc terminators must be followed by a newline or EOF to be valid. - if (start + ident_length == parser->end || start[ident_length] == '\n') { - parser->current.end = breakpoint + 1; - LEX(YP_TOKEN_STRING_CONTENT); - } - - // They can also be followed by a carriage return and then a - // newline. Be sure here that we don't accidentally read off the - // end. + // Heredoc terminators must be followed by a newline, CRLF, or EOF to be valid. if ( - (start + ident_length + 1 < parser->end) && - (start[ident_length] == '\r') && - (start[ident_length + 1] == '\n') + start + ident_length == parser->end || + match_eol_at(parser, start + ident_length) ) { parser->current.end = breakpoint + 1; LEX(YP_TOKEN_STRING_CONTENT); @@ -7225,17 +7013,14 @@ parser_lex(yp_parser_t *parser) { // stop looping before the newline and not after the // newline so that we can still potentially find the // terminator of the heredoc. - if (breakpoint + 1 < parser->end && breakpoint[1] == '\n') { - breakpoint++; - } else if (breakpoint + 2 < parser->end && breakpoint[1] == '\r' && breakpoint[2] == '\n') { - breakpoint += 2; + size_t eol_length = match_eol_at(parser, breakpoint + 1); + if (eol_length) { + breakpoint += eol_length; } else { yp_unescape_type_t unescape_type = (quote == YP_HEREDOC_QUOTE_SINGLE) ? YP_UNESCAPE_MINIMAL : YP_UNESCAPE_ALL; - size_t difference = yp_unescape_calculate_difference(breakpoint, parser->end, unescape_type, false, &parser->error_list); + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); - if (breakpoint[difference - 1] == '\n') { - yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); - } + yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); } @@ -7289,7 +7074,7 @@ yp_regular_expression_node_create_and_unescape(yp_parser_t *parser, const yp_tok assert((content->end - content->start) >= 0); yp_string_shared_init(&node->unescaped, content->start, content->end); - yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type, &parser->error_list); + yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type); return node; } @@ -7300,7 +7085,7 @@ yp_symbol_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *openin assert((content->end - content->start) >= 0); yp_string_shared_init(&node->unescaped, content->start, content->end); - yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type, &parser->error_list); + yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type); return node; } @@ -7311,7 +7096,7 @@ yp_string_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *openin assert((content->end - content->start) >= 0); yp_string_shared_init(&node->unescaped, content->start, content->end); - yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type, &parser->error_list); + yp_unescape_manipulate_string(parser, &node->unescaped, unescape_type); return node; } @@ -7322,7 +7107,7 @@ yp_xstring_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *openi assert((content->end - content->start) >= 0); yp_string_shared_init(&node->unescaped, content->start, content->end); - yp_unescape_manipulate_string(parser, &node->unescaped, YP_UNESCAPE_ALL, &parser->error_list); + yp_unescape_manipulate_string(parser, &node->unescaped, YP_UNESCAPE_ALL); return node; } @@ -8429,7 +8214,6 @@ parse_parameters( bool looping = true; yp_do_loop_stack_push(parser, false); - yp_parameters_order_t order = YP_PARAMETERS_ORDER_NONE; do { @@ -9021,7 +8805,7 @@ parse_conditional(yp_parser_t *parser, yp_context_t context) { } yp_token_t end_keyword = not_provided(parser); - yp_node_t *parent; + yp_node_t *parent = NULL; switch (context) { case YP_CONTEXT_IF: @@ -9031,7 +8815,6 @@ parse_conditional(yp_parser_t *parser, yp_context_t context) { parent = (yp_node_t *) yp_unless_node_create(parser, &keyword, predicate, statements); break; default: - parent = NULL; assert(false && "unreachable"); break; } @@ -9077,50 +8860,49 @@ parse_conditional(yp_parser_t *parser, yp_context_t context) { switch (context) { case YP_CONTEXT_IF: ((yp_if_node_t *) current)->consequent = (yp_node_t *) else_node; - // Recurse down if nodes setting the appropriate end location in - // all cases. - yp_node_t *recursing_node = parent; - bool recursing = true; - - while (recursing) { - switch (YP_NODE_TYPE(recursing_node)) { - case YP_NODE_IF_NODE: - yp_if_node_end_keyword_loc_set((yp_if_node_t *) recursing_node, &parser->previous); - recursing_node = ((yp_if_node_t *) recursing_node)->consequent; - break; - case YP_NODE_ELSE_NODE: - yp_else_node_end_keyword_loc_set((yp_else_node_t *) recursing_node, &parser->previous); - recursing = false; - break; - default: { - recursing = false; - break; - } - } - } break; case YP_CONTEXT_UNLESS: ((yp_unless_node_t *) parent)->consequent = else_node; - yp_unless_node_end_keyword_loc_set((yp_unless_node_t *) parent, &parser->previous); break; default: assert(false && "unreachable"); break; } } else { - expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close `if` statement."); + expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close conditional statement."); + } - switch (context) { - case YP_CONTEXT_IF: - yp_if_node_end_keyword_loc_set((yp_if_node_t *) parent, &parser->previous); - break; - case YP_CONTEXT_UNLESS: - yp_unless_node_end_keyword_loc_set((yp_unless_node_t *) parent, &parser->previous); - break; - default: - assert(false && "unreachable"); - break; + // Set the appropriate end location for all of the nodes in the subtree. + switch (context) { + case YP_CONTEXT_IF: { + yp_node_t *current = parent; + bool recursing = true; + + while (recursing) { + switch (YP_NODE_TYPE(current)) { + case YP_NODE_IF_NODE: + yp_if_node_end_keyword_loc_set((yp_if_node_t *) current, &parser->previous); + current = ((yp_if_node_t *) current)->consequent; + recursing = current != NULL; + break; + case YP_NODE_ELSE_NODE: + yp_else_node_end_keyword_loc_set((yp_else_node_t *) current, &parser->previous); + recursing = false; + break; + default: { + recursing = false; + break; + } + } + } + break; } + case YP_CONTEXT_UNLESS: + yp_unless_node_end_keyword_loc_set((yp_unless_node_t *) parent, &parser->previous); + break; + default: + assert(false && "unreachable"); + break; } return parent; @@ -9518,9 +9300,12 @@ parse_heredoc_common_whitespace(yp_parser_t *parser, yp_node_list_t *nodes) { const char *cur_char = content_loc->start; while (cur_char && cur_char < content_loc->end) { - // Any empty newlines aren't included in the minimum whitespace calculation - while (cur_char < content_loc->end && *cur_char == '\n') cur_char++; - while (cur_char + 1 < content_loc->end && *cur_char == '\r' && cur_char[1] == '\n') cur_char += 2; + // Any empty newlines aren't included in the minimum whitespace + // calculation. + size_t eol_length; + while ((eol_length = match_eol_at(parser, cur_char))) { + cur_char += eol_length; + } if (cur_char == content_loc->end) break; @@ -9535,11 +9320,12 @@ parse_heredoc_common_whitespace(yp_parser_t *parser, yp_node_list_t *nodes) { cur_char++; } - // If we hit a newline, then we have encountered a line that contains - // only whitespace, and it shouldn't be considered in the calculation of - // common leading whitespace. - if (*cur_char == '\n') { - cur_char++; + // If we hit a newline, then we have encountered a line that + // contains only whitespace, and it shouldn't be considered in + // the calculation of common leading whitespace. + eol_length = match_eol_at(parser, cur_char); + if (eol_length) { + cur_char += eol_length; continue; } @@ -9660,7 +9446,7 @@ parse_heredoc_dedent(yp_parser_t *parser, yp_node_t *node, yp_heredoc_quote_t qu yp_node_destroy(parser, node); } else { string->length = dest_length; - yp_unescape_manipulate_string(parser, string, (quote == YP_HEREDOC_QUOTE_SINGLE) ? YP_UNESCAPE_MINIMAL : YP_UNESCAPE_ALL, &parser->error_list); + yp_unescape_manipulate_string(parser, string, (quote == YP_HEREDOC_QUOTE_SINGLE) ? YP_UNESCAPE_MINIMAL : YP_UNESCAPE_ALL); nodes->nodes[write_index++] = node; } @@ -10559,7 +10345,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // fact a method call, not a constant read. if ( match_type_p(parser, YP_TOKEN_PARENTHESIS_LEFT) || - (binding_power <= YP_BINDING_POWER_ASSIGNMENT && (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 2, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR))) || + (binding_power <= YP_BINDING_POWER_ASSIGNMENT && (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 3, YP_TOKEN_UAMPERSAND, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR))) || (yp_accepts_block_stack_p(parser) && match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_DO, YP_TOKEN_BRACE_LEFT)) ) { yp_arguments_t arguments = YP_EMPTY_ARGUMENTS; @@ -10682,7 +10468,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // can still be a method call if it is followed by arguments or // a block, so we need to check for that here. if ( - (binding_power <= YP_BINDING_POWER_ASSIGNMENT && (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 2, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR))) || + (binding_power <= YP_BINDING_POWER_ASSIGNMENT && (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 3, YP_TOKEN_UAMPERSAND, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR))) || (yp_accepts_block_stack_p(parser) && match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_DO, YP_TOKEN_BRACE_LEFT)) ) { yp_arguments_t arguments = YP_EMPTY_ARGUMENTS; @@ -11100,7 +10886,12 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { return (yp_node_t *) yp_singleton_class_node_create(parser, &locals, &class_keyword, &operator, expression, statements, &parser->previous); } - yp_node_t *name = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected to find a class name after `class`."); + yp_node_t *constant_path = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected to find a class name after `class`."); + yp_token_t name = parser->previous; + if (name.type != YP_TOKEN_CONSTANT) { + yp_diagnostic_list_append(&parser->error_list, name.start, name.end, "Expected a constant name after `class`."); + } + yp_token_t inheritance_operator; yp_node_t *superclass; @@ -11141,7 +10932,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_constant_id_list_t locals = parser->current_scope->locals; yp_parser_scope_pop(parser); yp_do_loop_stack_pop(parser); - return (yp_node_t *) yp_class_node_create(parser, &locals, &class_keyword, name, &inheritance_operator, superclass, statements, &parser->previous); + return (yp_node_t *) yp_class_node_create(parser, &locals, &class_keyword, constant_path, &name, &inheritance_operator, superclass, statements, &parser->previous); } case YP_TOKEN_KEYWORD_DEF: { yp_token_t def_keyword = parser->current; @@ -11300,6 +11091,12 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { break; } case YP_CASE_PARAMETER: { + // If we're about to lex a label, we need to add the label + // state to make sure the next newline is ignored. + if (parser->current.type == YP_TOKEN_LABEL) { + lex_state_set(parser, parser->lex_state | YP_LEX_STATE_LABEL); + } + lparen = not_provided(parser); rparen = not_provided(parser); params = parse_parameters(parser, YP_BINDING_POWER_DEFINED, false, false, true); @@ -11529,13 +11326,14 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { parser_lex(parser); yp_token_t module_keyword = parser->previous; - yp_node_t *name = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected to find a module name after `module`."); + yp_node_t *constant_path = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected to find a module name after `module`."); + yp_token_t name; - // If we can recover from a syntax error that occurred while parsing the - // name of the module, then we'll handle that here. - if (YP_NODE_TYPE_P(name, YP_NODE_MISSING_NODE)) { - yp_token_t end_keyword = (yp_token_t) { .type = YP_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; - return (yp_node_t *) yp_module_node_create(parser, NULL, &module_keyword, name, NULL, &end_keyword); + // If we can recover from a syntax error that occurred while parsing + // the name of the module, then we'll handle that here. + if (YP_NODE_TYPE_P(constant_path, YP_NODE_MISSING_NODE)) { + yp_token_t missing = (yp_token_t) { .type = YP_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + return (yp_node_t *) yp_module_node_create(parser, NULL, &module_keyword, constant_path, &missing, NULL, &missing); } while (accept(parser, YP_TOKEN_COLON_COLON)) { @@ -11544,7 +11342,15 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { expect(parser, YP_TOKEN_CONSTANT, "Expected to find a module name after `::`."); yp_node_t *constant = (yp_node_t *) yp_constant_read_node_create(parser, &parser->previous); - name = (yp_node_t *)yp_constant_path_node_create(parser, name, &double_colon, constant); + constant_path = (yp_node_t *) yp_constant_path_node_create(parser, constant_path, &double_colon, constant); + } + + // Here we retrieve the name of the module. If it wasn't a constant, + // then it's possible that `module foo` was passed, which is a + // syntax error. We handle that here as well. + name = parser->previous; + if (name.type != YP_TOKEN_CONSTANT) { + yp_diagnostic_list_append(&parser->error_list, name.start, name.end, "Expected to find a module name after `module`."); } yp_parser_scope_push(parser, true); @@ -11571,7 +11377,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_diagnostic_list_append(&parser->error_list, module_keyword.start, module_keyword.end, "Module definition in method body"); } - return (yp_node_t *) yp_module_node_create(parser, &locals, &module_keyword, name, statements, &parser->previous); + return (yp_node_t *) yp_module_node_create(parser, &locals, &module_keyword, constant_path, &name, statements, &parser->previous); } case YP_TOKEN_KEYWORD_NIL: parser_lex(parser); @@ -12165,8 +11971,10 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { break; } case YP_CASE_PARAMETER: { + yp_accepts_block_stack_push(parser, false); yp_token_t opening = not_provided(parser); params = parse_block_parameters(parser, false, &opening, true); + yp_accepts_block_stack_pop(parser); break; } default: { @@ -12451,14 +12259,19 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + case YP_NODE_CLASS_VARIABLE_READ_NODE: + case YP_NODE_CONSTANT_PATH_NODE: + case YP_NODE_CONSTANT_READ_NODE: + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_global_variable_operator_and_write_node_create(parser, node, &token, value); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12476,12 +12289,11 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, message_loc.start, message_loc.end); - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_and_write_node_create(parser, node, &token, value, constant_id); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); } parser_lex(parser); @@ -12492,49 +12304,6 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); return (yp_node_t *) yp_call_operator_and_write_node_create(parser, (yp_call_node_t *) node, &token, value); } - case YP_NODE_CLASS_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_class_variable_operator_and_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_CONSTANT_PATH_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_constant_path_operator_and_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); - } - case YP_NODE_CONSTANT_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_constant_operator_and_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_and_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_constant_id_t constant_id = ((yp_local_variable_read_node_t *) node)->constant_id; - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_and_write_node_create(parser, node, &token, value, constant_id); - - yp_node_destroy(parser, node); - return result; - } case YP_NODE_MULTI_WRITE_NODE: { parser_lex(parser); yp_diagnostic_list_append(&parser->error_list, token.start, token.end, "Cannot use `&&=' on a multi-write."); @@ -12556,14 +12325,19 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + case YP_NODE_CLASS_VARIABLE_READ_NODE: + case YP_NODE_CONSTANT_PATH_NODE: + case YP_NODE_CONSTANT_READ_NODE: + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_global_variable_operator_or_write_node_create(parser, node, &token, value); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12581,12 +12355,11 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, message_loc.start, message_loc.end); - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_or_write_node_create(parser, node, &token, value, constant_id); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); } parser_lex(parser); @@ -12597,49 +12370,6 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); return (yp_node_t *) yp_call_operator_or_write_node_create(parser, (yp_call_node_t *) node, &token, value); } - case YP_NODE_CLASS_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_class_variable_operator_or_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_CONSTANT_PATH_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - return (yp_node_t *) yp_constant_path_operator_or_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); - } - case YP_NODE_CONSTANT_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_constant_operator_or_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_or_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_constant_id_t constant_id = ((yp_local_variable_read_node_t *) node)->constant_id; - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_or_write_node_create(parser, node, &token, value, constant_id); - - yp_node_destroy(parser, node); - return result; - } case YP_NODE_MULTI_WRITE_NODE: { parser_lex(parser); yp_diagnostic_list_append(&parser->error_list, token.start, token.end, "Cannot use `||=' on a multi-write."); @@ -12671,14 +12401,19 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + case YP_NODE_CLASS_VARIABLE_READ_NODE: + case YP_NODE_CONSTANT_PATH_NODE: + case YP_NODE_CONSTANT_READ_NODE: + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator"); - yp_node_t *result = (yp_node_t *) yp_global_variable_operator_write_node_create(parser, node, &token, value); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator"); + return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12696,12 +12431,11 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, message_loc.start, message_loc.end); - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, constant_id); + yp_token_t operator = not_provided(parser); + node = parse_target(parser, node, &operator, NULL); - yp_node_destroy(parser, node); - return result; + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); } yp_token_t operator = not_provided(parser); @@ -12711,49 +12445,6 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); return (yp_node_t *) yp_call_operator_write_node_create(parser, (yp_call_node_t *) node, &token, value); } - case YP_NODE_CLASS_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_class_variable_operator_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_CONSTANT_PATH_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - return (yp_node_t *) yp_constant_path_operator_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); - } - case YP_NODE_CONSTANT_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_constant_operator_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_write_node_create(parser, node, &token, value); - - yp_node_destroy(parser, node); - return result; - } - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { - parser_lex(parser); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_constant_id_t constant_id = ((yp_local_variable_read_node_t *) node)->constant_id; - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, constant_id); - - yp_node_destroy(parser, node); - return result; - } case YP_NODE_MULTI_WRITE_NODE: { parser_lex(parser); yp_diagnostic_list_append(&parser->error_list, token.start, token.end, "Unexpected operator."); @@ -12965,7 +12656,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t if ( (parser->current.type == YP_TOKEN_PARENTHESIS_LEFT) || - (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 2, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR)) + (token_begins_expression_p(parser->current.type) || match_any_type_p(parser, 3, YP_TOKEN_UAMPERSAND, YP_TOKEN_USTAR, YP_TOKEN_USTAR_STAR)) ) { // If we have a constant immediately following a '::' operator, then // this can either be a constant path or a method call, depending on @@ -13236,6 +12927,8 @@ yp_parser_metadata(yp_parser_t *parser, const char *metadata) { // Initialize a parser with the given start and end pointers. YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath) { + assert(source != NULL); + // Set filepath to the file that was passed if (!filepath) filepath = ""; yp_string_t filepath_string; @@ -13304,14 +12997,15 @@ yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char size_t newline_size = size / 22; yp_newline_list_init(&parser->newline_list, source, newline_size < 4 ? 4 : newline_size); - assert(source != NULL); + // Skip past the UTF-8 BOM if it exists. if (size >= 3 && (unsigned char) source[0] == 0xef && (unsigned char) source[1] == 0xbb && (unsigned char) source[2] == 0xbf) { - // If the first three bytes of the source are the UTF-8 BOM, then we'll skip - // over them. parser->current.end += 3; - } else if (size >= 2 && source[0] == '#' && source[1] == '!') { - // If the first two bytes of the source are a shebang, then we'll indicate - // that the encoding comment is at the end of the shebang. + parser->encoding_comment_start += 3; + } + + // If the first two bytes of the source are a shebang, then we'll indicate + // that the encoding comment is at the end of the shebang. + if (peek(parser) == '#' && peek_offset(parser, 1) == '!') { const char *encoding_comment_start = next_newline(source, (ptrdiff_t) size); if (encoding_comment_start) { parser->encoding_comment_start = encoding_comment_start + 1; diff --git a/yarp/yarp.h b/yarp/yarp.h index 787029fcc33ed0..078483193d58a5 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -30,6 +30,9 @@ void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buf void yp_print_node(yp_parser_t *parser, yp_node_t *node); +// Generate a scope node from the given node. +void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest); + // The YARP version and the serialization format. YP_EXPORTED_FUNCTION const char * yp_version(void); diff --git a/yjit.c b/yjit.c index 052346950f1883..badd46b99afebb 100644 --- a/yjit.c +++ b/yjit.c @@ -1169,6 +1169,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // Primitives used by yjit.rb VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self); +VALUE rb_yjit_print_stats_p(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self, VALUE context); VALUE rb_yjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self); diff --git a/yjit.rb b/yjit.rb index 30e353a5960e91..57c4efff8b896d 100644 --- a/yjit.rb +++ b/yjit.rb @@ -226,7 +226,9 @@ def self.simulate_oom! # :nodoc: # Avoid calling a method here to not interfere with compilation tests if Primitive.rb_yjit_stats_enabled_p at_exit do - _print_stats + if Primitive.rb_yjit_print_stats_p + _print_stats + end _dump_locations end end @@ -271,6 +273,8 @@ def _print_stats(out: $stderr) # :nodoc: # Number of failed compiler invocations compilation_failure = stats[:compilation_failure] + code_region_overhead = stats[:code_region_size] - (stats[:inline_code_size] + stats[:outlined_code_size]) + out.puts "num_send: " + format_number(13, stats[:num_send]) out.puts "num_send_known_class: " + format_number_pct(13, stats[:num_send_known_class], stats[:num_send]) out.puts "num_send_polymorphic: " + format_number_pct(13, stats[:num_send_polymorphic], stats[:num_send]) @@ -316,6 +320,8 @@ def _print_stats(out: $stderr) # :nodoc: out.puts "inline_code_size: " + format_number(13, stats[:inline_code_size]) out.puts "outlined_code_size: " + format_number(13, stats[:outlined_code_size]) out.puts "code_region_size: " + format_number(13, stats[:code_region_size]) + out.puts "code_region_overhead: " + format_number_pct(13, code_region_overhead, stats[:code_region_size]) + out.puts "freed_code_size: " + format_number(13, stats[:freed_code_size]) out.puts "yjit_alloc_size: " + format_number(13, stats[:yjit_alloc_size]) if stats.key?(:yjit_alloc_size) out.puts "live_context_size: " + format_number(13, stats[:live_context_size]) diff --git a/yjit/src/asm/x86_64/mod.rs b/yjit/src/asm/x86_64/mod.rs index 4ff89dafe3fbec..4a7c7e5a84474c 100644 --- a/yjit/src/asm/x86_64/mod.rs +++ b/yjit/src/asm/x86_64/mod.rs @@ -635,7 +635,7 @@ fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_r panic!("immediate value too large (num_bits={}, num={uimm:?})", num_bits); } }, - _ => unreachable!() + _ => panic!("unknown encoding combo: {opnd0:?} {opnd1:?}") }; } diff --git a/yjit/src/backend/x86_64/mod.rs b/yjit/src/backend/x86_64/mod.rs index c5f458cb205b65..60d0e8bf8d9611 100644 --- a/yjit/src/backend/x86_64/mod.rs +++ b/yjit/src/backend/x86_64/mod.rs @@ -291,7 +291,7 @@ impl Assembler *out = asm.next_opnd_out(Opnd::match_num_bits(&[*truthy, *falsy])); asm.push_insn(insn); }, - Insn::Mov { dest, src } => { + Insn::Mov { dest, src } | Insn::Store { dest, src } => { match (&dest, &src) { (Opnd::Mem(_), Opnd::Mem(_)) => { // We load opnd1 because for mov, opnd0 is the output diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 8e83f128af733f..6ccb14264bdf9f 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -1417,7 +1417,7 @@ fn guard_object_is_heap( asm.cmp(object, Qfalse.into()); asm.je(Target::side_exit(counter)); - if object_type.diff(Type::UnknownHeap) != TypeDiff::Incompatible { + if Type::UnknownHeap.diff(object_type) != TypeDiff::Incompatible { asm.ctx.upgrade_opnd_type(object_opnd, Type::UnknownHeap); } } @@ -1449,7 +1449,7 @@ fn guard_object_is_array( asm.cmp(flags_opnd, (RUBY_T_ARRAY as u64).into()); asm.jne(Target::side_exit(counter)); - if object_type.diff(Type::TArray) != TypeDiff::Incompatible { + if Type::UnknownHeap.diff(object_type) != TypeDiff::Incompatible { asm.ctx.upgrade_opnd_type(object_opnd, Type::TArray); } } @@ -1481,7 +1481,7 @@ fn guard_object_is_string( asm.cmp(flags_reg, Opnd::UImm(RUBY_T_STRING as u64)); asm.jne(Target::side_exit(counter)); - if object_type.diff(Type::TString) != TypeDiff::Incompatible { + if Type::UnknownHeap.diff(object_type) != TypeDiff::Incompatible { asm.ctx.upgrade_opnd_type(object_opnd, Type::TString); } } @@ -5063,9 +5063,7 @@ unsafe extern "C" fn build_kwhash(ci: *const rb_callinfo, sp: *const VALUE) -> V // at sp[-2]. Depending on the frame type, it can serve different purposes, // which are covered here by enum variants. enum SpecVal { - None, - BlockHandler(BlockHandler), - BlockParamProxy, + BlockHandler(Option), PrevEP(*const VALUE), PrevEPOpnd(Opnd), } @@ -5077,6 +5075,11 @@ pub enum BlockHandler { BlockISeq(IseqPtr), // invokesuper: GET_BLOCK_HANDLER() (GET_LEP()[VM_ENV_DATA_INDEX_SPECVAL]) LEPSpecVal, + // part of the allocate-free block forwarding scheme + BlockParamProxy, + // To avoid holding the block arg (e.g. proc and symbol) across C calls, + // we might need to set the block handler early in the call sequence + AlreadySet, } struct ControlFrame { @@ -5124,10 +5127,8 @@ fn gen_push_frame( // the outer environment depending on the frame type. // sp[-2] = specval; let specval: Opnd = match frame.specval { - SpecVal::None => { - VM_BLOCK_HANDLER_NONE.into() - } - SpecVal::BlockHandler(block_handler) => { + SpecVal::BlockHandler(None) => VM_BLOCK_HANDLER_NONE.into(), + SpecVal::BlockHandler(Some(block_handler)) => { match block_handler { BlockHandler::BlockISeq(block_iseq) => { // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). @@ -5142,17 +5143,18 @@ fn gen_push_frame( let lep_opnd = gen_get_lep(jit, asm); asm.load(Opnd::mem(64, lep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)) } - } - } - SpecVal::BlockParamProxy => { - let ep_opnd = gen_get_lep(jit, asm); - let block_handler = asm.load( - Opnd::mem(64, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL) - ); + BlockHandler::BlockParamProxy => { + let ep_opnd = gen_get_lep(jit, asm); + let block_handler = asm.load( + Opnd::mem(64, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL) + ); - asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), block_handler); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), block_handler); - block_handler + block_handler + } + BlockHandler::AlreadySet => 0.into(), // unused + } } SpecVal::PrevEP(prev_ep) => { let tagged_prev_ep = (prev_ep as usize) | 1; @@ -5160,9 +5162,13 @@ fn gen_push_frame( } SpecVal::PrevEPOpnd(ep_opnd) => { asm.or(ep_opnd, 1.into()) - }, + } }; - asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2), specval); + if let SpecVal::BlockHandler(Some(BlockHandler::AlreadySet)) = frame.specval { + asm.comment("specval should have been set"); + } else { + asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2), specval); + } // Write env flags at sp[-1] // sp[-1] = frame_type; @@ -5406,11 +5412,9 @@ fn gen_send_cfunc( let sp = asm.lea(asm.ctx.sp_opnd((SIZEOF_VALUE as isize) * 3)); let specval = if block_arg_type == Some(Type::BlockParamProxy) { - SpecVal::BlockParamProxy - } else if let Some(block_handler) = block { - SpecVal::BlockHandler(block_handler) + SpecVal::BlockHandler(Some(BlockHandler::BlockParamProxy)) } else { - SpecVal::None + SpecVal::BlockHandler(block) }; let mut frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL; @@ -5794,13 +5798,7 @@ fn gen_send_iseq( } let mut opts_missing: i32 = opt_num - opts_filled; - let block_arg = flags & VM_CALL_ARGS_BLOCKARG != 0; - let block_arg_type = if block_arg { - Some(asm.ctx.get_opnd_type(StackOpnd(0))) - } else { - None - }; exit_if_stack_too_large(iseq)?; exit_if_tail_call(asm, ci)?; @@ -5816,7 +5814,7 @@ fn gen_send_iseq( exit_if_wrong_number_arguments(asm, opts_filled, flags, opt_num, iseq_has_rest)?; exit_if_doing_kw_and_opts_missing(asm, doing_kw_call, opts_missing)?; exit_if_has_rest_and_optional_and_block(asm, iseq_has_rest, opt_num, iseq, block_arg)?; - exit_if_unsupported_block_arg_type(asm, block_arg_type)?; + let block_arg_type = exit_if_unsupported_block_arg_type(jit, asm, block_arg)?; // Block parameter handling. This mirrors setup_parameters_complex(). if iseq_has_block_param { @@ -6003,12 +6001,35 @@ fn gen_send_iseq( // We don't need the actual stack value asm.stack_pop(1); } + Some(Type::TProc) => { + // Place the proc as the block handler. We do this early because + // the block arg being at the top of the stack gets in the way of + // rest param handling later. Also, since there are C calls that + // come later, we can't hold this value in a register and place it + // near the end when we push a new control frame. + asm.comment("guard block arg is a proc"); + // Simple predicate, no need for jit_prepare_routine_call(). + let is_proc = asm.ccall(rb_obj_is_proc as _, vec![asm.stack_opnd(0)]); + asm.cmp(is_proc, Qfalse.into()); + jit_chain_guard( + JCC_JE, + jit, + asm, + ocb, + SEND_MAX_CHAIN_DEPTH, + Counter::guard_send_block_arg_type, + ); + + let proc = asm.stack_pop(1); + let callee_ep = -argc + num_locals + VM_ENV_DATA_SIZE as i32 - 1; + let callee_specval = callee_ep + VM_ENV_DATA_INDEX_SPECVAL; + let callee_specval = asm.ctx.sp_opnd(callee_specval as isize * SIZEOF_VALUE as isize); + asm.store(callee_specval, proc); + } None => { // Nothing to do } - _ => { - assert!(false); - } + _ => unreachable!(), } let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) }; @@ -6432,12 +6453,12 @@ fn gen_send_iseq( } else if let Some(captured_opnd) = captured_opnd { let ep_opnd = asm.load(Opnd::mem(64, captured_opnd, SIZEOF_VALUE_I32)); // captured->ep SpecVal::PrevEPOpnd(ep_opnd) - } else if block_arg_type == Some(Type::BlockParamProxy) { - SpecVal::BlockParamProxy - } else if let Some(block_handler) = block { - SpecVal::BlockHandler(block_handler) + } else if let Some(Type::TProc) = block_arg_type { + SpecVal::BlockHandler(Some(BlockHandler::AlreadySet)) + } else if let Some(Type::BlockParamProxy) = block_arg_type { + SpecVal::BlockHandler(Some(BlockHandler::BlockParamProxy)) } else { - SpecVal::None + SpecVal::BlockHandler(block) }; // Setup the new frame @@ -6642,20 +6663,35 @@ fn exit_if_has_rest_and_optional_and_block(asm: &mut Assembler, iseq_has_rest: b } #[must_use] -fn exit_if_unsupported_block_arg_type(asm: &mut Assembler, block_arg_type: Option) -> Option<()> { +fn exit_if_unsupported_block_arg_type( + jit: &mut JITState, + asm: &mut Assembler, + supplying_block_arg: bool +) -> Option> { + let block_arg_type = if supplying_block_arg { + asm.ctx.get_opnd_type(StackOpnd(0)) + } else { + // Passing no block argument + return Some(None); + }; + match block_arg_type { - Some(Type::Nil | Type::BlockParamProxy) => { + Type::Nil | Type::BlockParamProxy => { // We'll handle this later + Some(Some(block_arg_type)) } - None => { - // Nothing to do + _ if { + let sample_block_arg = jit.peek_at_stack(&asm.ctx, 0); + unsafe { rb_obj_is_proc(sample_block_arg) }.test() + } => { + // Speculate that we'll have a proc as the block arg + Some(Some(Type::TProc)) } _ => { gen_counter_incr(asm, Counter::send_block_arg); - return None + None } } - Some(()) } #[must_use] diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 89061ac4e5ecec..40646ebd342fdf 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -59,6 +59,8 @@ pub enum Type { TArray, // An object with the T_ARRAY flag set, possibly an rb_cArray CArray, // An un-subclassed string of type rb_cArray (can have instance vars in some cases) + TProc, // A proc object. Could be an instance of a subclass of ::rb_cProc + BlockParamProxy, // A special sentinel value indicating the block parameter should be read from // the current surrounding cfp } @@ -110,6 +112,8 @@ impl Type { RUBY_T_ARRAY => Type::TArray, RUBY_T_HASH => Type::Hash, RUBY_T_STRING => Type::TString, + #[cfg(not(test))] + RUBY_T_DATA if unsafe { rb_obj_is_proc(val).test() } => Type::TProc, _ => Type::UnknownHeap, } } @@ -155,6 +159,7 @@ impl Type { Type::TString => true, Type::CString => true, Type::BlockParamProxy => true, + Type::TProc => true, _ => false, } } @@ -189,6 +194,7 @@ impl Type { Type::Hash => Some(RUBY_T_HASH), Type::ImmSymbol | Type::HeapSymbol => Some(RUBY_T_SYMBOL), Type::TString | Type::CString => Some(RUBY_T_STRING), + Type::TProc => Some(RUBY_T_DATA), Type::Unknown | Type::UnknownImm | Type::UnknownHeap => None, Type::BlockParamProxy => None, } diff --git a/yjit/src/options.rs b/yjit/src/options.rs index f08217bfc2e7e0..cbed2718195b0a 100644 --- a/yjit/src/options.rs +++ b/yjit/src/options.rs @@ -26,9 +26,12 @@ pub struct Options { // The number of registers allocated for stack temps pub num_temp_regs: usize, - // Capture and print out stats + // Capture stats pub gen_stats: bool, + // Print stats on exit (when gen_stats is also true) + pub print_stats: bool, + // Trace locations of exits pub gen_trace_exits: bool, @@ -62,6 +65,7 @@ pub static mut OPTIONS: Options = Options { num_temp_regs: 5, gen_stats: false, gen_trace_exits: false, + print_stats: true, trace_exits_sample_rate: 0, pause: false, dump_insns: false, @@ -160,7 +164,7 @@ pub fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { } }, - ("dump-disasm", _) => match opt_val.to_string().as_str() { + ("dump-disasm", _) => match opt_val { "" => unsafe { OPTIONS.dump_disasm = Some(DumpDisasm::Stdout) }, directory => { let pid = std::process::id(); @@ -176,7 +180,16 @@ pub fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { ("greedy-versioning", "") => unsafe { OPTIONS.greedy_versioning = true }, ("no-type-prop", "") => unsafe { OPTIONS.no_type_prop = true }, - ("stats", "") => unsafe { OPTIONS.gen_stats = true }, + ("stats", _) => match opt_val { + "" => unsafe { OPTIONS.gen_stats = true }, + "quiet" => { + unsafe { OPTIONS.gen_stats = true } + unsafe { OPTIONS.print_stats = false } + }, + _ => { + return None; + } + }, ("trace-exits", "") => unsafe { OPTIONS.gen_trace_exits = true; OPTIONS.gen_stats = true; OPTIONS.trace_exits_sample_rate = 0 }, ("trace-exits-sample-rate", sample_rate) => unsafe { OPTIONS.gen_trace_exits = true; OPTIONS.gen_stats = true; OPTIONS.trace_exits_sample_rate = sample_rate.parse().unwrap(); }, ("dump-insns", "") => unsafe { OPTIONS.dump_insns = true }, diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index 9ef3e8e94ce6d2..27e3c6d08bb9fe 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -296,6 +296,7 @@ make_counters! { invokeblock_symbol, // Method calls that exit to the interpreter + guard_send_block_arg_type, guard_send_klass_megamorphic, guard_send_se_cf_overflow, guard_send_se_protected_check_failed, @@ -463,6 +464,17 @@ pub extern "C" fn rb_yjit_stats_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALU } } +/// Primitive called in yjit.rb +/// Check if stats generation should print at exit +#[no_mangle] +pub extern "C" fn rb_yjit_print_stats_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + if get_option!(print_stats) { + return Qtrue; + } else { + return Qfalse; + } +} + /// Primitive called in yjit.rb. /// Export all YJIT statistics as a Ruby hash. #[no_mangle]