Migrate token and AST testing into 'dzil test'

Author: ehuelsmann

perl/Makefile

@@ -43,7 +43,8 @@ clean: ## Remove all build artifacts and files generated by the acceptance tests
 
 .DELETE_ON_ERROR:
 
-acceptance: .built lib/Gherkin/Generated/Languages.pm $(TOKENS) $(ASTS) $(PICKLES) $(SOURCES) $(ERRORS) ## Build acceptance test dir and compare results with reference
+acceptance: .built lib/Gherkin/Generated/Languages.pm $(TOKENS) $(ASTS) $(PICKLES) $(SOURCES) $(ERRORS)
+	FEATURE_FILES_BASEDIR=$PWD/../testdata/ dzil test
 
 .built: perl5 $(SOURCE_FILES)
 	touch $@
@@ -57,16 +58,6 @@ lib/Gherkin/Generated/Languages.pm:
 $(GHERKIN_PARSER): $(GHERKIN_RAZOR) ../gherkin.berp
 	berp -g ../gherkin.berp -t $< -o $@ --noBOM
 
-acceptance/testdata/%.tokens: ../testdata/% ../testdata/%.tokens
-	mkdir -p $(@D)
-	$(GHERKIN_GENERATE_TOKENS) $< > $@
-	diff --unified $<.tokens $@
-
-acceptance/testdata/%.ast.ndjson: ../testdata/% ../testdata/%.ast.ndjson
-	mkdir -p $(@D)
-	$(GHERKIN) --no-source --no-pickles --predictable-ids $< | jq --sort-keys --compact-output "." > $@
-	diff --unified <(jq "." $<.ast.ndjson) <(jq "." $@)
-
 acceptance/testdata/%.pickles.ndjson: ../testdata/% ../testdata/%.pickles.ndjson
 	mkdir -p $(@D)
 	$(GHERKIN) --no-source --no-ast --predictable-ids $< | jq --sort-keys --compact-output "." > $@

perl/bin/gherkin-generate-tokens

@@ -15,5 +15,6 @@ on 'test' => sub {
 };
 
 on 'develop' => sub {
-   # there are no specific development dependencies...
+    requires "File::Find::Rule";
+    requires "Text::Diff";
 };

perl/cpanfile

@@ -0,0 +1,70 @@
+#!perl
+
+use strict;
+use warnings;
+
+BEGIN {
+    use Test2::V0;
+
+    skip_all 'No AUTHOR_TESTING'
+        unless $ENV{AUTHOR_TESTING};
+    skip_all 'Missing FEATURE_FILES_BASEDIR'
+        unless $ENV{FEATURE_FILES_BASEDIR};
+}
+
+use File::Find::Rule;
+use JSON::PP;
+
+use App::gherkin;
+use Gherkin::Parser;
+use Gherkin::TokenFormatterBuilder;
+
+use Test2::Tools::ClassicCompare qw/ is_deeply /;
+use Text::Diff;
+
+my @good_features =
+    File::Find::Rule
+    ->file()
+    ->name('*.feature')
+    ->in( File::Spec->catfile( $ENV{FEATURE_FILES_BASEDIR}, 'good' ) );
+
+skip_all 'No test files found'
+    unless @good_features;
+
+
+# Test tokens
+do {
+    my $tp = Gherkin::Parser->new( Gherkin::TokenFormatterBuilder->new() );
+    for my $feature (@good_features) {
+        subtest "$feature tokens" => sub {
+            my $tokens = $tp->parse( $feature );
+            my $str =  join( "\n", @{ $tokens } ) . "\n";
+            # compare $tokens string content with content of file "$feature.tokens"
+            my $diff = diff \$str, "$feature.tokens";
+
+            ok( not $diff )
+                or diag $diff;
+        };
+    }
+};
+
+# Test AST
+do {
+    my $app = App::gherkin->new;
+    $app->parse_options( qw/--no-source --no-pickles --predictable-ids/ );
+    for my $feature (@good_features) {
+        subtest "$feature AST" => sub {
+            my $content;
+            open my $fh, '>:encoding(UTF-8)', \$content;
+            local $app->{out_handle} = $fh;
+            $app->run( $feature );
+            close $fh;
+            my $diff = diff \$content, "$feature.ast.ndjson";
+
+            ok( not $diff )
+                or diag $diff;
+        };
+    }
+};
+
+done_testing;