diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0589f05..f835e24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,23 +12,17 @@ jobs: tests: name: Ruby ${{ matrix.ruby }} if: "contains(github.event.commits[0].message, '[ci skip]') == false" - runs-on: ubuntu-latest + runs-on: macos-latest env: CI: true ALLOW_FAILURES: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'jruby' }} strategy: fail-fast: false matrix: - ruby: - - 2.6 - - 2.7 - - "3.0" - - 3.1 - - ruby-head - - jruby + ruby: ['3.0', 3.1, 3.2, ruby-head, jruby] steps: - name: Clone repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -38,8 +32,8 @@ jobs: - name: Run tests run: ruby --version; bundle exec rspec spec || $ALLOW_FAILURES - name: Coveralls GitHub Action - uses: coverallsapp/github-action@v1.1.2 - if: "matrix.ruby == '3.0'" + uses: coverallsapp/github-action@v2 + if: "matrix.ruby == '3.2'" with: github-token: ${{ secrets.GITHUB_TOKEN }} wintests: @@ -56,7 +50,7 @@ jobs: - 3.1 steps: - name: Clone repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -65,8 +59,3 @@ jobs: run: bundle install --jobs 4 --retry 3 - name: Run tests run: ruby --version; bundle exec rspec spec || $ALLOW_FAILURES - - name: Coveralls GitHub Action - uses: coverallsapp/github-action@v1.1.2 - if: "matrix.ruby == '3.0'" - with: - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index b8d16ed..65aea93 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -10,7 +10,7 @@ jobs: name: Update gh-pages with docs steps: - name: Clone repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/README.md b/README.md index 1cf2b31..ce56435 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ Ruby [YAML-LD][] reader/writer for RDF.rb -[![Gem Version](https://badge.fury.io/rb/yaml-ld.png)](https://rubygems.org/gems/yaml-ld) -[![Build Status](https://secure.travis-ci.org/ruby-rdf/yaml-ld.png?branch=develop)](https://github.com/ruby-rdf/yaml-ld/actions?query=workflow%3ACI) +[![Gem Version](https://badge.fury.io/rb/yaml-ld.svg)](https://rubygems.org/gems/yaml-ld) +[![Build Status](https://github.com/ruby-rdf/yaml-ld/workflows/CI/badge.svg?branch=develop)](https://github.com/ruby-rdf/yaml-ld/actions?query=workflow%3ACI) [![Coverage Status](https://coveralls.io/repos/ruby-rdf/yaml-ld/badge.svg?branch=develop)](https://coveralls.io/github/ruby-rdf/yaml-ld?branch=develop) [![Gitter chat](https://badges.gitter.im/ruby-rdf.png)](https://gitter.im/gitterHQ/gitter) @@ -102,12 +102,17 @@ In addition to the input, both a `context` and `frame` may be specified using ei * [Psych](https://rubygems.org/gems/psych) (>= 4.0) * [RDF.rb](https://rubygems.org/gems/rdf) (~> 3.2) +### Ubuntu limitation + +As of October 2023, Ubuntu distributions are running with libyaml 0.2.1, which does not support YAML 1.2. The minimum version needed is libyaml 0.2.5. + ## Installation The recommended installation method is via [RubyGems](https://rubygems.org/). To install the latest official release of the `JSON-LD` gem, do: % [sudo] gem install yaml-ld + ## Download To get a local working copy of the development repository, do: diff --git a/VERSION b/VERSION index 4e379d2..bcab45a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.2 +0.0.3 diff --git a/examples/countries-alias.yamlld b/examples/countries-alias.yamlld new file mode 100644 index 0000000..bf5f0f3 --- /dev/null +++ b/examples/countries-alias.yamlld @@ -0,0 +1,14 @@ +%YAML 1.2 +--- +"@context": + "@vocab": "http://schema.org/" + "countries": "http://publication.europa.eu/resource/authority/country/" +"@graph": +- &ITA + "@id": countries:ITA +- "@id": http://people.example/Homer + name: Homer Simpson + nationality: *ITA +- "@id": http://people.example/Lisa + name: Lisa Simpson + nationality: *ITA diff --git a/examples/xsd.yamlld b/examples/xsd.yamlld index 9ab84a0..2cfc696 100644 --- a/examples/xsd.yamlld +++ b/examples/xsd.yamlld @@ -6,4 +6,10 @@ name: !string Gregg Kellogg homepage: https://greggkellogg.net/ depiction: http://www.gravatar.com/avatar/42f948adff3afaa52249d963117af7c8 -date: !date "2022-08-08" \ No newline at end of file +date: !date "2022-08-08" + +{ + "@context": {"@vocab": "http://xmlns.com/foaf/0.1/"}, + "name": { + "@value": "Gregg Kellogg", "@type": "http://www.w3.org/2001/XMLSchema#string",} +} \ No newline at end of file diff --git a/lib/yaml_ld.rb b/lib/yaml_ld.rb index d6c15e0..894d707 100644 --- a/lib/yaml_ld.rb +++ b/lib/yaml_ld.rb @@ -35,4 +35,11 @@ module YAML_LD # YAML-LD profiles YAML_LD_NS = "http://www.w3.org/ns/yaml-ld#" PROFILES = %w(extended).map {|p| YAML_LD_NS + p}.freeze + + class Error < JSON::LD::JsonLdError + class InvalidEncoding < YAML_LD::Error; @code = "invalid encoding"; end + class MappingKeyError < YAML_LD::Error; @code = "mapping-key-error"; end + class ProfileError < YAML_LD::Error; @code = "profile-error"; end + end + end diff --git a/lib/yaml_ld/representation.rb b/lib/yaml_ld/representation.rb index 570ddfc..8f37fb7 100644 --- a/lib/yaml_ld/representation.rb +++ b/lib/yaml_ld/representation.rb @@ -39,6 +39,16 @@ def load_stream(yaml, filename: nil, fallback: [], **options) end result.is_a?(Array) && result.empty? ? fallback : result + rescue Psych::SyntaxError => e + msg = filename ? "file: #{filename} #{e.message}" : e.message + if yaml.respond_to?(:read) + msg << "Content:\n" + yaml.tap(:rewind).read + end + if e.message.match?(/invalid leading UTF-8 octet/) + raise YAML_LD::Error::InvalidEncoding, msg + else + raise JSON::LD::JsonLdError::LoadingDocumentFailed, msg + end end module_function :load_stream @@ -100,18 +110,46 @@ def deep_dup(obj) def as_jsonld_ir(node, **options) # Scans scalars for built-in classes @ss ||= Psych::ScalarScanner.new(Psych::ClassLoader::Restricted.new([], %i())) + + # Record in-scope anchors to check for circular alias references. + in_scope_anchors = options[:in_scope_anchors] ||= {} + case node when Psych::Nodes::Stream node.children.map {|n| as_jsonld_ir(n, **options)} - when Psych::Nodes::Document then as_jsonld_ir(node.children.first, **options) - when Psych::Nodes::Sequence then node.children.map {|n| as_jsonld_ir(n, **options)} + when Psych::Nodes::Document + as_jsonld_ir(node.children.first, named_nodes: {}, **options) + when Psych::Nodes::Sequence + value = [] + if node.anchor + options = options.merge(in_scope_anchors: in_scope_anchors.merge(node.anchor => true)) + options[:named_nodes][node.anchor] = value + end + node.children.each {|n| value << as_jsonld_ir(n, **options)} + value when Psych::Nodes::Mapping - node.children.each_slice(2).inject({}) do |memo, (k,v)| - memo.merge(as_jsonld_ir(k) => as_jsonld_ir(v, **options)) + value = {} + if node.anchor + options = options.merge(in_scope_anchors: in_scope_anchors.merge(node.anchor => true)) + options[:named_nodes][node.anchor] = value + end + node.children.each_slice(2) do |k, v| + key = as_jsonld_ir(k) + raise YAML_LD::Error::MappingKeyError, "mapping key #{k} (#{key.inspect}) not a string" unless key.is_a?(String) + value[as_jsonld_ir(k)] = as_jsonld_ir(v, **options) + end + value + when ::Psych::Nodes::Scalar + value = scan_scalar(node, **options) + if node.anchor + options = options.merge(in_scope_anchors: in_scope_anchors.merge(node.anchor => true)) + options[:named_nodes][node.anchor] = value end - when ::Psych::Nodes::Scalar then scan_scalar(node, **options) + value when ::Psych::Nodes::Alias - # FIXME + raise JSON::LD::JsonLdError::LoadingDocumentFailed, "anchor for *#{node.anchor} not found" unless options[:named_nodes].key?(node.anchor) + raise JSON::LD::JsonLdError::LoadingDocumentFailed, "anchor for *#{node.anchor} creates a cycle" if in_scope_anchors.key?(node.anchor) + options[:named_nodes][node.anchor] end end module_function :as_jsonld_ir diff --git a/script/parse b/script/parse index 933e9aa..0df23f1 100755 --- a/script/parse +++ b/script/parse @@ -35,6 +35,7 @@ def run(input, options) puts "output saved in #{output_dir}" return end + options[:extendedYAML] = options[:parser_options][:extendedYAML] reader_class = RDF::Reader.for(options[:input_format].to_sym) raise "Reader not found for #{options[:input_format]}" unless reader_class @@ -42,23 +43,43 @@ def run(input, options) if options[:flatten] output = YAML_LD::API.flatten(input, options.delete(:context), **options) secs = Time.new - start - options[:output].puts output + if options[:output_format] == :jsonld + ir = YAML_LD::Representation.load(input, **options[:parser_options]) + options[:output].puts ir.to_json(JSON::LD::JSON_STATE) + else + options[:output].puts output + end puts "Flattened in #{secs} seconds." elsif options[:expand] options = options.merge(expandContext: options.delete(:context)) if options.key?(:context) output = YAML_LD::API.expand(input, **options) secs = Time.new - start - options[:output].puts output + if options[:output_format] == :jsonld + ir = YAML_LD::Representation.load(output, **options[:parser_options]) + options[:output].puts ir.to_json(JSON::LD::JSON_STATE) + else + options[:output].puts output + end puts "Expanded in #{secs} seconds." elsif options[:compact] output = YAML_LD::API.compact(input, options[:context], **options) secs = Time.new - start - options[:output].puts output + if options[:output_format] == :jsonld + ir = YAML_LD::Representation.load(output, **options[:parser_options]) + options[:output].puts ir.to_json(JSON::LD::JSON_STATE) + else + options[:output].puts output + end puts "Compacted in #{secs} seconds." elsif options[:frame] output = YAML_LD::API.frame(input, options[:frame], **options) secs = Time.new - start - options[:output].puts output + if options[:output_format] == :jsonld + ir = YAML_LD::Representation.load(output, **options[:parser_options]) + options[:output].puts ir.to_json(JSON::LD::JSON_STATE) + else + options[:output].puts output + end puts "Framed in #{secs} seconds." elsif options[:translate] && %i(jsonld yamlld).include?(options[:output_format]) # Translate between formats @@ -68,6 +89,9 @@ def run(input, options) ir.to_json(JSON::LD::JSON_STATE) : YAML_LD::Representation.dump(ir, version: [1,2], **options[:parser_options]) ) + elsif options[:inspect] + psych_internal = Psych.parse_stream(input) + options[:output].puts psych_internal.ai else r = reader_class.new(input, **options[:parser_options]) if options[:output_format] == :none @@ -124,6 +148,7 @@ OPT_ARGS = [ ["--frame", GetoptLong::REQUIRED_ARGUMENT, "Frame input, option value is frame to use"], ["--help", "-?", GetoptLong::NO_ARGUMENT, "This message"], ["--input-format", GetoptLong::REQUIRED_ARGUMENT, "Format of input, if not YAML-LD"], + ["--inspect", GetoptLong::NO_ARGUMENT, "Parse and dump IR format"], ["--output", "-o", GetoptLong::REQUIRED_ARGUMENT, "Where to store output (default STDOUT)"], ["--profile", GetoptLong::NO_ARGUMENT, "Run profiler with output to doc/profiles/"], ["--quiet", GetoptLong::NO_ARGUMENT, "Reduce output"], @@ -164,6 +189,7 @@ opts.each do |opt, arg| when '--frame' then options[:frame] = arg when "--help" then usage when '--input-format' then options[:input_format] = arg.to_sym + when '--inspect' then options[:inspect] = true when '--output' then options[:output] = File.open(arg, "w") when '--profile' then options[:profile] = true when '--quiet' diff --git a/script/tc b/script/tc new file mode 100755 index 0000000..5a022b1 --- /dev/null +++ b/script/tc @@ -0,0 +1,247 @@ +#!/usr/bin/env ruby +require 'rubygems' +$:.unshift(File.expand_path("../../lib", __FILE__)) +require "bundler/setup" +require 'logger' +require 'yaml_ld' +require 'rdf/nquads' +require 'rdf/isomorphic' +require File.expand_path("../../spec/spec_helper", __FILE__) +require File.expand_path("../../spec/suite_helper", __FILE__) +require 'getoptlong' + +ASSERTOR = "http://greggkellogg.net/foaf#me" +RUN_TIME = Time.now + +def earl_preamble(options) + options[:output].write File.read(File.expand_path("../../etc/doap.ttl", __FILE__)) + options[:output].puts %( + doap:release [ + doap:name "yaml_ld-#{YAML_LD::VERSION}"; + doap:revision "#{YAML_LD::VERSION}"; + doap:created "#{File.mtime(File.expand_path('../../VERSION', __FILE__)).strftime('%Y-%m-%d')}"^^xsd:date; +] . +<> foaf:primaryTopic ; + dc:issued "#{RUN_TIME.xmlschema}"^^xsd:dateTime; + foaf:maker <#{ASSERTOR}> . + +<#{ASSERTOR}> a earl:Assertor; + foaf:title "Implementor" . +) +end + +def compare_results(tc, result, expected) + if tc.evaluationTest? + if tc.testType == "jld:ToRDFTest" + result.equivalent_graph?(expected) ? "passed": "failed" + elsif tc.options[:ordered] + expected == result ? "passed": "failed" + else + if !result.equivalent_yamlld?(expected, extendedYAML: tc.options[:extendedYAML]) + "failed" + elsif result.to_s.include?('@context') + exp_expected = YAML_LD::API.expand(expected, **tc.options.merge(logger: false)) + exp_result = YAML_LD::API.expand(result, **tc.options.merge(logger: false)) + exp_result.equivalent_yamlld?(exp_expected, extendedYAML: tc.options[:extendedYAML]) ? "passed" : "failed" + else + "passed" + end + end + else + result.nil? ? "failed": "passed" + end +end + +def run_tc(man, tc, options) + tc.options[:logger] = options[:logger] + tc.options[:documentLoader] ||= Fixtures::SuiteTest.method(:documentLoader) + tc.options[:lowercaseLanguage] = true + + STDERR.write "run #{tc.property('input')}" + + if options[:verbose] + puts "\nTestCase: #{tc.inspect}" + puts "\nInput:\n" + tc.input + puts "\nContext:\n" + tc.context if tc.context + puts "\nFrame:\n" + tc.frame if tc.frame + puts "\nExpected:\n" + tc.expect if tc.expect && tc.positiveTest? + puts "\nExpected:\n" + tc.expectErrorCode if tc.negativeTest? + end + + output = "" + begin + puts "open #{tc.input_loc}" if options[:verbose] + + result = case tc.testType + when 'jld:CompactTest' + output = YAML_LD::API.compact(tc.input_loc, tc.context_json['@context'], validate: true, **tc.options) + expected = tc.evaluationTest? && tc.positiveTest? ? tc.expect : nnil + compare_results(tc, output, expected) + when 'jld:ExpandTest' + # MultiJson use OJ, by default, which doesn't handle native numbers the same as the JSON gem. + output = YAML_LD::API.expand(tc.input_loc, validate: true, **tc.options) + expected = tc.evaluationTest? && tc.positiveTest? ? tc.expect : nnil + compare_results(tc, output, expected) + when 'jld:FlattenTest' + output = YAML_LD::API.flatten(tc.input_loc, (tc.context_json['@context'] if tc.context_loc), validate: true, **tc.options) + expected = tc.evaluationTest? && tc.positiveTest? ? tc.expect : nnil + compare_results(tc, output, expected) + when 'jld:FrameTest' + output = YAML_LD::API.frame(tc.input_loc, tc.frame_loc, validate: true, **tc.options) + expected = tc.evaluationTest? && tc.positiveTest? ? tc.expect : nnil + compare_results(tc, output, expected) + when 'jld:FromRDFTest' + repo = RDF::Repository.load(tc.input_loc, format: :nquads, rdfstar: tc.options[:rdfstar]) + output = YAML_LD::API.fromRdf(repo, validate: true, **tc.options) + expected = tc.evaluationTest? && tc.positiveTest? ? tc.expect : nnil + compare_results(tc, output, expected) + when 'jld:ToRDFTest' + output = RDF::Repository.new.extend(RDF::Isomorphic) + YAML_LD::API.toRdf(tc.input_loc, **tc.options).map do |statement| + output << statement + end + + if tc.evaluationTest? && tc.positiveTest? + begin + if tc.options[:produceGeneralizedRdf] + quads = YAML_LD::API.toRdf(tc.input_loc, **tc.options.merge(validate: false)).map do |statement| + # Not really RDF, try different test method + tc.to_quad(statement) + end + output = quads.sort.uniq.join("") + output == tc.expect + else + expected = RDF::Repository.new << RDF::NQuads::Reader.new(tc.expect, rdfstar: tc.options[:rdfstar], validate: false, logger: []) + output.isomorphic?(expected) ? 'passed' : 'failed' + end + rescue RDF::ReaderError, YAML_LD::JsonLdError + quads = YAML_LD::API.toRdf(tc.input_loc, rdfstar: tc.options[:rdfstar], **tc.options.merge(validate: false)).map do |statement| + # Not really RDF, try different test method + tc.to_quad(statement) + end + + # FIXME: toRDF is outputing duplicate quads + output = quads.sort.uniq.join("") + output == tc.expect ? 'passed' : 'failed' + end + else + output.count > 0 ? 'passed' : 'failed' + end + end || "untested" + + output = output.dump(:nquads, validate: false) rescue output.to_s if output.is_a?(RDF::Enumerable) + puts "\nOutput:\n" + output if !tc.syntaxTest? && options[:verbose] + + result = result ? 'failed' : 'passed' unless tc.positiveTest? + options[:results][result] ||= 0 + options[:results][result] += 1 + rescue Interrupt + $stderr.puts "(interrupt)" + exit 1 + rescue StandardError => e + result = if tc.positiveTest? + STDERR.puts "#{" exception" unless options[:quiet]}: #{e}" + if options[:quiet] || !options[:verbose] + options[:results]['failed'] ||= 0 + options[:results]['failed'] += 1 + else + raise + end + "failed" + else + if e.message.include?(tc.property('expectErrorCode')) + options[:results]['passed'] ||= 0 + options[:results]['passed'] += 1 + "passed" + else + STDERR.puts("Expected exception: '#{tc.property('expectErrorCode')}' not '#{e}'") unless options[:quiet] + options[:results]['failed'] ||= 0 + options[:results]['failed'] += 1 + "failed" + end + end + end + + #options[:output].puts("\nOutput:\n" + output) unless options[:quiet] + + if options[:earl] + options[:output].puts %{ +[ a earl:Assertion; + earl:assertedBy <#{ASSERTOR}>; + earl:subject ; + earl:test <#{man}#{tc.id}>; + earl:result [ + a earl:TestResult; + earl:outcome earl:#{result}; + dc:date "#{RUN_TIME.xmlschema}"^^xsd:dateTime]; + earl:mode earl:automatic ] . +} + end + + puts "#{" test result:" unless options[:quiet]} #{result}" +end + +logger = Logger.new(STDERR) +logger.level = Logger::WARN +logger.formatter = lambda {|severity, datetime, progname, msg| "#{severity}: #{msg}\n"} + +options = { + output: STDOUT, + results: {}, + logger: logger +} + +opts = GetoptLong.new( + ["--help", "-?", GetoptLong::NO_ARGUMENT], + ["--debug", GetoptLong::NO_ARGUMENT], + ["--earl", GetoptLong::NO_ARGUMENT], + ["--quiet", "-q", GetoptLong::NO_ARGUMENT], + ["--output", "-o", GetoptLong::REQUIRED_ARGUMENT], + ["--stream", GetoptLong::NO_ARGUMENT], + ["--verbose", "-v", GetoptLong::NO_ARGUMENT] +) + +def help(options) + puts "Usage: #{$0} [options] [test-number ...]" + puts "Options:" + puts " --debug: Display detailed debug output" + puts " --earl: Generate EARL report" + puts " --quiet: Minimal output" + puts " --output: Output to specified file" + puts " --stream: Use streaming RDF reader/writer" + puts " --verbose: Verbose processing" + puts " --help,-?: This message" + exit(0) +end + +opts.each do |opt, arg| + case opt + when '--help' then help(options) + when '--debug' then logger.level = Logger::DEBUG + when '--earl' + options[:quiet] = options[:earl] = true + logger.level = Logger::FATAL + when '--output' then options[:output] = File.open(arg, "w") + when '--quiet' + options[:quiet] = true + logger.level = Logger::FATAL + when '--stream' then options[:stream] = true + when '--verbose' then options[:verbose] = true + end +end + +manifests = ["#{Fixtures::SuiteTest::SUITE}manifest.jsonld"] + +earl_preamble(options) if options[:earl] + +manifests.each do |man| + Fixtures::SuiteTest::Manifest.open(man) do |m| + m.entries.each do |tc| + next unless ARGV.empty? || ARGV.any? {|n| tc.property('@id').match(/#{n}/) || tc.property('input').match(/#{n}/)} + options = {stream: true}.merge(options) if man.include?('stream') + run_tc(man.sub(".jsonld", ""), tc, options) + end + end +end + +options[:results].each {|k, v| puts "#{k}: #{v}"} diff --git a/spec/suite_helper.rb b/spec/suite_helper.rb new file mode 100644 index 0000000..4c413c7 --- /dev/null +++ b/spec/suite_helper.rb @@ -0,0 +1,410 @@ +require 'yaml_ld' + +# For now, override RDF::Utils::File.open_file to look for the file locally before attempting to retrieve it +module RDF::Util + module File + LOCAL_PATHS = { + "https://json-ld.github.io/yaml-ld/tests/" => ::File.expand_path("../../../w3c-yaml-ld/tests", __FILE__) + '/', + "file:" => "" + } + + class << self + alias_method :original_open_file, :open_file + end + + ## + # Override to use Patron for http and https, Kernel.open otherwise. + # + # @param [String] filename_or_url to open + # @param [Hash{Symbol => Object}] options + # @option options [Array, String] :headers + # HTTP Request headers. + # @return [IO] File stream + # @yield [IO] File stream + def self.open_file(filename_or_url, **options, &block) + LOCAL_PATHS.each do |r, l| + next unless Dir.exist?(l) && filename_or_url.start_with?(r) + #puts "attempt to open #{filename_or_url} locally" + url_no_frag_or_query = RDF::URI(filename_or_url).dup + url_no_frag_or_query.query = nil + url_no_frag_or_query.fragment = nil + localpath = url_no_frag_or_query.to_s.sub(r, l) + response = begin + ::File.open(localpath) + rescue Errno::ENOENT => e + raise IOError, e.message + end + + document_options = { + base_uri: RDF::URI(filename_or_url), + charset: Encoding::UTF_8, + code: 200, + headers: options.fetch(:headers, {}) + } + #puts "use #{filename_or_url} locally" + document_options[:headers][:content_type] = case localpath + when /\.ttl$/ then 'text/turtle' + when /\.nq$/ then 'application/n-quads' + when /\.nt$/ then 'application/n-triples' + when /\.html$/ then 'text/html' + when /\.jsonld$/ then 'application/ld+json' + when /\.json$/ then 'application/json' + when /\.yamlld$/ then 'application/ld+yaml' + when /\.yaml$/ then 'application/yaml' + else 'unknown' + end + + document_options[:headers][:content_type] = response.content_type if response.respond_to?(:content_type) + # For overriding content type from test data + document_options[:headers][:content_type] = options[:contentType] if options[:contentType] + + remote_document = RDF::Util::File::RemoteDocument.new(response.read, **document_options) + response.close + if block_given? + return yield remote_document + else + return remote_document + end + end + + original_open_file(filename_or_url, **options, &block) + end + end +end + +module Fixtures + module SuiteTest + SUITE = RDF::URI("https://json-ld.github.io/yaml-ld/tests/") + + class Manifest < JSON::LD::Resource + attr_accessor :manifest_url + + def self.open(file) + RDF::Util::File.open_file(file) do |remote| + json = JSON.parse(remote.read) + if block_given? + yield self.from_jsonld(json, manifest_url: RDF::URI(file)) + else + self.from_jsonld(json, manifest_url: RDF::URI(file)) + end + end + end + + def initialize(json, manifest_url:) + @manifest_url = manifest_url + super + end + + # @param [Hash] json framed JSON-LD + # @return [Array] + def self.from_jsonld(json, manifest_url: ) + Manifest.new(json, manifest_url: manifest_url) + end + + def entries + # Map entries to resources + attributes['sequence'].map do |e| + e.is_a?(String) ? Manifest.open(manifest_url.join(e).to_s) : Entry.new(e, manifest_url: manifest_url) + end + end + end + + class Entry < JSON::LD::Resource + attr_accessor :logger + attr_accessor :manifest_url + + def initialize(json, manifest_url:) + @manifest_url = manifest_url + super + end + + # Base is expanded input if not specified + def base + options.fetch('base', manifest_url.join(property('input')).to_s) + end + + def options + @options ||= begin + opts = { + documentLoader: Fixtures::SuiteTest.method(:documentLoader), + validate: true, + lowercaseLanguage: true, + } + (property('option') || {}).each do |k, v| + opts[k.to_sym] = v + end + if opts[:expandContext] && !RDF::URI(opts[:expandContext]).absolute? + # Resolve relative to manifest location + opts[:expandContext] = manifest_url.join(opts[:expandContext]).to_s + end + opts + end + end + + # Alias input, context, expect and frame + %w(input context expect frame).each do |m| + define_method(m.to_sym) do + return nil unless property(m) + res = nil + file = self.send("#{m}_loc".to_sym) + + dl_opts = {safe: true} + dl_opts[:contentType] = options[:contentType] if m == 'input' && options[:contentType] + RDF::Util::File.open_file(file, **dl_opts) do |remote_doc| + res = remote_doc.read + end + res + end + + define_method("#{m}_loc".to_sym) do + file = property(m) + + # Handle redirection internally + if m == "input" && options[:redirectTo] + file = options[:redirectTo] + end + + property(m) && manifest_url.join(file).to_s + end + + define_method("#{m}_json".to_sym) do + JSON.parse(self.send(m)) if property(m) + end + end + + def testType + property('@type').reject {|t| t =~ /EvaluationTest|SyntaxTest/}.first + end + + def evaluationTest? + property('@type').to_s.include?('EvaluationTest') + end + + def positiveTest? + property('@type').to_s.include?('Positive') + end + + def syntaxTest? + property('@type').to_s.include?('Syntax') + end + + + # Execute the test + def run(rspec_example = nil) + logger = @logger = RDF::Spec.logger + logger.info "test: #{inspect}" + logger.info "purpose: #{purpose}" + logger.info "source: #{input rescue nil}" + logger.info "context: #{context}" if context_loc + logger.info "options: #{options.inspect}" unless options.empty? + logger.info "frame: #{frame}" if frame_loc + + options = self.options + if options[:specVersion] == "json-ld-1.0" + skip "1.0 test" + return + end + + # Because we're doing exact comparisons when ordered. + options[:lowercaseLanguage] = true if options[:ordered] + + if positiveTest? + logger.info "expected: #{expect rescue nil}" if expect_loc + begin + result = case testType + when "jld:ExpandTest" + YAML_LD::API.expand(input_loc, logger: logger, **options) + when "jld:CompactTest" + YAML_LD::API.compact(input_loc, context_json['@context'], logger: logger, **options) + when "jld:FlattenTest" + YAML_LD::API.flatten(input_loc, (context_json['@context'] if context_loc), logger: logger, **options) + when "jld:FrameTest" + YAML_LD::API.frame(input_loc, frame_loc, logger: logger, **options) + when "jld:FromRDFTest" + # Use an array, to preserve input order + repo = RDF::NQuads::Reader.open(input_loc, rdfstar: options[:rdfstar]) do |reader| + reader.each_statement.to_a + end.to_a.uniq.extend(RDF::Enumerable) + logger.info "repo: #{repo.dump(self.id == '#t0012' ? :nquads : :trig)}" + YAML_LD::API.fromRdf(repo, logger: logger, **options) + when "jld:ToRDFTest" + repo = RDF::Repository.new + if manifest_url.to_s.include?('stream') + YAML_LD::Reader.open(input_loc, stream: true, logger: logger, **options) do |statement| + repo << statement + end + else + YAML_LD::API.toRdf(input_loc, rename_bnodes: false, logger: logger, **options) do |statement| + repo << statement + end + end + logger.info "nq: #{repo.map(&:to_nquads)}" + repo + when "jld:HttpTest" + res = input_json + rspec_example.instance_eval do + # use the parsed input file as @result for Rack Test application + @results = res + get "/", {}, "HTTP_ACCEPT" => options.fetch(:httpAccept, ""), "HTTP_LINK" => options.fetch(:httpLink, nil) + expect(last_response.status).to eq 200 + expect(last_response.content_type).to eq options.fetch(:contentType, "") + last_response.body + end + else + fail("Unknown test type: #{testType}") + end + + result = YAML_LD::Representation.load(result, extendedYAML: options[:extendedYAML]) if result.is_a?(String) + + if evaluationTest? + if testType == "jld:ToRDFTest" + expected = RDF::Repository.new << RDF::NQuads::Reader.new(expect, rdfstar: options[:rdfstar], logger: []) + rspec_example.instance_eval { + expect(result).to be_equivalent_graph(expected, logger) + } + else + expected = YAML_LD::Representation.load(expect, extendedYAML: options[:extendedYAML]) + + # If called for, remap bnodes + result = remap_bnodes(result, expected) if options[:remap_bnodes] + + if options[:ordered] + # Compare without transformation + rspec_example.instance_eval { + expect(result).to produce(expected, logger) + } + else + # Without key ordering, reorder result and expected embedded array values and compare + # If results are compacted, expand both, reorder and re-compare + rspec_example.instance_eval { + expect(result).to produce_yamlld(expected, logger) + } + + # If results are compacted, expand both, reorder and re-compare + if result.to_s.include?('@context') + exp_expected = JSON::LD::API.expand(expected, **options) + exp_result = JSON::LD::API.expand(result, **options) + rspec_example.instance_eval { + expect(exp_result).to produce_yamlld(exp_expected, logger) + } + end + end + end + else + rspec_example.instance_eval { + expect(result).to_not be_nil + } + end + rescue JSON::LD::JsonLdError => e + fail("Processing error: #{e.message}") + end + else + logger.info "expected: #{property('expect')}" if property('expect') + t = self + rspec_example.instance_eval do + if t.evaluationTest? + expect do + case t.testType + when "jld:ExpandTest" + JSON::LD::API.expand(t.input_loc, logger: logger, **options) + when "jld:CompactTest" + JSON::LD::API.compact(t.input_loc, t.context_json['@context'], logger: logger, **options) + when "jld:FlattenTest" + JSON::LD::API.flatten(t.input_loc, t.context_loc, logger: logger, **options) + when "jld:FrameTest" + JSON::LD::API.frame(t.input_loc, t.frame_loc, logger: logger, **options) + when "jld:FromRDFTest" + repo = RDF::Repository.load(t.input_loc, rdfstar: options[:rdfstar]) + logger.info "repo: #{repo.dump(t.id == '#t0012' ? :nquads : :trig)}" + JSON::LD::API.fromRdf(repo, logger: logger, **options) + when "jld:HttpTest" + rspec_example.instance_eval do + # use the parsed input file as @result for Rack Test application + @results = t.input_json + get "/", {}, "HTTP_ACCEPT" => options.fetch(:httpAccept, "") + expect(last_response.status).to eq t.property('expect') + expect(last_response.content_type).to eq options.fetch(:contentType, "") + raise "406" if t.property('expect') == 406 + raise "Expected status #{t.property('expectErrorCode')}, not #{last_response.status}" + end + when "jld:ToRDFTest" + if t.manifest_url.to_s.include?('stream') + JSON::LD::Reader.open(t.input_loc, stream: true, logger: logger, **options).each_statement {} + else + JSON::LD::API.toRdf(t.input_loc, rename_bnodes: false, logger: logger, **options) {} + end + else + success("Unknown test type: #{testType}") + end + end.to raise_error(/#{t.property('expectErrorCode')}/) + else + fail("No support for NegativeSyntaxTest") + end + end + end + end + + # Don't use NQuads writer so that we don't escape Unicode + def to_quad(thing) + case thing + when RDF::URI + thing.to_ntriples + when RDF::Node + escaped(thing) + when RDF::Literal::Double + thing.canonicalize.to_ntriples + when RDF::Literal + v = quoted(escaped(thing.value)) + case thing.datatype + when nil, "http://www.w3.org/2001/XMLSchema#string", "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString" + # Ignore these + else + v += "^^#{to_quad(thing.datatype)}" + end + v += "@#{thing.language}" if thing.language + v + when RDF::Statement + thing.to_quad.map {|r| to_quad(r)}.compact.join(" ") + " .\n" + end + end + + ## + # @param [String] string + # @return [String] + def quoted(string) + "\"#{string}\"" + end + + ## + # @param [String, #to_s] string + # @return [String] + def escaped(string) + string.to_s.gsub('\\', '\\\\').gsub("\t", '\\t'). + gsub("\n", '\\n').gsub("\r", '\\r').gsub('"', '\\"') + end + end + + ## + # Document loader to use for tests having `useDocumentLoader` option + # + # @param [RDF::URI, String] url + # @param [Hash Object>] options + # @option options [Boolean] :validate + # Allow only appropriate content types + # @return [RDF::Util::File::RemoteDocument] retrieved remote document and context information unless block given + # @yield remote_document + # @yieldparam [RDF::Util::File::RemoteDocument] remote_document + # @raise [JsonLdError] + def documentLoader(url, **options, &block) + options[:headers] ||= JSON::LD::API::OPEN_OPTS[:headers].dup + options[:headers][:link] = Array(options[:httpLink]).join(',') if options[:httpLink] + + url = url.to_s[5..-1] if url.to_s.start_with?("file:") + YAML_LD::API.documentLoader(url, **options, &block) + rescue JSON::LD::JsonLdError::LoadingDocumentFailed, JSON::LD::JsonLdError::MultipleContextLinkHeaders + raise unless options[:safe] + "don't raise error" + end + module_function :documentLoader + end +end diff --git a/spec/suite_spec.rb b/spec/suite_spec.rb new file mode 100644 index 0000000..2e0e7a5 --- /dev/null +++ b/spec/suite_spec.rb @@ -0,0 +1,22 @@ +# coding: utf-8 +require_relative 'spec_helper' + +describe YAML_LD do + describe "test suite" do + require_relative 'suite_helper' + m = Fixtures::SuiteTest::Manifest.open("#{Fixtures::SuiteTest::SUITE}manifest.jsonld") + describe m.name do + m.entries.each do |t| + specify "#{t.property('@id')}: #{t.name} #{t.positiveTest? ? 'unordered' : '(negative test)'}" do + t.options[:ordered] = false + expect{t.run self}.not_to write.to(:error) + end + + specify "#{t.property('@id')}: #{t.name} ordered" do + t.options[:ordered] = true + expect {t.run self}.not_to write.to(:error) + end if t.positiveTest? + end + end + end +end unless ENV['CI'] \ No newline at end of file diff --git a/spec/support/extensions.rb b/spec/support/extensions.rb index 84678b5..262b86e 100644 --- a/spec/support/extensions.rb +++ b/spec/support/extensions.rb @@ -42,3 +42,11 @@ def equivalent_structure?(other, ordered: false) end end end + +class String + def equivalent_yamlld?(other, ordered: false, extendedYAML: false) + actual = YAML_LD::Representation.load(self, aliases: true, extendedYAML: extendedYAML) + expected = other.is_a?(String) ? YAML_LD::Representation.load(other, aliases: true, extendedYAML: extendedYAML) : other + actual.equivalent_structure?(expected, ordered: ordered) + end +end \ No newline at end of file diff --git a/yaml-ld.gemspec b/yaml-ld.gemspec index 9bd966d..bf23ea7 100755 --- a/yaml-ld.gemspec +++ b/yaml-ld.gemspec @@ -27,19 +27,19 @@ Gem::Specification.new do |gem| gem.require_paths = %w(lib) gem.test_files = Dir.glob('spec/**/*.rb') + Dir.glob('spec/test-files/*') - gem.required_ruby_version = '>= 2.6' + gem.required_ruby_version = '>= 3.0' gem.requirements = [] - gem.add_runtime_dependency 'json-ld', '~> 3.2', '>= 3.2.3' + gem.add_runtime_dependency 'json-ld', '~> 3.3' gem.add_runtime_dependency 'psych', '>= 3.3' # Rails 6.0 cannot use psych 4.0 - gem.add_runtime_dependency 'rdf', '~> 3.2', '>= 3.2.9' - gem.add_runtime_dependency 'rdf-xsd', '~> 3.2' - gem.add_development_dependency 'rdf-isomorphic', '~> 3.2' - gem.add_development_dependency 'rdf-spec', '~> 3.2' - gem.add_development_dependency 'rdf-trig', '~> 3.2' - gem.add_development_dependency 'rdf-turtle', '~> 3.2' - gem.add_development_dependency 'rdf-vocab', '~> 3.2' - gem.add_development_dependency 'rdf-xsd', '~> 3.2' - gem.add_development_dependency 'rspec', '~> 3.10' + gem.add_runtime_dependency 'rdf', '~> 3.3' + gem.add_runtime_dependency 'rdf-xsd', '~> 3.3' + gem.add_development_dependency 'rdf-isomorphic', '~> 3.3' + gem.add_development_dependency 'rdf-spec', '~> 3.3' + gem.add_development_dependency 'rdf-trig', '~> 3.3' + gem.add_development_dependency 'rdf-turtle', '~> 3.3' + gem.add_development_dependency 'rdf-vocab', '~> 3.3' + gem.add_development_dependency 'rdf-xsd', '~> 3.3' + gem.add_development_dependency 'rspec', '~> 3.12' gem.add_development_dependency 'rspec-its', '~> 1.3' gem.add_development_dependency 'yard' , '~> 0.9'