From 17b3ff4f0ae1a2c9655f447711796b00024a7476 Mon Sep 17 00:00:00 2001 From: Brandon Mitchell Date: Fri, 13 Mar 2026 10:34:54 -0400 Subject: [PATCH] [WIP] Support for conformance results.yaml input This reworks the page to show a collapsed list under the version because there are a lot of workflows in the new conformance tests. The instructions have also been updated and adjusted for markdown linting. Signed-off-by: Brandon Mitchell --- hack/build-static-site.sh | 2 +- instructions.md | 113 ++++++------ netlify.toml | 4 +- products-page-generator/README.md | 4 +- products-page-generator/go.mod | 2 +- products-page-generator/go.sum | 3 +- products-page-generator/index.md.tpl | 133 ++++++++++---- products-page-generator/main.go | 249 +++++++++++++++++++-------- 8 files changed, 336 insertions(+), 174 deletions(-) diff --git a/hack/build-static-site.sh b/hack/build-static-site.sh index a629e20..e9fa0f5 100755 --- a/hack/build-static-site.sh +++ b/hack/build-static-site.sh @@ -28,7 +28,7 @@ go run main.go cd jekyll/ rm -f Gemfile.lock bundle config set --local path 'vendor/bundle' -bundle install --path 'vendor/bundle' +bundle install --path 'vendor/bundle' || bundle install bundle exec jekyll build cp favicon.ico _site/ echo '/** Disable unnecessary site navbar/menu */' >> _site/assets/main.css diff --git a/instructions.md b/instructions.md index 0ce3c4a..a81ef4a 100644 --- a/instructions.md +++ b/instructions.md @@ -2,63 +2,56 @@ ## Running the tests -Each spec will provide its own testing instructions, and each will produce -the following files that contain the test results and output: +Each spec will provide its own testing instructions, and each will produce the following files that contain the test results and output: + +- `results.yaml` - `report.html` - `junit.xml` ### OCI Distribution Specification -Please see instructions [here](https://github.com/opencontainers/distribution-spec/blob/main/conformance/README.md). +Please see the [distribution-spec instructions](https://github.com/opencontainers/distribution-spec/blob/main/conformance/README.md). ## Uploading -Prepare a PR to -[https://github.com/opencontainers/oci-conformance](https://github.com/opencontainers/oci-conformance). -Here are [directions](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) to -prepare a pull request from a fork. -In the descriptions below, `$spec.X.Y` refers to the spec and its major and minor -version, and `$dir` is a short subdirectory name to hold the results for your -product. Examples would be `gcr` or `dockerhub`. +Prepare a PR to [https://github.com/opencontainers/oci-conformance](https://github.com/opencontainers/oci-conformance). +Here are [directions](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) to prepare a pull request from a fork. +In the descriptions below, `$spec.X.Y` refers to the spec and its major and minor version, and `$dir` is a short subdirectory name to hold the results for your product. +Examples would be `gcr` or `dockerhub`. Description: `Conformance results for $spec/vX.Y/$dir` ### Contents of the PR -For simplicity you can submit the tarball or extract the relevant information from the tarball to compose your submission. +For simplicity you can submit the tarball or extract the relevant information from the tarball to compose your submission. -``` -$spec/vX.Y/$dir/README.md: Description of how to reproduce your results. -$spec/vX.Y/$dir/report.html: Human-readable HTML test report. -$spec/vX.Y/$dir/junit.xml: Machine-readable JUnit test report. -$spec/vX.Y/$dir/PRODUCT.yaml: See below. -``` +- `$spec/vX.Y/$dir/README.md`: Description of how to reproduce your results. +- `$spec/vX.Y/$dir/results.yaml`: Machine-readable test results in yaml. +- `$spec/vX.Y/$dir/report.html`: Human-readable HTML test report. +- `$spec/vX.Y/$dir/junit.xml`: Machine-readable JUnit test report. +- `$spec/vX.Y/$dir/PRODUCT.yaml`: See below. -Entirely optional, but encouraged, you can also include the following files -(if you have not already submitted them previously): +Entirely optional, but encouraged, you can also include the following files (if you have not already submitted them previously): -``` -$spec/live/$dir/badges.md: See below. -``` +- `$spec/live/$dir/badges.md`: See below. #### PRODUCT.yaml This file describes your product. It is YAML formatted with the following root-level fields. Please fill in as appropriate. -| Field | Description | -| ------------------- | ----------- | -| `vendor` | Name of the legal entity that is certifying. This entity must have a signed participation form on file with the OCI | -| `name` | Name of the product being certified. | -| `version` | The version of the product being certified (not the version of OCI spec). | -| `website_url` | URL to the product information website | -| `repo_url` | If your product is open source, this field is necessary to point to the primary GitHub repo containing the source. It's OK if this is a mirror. OPTIONAL | -| `documentation_url` | URL to the product documentation | -| `product_logo_url` | URL to the product's logo, (must be in SVG, AI or EPS format -- not a PNG -- and include the product name). OPTIONAL. If not supplied, we'll use your company logo. Please see logo guidelines (TODO: link) | +| Field | Description | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vendor` | Name of the legal entity that is certifying. This entity must have a signed participation form on file with the OCI | +| `name` | Name of the product being certified. | +| `version` | The version of the product being certified (not the version of OCI spec). | +| `website_url` | URL to the product information website | +| `repo_url` | If your product is open source, this field is necessary to point to the primary GitHub repo containing the source. It's OK if this is a mirror. OPTIONAL | +| `documentation_url` | URL to the product documentation | +| `product_logo_url` | URL to the product's logo, (must be in SVG, AI or EPS format -- not a PNG -- and include the product name). OPTIONAL. If not supplied, we'll use your company logo. | | `type` | Is your product a distribution, hosted platform, or installer (see [definitions](https://github.com/opencontainers/oci-conformance/blob/main/faq.md#what-is-a-distribution-and-what-is-a-platform)) | -| `description` | One sentence description of your offering | +| `description` | One sentence description of your offering | -Examples below are for a fictional OCI implementation called _Turbo -Encabulator_ produced by a company named _Yoyodyne_. +Examples below are for a fictional OCI implementation called _Turbo Encabulator_ produced by a company named _Yoyodyne_. ```yaml vendor: Yoyodyne @@ -74,23 +67,18 @@ description: 'The Yoyodyne Turbo Encabulator is a superb OCI distribution for al #### badges.md (Optional) -If you are running live tests (for example, in GitHub Actions), you are -encouraged to include a `badges.md` files which contains Markdown badges pointing to your test results. +If you are running live tests (for example, in GitHub Actions), you are encouraged to include a `badges.md` files which contains Markdown badges pointing to your test results. -These badges will be displayed on a web-based dashboard showing live -conformance results for various products. +These badges will be displayed on a web-based dashboard showing live conformance results for various products. -Since you are likely testing conformance on the latest -changeset to your product, this file should be submitted to the directory -`$spec/live/$dir` (vs. `$spec/vX.Y/$dir`). *If you have previously submitted -this file, you do not need to do so again to submit conformance for a new -spec version.* +Since you are likely testing conformance on the latest changeset to your product, this file should be submitted to the directory `$spec/live/$dir` (vs. `$spec/vX.Y/$dir`). +_If you have previously submitted this file, you do not need to do so again to submit conformance for a new spec version._ This file should simply contain Markdown badges, each on a new line. Here is an example `badges.md` file showing 4 badges: -``` +```markdown [![](https://github.com/myorg/myproduct/workflows/oci-pull/badge.svg)](https://github.com/myorg/myproduct/actions?query=workflow%3Aoci-pull) [![](https://github.com/myorg/myproduct/workflows/oci-push/badge.svg)](https://github.com/myorg/myproduct/actions?query=workflow%3Aoci-push) [![](https://github.com/myorg/myproduct/workflows/oci-content-discovery/badge.svg)](https://github.com/myorg/myproduct/actions?query=workflow%3Aoci-content-discovery) @@ -99,42 +87,43 @@ Here is an example `badges.md` file showing 4 badges: ## Amendment for Private Review -If you need a private review for an unreleased product, please email a zip file containing what you would otherwise submit -as a pull request to certification@opencontainers.org. We'll review and confirm that you are ready to be OCI Certified -as soon as you open the pull request. We can then often arrange to accept your pull request soon after you make it, at which point you become OCI Certified. +If you need a private review for an unreleased product, please email a zip file containing what you would otherwise submit as a pull request to . +We'll review and confirm that you are ready to be OCI Certified as soon as you open the pull request. +We can then often arrange to accept your pull request soon after you make it, at which point you become OCI Certified. ## Review -A reviewer will shortly comment on and/or accept your pull request, following this [process](reviewing.md). -If you don't see a response within 3 business days, please contact certification@opencontainers.org. +A reviewer will shortly comment on and/or accept your pull request, following the [reviewer process](reviewing.md). +If you don't see a response within 3 business days, please contact . ## Example Script -Combining the steps provided here, the process looks like this (Example: `distribution-spec/v1.0`): +Combining the steps provided here, the process looks like this (Example: `distribution-spec/v1.1`): -``` +```shell spec_name=distribution-spec -spec_version=v1.0 +spec_version=1.1 prod_name=example rm -rf tmp && git clone https://github.com/opencontainers/${spec_name}.git tmp (cd tmp && docker build -t conformance:latest -f Dockerfile.conformance .) rm -rf tmp results -docker run --rm \ +docker run -it --rm --net=host \ + -u "$(id -u):$(id -g)" \ -v $(pwd)/results:/results \ - -w /results \ - -e OCI_ROOT_URL="https://r.myreg.io" \ - -e OCI_NAMESPACE="myorg/myrepo" \ + -e OCI_VERSION="$spec_version$" \ + -e OCI_REGISTRY="r.myreg.io" \ + -e OCI_REPO1="myorg/myrepo" \ + -e OCI_REPO2="myorg/myrepo2" \ -e OCI_USERNAME="myuser" \ -e OCI_PASSWORD="mypass" \ - -e OCI_DEBUG="true" \ conformance:latest -mkdir -p ./${spec_name}/${spec_version}/${prod_name} -cp ./results/* ./${spec_name}/${spec_version}/${prod_name}/ +mkdir -p ./${spec_name}/v${spec_version}/${prod_name} +cp ./results/* ./${spec_name}/v${spec_version}/${prod_name}/ rm -rf results -cat << EOF > ./${spec_name}/${spec_version}/${prod_name}/PRODUCT.yaml +cat << EOF > ./${spec_name}/v${spec_version}/${prod_name}/PRODUCT.yaml vendor: Yoyodyne name: Turbo Encabulator version: v1.7.4 @@ -149,6 +138,4 @@ EOF ## Issues -If you have problems certifying that you feel are an issue with the conformance -program itself (and not just your own implementation), you can file an issue in -the [repository](https://github.com/opencontainers/oci-conformance). +If you have problems certifying that you feel are an issue with the conformance program itself (and not just your own implementation), you can [file an issue](https://github.com/opencontainers/oci-conformance/issues). diff --git a/netlify.toml b/netlify.toml index 002addf..7e55ffa 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,4 +1,6 @@ [build] command = "./hack/build-static-site.sh" publish = "_site/" - environment = { GO_VERSION = "1.22.x" } +[build.environment] + GO_VERSION = "1.26.x" + RUBY_VERSION = "3.3" diff --git a/products-page-generator/README.md b/products-page-generator/README.md index 9494347..3cbbfe5 100644 --- a/products-page-generator/README.md +++ b/products-page-generator/README.md @@ -5,7 +5,7 @@ used to display conformance across all specs and products. To run: -``` +```shell go run main.go ``` @@ -14,7 +14,7 @@ This will produce a new `output/` directory. You can then use [Jekyll](https://jekyllrb.com/) (or something similar) to produce a static HTML website: -``` +```shell cd jekyll/ bundle config set --local path 'vendor/bundle' bundle install diff --git a/products-page-generator/go.mod b/products-page-generator/go.mod index 40e3dfa..f9fb7a5 100644 --- a/products-page-generator/go.mod +++ b/products-page-generator/go.mod @@ -10,8 +10,8 @@ replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 require ( github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig v2.22.0+incompatible + github.com/goccy/go-yaml v1.19.2 github.com/joshdk/go-junit v1.0.0 - gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/products-page-generator/go.sum b/products-page-generator/go.sum index d02a5e3..3bd963a 100644 --- a/products-page-generator/go.sum +++ b/products-page-generator/go.sum @@ -6,6 +6,8 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= @@ -25,7 +27,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/products-page-generator/index.md.tpl b/products-page-generator/index.md.tpl index 6870205..95203ac 100644 --- a/products-page-generator/index.md.tpl +++ b/products-page-generator/index.md.tpl @@ -37,9 +37,10 @@ technical decisions when choosing a product to use as an OCI registry. ### Open Source -{{ with .Submissions -}} +{{- with .Submissions -}} {{- range $key, $value := . }} {{- if .IsOSS }} + #### {{ $value.Meta.Name }} @@ -48,41 +49,72 @@ technical decisions when choosing a product to use as an OCI registry. __Homepage:__ [{{ $value.Meta.WebsiteURL }}]({{ $value.Meta.WebsiteURL }}) +{{- if ne $value.Meta.DocumentationURL $value.Meta.RepoURL }} -{{ if ne $value.Meta.DocumentationURL $value.Meta.RepoURL }} __Documentation:__ [{{ $value.Meta.DocumentationURL }}]({{ $value.Meta.DocumentationURL }}) -{{ end }} +{{- end }} + +{{- if $value.Meta.RepoURL }} -{{ if $value.Meta.RepoURL }} __Repository:__ [{{ $value.Meta.RepoURL }}]({{ $value.Meta.RepoURL }}) -{{ end }} +{{- end }} __Vendor:__ {{ $value.Meta.Vendor }} -__Latest software version tested__: `{{ $value.Meta.Version }}` +__Latest software version tested__: {{ $value.Meta.Version }} -__Latest spec version supported__: `{{ $value.LatestVersion }}`
**[Test report](./static/{{ $value.LatestVersion }}/reports/{{ $key }}/)** -| **[How to reproduce](./static/{{ $value.LatestVersion }}/instructions/{{ $key }}/)** +__Latest spec version supported__: {{ $value.LatestVersion }} -{{ if $value.BadgesMarkdown }} -__Live results:__ {{ $value.BadgesMarkdown }} -{{ end }} +{{- if $value.BadgesMarkdown }} -| Supported workflows: | {{ range $_, $version := $value.AllVersions }}{{ $version }} | {{ end }} -| ---------------------- | ------ | -{{- range $_, $v := (index $value.Workflows $value.LatestVersion) }} -| {{ $v.Name }} {{ if $v.Required }}(required){{ end }} | {{ range $_, $version := $value.AllVersions }}{{ range $_, $workflow := (index $value.Workflows $version) }}{{ if eq $workflow.Name $v.Name }}{{ if $workflow.Supported }} ✅ {{ else }} ✖️ {{ end }} | {{ end }}{{ end }}{{ end }} +__Live results:__ {{ $value.BadgesMarkdown }} {{- end }} -{{ end }} -{{- end }} -{{ end }} +{{- range $_, $version := $value.AllVersions }} + +
+
+Version: {{ $version }} + + + +
+Workflows:
+{{- range $_, $workflow := (index $value.Workflows $version) }} +{{- if or (eq $workflow.Supported.String "Skip") (eq $workflow.Supported.String "Disabled") }} +✖️ {{ $workflow.Name }}
+{{- else if ne $workflow.Supported.String "Pass" }} +❗ {{ $workflow.Name }}
+{{- end }}{{/* non-passing workflows */}} +{{- end }}{{/* range over Workflows */}} + +{{- if gt (index (index $value.Summary $version).Counts "Pass") 4 }}
+✅✅...✅ {{- (index (index $value.Summary $version).Counts "Pass") }} of {{ len (index $value.Workflows $version) }} workflows passed{{- end }} +{{- range $_, $workflow := (index $value.Workflows $version) }} +{{- if eq $workflow.Supported.String "Pass" }} +✅ {{ $workflow.Name }}
+{{- end }}{{/* passing workflows */}} +{{- end }}{{/* range over Workflows */}} +{{- if gt (index (index $value.Summary $version).Counts "Pass") 4 }}
{{- end }} +
+ +
+
+{{- end }}{{/* range over AllVersions */}} +
+{{- end }}{{/* if OSS */}} +{{- end }}{{/* range over submissions */}} +{{- end }}{{/* with .Submissions */}} ### Hosted -{{ with .Submissions -}} +{{- with .Submissions -}} {{- range $key, $value := . }} {{- if not .IsOSS }} + #### {{ $value.Meta.Name }} @@ -91,26 +123,65 @@ __Live results:__ {{ $value.BadgesMarkdown }} __Homepage:__ [{{ $value.Meta.WebsiteURL }}]({{ $value.Meta.WebsiteURL }}) +{{- if ne $value.Meta.DocumentationURL $value.Meta.RepoURL }} + __Documentation:__ [{{ $value.Meta.DocumentationURL }}]({{ $value.Meta.DocumentationURL }}) +{{- end }} + +{{- if $value.Meta.RepoURL }} + +__Repository:__ [{{ $value.Meta.RepoURL }}]({{ $value.Meta.RepoURL }}) +{{- end }} __Vendor:__ {{ $value.Meta.Vendor }} -__Latest spec version supported__: `{{ $value.LatestVersion }}`
**[Test report](./static/{{ $value.LatestVersion }}/reports/{{ $key }}/)** -| **[How to reproduce](./static/{{ $value.LatestVersion }}/instructions/{{ $key }}/)** +__Latest software version tested__: {{ $value.Meta.Version }} -{{ if $value.BadgesMarkdown }} -__Live results:__ {{ $value.BadgesMarkdown }} -{{ end }} +__Latest spec version supported__: {{ $value.LatestVersion }} -| Supported workflows: | {{ range $_, $version := $value.AllVersions }}{{ $version }} | {{ end }} -| ---------------------- | ------ | -{{- range $_, $v := (index $value.Workflows $value.LatestVersion) }} -| {{ $v.Name }} {{ if $v.Required }}(required){{ end }} | {{ range $_, $version := $value.AllVersions }}{{ range $_, $workflow := (index $value.Workflows $version) }}{{ if eq $workflow.Name $v.Name }}{{ if $workflow.Supported }} ✅ {{ else }} ✖️ {{ end }} | {{ end }}{{ end }}{{ end }} -{{- end }} +{{- if $value.BadgesMarkdown }} -{{ end }} +__Live results:__ {{ $value.BadgesMarkdown }} {{- end }} -{{ end }} + +{{- range $_, $version := $value.AllVersions }} + +
+
+Version: {{ $version }} + + + +
+Workflows:
+{{- range $_, $workflow := (index $value.Workflows $version) }} +{{- if or (eq $workflow.Supported.String "Skip") (eq $workflow.Supported.String "Disabled") }} +✖️ {{ $workflow.Name }}
+{{- else if ne $workflow.Supported.String "Pass" }} +❗ {{ $workflow.Name }}
+{{- end }}{{/* non-passing workflows */}} +{{- end }}{{/* range over Workflows */}} + +{{- if gt (index (index $value.Summary $version).Counts "Pass") 4 }}
+✅✅...✅ {{- (index (index $value.Summary $version).Counts "Pass") }} of {{ len (index $value.Workflows $version) }} workflows passed{{- end }} +{{- range $_, $workflow := (index $value.Workflows $version) }} +{{- if eq $workflow.Supported.String "Pass" }} +✅ {{ $workflow.Name }}
+{{- end }}{{/* passing workflows */}} +{{- end }}{{/* range over Workflows */}} +{{- if gt (index (index $value.Summary $version).Counts "Pass") 4 }}
{{- end }} +
+ +
+
+{{- end }}{{/* range over AllVersions */}} +
+{{- end }}{{/* if OSS */}} +{{- end }}{{/* range over submissions */}} +{{- end }}{{/* with .Submissions */}} --- diff --git a/products-page-generator/main.go b/products-page-generator/main.go index d723231..99e3136 100644 --- a/products-page-generator/main.go +++ b/products-page-generator/main.go @@ -2,29 +2,36 @@ package main import ( "bytes" + _ "embed" "fmt" "html/template" "log" "os" "path/filepath" + "slices" "sort" "strings" "github.com/Masterminds/semver" "github.com/Masterminds/sprig" + "github.com/goccy/go-yaml" "github.com/joshdk/go-junit" - "gopkg.in/yaml.v3" ) type ( + results struct { + APIs map[string]status `yaml:"apis"` + Data map[string]status `yaml:"data"` + } + submission struct { IsOSS bool LatestVersion string AllVersions []string Meta submissionMeta BadgesMarkdown string - ReadmeMarkdown string Workflows map[string][]workflow + Summary map[string]summary } submissionMeta struct { @@ -39,9 +46,13 @@ type ( Description string `yaml:"description"` } + summary struct { + Counts map[string]int + } + workflow struct { Name string - Supported bool + Supported status Required bool } ) @@ -53,10 +64,10 @@ const ( badgesFilename = "badges.md" readmeFilename = "README.md" reportFilename = "report.html" + resultsFilename = "results.yaml" junitFilename = "junit.xml" junitTestPrefix = "OCI Distribution Conformance Tests" metaTypeOSS = "distribution" - templateFilename = "index.md.tpl" outputDir = "output" outputFilename = "README.md" staticDir = "static" @@ -69,6 +80,7 @@ const ( indexPermalinkTemplate = `--- permalink: %s/index.html --- +%s ` workflowPull = "Pull" @@ -77,6 +89,53 @@ permalink: %s/index.html workflowContentManagement = "Content Management" ) +//go:embed index.md.tpl +var templateEmbed string + +type status int + +const ( + statusUnknown status = iota // status is undefined + statusDisabled // test was disabled by configuration + statusSkip // test was skipped + statusPass // test passed +) + +func (s status) String() string { + switch s { + case statusPass: + return "Pass" + case statusSkip: + return "Skip" + case statusDisabled: + return "Disabled" + default: + return "Unknown" + } +} + +func (s status) MarshalText() ([]byte, error) { + ret := s.String() + if ret == "Unknown" { + return []byte(ret), fmt.Errorf("unknown status %d", s) + } + return []byte(ret), nil +} + +func (s *status) UnmarshalText(text []byte) error { + switch strings.ToLower(string(text)) { + case "pass": + *s = statusPass + case "skip": + *s = statusSkip + case "disabled": + *s = statusDisabled + default: + return fmt.Errorf("unknown status %s", string(text)) + } + return nil +} + func main() { os.RemoveAll(outputDir) os.Mkdir(outputDir, 0755) @@ -85,8 +144,10 @@ func main() { if err != nil { log.Fatal(err) } - tpl, err := template.New(templateFilename). - Funcs(sprig.FuncMap()).ParseFiles(templateFilename) + // TODO: update template to compress workflows tables to only exceptions/highlights with an expansion option + // TODO: is sprig needed? + tpl, err := template.New("index"). + Funcs(sprig.FuncMap()).Parse(templateEmbed) if err != nil { log.Fatal(err) } @@ -118,71 +179,57 @@ func getSubmissions() (map[string]submission, error) { if err != nil { return nil, err } - for i := len(minorVersions) - 1; i >= 0; i-- { - minorVersion := minorVersions[i] - dirEntries, err := os.ReadDir(filepath.Join(rootDir, minorVersion)) + slices.Reverse(minorVersions) + for _, minorVersion := range minorVersions { + dirProjects, err := os.ReadDir(filepath.Join(rootDir, minorVersion)) if err != nil { return nil, err } - for _, dirEntry := range dirEntries { - k := dirEntry.Name() - readmeRaw, err := os.ReadFile(filepath.Join(rootDir, minorVersion, k, readmeFilename)) + for _, dirProject := range dirProjects { + projectName := dirProject.Name() + // import the readme with a template + readmeOrig, err := os.ReadFile(filepath.Join(rootDir, minorVersion, projectName, readmeFilename)) if err != nil { return nil, err } - readmeParentDirPermalink := filepath.Join(staticDir, minorVersion, instructionsDir, k) - readmeRaw = append([]byte(fmt.Sprintf(indexPermalinkTemplate, readmeParentDirPermalink)), readmeRaw...) - readmeParentDir := filepath.Join(outputDir, readmeParentDirPermalink, readmeFilename) - os.MkdirAll(readmeParentDir, 0755) - err = os.WriteFile( - filepath.Join(readmeParentDir, readmeFilename), - readmeRaw, 0644) + projectInstructionsDir := filepath.Join(staticDir, minorVersion, instructionsDir, projectName) + projectReadmeRaw := fmt.Appendf(nil, indexPermalinkTemplate, projectInstructionsDir, readmeOrig) + projectReadmeFilename := filepath.Join(outputDir, projectInstructionsDir, readmeFilename) + err = os.MkdirAll(filepath.Dir(projectReadmeFilename), 0755) if err != nil { return nil, err } - reportRaw, err := os.ReadFile(filepath.Join(rootDir, minorVersion, k, reportFilename)) + err = os.WriteFile(projectReadmeFilename, projectReadmeRaw, 0644) if err != nil { return nil, err } - os.MkdirAll(filepath.Join(outputDir, staticDir, minorVersion, reportsDir, k), 0755) - err = os.WriteFile( - filepath.Join(outputDir, staticDir, minorVersion, reportsDir, k, staticIndex), - reportRaw, 0644) + // copy the report + projectReportRaw, err := os.ReadFile(filepath.Join(rootDir, minorVersion, projectName, reportFilename)) if err != nil { return nil, err } - if s, ok := submissions[k]; ok { - allVersions := append(s.AllVersions, minorVersion) - s.AllVersions = allVersions - junitPath := filepath.Join(rootDir, minorVersion, k, junitFilename) - b, err := os.ReadFile(junitPath) - if err != nil { - return nil, err - } - workflows, err := workflowsFromJunitBytes(b) - if err != nil { - return nil, err - } - s.Workflows[minorVersion] = workflows - submissions[k] = s - } else { + projectReportFilename := filepath.Join(outputDir, staticDir, minorVersion, reportsDir, projectName, staticIndex) + err = os.MkdirAll(filepath.Dir(projectReportFilename), 0755) + if err != nil { + return nil, err + } + err = os.WriteFile(projectReportFilename, projectReportRaw, 0644) + if err != nil { + return nil, err + } + // load metadata yaml if this is the first submission for the project + if _, ok := submissions[projectName]; !ok { var meta submissionMeta - b, err := os.ReadFile(filepath.Join(rootDir, minorVersion, k, metaFilename)) + metaRaw, err := os.ReadFile(filepath.Join(rootDir, minorVersion, projectName, metaFilename)) if err != nil { return nil, err } - err = yaml.Unmarshal(b, &meta) + err = yaml.Unmarshal(metaRaw, &meta) if err != nil { return nil, err } - readmePath := filepath.Join(rootDir, minorVersion, k, readmeFilename) - b, err = os.ReadFile(readmePath) - if err != nil { - return nil, err - } - readmeMarkdown := string(b) var badgesMarkdown string - badgesPath := filepath.Join(rootDir, liveSubdir, k, badgesFilename) + badgesPath := filepath.Join(rootDir, liveSubdir, projectName, badgesFilename) if _, err := os.Stat(badgesPath); err == nil { b, err := os.ReadFile(badgesPath) if err != nil { @@ -190,25 +237,74 @@ func getSubmissions() (map[string]submission, error) { } badgesMarkdown = string(b) } - junitPath := filepath.Join(rootDir, minorVersion, k, junitFilename) - b, err = os.ReadFile(junitPath) + submissions[projectName] = submission{ + IsOSS: meta.Type == metaTypeOSS, + AllVersions: []string{}, + LatestVersion: minorVersion, + Meta: meta, + BadgesMarkdown: badgesMarkdown, + Workflows: map[string][]workflow{}, + Summary: map[string]summary{}, + } + } + s := submissions[projectName] + s.AllVersions = append(s.AllVersions, minorVersion) + // attempt to read results.yaml + resultsRaw, err := os.ReadFile(filepath.Join(rootDir, minorVersion, projectName, resultsFilename)) + if err == nil { + projectResults := results{} + err = yaml.Unmarshal(resultsRaw, &projectResults) if err != nil { return nil, err } - workflows, err := workflowsFromJunitBytes(b) + workflows := []workflow{} + for api, s := range projectResults.APIs { + name := fmt.Sprintf("API: %s", api) + w := workflow{ + Name: name, + Supported: s, + } + // required API workflows + switch api { + case "Blob get", "Blob head", "Manifest get by digest", "Manifest get by tag", "Manifest head by digest", "Manifest head by tag": + w.Required = true + } + workflows = append(workflows, w) + } + for data, s := range projectResults.Data { + name := fmt.Sprintf("Data: %s", data) + w := workflow{ + Name: name, + Supported: s, + } + // required data workflows + switch data { + case "Blobs sha256", "Image", "Index": + w.Required = true + } + workflows = append(workflows, w) + } + // sort the workflows to undo random range over maps + slices.SortFunc(workflows, func(a, b workflow) int { + return strings.Compare(a.Name, b.Name) + }) + s.Workflows[minorVersion] = workflows + s.Summary[minorVersion] = genSummary(workflows) + } else { + // fallback to reading original conformance junit and workflows + junitPath := filepath.Join(rootDir, minorVersion, projectName, junitFilename) + b, err := os.ReadFile(junitPath) if err != nil { return nil, err } - submissions[k] = submission{ - IsOSS: meta.Type == metaTypeOSS, - LatestVersion: minorVersion, - AllVersions: []string{minorVersion}, - Meta: meta, - BadgesMarkdown: badgesMarkdown, - ReadmeMarkdown: readmeMarkdown, - Workflows: map[string][]workflow{minorVersion: workflows}, + workflows, err := workflowsFromJunitBytes(b) + if err != nil { + return nil, err } + s.Workflows[minorVersion] = workflows + s.Summary[minorVersion] = genSummary(workflows) } + submissions[projectName] = s } } return submissions, nil @@ -234,6 +330,7 @@ func getMinorVersionsSorted() ([]string, error) { } vs[i] = v } + // TODO: make a simple semver sort to remove MasterMinds dependency sort.Sort(semver.Collection(vs)) minorVersions := make([]string, numVersions) for i, v := range vs { @@ -242,35 +339,39 @@ func getMinorVersionsSorted() ([]string, error) { return minorVersions, nil } +func genSummary(workflows []workflow) summary { + ret := summary{ + Counts: map[string]int{}, + } + for _, w := range workflows { + ret.Counts[w.Supported.String()]++ + } + return ret +} + func workflowsFromJunitBytes(b []byte) ([]workflow, error) { suites, err := junit.Ingest(b) if err != nil { return nil, err } - skippedTests := map[string]bool{ - workflowPull: true, - workflowPush: true, - workflowContentDiscovery: true, - workflowContentManagement: true, + workflows := []workflow{ + {workflowPull, statusSkip, true}, + {workflowPush, statusSkip, false}, + {workflowContentDiscovery, statusSkip, false}, + {workflowContentManagement, statusSkip, false}, } for _, suite := range suites { for _, test := range suite.Tests { if test.Status != junit.StatusSkipped { - for k, v := range skippedTests { - if v { - if strings.HasPrefix(test.Name, fmt.Sprintf("%s %s ", junitTestPrefix, k)) { - skippedTests[k] = false + for i, w := range workflows { + if w.Supported == statusSkip { + if strings.HasPrefix(test.Name, fmt.Sprintf("%s %s ", junitTestPrefix, w.Name)) { + workflows[i].Supported = statusPass } } } } } } - workflows := []workflow{ - {workflowPull, !skippedTests[workflowPull], true}, - {workflowPush, !skippedTests[workflowPush], false}, - {workflowContentDiscovery, !skippedTests[workflowContentDiscovery], false}, - {workflowContentManagement, !skippedTests[workflowContentManagement], false}, - } return workflows, nil }