diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 497ed948e..61d518d4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,4 +203,4 @@ jobs: deployment_repo: ${{ inputs.deployment_repo }} images: ${{ inputs.deployment_images }} secrets: - token: ${{ secrets.deployment_token }} \ No newline at end of file + token: ${{ secrets.deployment_token }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2f1402a48..414b179aa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,4 +50,4 @@ jobs: author_name: catalyst-cibot author_email: ci@projectcatalyst.io message: "chore: updates ${{ github.event.repository.name }} deployments" - push: true \ No newline at end of file + push: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a9a5fc91b..a631fe7bf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -172,4 +172,4 @@ jobs: ${{ inputs.aws_ecr_registry }} ghcr.io/${{ github.repository }} tags: | - ${{ steps.fetch_tag.outputs.tag }} \ No newline at end of file + ${{ steps.fetch_tag.outputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43433fd2f..4f9613777 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -170,4 +170,4 @@ jobs: - name: Create release uses: softprops/action-gh-release@v1 with: - files: ${{ steps.collect.outputs.artifacts }} \ No newline at end of file + files: ${{ steps.collect.outputs.artifacts }} diff --git a/Earthfile b/Earthfile index 070b9a50f..8ecf21e4b 100644 --- a/Earthfile +++ b/Earthfile @@ -1,5 +1,5 @@ # Set the Earthly version to 0.7 -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words livedocs sitedocs @@ -48,4 +48,4 @@ repo-config: edit-docs: LOCALLY - RUN ./earthly/docs/dev/local.py cat-ci-docs:latest \ No newline at end of file + RUN ./earthly/docs/dev/local.py cat-ci-docs:latest diff --git a/cli/go.mod b/cli/go.mod index fec7b7495..f3466188a 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -6,11 +6,10 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/alecthomas/kong v0.7.1 github.com/aws/aws-sdk-go v1.44.274 - github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b + github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb github.com/onsi/ginkgo/v2 v2.9.7 github.com/onsi/gomega v1.27.7 github.com/spf13/afero v1.9.5 - gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -24,8 +23,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.9.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/go.sum b/cli/go.sum index 33bb07292..687fd8a03 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -61,8 +61,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b h1:kwdLqircFHqYNLrBuSOybPpGPnqaqXEcRvJOWiSqRZw= -github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b/go.mod h1:OoxU/3mRER8gdd4l1lxF6EgK+0vf7zQ8m9g269y6hWI= +github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb h1:dP0Wcrfbq5whzPCT/ZHGTvWVvx7S6BoF32nRHqhKt7E= +github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb/go.mod h1:UX9246125xgKvMIKim7WTxR8nookovxWx8/Ze8kpYlQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -164,7 +164,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/poy/onpar v0.3.1 h1:iqwl2S3U08PY8Bm9Hfh9v3fxdig7hZy9bsZfTephng8= +github.com/poy/onpar v0.3.2 h1:yo8ZRqU3C4RlvkXPWUWfonQiTodAgpKQZ1g8VTNU9xU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -325,8 +325,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/cli/pkg/earthfile.go b/cli/pkg/earthfile.go index 208a60dce..8182daa71 100644 --- a/cli/pkg/earthfile.go +++ b/cli/pkg/earthfile.go @@ -14,7 +14,7 @@ type Earthfile struct { Path string `json:"path,omitempty"` SourceLocation *spec.SourceLocation `json:"sourceLocation,omitempty"` Targets []spec.Target `json:"targets,omitempty"` - UserCommands []spec.UserCommand `json:"userCommands,omitempty"` + Functions []spec.Function `json:"functions,omitempty"` Version *spec.Version `json:"version,omitempty"` } diff --git a/cli/pkg/parsers/earthly_parser.go b/cli/pkg/parsers/earthly_parser.go index 56586188f..3befdfdf1 100644 --- a/cli/pkg/parsers/earthly_parser.go +++ b/cli/pkg/parsers/earthly_parser.go @@ -24,7 +24,7 @@ func (e EarthlyParser) Parse(path string) (pkg.Earthfile, error) { Path: path, SourceLocation: ef.SourceLocation, Targets: ef.Targets, - UserCommands: ef.UserCommands, + Functions: ef.Functions, Version: ef.Version, }, nil } diff --git a/docs/src/guides/docs.md b/docs/src/guides/docs.md index 1c5831f9a..890cc27e4 100644 --- a/docs/src/guides/docs.md +++ b/docs/src/guides/docs.md @@ -52,7 +52,7 @@ src: The first step of building process it preparing a source files. It is mandatory to have a `src` directory with all documentation md files in it and `mkdocs.yml` file. -This directory and file will be picked during the execution of `+SRC` UDC target. +This directory and file will be picked during the execution of `+SRC` Function. Also it is possible to replace defined `includes`, `macros` and `overrides` dirs to customize some docs appearance and configuration. @@ -78,7 +78,7 @@ docs: DO ../earthly/docs+BUILD ``` -To build a docs artifact which will be used later just invoke `+BUILD` UDC target +To build a docs artifact which will be used later just invoke `+BUILD` Function on the already prepared docs environment `+src` target which we have discussed before. ```Earthfile @@ -93,7 +93,7 @@ local: ``` To finally build a docker image which is pretty strait forward process, -you should firstly invoke `+PACKAGE` UDC target which will prepare an environment for future docs image, +you should firstly invoke `+PACKAGE` Function which will prepare an environment for future docs image, next step is to copy builded artifact from the previous step to `/usr/share/nginx/html` folder. And the last step is to save a docker image with the specified name, tag and registry if it is needed. diff --git a/docs/src/guides/languages/go.md b/docs/src/guides/languages/go.md index 595cdb279..1d35e90df 100644 --- a/docs/src/guides/languages/go.md +++ b/docs/src/guides/languages/go.md @@ -56,7 +56,7 @@ deps: # Any build dependencies should also be captured in this target. RUN apk add --no-cache gcc musl-dev - # This UDC automatically copies the go.mod and go.sum files and runs + # This Function automatically copies the go.mod and go.sum files and runs # `go mod download` to install the dependencies. DO ../../earthly/go+DEPS --ginkgo="false" ``` @@ -75,11 +75,11 @@ This target is also going to build responsible for installing external build dep These are dependencies that are not specific to a language and usually get installed system-wide. In our case, since we're building a static binary, we need `gcc` and `musl`. -Finally, the actual logic we will be using is encapsulate in a UDC. +Finally, the actual logic we will be using is encapsulate in a Function. This is a very common pattern, as an `Earthfile` can get repetitive across a repository. -In our case, we use the `go+DEPS` UDC that will automatically copy our `go.mod` and `go.sum` files and then execute +In our case, we use the `go+DEPS` Function that will automatically copy our `go.mod` and `go.sum` files and then execute `go mod download`. -The UDC will also establish a cache for the Go tooling. +The Function will also establish a cache for the Go tooling. This means that, even if our source code changes, we'll see a substantial speed boost when compiling because the cache is preserved across Earthly runs. @@ -96,10 +96,10 @@ check: # This target checks the overall health of the source code. FROM +src - # This UDC validates the code is formatted according to Go standards. + # This Function validates the code is formatted according to Go standards. DO ../../earthly/go+FMT --src="go.mod go.sum cmd" - # This UDC runs golangci-lint to check for common errors. + # This Function runs golangci-lint to check for common errors. DO ../../earthly/go+LINT --src="go.mod go.sum cmd" ``` @@ -111,8 +111,8 @@ Any future targets which need access to the source code will inherit from this t Now that the source code is available, we can begin performing static checks. These checks are intended to verify the code is healthy and conforms to a certain standard. -As we did in the previous section, here we rely on UDCs again to perform these checks. -These two UDCs will validate the code formatting is correct and also perform a series of lints to validate code quality. +As we did in the previous section, here we rely on Functions again to perform these checks. +These two Functions will validate the code formatting is correct and also perform a series of lints to validate code quality. Note that these checks are fast (compared to later steps) and perform quick feedback on code quality. Since this is the first target run in CI, we want to fail the CI as quickly as possible if we can easily find code quality issues. diff --git a/docs/src/guides/languages/postgresql.md b/docs/src/guides/languages/postgresql.md index b6d86ac13..700c913ba 100644 --- a/docs/src/guides/languages/postgresql.md +++ b/docs/src/guides/languages/postgresql.md @@ -62,7 +62,7 @@ builder: The first target we are going to consider will be responsible to prepare a PostgreSQL environment (Earthly `+postgres-base` target), migrations, migrations configuration and seed data (`COPY --dir ./migrations ./data ./refinery.toml .`), -doing some final build step (Earthly `+BUILDER` UDC target). +doing some final build step (Earthly `+BUILDER` Function). In the next steps we are going to inheriting from this `+builder` target which contains all necessary data, dependencies, environment to properly run PostgreSQL database. @@ -92,8 +92,8 @@ These checks are intended to verify the code is healthy and well formatted to a and done with the help of the `sqlfluff` tool which is already configured during the `+postgres-base` target. To apply and fix some formatting issues you can run `+format` target which will picks up directory -where your Earthly file lies in as a source dir for formatting and run `+FORMAT` UDC target. -Under the hood `+FORMAT` UDC target runs `sqlfluff-image` docker image, +where your Earthly file lies in as a source dir for formatting and run `+FORMAT` Function. +Under the hood `+FORMAT` Function runs `sqlfluff-image` docker image, which contains the same configuration and setup which is applied during the `+check`. @@ -118,7 +118,7 @@ With the `*.sql` files validation out of the way, we can finally build our Postg Since we need migration and seed data files, we'll inherit from the `builder` target. The actual image build process is pretty straight-forward -and fully defined under the `+BUILD` UDC target. +and fully defined under the `+BUILD` Function. The only thing it is needed to specify is a few arguments: * `tag` - the tag of the image, default value `latest`. diff --git a/docs/src/guides/languages/python.md b/docs/src/guides/languages/python.md index a36d2ee24..b6ae6e6a5 100644 --- a/docs/src/guides/languages/python.md +++ b/docs/src/guides/languages/python.md @@ -62,7 +62,7 @@ so it is mandatory to have `pyproject.toml` and `poetry.lock` files in the root The fist step of the `builder` target is prepare a Python environment with poetry via `+python-base` target. Next step is to copy source code of the project and finally finalize the build -with some poetry project setup which is done with `+BUILDER` UDC target. +with some poetry project setup which is done with `+BUILDER` Function. ### Running checks diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index c2f81932f..f43f85197 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -69,8 +69,8 @@ The fist step of the `builder` target is to prepare a Rust environment via `+rus Next step is to copy source code of the project. Note that you need to copy only needed files for Rust build process, any other irrelevant stuff should omitted. -And finally finalize the build with `+SETUP` UDC target. -The `+SETUP` UDC target requires `rust-toolchain.toml` file, +And finally finalize the build with `+SETUP` Function. +The `+SETUP` Function requires `rust-toolchain.toml` file, with the specified `channel` option in it. This `rust-toolchain.toml` file could be specified via the `toolchain` argument of the `+SETUP` target like this @@ -113,8 +113,8 @@ check: With prepared environment and all data, we're now ready to start operating with the source code and configuration files. The `hosted-check` target which actually performs all checks and validation -with the help of `+CHECK` UDC target. -The `+CHECK` UDC target performs static checks of the Rust project as +with the help of `+CHECK` Function. +The `+CHECK` Function performs static checks of the Rust project as `cargo fmt`, `cargo machete`, `cargo deny` which will validate formatting, find unused dependencies and any supply chain issues with dependencies. Here is the list of steps (look at `./earthly/rust/scripts/std_checks.sh`): @@ -181,8 +181,8 @@ Obviously it inherits `builder` target environment and than performs build of th Important to note that in this particular example we are dealing with the executable Rust project, so it produces binary as a final artifact. Another case of the building Rust library we will consider later. -Actual build process is done with `+BUILD` UDC target. -The `+BUILD` UDC have few arguments `libs` and `bins`, +Actual build process is done with `+BUILD` Function. +The `+BUILD` Function have few arguments `libs` and `bins`, they should be specified to properly generate `cargo-modules` docs (see description below). The `libs` argument takes a list of library crate's names in your Rust project, e.g. `--libs="crate1 crate2"`. @@ -209,11 +209,11 @@ Generated artifacts are `doc/workspace.dot`, `doc/full.dot`, `doc/all.dot` files Generated artifacts are `doc/$crate.$bin.bin.modules.tree`, `doc/$crate.$bin.bin.modules.dot` for the specified `--bins="crate1/bin1"` argument and `target/doc/$crate.lib.modules.tree`, `target/doc/$crate.lib.modules.dot` -for the specified `--libs="crate1"` argument of the `+BUILD` UDC. +for the specified `--libs="crate1"` argument of the `+BUILD` Function. Next steps is mandatory if you are going to produce a binary as an artifact, for Rust libraries the are not mandatory and could be omitted. -The `+SMOKE_TEST` UDC target checks that produced binary with the specified name (`--bin` argument) +The `+SMOKE_TEST` Function checks that produced binary with the specified name (`--bin` argument) is executable, isn't a busted mess. Final step is to provide desired artifacts: docs and binary. diff --git a/docs/src/guides/markdown.md b/docs/src/guides/markdown.md index 148f33b35..d9819a906 100644 --- a/docs/src/guides/markdown.md +++ b/docs/src/guides/markdown.md @@ -4,14 +4,14 @@ icon: simple/markdown # Markdown Check -This Earthly Target and UDC enables uniform linting of Markdown files to maintain consistency and quality. +This Earthly Target and Function enables uniform linting of Markdown files to maintain consistency and quality. -This UDC is **NOT** intended to be used inside container builds. +This Function is **NOT** intended to be used inside container builds. Its sole purpose is to enforce uniform style rules for all markdown files in a repository. It makes no assumptions about which files may or may not end up inside a container or are part of a build. This is *INTENTIONAL*. -IF this UDC is used inside a container build, it is **NOT** a bug if it does not do the correct thing. +IF this Function is used inside a container build, it is **NOT** a bug if it does not do the correct thing. ## Introduction @@ -83,7 +83,7 @@ markdown-check-fix: DO github.com/input-output-hk/catalyst-ci/earthly/mdlint:+MDLINT_LOCALLY --src=$(echo ${PWD}) --fix=--fix ``` -In this use case, the UDC is run Locally, so that the markdown in the repo can be directly checked. +In this use case, the Function is run Locally, so that the markdown in the repo can be directly checked. !!! Note @@ -105,4 +105,4 @@ For more example, please refer to this [doc](https://github.com/DavidAnson/markd mdlint-cli2 is integrated into VSCode and may be integrated into other Editors. The editor integration should pick up both the `.markdownlint.jsonc` and `.markdownlint-cli2.jsonc` configuration files. -It will then behave exactly the same as the Earthly UDC. +It will then behave exactly the same as the Earthly Function. diff --git a/docs/src/guides/spellcheck.md b/docs/src/guides/spellcheck.md index 053356e1a..02e30f501 100644 --- a/docs/src/guides/spellcheck.md +++ b/docs/src/guides/spellcheck.md @@ -142,4 +142,4 @@ For these files/paths, exclude them from the spell check by adding their filenam `cspell` is integrated into VSCode and may be integrated into other Editors. -The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly UDC. +The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly Function. diff --git a/docs/src/reference/function.md b/docs/src/reference/function.md new file mode 100644 index 000000000..cb954c567 --- /dev/null +++ b/docs/src/reference/function.md @@ -0,0 +1,38 @@ +# Functions + +## Overview + +The Catalyst CI repository provides a number of Earthly +[Functions](https://docs.earthly.dev/docs/guides/functions). +You can think of a Function as a reusable snippet of Earthly code that serves the same purpose as functions in a common programming +language. +Functions are helpful for several reasons: + +1. They keep Earthfiles DRY +2. They can encapsulate complex logic into a simple contractual interface +3. They enforce standardization and prevent solving the same problem in multiple different ways + +The third reason is particularly useful for Catalyst as we have multiple repositories with dozens of Earthfiles often solving +similar problems. + +## Usage + +You are highly encouraged to review the currently available Functions in the +[Catalyst CI repository](https://github.com/input-output-hk/catalyst-ci/tree/master/earthly). +The folder structure is broken out by language/technology and should be relatively easy to navigate. +You can incorporate these Functions into your Earthfiles by using something like below: + +```Earthfile +DO github.com/input-output-hk/catalyst-ci/earthly/+ --arg1=value1 +``` + +Replacing `folder` and `Function_NAME` respectively. +The passing of an argument is optional, as some Functions do not require any input arguments. + +## Contributing + +Please feel encouraged to contribute Functions to the repository. +If you're seeing the same logic being re-used across multiple Earthfiles in a +repository, this is a good candidate for refactoring into a Function. +Additionally, for SMEs who are aware of language best practices, encoding those +into a Function will help increase the overall health of the CI process. diff --git a/docs/src/reference/udc.md b/docs/src/reference/udc.md deleted file mode 100644 index 8dfc85249..000000000 --- a/docs/src/reference/udc.md +++ /dev/null @@ -1,37 +0,0 @@ -# UDCs - -## Overview - -The Catalyst CI repository provides a number of Earthly [User Defined Commands](https://docs.earthly.dev/docs/guides/udc) (UDCs). -You can think of a UDC as a reusable snippet of Earthly code that serves the same purpose as functions in a common programming -language. -UDCs are helpful for several reasons: - -1. They keep Earthfiles DRY -2. They can encapsulate complex logic into a simple contractual interface -3. They enforce standardization and prevent solving the same problem in multiple different ways - -The third reason is particularly useful for Catalyst as we have multiple repositories with dozens of Earthfiles often solving -similar problems. - -## Usage - -You are highly encouraged to review the currently available UDCs in the -[Catalyst CI repository](https://github.com/input-output-hk/catalyst-ci/tree/master/earthly). -The folder structure is broken out by language/technology and should be relatively easy to navigate. -You can incorporate these UDCs into your Earthfiles by using something like below: - -```Earthfile -DO github.com/input-output-hk/catalyst-ci/earthly/+ --arg1=value1 -``` - -Replacing `folder` and `UDC_NAME` respectively. -The passing of an argument is optional, as some UDCs do not require any input arguments. - -## Contributing - -Please feel encouraged to contribute UDCs to the repository. -If you're seeing the same logic being re-used across multiple Earthfiles in a repository, this is a good candidate for refactoring -into a UDC. -Additionally, for SMEs who are aware of language best practices, encoding those into a UDC will help increase the overall health of -the CI process. diff --git a/docs/src/style/index.md b/docs/src/style/index.md index 5d55acc97..59725b1e0 100644 --- a/docs/src/style/index.md +++ b/docs/src/style/index.md @@ -12,6 +12,13 @@ In most circumstances, the standards provided by this style guide should *not* b If an exception must me made, the rationale should be included in the respective PR. Any `Earthfile` which does not adhere to this style guide will be rejected if no further justification is made. +### Earthly User Defined Commands (Functions) deprecated starting from Earthly 0.8 + + +!!! Warning +Please be advised that Earthly's File Descriptor Cache (FDC) is deprecated, effective upon updating to Earthly 0.8. + + ## Organization ### Adhere to a consistent structure @@ -156,13 +163,14 @@ This can be through inheriting from it or by directly copying artifacts. The main point is that a subproject should not have multiple builds scattered in various places. Each subproject has an authoritative `build` target that *all* targets use when fetching build artifacts. -### Prefer User Defined Commands (UDCs) +### Prefer FUNCTION -The primary purpose of a UDC is to reduce boilerplate and promote reusing common workflows. +The primary purpose of a Function is to reduce boilerplate and promote reusing common workflows. Many build patterns tend to be repetitive. For example, copying a package lockfile and installing dependencies is very common. - -In these cases, a UDC should be preferred. -The `catalyst-ci` repository provides a number of UDCs in the `earthly` subdirectory. +In these cases, a FUNCTION should be preferred. +The catalyst-ci repository provides a number of FUNCTIONs in the earthly subdirectory. These should be used prior to writing a new one. If a common use case is not covered in this subdirectory, a PR should be opened to add it. +The use of functions in Earthly contributes to a more modular and organized build system, +enhancing code readability and maintainability. diff --git a/earthly/bash/Earthfile b/earthly/bash/Earthfile index b1d09a400..93617567f 100644 --- a/earthly/bash/Earthfile +++ b/earthly/bash/Earthfile @@ -1,5 +1,5 @@ # cspell UDCs and Containers. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # Internal: builder creates a container we can use to execute shellcheck builder: @@ -22,7 +22,7 @@ builder: # shellcheck - Check all shell files recursively in the src with shellcheck. SHELLCHECK: - COMMAND + FUNCTION ARG --required src FROM +builder @@ -30,3 +30,4 @@ SHELLCHECK: COPY --dir $src /src RUN ./check-all.sh /src + diff --git a/earthly/cspell/Earthfile b/earthly/cspell/Earthfile index 5b9932233..8ef1fb488 100644 --- a/earthly/cspell/Earthfile +++ b/earthly/cspell/Earthfile @@ -1,10 +1,10 @@ # cspell UDCs and Containers. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 CHECK: # Spell checking all docs and code is done with cspell # See: cspell.org - COMMAND + FUNCTION # Where we want to run the `lint` from. Use `.` to check the whole repo. ARG src=. @@ -17,4 +17,4 @@ CHECK: COPY $src . - RUN cspell-cli lint . --dot + RUN cspell-cli lint . --dot \ No newline at end of file diff --git a/earthly/docs/Earthfile b/earthly/docs/Earthfile index e26dd20f1..74b8a16e0 100644 --- a/earthly/docs/Earthfile +++ b/earthly/docs/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words libgcc freetype lcms openjpeg etag @@ -64,7 +64,7 @@ common: # Common src setup SRC: - COMMAND + FUNCTION FROM +deps @@ -81,7 +81,7 @@ SRC: # Build the docs - We always do this in a `docs` target. # The only target that needs customizing is the `src` target. BUILD: - COMMAND + FUNCTION RUN /scripts/build.sh @@ -95,7 +95,7 @@ BUILD: # We do however build a docs package which is used for locally serving the docs # which assists in documentation writing. PACKAGE: - COMMAND + FUNCTION # Use the official Nginx base image FROM nginx:alpine3.18-slim @@ -121,4 +121,4 @@ PACKAGE: # And then save the image # SAVE IMAGE image_name - \ No newline at end of file + diff --git a/earthly/go/Earthfile b/earthly/go/Earthfile index e459ae29e..df5c8e5e2 100644 --- a/earthly/go/Earthfile +++ b/earthly/go/Earthfile @@ -1,7 +1,7 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 DEPS: - COMMAND + FUNCTION ARG ginkgo="true" @@ -24,7 +24,7 @@ DEPS: SAVE ARTIFACT go.sum AS LOCAL go.sum FMT: - COMMAND + FUNCTION # latest linux/amd64 FROM cytopia/gofmt@sha256:ae76abdbc96780819ce175f7ba4bf1fec98e9f708eb680d0b949d1737a959983 @@ -39,7 +39,7 @@ FMT: RUN gofmt -l . | grep . && exit 1 || exit 0 LINT: - COMMAND + FUNCTION # v1.54-alpine FROM golangci/golangci-lint@sha256:a8423cab177c2863b863ee6d9f00c58f101cadd6120e195b2bb59fdfa189c1ae @@ -55,6 +55,8 @@ LINT: ENV GOLANGCI_LINT_CACHE=/cache/golangci-lint ENV GOCACHE=/cache/go-build + RUN go get ./... + RUN golangci-lint run \ --timeout 5m \ -E errchkjson \ @@ -64,4 +66,4 @@ LINT: -E gosec \ -E nilnil \ -E revive \ - -E testpackage \ No newline at end of file + -E testpackage diff --git a/earthly/mdlint/Earthfile b/earthly/mdlint/Earthfile index a41b5a877..edb0555f4 100644 --- a/earthly/mdlint/Earthfile +++ b/earthly/mdlint/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words markdownlint MDLINT_LOCALLY: @@ -14,7 +14,7 @@ MDLINT_LOCALLY: # in the `next` branch of `markdownlint`. # Container is built by: # https://github.com/DavidAnson/markdownlint-cli2/blob/next/docker/Dockerfile-rules - COMMAND + FUNCTION # Directory we need to run lint checks against. ARG --required src @@ -53,7 +53,7 @@ CHECK: # in the `next` branch of `markdownlint`. # Container is built by: # https://github.com/DavidAnson/markdownlint-cli2/blob/next/docker/Dockerfile-rules - COMMAND + FUNCTION ARG src = . @@ -84,4 +84,4 @@ mdlint-test: ARG src=$(echo ${PWD}/../../) - DO +MDLINT_LOCALLY --src=${src} + RUN +MDLINT_LOCALLY --src=${src} diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index 0a0e3647f..ea494a6e4 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,5 +1,5 @@ # Common PostgreSQL Earthly builders -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words colordiff nushell psycopg dbviz @@ -59,7 +59,7 @@ postgres-base: # seed - OPTIONAL - Location of Seed data directory - DEFAULT: ./seed # refinery_toml - OPTIONAL - Location of refinery,toml which configures migrations. DEFAULT: ./refinery.toml BUILDER: - COMMAND + FUNCTION ARG migrations=./migrations ARG seed=./seed @@ -88,7 +88,7 @@ BUILDER: # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILD --image_name= # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+DOCS DOCS: - COMMAND + FUNCTION ARG diagrams=./diagrams.json ARG migrations=./migrations @@ -109,7 +109,7 @@ DOCS: # Linter checks for sql files CHECK: - COMMAND + FUNCTION RUN /scripts/std_checks.sh @@ -118,7 +118,7 @@ CHECK: # Can not do it LOCALLY as the command will change ownership of the fixed files # if run within docker. FORMAT: - COMMAND + FUNCTION # Can not run format LOCALLY as it changes permissions of files to root:root RUN --no-cache printf "%s\n%s\n%s\n" \ @@ -143,7 +143,7 @@ integration-test: INTEGRATION_TEST_SETUP: - COMMAND + FUNCTION FROM +integration-test @@ -156,7 +156,7 @@ INTEGRATION_TEST_SETUP: # Arguments: # * image_name: The name of the image (required). BUILD: - COMMAND + FUNCTION ARG --required image_name USER postgres:postgres diff --git a/earthly/python/Earthfile b/earthly/python/Earthfile index 2b078886c..dbe99a9a0 100644 --- a/earthly/python/Earthfile +++ b/earthly/python/Earthfile @@ -1,5 +1,5 @@ # Common Python UDCs and Builders. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words libgcc @@ -33,7 +33,7 @@ python-base: # Parameters: # opts : Options passed to the `poetry install` command. BUILDER: - COMMAND + FUNCTION ARG opts # Copy our dependencies. diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index ae702cb41..2164b8ea9 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,5 +1,5 @@ # Common Rust UDCs and Builders. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset readelf depgraph lcov # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT @@ -96,7 +96,7 @@ rust-base-all-hosts: # Parameters: # * toolchain : The `rust-toolchain` toml file. SETUP: - COMMAND + FUNCTION ARG toolchain=./rust-toolchain.toml # Copy our toolchain dependency. @@ -116,7 +116,7 @@ SETUP: # Steps we do during the `check` CI phase for all Rust programs CHECK: - COMMAND + FUNCTION # This is set up so that ALL checks are run and it will fail if any fail. # This improves visibility into all issues that need to be corrected for `check` @@ -128,7 +128,7 @@ CHECK: # * libs : The list of lib crates `cargo-modules` docs to build. # * bins : The list of binaries `cargo-modules` docs to build. BUILD: - COMMAND + FUNCTION ARG libs="" ARG bins="" @@ -136,7 +136,7 @@ BUILD: # Run unit tests and generates test and coverage report artifacts UNIT_TEST: - COMMAND + FUNCTION # remove artifacts that may affect the coverage results RUN cargo llvm-cov clean --workspace # run unit tests and display test result and test coverage @@ -146,7 +146,7 @@ UNIT_TEST: # Check if the build executable, isn't a busted mess. SMOKE_TEST: - COMMAND + FUNCTION ARG --required bin RUN ldd target/$TARGETARCH/release/$bin diff --git a/examples/go/Earthfile b/examples/go/Earthfile index 74a9062e3..8e102e6d9 100644 --- a/examples/go/Earthfile +++ b/examples/go/Earthfile @@ -1,6 +1,6 @@ # WARNING: If you modify this file, please update the guide that it is dependent # on in docs/guides/languages/go.md. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # The structure of this Earthfile is derived from the style guide: # https://input-output-hk.github.io/catalyst-ci/style/#adhere-to-a-consistent-structure @@ -73,4 +73,4 @@ publish-example: COPY +build/hello . # Use the cached artifact from the build target. ENTRYPOINT ["/app/hello"] - SAVE IMAGE hello:${tag} \ No newline at end of file + SAVE IMAGE hello:${tag} diff --git a/examples/postgresql/Earthfile b/examples/postgresql/Earthfile index b3aac3253..de8830be3 100644 --- a/examples/postgresql/Earthfile +++ b/examples/postgresql/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words psql @@ -43,7 +43,7 @@ all-tests: # Internal Function to run an specific integration test script. INTEGRATION_TEST_RUN: - COMMAND + FUNCTION ARG seed_data ARG test_script diff --git a/utilities/scripts/Earthfile b/utilities/scripts/Earthfile index b9b32b5db..a52567a91 100644 --- a/utilities/scripts/Earthfile +++ b/utilities/scripts/Earthfile @@ -1,5 +1,5 @@ # Earthfile containing common scripts for easy reference -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # Internal: bash-scripts : Common bash scripts. bash-scripts: @@ -19,12 +19,12 @@ python-scripts: # UDC to add our common bash scripts to a container image. ADD_BASH_SCRIPTS: - COMMAND + FUNCTION COPY --dir +bash-scripts/include /scripts/include # UDC to add our common bash scripts to a container image. ADD_PYTHON_SCRIPTS: - COMMAND + FUNCTION - COPY --dir +python-scripts/python /scripts/python \ No newline at end of file + COPY --dir +python-scripts/python /scripts/python \ No newline at end of file