From 5f63578b248895c26240634fc14dd6a378c63bce Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 17:48:18 -0500 Subject: [PATCH 01/11] ci: cargo deny on push/pull requests --- .github/workflows/audit.yml | 7 ------- .github/workflows/deny.yml | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/deny.yml diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index fc4a413d..f1a024db 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -19,10 +19,3 @@ jobs: - uses: rustsec/audit-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - cargo-deny: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: EmbarkStudios/cargo-deny-action@v2 - with: - command: check advisories bans licenses sources diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml new file mode 100644 index 00000000..24ad229f --- /dev/null +++ b/.github/workflows/deny.yml @@ -0,0 +1,19 @@ +name: CI +on: [push, pull_request] +jobs: + cargo-deny: + runs-on: ubuntu-22.04 + strategy: + matrix: + checks: + - advisories + - bans licenses sources + + # Prevent sudden announcement of a new advisory from failing ci: + continue-on-error: ${{ matrix.checks == 'advisories' }} + + steps: + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check ${{ matrix.checks }} From 6bd1d740c6c364730a4b319c07c6585ddda56a7f Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 18:10:48 -0500 Subject: [PATCH 02/11] ci: update action version --- .github/workflows/deny.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 24ad229f..593f955c 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -13,7 +13,7 @@ jobs: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 + - uses: actions/checkout@v4.2.2 + - uses: EmbarkStudios/cargo-deny-action@v2.0.4 with: command: check ${{ matrix.checks }} From f4afb426e945489be8296df7d3913cf7527881bc Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 18:18:08 -0500 Subject: [PATCH 03/11] ci: exclude addchain as it's an old crate --- .github/workflows/deny.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 593f955c..df4c5e41 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -17,3 +17,4 @@ jobs: - uses: EmbarkStudios/cargo-deny-action@v2.0.4 with: command: check ${{ matrix.checks }} + arguments: --exclude addchain From 7ee32f17fae0fe95972599b5e6d4d8e2c64f336b Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 18:31:55 -0500 Subject: [PATCH 04/11] build: add deny.toml --- deny.toml | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 deny.toml diff --git a/deny.toml b/deny.toml new file mode 100644 index 00000000..c4ba5216 --- /dev/null +++ b/deny.toml @@ -0,0 +1,126 @@ +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# Version of the advisory config. See https://github.com/EmbarkStudios/cargo-deny/pull/611 +version = 2 +# The path where the advisory database is cloned/fetched into +db-path = "~/cargo/advisory-db" +# The url of the advisory database to use +db-urls = ["https://github.com/rustsec/advisory-db"] +# The lint level for crates that have been yanked from their source registry +yanked = "warn" +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", +] +# Threshold for security vulnerabilities, any vulnerability with a CVSS score +# lower than the range specified will be ignored. Note that ignored advisories +# will still output a note when they are encountered. +# * None - CVSS Score 0.0 +# * Low - CVSS Score 0.1 - 3.9 +# * Medium - CVSS Score 4.0 - 6.9 +# * High - CVSS Score 7.0 - 8.9 +# * Critical - CVSS Score 9.0 - 10.0 +#severity-threshold = + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# Version of the license config. See https://github.com/EmbarkStudios/cargo-deny/pull/611 +version = 2 +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "Apache-2.0 WITH LLVM-exception", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "MPL-2.0", + "Unlicense", + "Unicode-3.0", + "Zlib", +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # OpenSSL is Apache License v2.0 (ASL v2) + # https://www.openssl.org/blog/blog/2017/03/22/license/ + # ring crate is ISC & MIT + { allow = ["ISC", "MIT", "OpenSSL"], name = "ring" }, + # libp2p is not re-distributing unicode tables data by itself + { allow = ["MIT", "Apache-2.0", "Unicode-DFS-2016"], name = "unicode-ident" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +[[licenses.clarify]] +name = "ring" +expression = "ISC AND MIT AND OpenSSL" +license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +#registries = [ +#] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# List of crates that are allowed. Use with care! +#allow = [ +#] +# List of crates to deny +#deny = [ +#] +# Certain crates/versions that will be skipped when doing duplicate detection. +#skip = [ +#] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite +#skip-tree = [ +#] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "deny" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "deny" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +#allow-git = [ +#] From a8cd208e68d00c083ceac13c728f83cfd7c9c6a2 Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 18:36:26 -0500 Subject: [PATCH 05/11] ci: make an exception for anemo --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index c4ba5216..7901aa5b 100644 --- a/deny.toml +++ b/deny.toml @@ -46,6 +46,7 @@ allow = [ "Unicode-3.0", "Zlib", ] +allow-git = ["https://github.com/mystenlabs/anemo.git"] # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the # canonical license text of a valid SPDX license file. From 048773292265f6dcf20bf62be07d388faf09aec9 Mon Sep 17 00:00:00 2001 From: chad Date: Mon, 27 Jan 2025 18:41:39 -0500 Subject: [PATCH 06/11] build: add more crates to allow --- deny.toml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/deny.toml b/deny.toml index 7901aa5b..48d62810 100644 --- a/deny.toml +++ b/deny.toml @@ -46,7 +46,6 @@ allow = [ "Unicode-3.0", "Zlib", ] -allow-git = ["https://github.com/mystenlabs/anemo.git"] # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the # canonical license text of a valid SPDX license file. @@ -55,12 +54,16 @@ confidence-threshold = 0.8 # Allow 1 or more licenses on a per-crate basis, so that particular licenses # aren't accepted for every possible crate as with the normal allow list exceptions = [ - # OpenSSL is Apache License v2.0 (ASL v2) - # https://www.openssl.org/blog/blog/2017/03/22/license/ - # ring crate is ISC & MIT - { allow = ["ISC", "MIT", "OpenSSL"], name = "ring" }, - # libp2p is not re-distributing unicode tables data by itself - { allow = ["MIT", "Apache-2.0", "Unicode-DFS-2016"], name = "unicode-ident" }, + + { allow = [ + "MIT", + ], name = "addchain" }, + { allow = [ + "MIT", + ], name = "anemo" }, + { allow = [ + "MIT", + ], name = "addr2line" }, ] # Some crates don't have (easily) machine readable licensing information, From 380a80e84e3935f9b24922349cf210bcd4d05133 Mon Sep 17 00:00:00 2001 From: chad Date: Tue, 28 Jan 2025 21:01:35 -0500 Subject: [PATCH 07/11] deps: update dependencies + deny.toml for Sui crates --- Cargo.lock | 2 +- Cargo.toml | 2 +- atoma-bin/atoma_node.rs | 5 ++--- atoma-service/Cargo.toml | 2 +- deny.toml | 24 +++++++++++++++++++++--- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 985791b9..7ece6ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -793,7 +793,7 @@ dependencies = [ "base64 0.22.1", "clap 4.5.27", "config", - "dotenv", + "dotenvy", "flume", "futures", "hex", diff --git a/Cargo.toml b/Cargo.toml index dc797579..d21c921a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ blake2 = "0.10.6" clap = "4.5.4" config = "0.14.0" dcap-rs = { git = "https://github.com/automata-network/dcap-rs.git" } -dotenv = "0.15.0" +dotenvy = "0.15.7" fastcrypto = "0.1.8" flume = "0.11.1" futures = "0.3.30" diff --git a/atoma-bin/atoma_node.rs b/atoma-bin/atoma_node.rs index f40c9913..1b2707d4 100644 --- a/atoma-bin/atoma_node.rs +++ b/atoma-bin/atoma_node.rs @@ -16,7 +16,6 @@ use atoma_state::{config::AtomaStateManagerConfig, AtomaState, AtomaStateManager use atoma_sui::{client::AtomaSuiClient, AtomaSuiConfig, SuiEventSubscriber}; use atoma_utils::spawn_with_shutdown; use clap::Parser; -use dotenv::dotenv; use futures::future::try_join_all; use hf_hub::{api::sync::ApiBuilder, Repo, RepoType}; use sui_keys::keystore::FileBasedKeystore; @@ -127,7 +126,7 @@ impl Config { /// async fn example() -> Result<()> { /// let models = vec!["facebook/opt-125m".to_string()]; /// let revisions = vec!["main".to_string()]; -/// +/// /// let tokenizers = initialize_tokenizers(&models, &revisions).await?; /// Ok(()) /// } @@ -177,7 +176,7 @@ async fn initialize_tokenizers( async fn main() -> Result<()> { let _log_guards = setup_logging(LOGS).context("Failed to setup logging")?; - dotenv().ok(); + dotenvy::dotenv().ok(); let args = Args::parse(); let config = Config::load(&args.config_path).await?; diff --git a/atoma-service/Cargo.toml b/atoma-service/Cargo.toml index 2106fad6..1dbd5666 100644 --- a/atoma-service/Cargo.toml +++ b/atoma-service/Cargo.toml @@ -19,7 +19,7 @@ axum = { workspace = true } base64 = { workspace = true } clap = { workspace = true } config = { workspace = true } -dotenv = { workspace = true } +dotenvy = { workspace = true } flume = { workspace = true } futures = { workspace = true } hex = { workspace = true } diff --git a/deny.toml b/deny.toml index 48d62810..0bd955b6 100644 --- a/deny.toml +++ b/deny.toml @@ -13,7 +13,19 @@ yanked = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - #"RUSTSEC-0000-0000", + # These are all related to the sui SDK, which is outside of our control + # difference 2.0.0 is unmaintained + "RUSTSEC-2020-0095", + # rusoto is unmaintained, use aws crates + "RUSTSEC-2022-0071", + # we don't do RSA signing on Sui (only verifying for zklogin) + "RUSTSEC-2023-0071", + # A few dependencies use unpatched rustls. + "RUSTSEC-2024-0336", + # allow yaml-rust being unmaintained + "RUSTSEC-2024-0320", + # allow unmaintained proc-macro-error used in transitive dependencies + "RUSTSEC-2024-0370", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories @@ -126,5 +138,11 @@ unknown-git = "deny" # if not specified. If it is specified but empty, no registries are allowed. allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories -#allow-git = [ -#] +# Allow the mysten labs repositories for Sui +allow-git = [ + "https://github.com/asonnino/prometheus-parser", + "https://github.com/zhiburt/tabled", +] + +[sources.allow-org] +github = ["mystenmark", "bmwill", "mystenlabs", "nextest-rs"] From 982e2f7c21db59c3515919d08f4bd5256e539c10 Mon Sep 17 00:00:00 2001 From: chad Date: Tue, 28 Jan 2025 21:02:51 -0500 Subject: [PATCH 08/11] build: update lockfile --- Cargo.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ece6ec5..0f23a43a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2401,12 +2401,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - [[package]] name = "dotenvy" version = "0.15.7" From f58811eec7344285c62bf5e61524746a1ef9799e Mon Sep 17 00:00:00 2001 From: chad Date: Tue, 28 Jan 2025 21:07:30 -0500 Subject: [PATCH 09/11] chore: other license updates --- deny.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index 0bd955b6..397df886 100644 --- a/deny.toml +++ b/deny.toml @@ -14,6 +14,8 @@ yanked = "warn" # output a note when they are encountered. ignore = [ # These are all related to the sui SDK, which is outside of our control + # derivative is not maintained + "RUSTSEC-2024-0388", # difference 2.0.0 is unmaintained "RUSTSEC-2020-0095", # rusoto is unmaintained, use aws crates @@ -54,6 +56,7 @@ allow = [ "ISC", "MIT", "MPL-2.0", + "CC0-1.0", "Unlicense", "Unicode-3.0", "Zlib", @@ -145,4 +148,4 @@ allow-git = [ ] [sources.allow-org] -github = ["mystenmark", "bmwill", "mystenlabs", "nextest-rs"] +github = ["mystenmark", "bmwill", "mystenlabs", "nextest-rs", "rustls"] From abd583de476c0e931af1194b43a2af171ab8b4f4 Mon Sep 17 00:00:00 2001 From: chad Date: Tue, 28 Jan 2025 21:10:02 -0500 Subject: [PATCH 10/11] chore: added more license updates --- deny.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deny.toml b/deny.toml index 397df886..c8db0e0d 100644 --- a/deny.toml +++ b/deny.toml @@ -56,6 +56,8 @@ allow = [ "ISC", "MIT", "MPL-2.0", + "BSL-1.0", + "OpenSSL", "CC0-1.0", "Unlicense", "Unicode-3.0", From 4e70133c0bfd9a9fa80e6d6326dcc8f9833f85ca Mon Sep 17 00:00:00 2001 From: chad Date: Tue, 28 Jan 2025 21:12:42 -0500 Subject: [PATCH 11/11] chore: remove unused advisories --- deny.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deny.toml b/deny.toml index c8db0e0d..f8a2c135 100644 --- a/deny.toml +++ b/deny.toml @@ -18,12 +18,8 @@ ignore = [ "RUSTSEC-2024-0388", # difference 2.0.0 is unmaintained "RUSTSEC-2020-0095", - # rusoto is unmaintained, use aws crates - "RUSTSEC-2022-0071", # we don't do RSA signing on Sui (only verifying for zklogin) "RUSTSEC-2023-0071", - # A few dependencies use unpatched rustls. - "RUSTSEC-2024-0336", # allow yaml-rust being unmaintained "RUSTSEC-2024-0320", # allow unmaintained proc-macro-error used in transitive dependencies