From cb0046561bcef906bb533893e919e564990974a7 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Thu, 13 Jul 2023 09:13:32 -0700 Subject: [PATCH] A smattering of commits to get the build / deploy flow working again (#1323) * Reconfigure cloudflare worker * Fix code coverage api * Fix graphviz rendering issues * Update mermaid / puppeteer build process * fix: disable MinerConstructorParams * Commenting it out still generates the error * Ignore errors due to lack of rusl-fil-proofs cov * Remove hint with a broken link * Revert to CommonJS * Update jsdom version to avoid build error * Disable minification The build breaks when minified, but works fine without it. This shoud be re-enabled later, nut for now I just want to get this site building again. * Add .nvmrc * Remove puppeteer config * Check generated mermaid files into src These files are very hard to generate via CI because they require puppeteer, but easy to generate locally. Eventually this should be re-enabled in CI * Modify build image to be node:20.2 * Update .fleek.json * Update .nvmrc * revert node image, git bin required at ci --------- Co-authored-by: Chris Waring <106938+cwaring@users.noreply.github.com> --- .fleek.json | 2 +- .gitignore | 3 +- .nvmrc | 1 + api/index.js | 42 +- api/package.json | 3 +- api/wrangler.toml | 8 +- config.toml | 6 +- .../algorithms/expected_consensus/_index.md | 4 - .../storage_power_actor.md | 2 - package.json | 10 +- .../diagrams/double_fork.svg | 72 ++ .../diagrams/parent_grinding.svg | 89 ++ .../diagrams/time_offset.svg | 60 ++ .../intro/diagrams/buildsys/buildsys.svg | 257 +++++ .../intro/diagrams/orient/filecoin.svg | 906 ++++++++++++++++++ .../intro/diagrams/overview1/overview.svg | 730 ++++++++++++++ .../intro/diagrams/overview2/overview2.svg | 1 + .../diagrams/sequence/full-deals-on-chain.svg | 1 + .../_gen/diagrams/intro/new-state-diagram.svg | 1 + .../chainsync/chainsync_fsm.svg | 150 +++ .../data_transfer/alternate-pull-flow.svg | 1 + .../data_transfer/pull-flow.svg | 1 + .../data_transfer/push-flow.svg | 1 + .../diagrams/deal-flow.svg | 1 + .../diagrams/deal-payment.svg | 1 + .../retrieval_market/retrieval_flow_v0.svg | 1 + .../retrieval_market/retrieval_flow_v1.svg | 1 + .../storage_market/storage_market_flow.svg | 1 + .../diagrams/miner_post_state_machine.svg | 62 ++ .../miner_post_state_machine_legend.svg | 51 + .../storage_mining/diagrams/mining_cycle.svg | 123 +++ .../diagrams/sector_chain_fsm.svg | 225 +++++ .../storage_mining/diagrams/sector_fsm.svg | 335 +++++++ .../diagrams/sector_offchain_fsm.svg | 157 +++ .../diagrams/sector_state_fsm.svg | 140 +++ .../diagrams/sector_state_legend.svg | 81 ++ .../diagrams/sector_state_machine.svg | 82 ++ .../diagrams/sector_state_machine_legend.svg | 69 ++ tools/diagrams.js | 24 +- tools/pptr.config | 4 - 40 files changed, 3661 insertions(+), 48 deletions(-) create mode 100644 .nvmrc create mode 100644 static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg create mode 100644 static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg create mode 100644 static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg create mode 100644 static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg create mode 100644 static/_gen/diagrams/intro/diagrams/orient/filecoin.svg create mode 100644 static/_gen/diagrams/intro/diagrams/overview1/overview.svg create mode 100644 static/_gen/diagrams/intro/diagrams/overview2/overview2.svg create mode 100644 static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg create mode 100644 static/_gen/diagrams/intro/new-state-diagram.svg create mode 100644 static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg create mode 100644 static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg create mode 100644 static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg create mode 100644 static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg create mode 100644 static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg create mode 100644 static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg create mode 100644 static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg create mode 100644 static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg create mode 100644 static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg create mode 100644 static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg delete mode 100644 tools/pptr.config diff --git a/.fleek.json b/.fleek.json index 0dae6993b..5dadc440c 100644 --- a/.fleek.json +++ b/.fleek.json @@ -1,6 +1,6 @@ { "build": { - "image": "hugomrdias/puppeteer", + "image": "node:20.2", "command": "npm install && npm run build", "publicDir": "public" } diff --git a/.gitignore b/.gitignore index 57dea231d..b82708cff 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ yarn.lock package-lock.json node_modules resources -static/_gen data/toc.json api/dist api/worker +.hugo_build.lock +assets/jsconfig.json diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..6ed5da955 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v20.2.0 diff --git a/api/index.js b/api/index.js index 4fb6e7538..a7868699e 100644 --- a/api/index.js +++ b/api/index.js @@ -31,21 +31,41 @@ async function handleRequest(event) { async function cov(event) { const url = new URL(event.request.url) // https://github.com/filecoin-project/lotus - const repo = url.searchParams.get('repo').split('/').slice(3).join('/') - const data = await get(event, { - url: `https://codecov.io/api/gh/${repo}`, + const [owner, repo] = url.searchParams.get('repo').split('/').slice(3) + const headers = { + 'User-Agent': 'ianconsolata', + Accept: 'application/json', + Authorization: `Bearer ${CODECOV_TOKEN}`, + } + const repo_resp = await get(event, { + url: `https://api.codecov.io/api/v2/github/${owner}/repos/${repo}/`, transform: (data) => { const out = { - cov: dlv(data, 'commit.totals.c', 0), - ci: dlv(data, 'commit.ci_passed', false), - repo: dlv(data, 'repo.name', 'N/A'), - org: dlv(data, 'owner.username', 'N/A'), - lang: dlv(data, 'repo.language', 'N/A'), + branch: dlv(data, 'branch', 'master'), + lang: dlv(data, 'language', 'N/A'), } return out }, + headers, }) - return data + const repo_data = await repo_resp.json() + + const cov_data = await get(event, { + url: `https://api.codecov.io/api/v2/github/${owner}/repos/${repo}/branches/${repo_data.branch}/`, + transform: (data) => { + const out = { + cov: dlv(data, 'head_commit.totals.coverage', 0), + ci: dlv(data, 'head_commit.ci_passed', false), + repo: repo, + org: owner, + lang: repo_data.lang, + branch: repo_data.branch, + } + return out + }, + headers, + }) + return cov_data } async function github(event) { @@ -56,7 +76,7 @@ async function github(event) { const path = file.slice(7).join('/') const ref = file[6] const headers = { - 'User-Agent': 'hugomrdias', + 'User-Agent': 'ianconsolata', Authorization: `token ${GITHUB_TOKEN}`, } @@ -82,7 +102,7 @@ async function github(event) { async function releases(event) { const headers = { - 'User-Agent': 'hugomrdias', + 'User-Agent': 'ianconsolata', Authorization: `token ${GITHUB_TOKEN}`, } const rsp = await get(event, { diff --git a/api/package.json b/api/package.json index 9d1ae7c58..89fa46913 100644 --- a/api/package.json +++ b/api/package.json @@ -12,6 +12,7 @@ "dlv": "^1.1.3", "merge-options": "^3.0.3", "nanoid": "^3.1.16", - "p-map": "^4.0.0" + "p-map": "^4.0.0", + "wrangler": "^2.17.0" } } diff --git a/api/wrangler.toml b/api/wrangler.toml index 96b98f576..ffbd9ff42 100644 --- a/api/wrangler.toml +++ b/api/wrangler.toml @@ -1,7 +1,5 @@ name = "specs-api" -type = "webpack" -account_id = "fffa4b4363a7e5250af8357087263b3a" +account_id = "e85b6fdda17f8c464c343dc84d1373e8" workers_dev = true -route = "" -zone_id = "" -webpack_config = "webpack.config.js" +main = "index.js" +compatibility_date = "2023-04-27" diff --git a/config.toml b/config.toml index 18de27b79..297d7771c 100644 --- a/config.toml +++ b/config.toml @@ -5,6 +5,10 @@ canonifyurls = false theme = "book" # relativeURLs = true +# allow building even if getting remote JSON fails +# needed because rust-fil-proofs has no coverage +ignoreErrors = ["error-remote-getjson"] + # Book configuration disablePathToLower = true enableGitInfo = true @@ -41,7 +45,7 @@ enableGitInfo = true # weight = 10 [params] - API = 'https://specs-api.protocol-labs.workers.dev' + API = 'https://specs-api.filecoin-foundation.workers.dev' # (Optional, default true) Controls table of contents visibility on right side of pages. # Start and end levels can be controlled with markup.tableOfContents setting. # You can also specify this parameter per page in front matter. diff --git a/content/algorithms/expected_consensus/_index.md b/content/algorithms/expected_consensus/_index.md index 3d8d39263..8597b0792 100644 --- a/content/algorithms/expected_consensus/_index.md +++ b/content/algorithms/expected_consensus/_index.md @@ -341,10 +341,6 @@ The exact value for these parameters remain to be determined, but for testing pu - `wFunction = log2b` with - `log2b(X) = floor(log2(x)) = (binary length of X) - 1` and `log2b(0) = 0`. Note that that special case should never be used (given it would mean an empty power table). -{{< hint warning >}} -**Note that if your implementation does not allow for rounding to the fourth decimal**, miners should apply the [tie-breaker below](selecting-between-tipsets-with-equal-weight). Weight changes will be on the order of single digit numbers on expectation, so this should not have an outsized impact on chain consensus across implementations. -{{< /hint >}} - `ParentWeight` is the aggregate chain weight of a given block's parent set. It is calculated as the `ParentWeight` of any of its parent blocks (all blocks in a given Tipset should have the same `ParentWeight` value) plus the delta weight of each parent. To make the diff --git a/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md b/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md index a050d25a3..d15b17ebc 100644 --- a/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md +++ b/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md @@ -16,8 +16,6 @@ dashboardTests: 0 {{}} -{{}} - ## The Power Table The portion of blocks a given miner generates through leader election in EC (and so the block rewards they earn) is proportional to their `Quality-Adjusted Power Fraction` over time. That is, a miner whose quality adjusted power represents 1% of total quality adjusted power on the network should mine 1% of blocks on expectation. diff --git a/package.json b/package.json index 228da67cf..6999df601 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "serve": "npm start", "watch-hugo": "npm run build-diagrams && hugo server --bind=0.0.0.0 --disableFastRender --renderToDisk", "watch-assets": "tools/watch.js --watch", - "build": "npm run build-diagrams && hugo && npm run build-toc && hugo --gc --minify", + "build": "hugo && npm run build-toc && hugo --gc", "build-diagrams": "tools/diagrams.js", "build-toc": "tools/toc.js", "clean": "premove public resources static/_gen", @@ -27,16 +27,16 @@ "tablesort": "^5.2.1" }, "devDependencies": { - "@mermaid-js/mermaid-cli": "^8.8.1", + "@mermaid-js/mermaid-cli": "^10.2.2", "chokidar": "^3.4.3", "concurrently": "^5.3.0", "conventional-github-releaser": "^3.1.5", "execa": "^4.0.3", "globby": "^11.0.1", - "graphviz-cli": "^1.0.0", - "hugo-extended": "^0.76.5", + "graphviz-cli": "^2.0.0", + "hugo-extended": "^0.113.0", "husky": ">=4", - "jsdom": "^16.4.0", + "jsdom": "^22.1.0", "lint-staged": ">=10", "np": "^6.5.0", "premove": "^3.0.1", diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg new file mode 100644 index 000000000..8aa99723c --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg @@ -0,0 +1,72 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerAny + + + +B3->B1 + + + + + +B4 + +B4 +MinerE + + + +B4->B2 + + + + + +B5 + +B5 +MinerE + + + +B5->B3 + + + + + diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg new file mode 100644 index 000000000..a4d8772a3 --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg @@ -0,0 +1,89 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerE + + + +B3->B1 + + + + + +B4 + +B4 +MinerE + + + +B4Null + + + + +B4->B4Null + + + + + +B5 + +B5 +MinerAny + + + +B5->B2 + + + + + +B5->B3 + + + + + +B4Null->B2 + + + + + diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg new file mode 100644 index 000000000..c1206f094 --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg @@ -0,0 +1,60 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerE + + + +B3->B2 + + + + + +B4 + +B4 +MinerE + + + +B4->B2 + + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg b/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg new file mode 100644 index 000000000..f33e900e8 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg @@ -0,0 +1,257 @@ + + + + + + +G + + + +.md + +.md + + + +make pdf +make pdf + + + +.md->make pdf + + + + + +hugo build + +hugo build + + + +.md->hugo build + + + + + +.ipld + +.ipld + + + +bin/codeGen +bin/codeGen + + + +.ipld->bin/codeGen + + + + + +.go + +.go + + + +go test +go test + + + +.go->go test + + + + + +.go->hugo build + + + + + +.orient + +.orient + + + +orient +orient + + + +.orient->orient + + + + + +.org + +.org + + + +<emacs> + +<emacs> + + + +.org-><emacs> + + + + + +.dot + +.dot + + + +dot +dot + + + +.dot->dot + + + + + +[website] + +[website] + + + +spec.pdf + +spec.pdf + + + +emacs +emacs + + + +.svg + +.svg + + + +dot->.svg + + + + + +orient->.dot + + + + + +orient-><emacs> + + + + + +.html + +.html + + + +orient->.html + + + + + +bin/codeGen->.go + + + + + +hugo +hugo + + + +[ok] + +[ok] + + + +go test->[ok] + + + + + +make website +make website + + + +make website->[website] + + + + + +make pdf->spec.pdf + + + + + +<emacs>->.md + + + + + +hugo build->.html + + + + + +.html->make website + + + + + +.svg->make website + + + + + +.svg->make pdf + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg b/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg new file mode 100644 index 000000000..5db7eeac4 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg @@ -0,0 +1,906 @@ + + + + + + + + + +69 + +ACTORS-MESSAGES-FRACTION + + + +68 + +ACTORS-MESSAGES-PER-BLOCK + + + +68->69 + + + + + +62 + +MESSAGES + + + +62->69 + + + + + +62->68 + + + + + +66 + +TX-MESSAGES-PER-BLOCK + + + +62->66 + + + + + +64 + +RECEIPTS + + + +62->64 + + + + + +63 + +MESSAGES-SIZE + + + +62->63 + + + + + +67 + +TX-MESSAGES-FRACTION + + + +67->66 + + + + + +66->62 + + + + + +65 + +RECEIPTS-SIZE + + + +64->65 + + + + + +61 + +PROOF-MESSAGES-FRACTION + + + +61->62 + + + + + +60 + +AVG-PROOFS-MESSAGES-PER-BLOCK + + + +60->62 + + + + + +59 + +AVG-POSTS-MESSAGES-PER-BLOCK + + + +59->60 + + + + + +58 + +AVG-SEALS-MESSAGES-PER-BLOCK + + + +58->60 + + + + + +56 + +ALL-POST-MESSAGES-PER-YEAR + + + +56->59 + + + + + +57 + +MINERS + + + +57->56 + + + + + +55 + +ALL-SEAL-SIZE-PER-YEAR + + + +54 + +ALL-SEAL-MESSAGES-PER-YEAR + + + +54->58 + + + + + +54->55 + + + + + +53 + +SECTORS-COUNT + + + +53->56 + + + + + +53->54 + + + + + +52 + +STORAGE-NETWORK-CAPACITY + + + +52->53 + + + + + +48 + +PROVING-PERIOD-SECONDS + + + +51 + +POSTS-PER-SECTOR-PER-YEAR + + + +48->51 + + + + + +51->56 + + + + + +50 + +SEALS-PER-SECTOR-PER-YEAR + + + +50->54 + + + + + +49 + +RESEAL + + + +49->50 + + + + + +47 + +PROVING-PERIOD-HOURS + + + +47->48 + + + + + +46 + +MAX-TICKETS + + + +45 + +ONE-BLOCK-IN-TEN-YEARS + + + +45->46 + + + + + +44 + +BLOCKS-IN-TEN-YEARS + + + +44->45 + + + + + +41 + +BLOCKS-IN-A-YEAR + + + +41->59 + + + + + +41->58 + + + + + +41->44 + + + + + +43 + +EXPECTED-WINNING-MINERS + + + +43->41 + + + + + +42 + +YEAR-IN-SECONDS + + + +42->51 + + + + + +42->41 + + + + + +40 + +TICKETS + + + +39 + +AVG-TICKETS + + + +39->40 + + + + + +36 + +POST-CHALLENGE-TIME + + + +38 + +POST-CHALLENGE-BLOCKS + + + +36->38 + + + + + +37 + +BLOCK-TIME + + + +37->41 + + + + + +37->38 + + + + + +35 + +POST-CHALLENGE-HOURS + + + +35->36 + + + + + +33 + +ENCODING-TIME + + + +34 + +KDF-HASH-SIZE + + + +34->33 + + + + + +32 + +KDF-CONTENT + + + +32->33 + + + + + +31 + +NODES + + + +30 + +NODE-SIZE + + + +30->33 + + + + + +30->31 + + + + + +29 + +SECTOR-SIZE-GIB + + + +28 + +SECTOR-SIZE + + + +29->28 + + + + + +28->53 + + + + + +28->31 + + + + + +27 + +DEGREE-EXPANDER + + + +26 + +DEGREE + + + +27->26 + + + + + +26->32 + + + + + +25 + +DEGREE-BASE + + + +25->26 + + + + + +23 + +MESSAGE-RECEIPT + + + +23->65 + + + + + +24 + +GAS-USED + + + +24->23 + + + + + +22 + +RETURN + + + +22->23 + + + + + +21 + +EXIT-CODE + + + +21->23 + + + + + +20 + +MESSAGE-RECEIPTS-CID + + + +12 + +MESSAGE-SIZE + + + +12->63 + + + + + +19 + +ACTOR-METHOD + + + +19->12 + + + + + +18 + +GAS-LIMIT + + + +18->12 + + + + + +17 + +GAS-PRICE + + + +17->12 + + + + + +16 + +VALUE + + + +16->12 + + + + + +15 + +U64 + + + +15->24 + + + + + +15->19 + + + + + +15->18 + + + + + +15->17 + + + + + +15->16 + + + + + +14 + +MESSAGE-NONCE + + + +14->12 + + + + + +13 + +VARINT + + + +13->22 + + + + + +13->21 + + + + + +13->14 + + + + + +11 + +FROM-ADDRESS + + + +11->12 + + + + + +10 + +TO-ADDRESS + + + +10->12 + + + + + +9 + +MESSAGES-ROOT-CID + + + +8 + +ADDRESS-SIZE + + + +8->11 + + + + + +8->10 + + + + + +7 + +CID-SIZE + + + +7->20 + + + + + +7->9 + + + + + +7->8 + + + + + +6 + +EIX + + + +6->52 + + + + + +5 + +PIB + + + +5->6 + + + + + +4 + +TIB + + + +4->5 + + + + + +3 + +GIB + + + +3->28 + + + + + +3->4 + + + + + +2 + +MIB + + + +2->3 + + + + + +1 + +KIB + + + +1->2 + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/overview1/overview.svg b/static/_gen/diagrams/intro/diagrams/overview1/overview.svg new file mode 100644 index 000000000..f5e2703da --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/overview1/overview.svg @@ -0,0 +1,730 @@ + + + + + + +Filecoin + + +cluster_blockchain_subgroup + + + +cluster_blockchain + +Blockchain + + +cluster_vm + +Filecoin VM + + +cluster_messagepool + +Message Pool + + +cluster_storagepowerconsensus + +Storage Power Consensus + + +cluster_markets + + + +cluster_storagemarket + +Storage Market + + +cluster_retrievalmarket + +Retrieval Market + + +cluster_filesdata + + + +cluster_pieces + +Data Transfers + + +cluster_files + +Local File Storage + + +cluster_storageminerprocess + + + +cluster_storagemining + +Storage Mining + + +cluster_sectorindexer + +Sector Indexer + + +cluster_filproofs_subsystem + +Storage Proving + + +cluster_systems + + + +cluster_repository + +Repository + + +cluster_filproofs_lib + +Filecoin Proofs + + +cluster_payments + +Payments + + +cluster_net + + + +cluster_clock + +Clock + + +cluster_network + +Network Interface + + +cluster_ipfs + +IPFS + + +cluster_libp2p + +libp2p + + + +Chain + +Chain + + + +Block Producer + +Block Producer + + + + +VM Interpreter + +VM Interpreter + + + +Chain->VM Interpreter + + + + + +IPLD DagStore + +IPLD DagStore + + + +Chain->IPLD DagStore + + + + + +Message Storage + +Message Storage + + + +Block Producer->Message Storage + + + + + +Block Syncer + +Block Syncer + + + +Gossipsub + + + + + +Gossipsub + + + +Block Syncer->Gossipsub + + + + + +Messages + +Messages + + + + +Init Actor + + +Init +Actor + + + +Account Actor + + +Account +Actor + + + + +Cron Actor + + +Cron +Actor + + + +Message Syncer + +Message Syncer + + + + +Filecoin Network Node + +Filecoin Network Node + + + + +Message Syncer->Gossipsub + + + + + +Expected Consensus + +Expected Consensus + + + +Storage Power Actor + + +Storage +Power +Actor + + + + +Storage Provider + +Storage +Provider + + + +Storage Provider->Chain + + + + + +Data Transfers + +Data Transfers + + + +Storage Provider->Data Transfers + + + + + +Storage Client + +Storage +Client + + + +Storage Deal Protocol + + + + + +Storage +Deal +Protocol + + + +Storage Market Actor + + +Storage +Market +Actor + + + + + + +Retrieval Provider + +Retrieval +Provider + + + +Retrieval Provider->Chain + + + + + +Retrieval Provider->Data Transfers + + + + + +Retrieval Client + +Retrieval +Client + + + +Retrieval Deal Protocol + + + + + +Retrieval +Deal +Protocol + + + +PieceStore + +PieceStore + + + +Data Transfers->PieceStore + + + + + +Data Transfers Scheduler + +Data Transfers Scheduler + + + +AuthSwap + + + + + +data +transfer +protocol +(unnamed) + + + +Graphsync + + + + + +Graphsync + + + +AuthSwap->Graphsync + + + + + +FileStore + +FileStore + + + +Disk2 + + +Data Storage System +(Disks, etc) + + + +FileStore->Disk2 + + + + + +PieceStore->FileStore + + + + + +Mining Scheduler + +Mining Scheduler + + + +Sector Builder + +Sector Builder + + + +Mining Scheduler->Sector Builder + + + + + +Pledge Prover + +Pledge Prover + + + +Mining Scheduler->Pledge Prover + + + + + +Storage Miner Actor + + +Storage +Miner +Actor + + + +Sector Store + +Sector Store + + + +Sector Store->PieceStore + + + + + +Seal Prover + +Seal Prover + + + +PoSt Generator + +PoSt Generator + + + +PoSt Generator->Chain + + + + + +Pledge Prover->Storage Power Actor + + + + + +Stacked-DRG-Porep + +Stacked-DRG-Porep + + + +Pledge Prover->Stacked-DRG-Porep + + + + + +Config + +Config + + + +Local Storage + + +Local Storage +(Disk) + + + +Config->Local Storage + + + + + +KeyStore + +KeyStore + + + +KeyStore->Local Storage + + + + + +IPLD DagStore->Local Storage + + + + + +Rational PoSt + +Rational PoSt + + + + +Piece Inclusion Proofs + +Piece +Inclusion +Proofs + + + +VRFs + +VRFs + + + +VDFs + +VDFs + + + + + +Wallet + +Wallet + + + +Wallet->KeyStore + + + + + +Payment Channel Actor + + +Payment +Channel +Actor + + + +Wall Clock + +Wall Clock + + + +NTP + +NTP + + + +Wall Clock->NTP + + + + + +Switch + +Switch + + + +NTP->Switch + + + + + + +p2p.Node + +p2p.Node + + + +Graphsync->p2p.Node + + + + + +Bitswap + + + + + +Bitswap + + + +Bitswap->p2p.Node + + + + + +p2p.Node->Switch + + + + + +Switch->KeyStore + + + + + +Conn + +Conn + + + +Switch->Conn + + + + + +Identify + + + + + +Identify + + + +Identify->p2p.Node + + + + + +Gossipsub->p2p.Node + + + + + +Bootstrap List + + + + + +Bootstrap +List + + + +Bootstrap List->p2p.Node + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg b/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg new file mode 100644 index 000000000..64cc727a8 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg @@ -0,0 +1 @@ +
libraries
FilecoinNode
FilesAndData
multiformats
libp2p
ipld
ipfs
libp2pCore
libp2pProtocols
Repository
Pos0
Bitswap
Graphsync
UnixFS
DagImports
ipld.CID
ipld.Selector
ipld.GraphStore
Gossipsub
BootstrapList
p2p.Node
p2p.PeerID
p2p.PeerInfo
multihash
multiaddr
Pos1
P2PNode
Clock
FileStore
Config
IpldStore
KeyStore
Pos2
File
Piece
DataTransfer
 \ No newline at end of file diff --git a/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg b/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg new file mode 100644 index 000000000..8a2d634a3 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg @@ -0,0 +1 @@ +RetrievalClientRetrievalProviderStorageMarketParticipant1StorageMarketParticipant2StorageMarketActorPaymentChannelActorPaymentSubsystemBlockchainSubsystemBlockSyncerBlockProducerStoragePowerConsensusSubsystemStoragePowerActorStorageMiningSubsystemStorageMinerActorSectorIndexerSubsystemStorageProvingSubsystemFilecoinProofsSubsystemClockSubsystemlibp2pStorageMarketParticipantSectorSealerWallClockBlockchainMessagePoolMiningSchedulerStorageMinerSubsystemRetrievalMarketSubsystemStorageMarketSubsystemBlockchainGroupMiningGroupopt[RetrievalMarket]opt[RegisterStorageMiner]opt[StorageDealMake]opt[AddingDealToSector]opt[ClientQuery]alt[1 - success][0 - failure]opt[CommitSector]opt[SealingSector]opt[Round Cutoff]loop[BlockReception]epoch is incremented by 1alt[New Tipset][Null block last round]Using tickets derived in failed election proof in last epochalt[New Tipset][Null block last round]alt[1 - success][0 - failure]opt[SubmitPoSt]opt[1- success]loop[forEach StorageMiningSubsystem.StorageMiner]loop[BlockProduction]opt[Expired deals]Schedule and resume PoStsSchedule and resume SEALsMaintain FaultSetMaintain DoneSetopt[MiningScheduler]PoSt challenge if a node has not mined a block in a ProvingPeriodalt[1 - success][0 - failure]opt[SubmitPoSt]opt[ClockSubsystem]SectorStateSets := (FaultSet, RecoverSet, ExpireSet)opt[Declaration before receiving a challenge]FaultSet is all sectors if no post submittedsectors Faulted longer than threshold proving periods are destroyedopt[if miner ProvingPeriod ends]loop[forEach StorageMinerActor in StoragePowerActor.Miners]loop[EveryBlock]opt[Storage Fault]opt[Valid Fault]opt[Consensus Fault]NewRetrievalQuery(RetreivalQuery)RetrievalQueryResponseNewRetrievalDealProposal(RetrievalDealProposal)AcceptRetrievalDealProposal(RetrievalDealPropsal)NewPaymentChannelCreateMiner(ownerPubKey PubKey, workerPubKey PubKey, pledgeAmt TokenAmount)CreateStorageMiner(OwnerAddr, WorkerPubKey)StorageMinerActorRegisterParticipant(TokenAmount)AddBalance(TokenAmount)WithdrawBalance(TokenAmount)CheckLockedBalance(Address)TokenAmountNewStorageDealProposal(StorageDealProposal)VerifyBalance(StorageMarketParticipant1)SignStorageDealProposal(StorageDealProposal)NewStorageDeal(StorageDeal)PublishStorageDeal(StorageDeal)PublishStorageDeal(StorageDeal)HandleStorageDeal(Deal)AddDealToSector(Deal, SectorID)IndexSectorByDealExpiration(SectorID, Deal)(SectorID, Deal)NotifyStorageDealStaged(Deal,SectorID)QueryStorageDealProposalStatus(StorageDealProposalQuery)StorageDealProposalQueryResponseQueryStorageDealStatus(StorageDealQuery)StorageDealQueryResponseGetSealSeed(Chain, Epoch)SeedSealSector(Seed, SectorID, ReplicaCfg)Seal(Seed, SectorID, ReplicaCfg)SealOutputsSealOutputsCommitSector(Seed, SectorID, SealCommitment, SealProof, [&Deal], [Deal])VerifySeal(SectorID, OnSectorInfo){1,0}AddDeal(SectorID, [&Deal], DealStatusOnChain)AddDeal(SectorID, [Deal], DealStatusPending)IncrementPower(StorageMiner.WorkerPubKey)CommitSectorErrorSubscribe(OnNewBlock)Event(OnNewBlock, block)ValidateBlockSyntax(block)HandleBlock(block)ValidateBlock(block)ValidateBlock(block)ValidateBlock(block)StateTree ← TryGenerateStateTree(block)AssembleTipsets()[Tipset] ← AssembleTipsets()Tipset ← ChooseTipset([Tipset])ApplyStateTree(StateTree)OnNewTipset(Chain, Epoch)OnNewRound()GetElectionArtifacts(Chain, Epoch)TK ← TicketAtEpoch(Chain, Epoch-k)T1 ← TicketAtEpoch(Chain, Epoch-1)TK, T1EP ← DrawElectionProof(TK.randomness(), StorageMiner.WorkerKey)T0 ← GenerateNextTicket(T1.randomness(), StorageMiner.WorkerKey)T1 ← GenerateNextTicket(T0.randomness(), StorageMiner.WorkerKey)TryLeaderElection(EP){1, 0}GetPoStChallenge(Chain, Epoch)challengeGeneratePoSt(challenge, [SectorID])PoStProofSubmitPoSt(PoStProof, DoneSet)VerifyPoSt(PoStProof){1,0}UpdateDoneSet()HandleStorageDealPayment()CloseExpiredStorageDeal()PoStErrorSlashStorageCollateral()GenerateBlock(EP, T0, Tipset, StorageMiner.Address)GetMostProfitableMessages(StorageMiner.Address)[Message]block ← AssembleBlock([Message], Tipset, EP, T0, StorageMiner.Address)PropagateBlock(block)OnNewTipset(Chain, Epoch)[SectorID] ← LookupSectorByDealExpiry(Epoch)PurgeSectorsWithNoLiveDeals([SectorID])HandleExpiredDeals([SectorID])CloseExpiredStorageDeal(StorageDeal)GetPoStChallenge(Chain, Epoch)challengeGeneratePoSt(challenge, [SectorID])PoStProofSubmitPoSt(PoStProof, DoneSet)VerifyPoSt(PoStProof){1,0}UpdateDoneSet()HandleStorageDealPayment()CloseExpiredStorageDeal()PoStErrorSlashStorageCollateral()UpdateSectorStatus([FaultSet], SectorStateSets)RecomputeMinerPower()SlashStorageCollateral()ProvingPeriodUpdate()computeProvingPeriodEndSectorState()UpdateSectorStatus(newSectorState)RecomputeMinerPower()HandleFailedDeals([newSectorState.DestroyedSet])DeclareConsensusFault(ConsensusFaultProof)ValidateFault(ConsensusFaultProof)TerminateMiner(Address)SlashPledgeCollateral(Address)UpdateBalance(Reward)RetrievalClientRetrievalProviderStorageMarketParticipant1StorageMarketParticipant2StorageMarketActorPaymentChannelActorPaymentSubsystemBlockchainSubsystemBlockSyncerBlockProducerStoragePowerConsensusSubsystemStoragePowerActorStorageMiningSubsystemStorageMinerActorSectorIndexerSubsystemStorageProvingSubsystemFilecoinProofsSubsystemClockSubsystemlibp2pStorageMarketParticipantSectorSealerWallClockBlockchainMessagePoolMiningSchedulerStorageMinerSubsystem \ No newline at end of file diff --git a/static/_gen/diagrams/intro/new-state-diagram.svg b/static/_gen/diagrams/intro/new-state-diagram.svg new file mode 100644 index 000000000..1c810ae48 --- /dev/null +++ b/static/_gen/diagrams/intro/new-state-diagram.svg @@ -0,0 +1 @@ +
PreCommitSectors
CommitSectors
CronPreCommitExpiry (PCD)
SubmittedWindowPoSt
DeclareFault
SubmitWindowPoSt (SP)
ProvingDeadline (SP)
TerminateSectors
(TF)
SubmittedWindowPoSt (FF)
ProvingDeadline (FF)
DeclareFaultRecovered
EarlyExpiration (TF)
TerminateSectors (TF)
SubmittedWindowPoSt (FF)
DeclareFault
ProvingDeadline (SP)
TerminateSectors (TF)
SubmittedWindowPoSt
DeclareFault
SubmitWindowPoSt (SP)
ProvingDeadline (SP)
CronExpiration
TerminateSectors (TF)
CompactSectors
Null
Precommitted
Committed
Deleted
Active
Faulty
Terminated
Recovering
 \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg b/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg new file mode 100644 index 000000000..740918629 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg @@ -0,0 +1,150 @@ + + + + + + +sector + + + +INIT + +INIT + + + +init2boot +init2boot + + + +INIT->init2boot + + + + +follow2init +follow2init + + + +INIT->follow2init + + + + + +BOOTSTRAP + +BOOTSTRAP + + + +boot2check +boot2check + + + +BOOTSTRAP->boot2check + + + + +boot2catch +boot2catch + + + +BOOTSTRAP->boot2catch + + + + +SYNC_CHECKPOINT + +SYNC_CHECKPOINT + + + +check2catch +check2catch + + + +SYNC_CHECKPOINT->check2catch + + + + +CHAIN_CATCHUP + +CHAIN_CATCHUP + + + +catch2follow +catch2follow + + + +CHAIN_CATCHUP->catch2follow + + + + +follow2catch +follow2catch + + + +CHAIN_CATCHUP->follow2catch + + + + + +CHAIN_FOLLOW + +CHAIN_FOLLOW + + + +init2boot->BOOTSTRAP + + + + + +boot2check->SYNC_CHECKPOINT + + + + + +boot2catch->CHAIN_CATCHUP + + + + + +check2catch->CHAIN_CATCHUP + + + + + +catch2follow->CHAIN_FOLLOW + + + + + +follow2catch->CHAIN_FOLLOW + + + + diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg new file mode 100644 index 000000000..6214020d4 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne System. Likely A ClientOne System. Likely A MinerInitiate PullSchedule TransferMake Graphsync Request w/ Data Transfer Request Piggy BackedSend Graphsync Request (w/ Data Transfer Request)Verify Request (validate & schedule)Validate Pull RequestPull Request validatedSchedule TransferSend response w/ DTR Accepted Piggy BackedSend response w/ DTR Accepted Piggy BackedResponse Progress (to end) (include DT Accepted)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg new file mode 100644 index 000000000..2934c535d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne System. Likely A ClientOne System. Likely A MinerInitiate PullSend Data Transfer RequestValidate Pull RequestPull Request validatedSchedule TransferSend Data Transfer Request AcceptedSchedule TransferMake Graphsync RequestSend Graphsync RequestVerify Transfer ScheduledRequest is scheduledSend ResponseResponse Progress (to end)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg new file mode 100644 index 000000000..5eed429d2 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne system. Likely A ClientOne system. Likely A MinerInitiate PushSchedule TransferSend Data Transfer RequestValidate Push RequestPush Request validatedSchedule TransferMake Graphsync RequestSend Graphsync RequestVerify Transfer ScheduledRequest is scheduledSend ResponseResponse Progress (to end)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg new file mode 100644 index 000000000..11f710f2c --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg @@ -0,0 +1 @@ +StorageClientStorageProviderStorageMiningSubsystemStorageMarketActorDataTransferStorageMinerActorCronActorAddBalance()AddBalance()OnStorageDealProposal(DealRequest)DealResponseOpenPushTransferRequest(provider, voucher, payload, selector)ValidatePush(client, voucher, payload, selector)acceptOnDataTransferEvent(DataTransferComplete, state)verifyStorageDealProposal(proposal)verifyPieceCID(proposal, payloadCID)signStorageDealProposal(proposal)DealResponse (updated)publishStorageDealMessage(message)PublishStorageDeal(message)generateStorageDealID()lockFund()dealIDDealResponse (updated)HandleStorageDeal(deal)CommitSector()DealResponse (updated)NotifyOfPoStChallenge()SubmitPoSt()activateSectorNotifyOfOnChainDealStatus(DealActive)DealResponse (updated)NotifyOfPoStChallenge()/CheckSurprisePoStSubmissionHappened()NotifyOfOnChainDealStatus(DealExpired/DealFailing)DealResponse (updated)StorageClientStorageProviderStorageMiningSubsystemStorageMarketActorDataTransferStorageMinerActorCronActor \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg new file mode 100644 index 000000000..fd660086d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg @@ -0,0 +1 @@ +StorageMiningSubsystemStorageMinerActorStorageMarketActorPublishStorageDeals_validateNewStorageDeal_assertValidClienSignature_assertValidDealTimingAtPublish_assertValidDealMinimum_assertSufficientBalanceAvailableForDeal_lockFundsForStorageDeal_generateStorageDealIDDealIDsPreCommitSectorVerifyPublishedDealIDs(dealIDs)_assertPublishedDealState_assertDealStartAfterCurrEpoch_assertDealExpireAfterMaxProveCommitWindowSuccessProveCommitSectorActivateDeals(dealIDs)_assertPublishedDealState_assertDealNotYetExpired_activateDeal(publishedDeal)Deals_initializeUtilizationInfo_onSuccessfulPoStProcessDealPayment_assertActiveDealState_getStorageFeeSinceLastPayment_transferBalance_submitPowerReport (if any deal expires)ProcessDealExpiration_assertActiveDealState_getStorageFeeSinceLastPayment_transferBalance_unlockCollateral_submitFaultReport(if TerminatedFault)ProcessDealSlash_slashTerminatedFaults_assertActiveDealState_terminateDealWithdrawBalancefundStorageMiningSubsystemStorageMinerActorStorageMarketActor \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg new file mode 100644 index 000000000..58c322d5b --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg @@ -0,0 +1 @@ +RetrievalClientNodeRetrievalProviderBlockStore (Client)BlockStore (Provider)Storage Miner Subsystemopt[Discovery]opt[Create Payment Channel]opt[Unsealing]loop[Sending Data until Payment Required]loop[Retrieve In Pieces]Query Chain For Providers w/ PieceIDProviders With PieceIDSend Query for PieceSend information about Piece (price, size, etc)Send RetrievalDealProposalSend RetrievalDealResponse, acceptingCreate Payment ChannelMessage For Channel Gets On ChainUnseal SectorUnsealed Piece DataWrite Piece Blocks to BlockStoreBlocks writtenRead Blocks from BlockStoreBlocksSend BlocksVerify BlocksWrite BlocksBlocks writtenRequest PaymentCreate Voucher On Payment ChannelMessage For Voucher Gets On ChainSend Reference To Payment VoucherRedeem Voucher On Payment ChannelMessage For Voucher Redeemed On ChainRetrievalClientNodeRetrievalProviderBlockStore (Client)BlockStore (Provider)Storage Miner Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg new file mode 100644 index 000000000..14955ac62 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg @@ -0,0 +1 @@ +RetrievalClientData Transfer SubsystemRetrievalProviderNodeBlockStore (Client)BlockStore (Provider)Storage Miner Subsystemopt[Discovery]opt[Create Payment Channel]opt[Unsealing]loop[Sending Data until Payment Required]loop[Retrieve In Pieces]Query Chain For Providers w/ PieceIDProviders With PieceIDSend Query for PieceSend information about Piece (price, size, etc)Open Pull Data Transfer w/ RetrievalDealProposal voucherValidate RetrievalDealProposalData Transfer AcceptedUpdate Transfer In ProgressPauseCreate Payment ChannelMessage For Channel Gets On ChainUnpausePauseUnseal SectorUnsealed Piece DataWrite Piece Blocks to BlockStoreBlocks writtenUnpauseRead Blocks from BlockStoreBlocksHandle Send+Verify Of BlocksWrite BlocksBlocks writtenPauseSend Request Payment Intermediate VoucherValidate Request Payment VoucherCreate Voucher On Payment ChannelMessage For Voucher Gets On ChainSend Reference To Payment VoucherValidate Reference To Payment VoucherRedeem Voucher On Payment ChannelMessage For Voucher Redeemed On ChainUnpauseUpdate Transfer CompleteRetrievalClientData Transfer SubsystemRetrievalProviderNodeBlockStore (Client)BlockStore (Provider)Storage Miner Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg b/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg new file mode 100644 index 000000000..73532418e --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg @@ -0,0 +1 @@ +StorageClientData TransferStorageProviderNodeStorageMarketActorStorage Mining SubsystemDiscoveryDiscovering Minersloop[Each Miner Actor]Querying For Asksloop[each StorageProvider]NegotiationProposing a dealVerifying a dealTransferring DataEnsuring CollateralPublishingPutting Deal On ChainVerify Deal PublishedHandoffStorageMiner Handoff And SealingStorage Miner adds piece to sector precommits, seals sector, activates deal See Storage Mining Subsystem for more detailList Miner Actors from chain stateList of Miner ActorsQuery actor properties from chain stateMiner actor properties from chain stateSend Query For Current AskCurrent Storage AskAdd funds for a deal as neccesaryFunds addedCalculate piece commitmentPropose a dealVerify deal parameters (price, size, etc)Check StorageClient balanceClient balanceVerify balance greater that total storage priceIndicate Intent To Accept DealOpen push request to clientSend push request to providerAccept and request dataSend request for dataSend DataFinished Sending DataNotify data transfer completeVerify data matches piece commitmentAdd collateral for deal as neededCollateral addedSign Deal ProposalPublish Deal On ChainDeal Accepted (w/ Publish Message CID)Deal PublishedVerify Publish Message Appears On ChainDeal PublishedAdd PieceStorageClientData TransferStorageProviderNodeStorageMarketActorStorage Mining Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg new file mode 100644 index 000000000..94309cd54 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg @@ -0,0 +1,62 @@ + + + + + + +miner_post + + +cluster_0 + +Lose power on all sectors + + + +OK + + +OK + + + +Challenged + +Challenged + + + +OK->Challenged + + + + + +Challenged->OK + + + + + +DetectedFault + +DetectedFault + + + +Challenged->DetectedFault + + + + + + +DetectedFault->OK + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg new file mode 100644 index 000000000..ac79bd504 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg @@ -0,0 +1,51 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg new file mode 100644 index 000000000..fd4b00e4d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg @@ -0,0 +1,123 @@ + + + + + + +mining_cycle + + +cluster_local + +Storage Miner +(off chain) + + +cluster_local_mining + +Mining + + +cluster_chain + +Storage Miner Actor +(on chain) + + + +ActiveL + +ActiveL + + + +ChallengedL + +ChallengedL + + + + +ActiveL->ChallengedL + + +Chain issues challenge +(implicit) + + + +ActiveC + +ActiveC + + + +ActiveL->ActiveC + + +Add Sectors +(sm.CommitSector) + + + +ProducePost +Produce & submit PoSt +(sm.SubmitPost) + + + +ChallengedL->ProducePost + + + + +OwnerAccountC + +OwnerAccountC + + + + +OwnerAccountC->ActiveC + + +Create Miner +(spa.CreateMiner) + + + +ActiveC->ActiveL + + +Miner Created + + + +OwnerAccL +OwnerAccL + + + +OwnerAccL->ActiveC + + +Create Miner +(spa.CreateMiner) + + + +ProducePost->ActiveL + + + + + +ProducePost->ActiveC + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg new file mode 100644 index 000000000..15eccb32c --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg @@ -0,0 +1,225 @@ + + + + + + +sector + + +cluster_chain + +Sector States in StorageMinerActor +(on chain) + + +cluster_postproving + +Proving Set + + +cluster_nopowerC1 + +Does not count for power + + +cluster_nopowerC2 + +Does not count for power + + +cluster_activate + + + +cluster_powerC + +Counts for power + + +cluster_nopowerC2 + +Does not count for power + + +cluster_expire_by_post + + + + +CommittedC + +Committed + + + +FirstPoSt + +1st PoSt +(sm.SubmitPost) +ActivateSectors + + + +CommittedC->FirstPoSt + + + + +ActiveC + +Active + + + + + +ExpireByPost + +Expiration Reached +(sma.CronTick +-> sm.ExpireSectors) + + + +CommittedC->ExpireByPost + + + + +FaultsDeclared + +Fault Declared +(sm.DeclareFaults) + + + +CommittedC->FaultsDeclared + + + + +FaultSpotted + +Fault Spotted +(ca.CronTick +-> sm.PostFailed) + + + +CommittedC->FaultSpotted + + + + +RecoveringC + +Recovering + + + +RecoveringC->FirstPoSt + + + + +RecoveringC->ExpireByPost + + + + +RecoveringC->FaultsDeclared + + + + +RecoveringC->FaultSpotted + + + + +FirstPoSt->ActiveC + + + + + +ActiveC->ExpireByPost + + + + +ActiveC->FaultsDeclared + + + + + +ActiveC->FaultSpotted + + + + + +FaultedC + +Faulted + + + +FaultedC->RecoveringC + + +sm.RecoverFaults + + + +ExpiredC + +Cleared + + + +FaultedC->ExpiredC + + +Expiry epoch reached +(sma.CronTick +-> sm.ExpireSectors) + + + +FaultedC->ExpiredC + + +Faulted 3+ times +(sma.CronTick +-> sm.SectorCleanup) + + + + + +ExpireByPost->ExpiredC + + + + + +FaultsDeclared->FaultedC + + + + + + +FaultSpotted->FaultedC + + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg new file mode 100644 index 000000000..6fcb27eb8 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg @@ -0,0 +1,335 @@ + + + + + + +sector + + +cluster_local + +Sector States in Storage Miner +(off chain) + + +cluster_power + + + +cluster_chain + +Sector States in StorageMinerActor +(on chain) + + +cluster_postproving + +must be proved in post + + +cluster_nopowerC1 + +Does not count for power + + +cluster_powerC + +Counts for power + + +cluster_nopowerC2 + +Does not count for power + + + +Empty + +Empty + + + +Packing + +Packing + + + + +Empty->Packing + + +Add Deal + + + +Packing->Packing + + + + + +Unsealed + +Unsealed + + + + +Packing->Unsealed + + +Sector is full + + + +Sealed + +Sealed + + + + +Unsealed->Sealed + + +Seal + + + +NotProving + +Not Proving + + + + +Proving + +Proving + + + +Sealed->Proving + + +Commit +(sm.CommitSector) + + + +CommittedC + +Unproven + + + +Sealed->CommittedC + + + + + +Deleted + +Deleted + + + + +NotProving->Deleted + + +Give up on recovery +or +Expiry + + + +NotProving->Proving + + +Recover +(sm.SubmitPost) + + + +FaultsDeclared + +Fault Declared +(sm.DeclareFaults) + + + +NotProving->FaultsDeclared + + + + +FaultSpotted + +Fault Spotted +(sma.CronTick +-> sm.PostFailed) + + + +NotProving->FaultSpotted + + + + + + +Proving->NotProving + + +Lost data +or +Stopped Proving + + + +Proving->Proving + + + + + +FirstPost +1st PoSt +(sm.SubmitPost +->sm.ActivateSector) + + + +Proving->FirstPost + + + + + + +CommittedC->Proving + + +Commit Sector +(wait for commit) + + + + + +CommittedC->FirstPost + + + + +CommittedC->FaultsDeclared + + + + +CommittedC->FaultSpotted + + + + +ActiveC + +Active + + + +ExpiredC + +Deleted + + + +ActiveC->ExpiredC + + +Expiry epoch reached +(sm.SubmitPost +-> sm.ExpireSectors) + + + + +ActiveC->FaultsDeclared + + + + +ActiveC->FaultSpotted + + + + + +FirstPost->ActiveC + + + + + + + +FaultedC + +Faulted + + + +FaultedC->CommittedC + + +Recover +(sm.SubmitPost) + + + +FaultedC->ExpiredC + + +Expiry epoch reached +(sma.CronTick +-> sm.ExpireSectors) + + + +FaultedC->ExpiredC + + +Faulted 3+ times +(sma.CronTick +-> sm.SectorCleanup) + + + + + + + + +FaultsDeclared->FaultedC + + + + + +FaultSpotted->FaultedC + + + + + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg new file mode 100644 index 000000000..9d3a3fd05 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg @@ -0,0 +1,157 @@ + + + + + + +sector + + +cluster_local + +Sector States in Storage Miner +(off chain) + + +cluster_power + + + + +Empty + +Empty + + + +Packing + +Packing + + + + +Empty->Packing + + +Add Deal + + + +Packing->Packing + + +Add Deal + + + +Unsealed + +Unsealed + + + + +Packing->Unsealed + + +Sector is full + + + +Sealed + +Sealed + + + + +Unsealed->Sealed + + +Seal + + + +NotProving + +Not Proving + + + + +Proving + +Proving + + + +Sealed->Proving + + +Commit Sector +(sm.CommitSector) + + + +NotProving->NotProving + + +Fault Declared +(sm.DeclareFaults) + + + +NotProving->NotProving + + +Fault Spotted +(sma.CronTick +-> sm.PostFailed) + + + +Deleted + +Deleted + + + + +NotProving->Deleted + + +Give up on recovery +or +Expiry + + + +NotProving->Proving + + +Recover +(sm.SubmitPost) + + + +Proving->NotProving + + +Lost data +or +Stopped Proving + + + +Proving->Proving + + +Produce Post +(sm.SubmitPost) + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg new file mode 100644 index 000000000..9607d61f0 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg @@ -0,0 +1,140 @@ + + + + + + +sectoRecovering + + +cluster_0 + +Proving Set + + +cluster_1 + +Not Counted for Power + + + +Cleared + + +Cleared + + + +Precommitted + +Precommitted + + + +Cleared->Precommitted + + + + + +Precommitted->Cleared + + + + + + + +Committed + +Committed + + + +Precommitted->Committed + + + + + +Active + +Active +(Counted +for Power) + + + +Committed->Active + + + + + +Failing + +Failing + + + +Committed->Failing + + + + + + + +Active->Cleared + + + + + +Active->Failing + + + + + + + +Recovering + +Recovering + + + +Recovering->Active + + + + + +Recovering->Failing + + + + + + + +Failing->Cleared + + + + + + + + + +Failing->Recovering + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg new file mode 100644 index 000000000..2495d094f --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg @@ -0,0 +1,81 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg new file mode 100644 index 000000000..6b13ef76f --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg @@ -0,0 +1,82 @@ + + + + + + +sector_state + + +cluster_0 + +Proving Set (Counted for Power) + + + +Cleared + + +Cleared + + + +PreCommit + +PreCommit + + + +Cleared->PreCommit + + + + + +PreCommit->Cleared + + + + + +Active + +Active + + + +PreCommit->Active + + + + + + +Active->Cleared + + + + + + + +TempFault + +TempFault + + + +Active->TempFault + + + + + +TempFault->Active + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg new file mode 100644 index 000000000..a4388337a --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg @@ -0,0 +1,69 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/tools/diagrams.js b/tools/diagrams.js index a2d5abead..71e7db06c 100755 --- a/tools/diagrams.js +++ b/tools/diagrams.js @@ -1,19 +1,15 @@ #!/usr/bin/env node - const globby = require('globby') -const execa = require('execa') const path = require('path') const fs = require('fs') +const graphviz = require('graphviz-cli') -const runMmd = (p) => { +const runMmd = async (p) => { + const mmdc = await import('@mermaid-js/mermaid-cli') const outDir = path.dirname(p).replace('content/', 'static/_gen/diagrams/') const outFile = path.basename(p).replace('.mmd', '.svg') - fs.mkdirSync(outDir, { recursive: true }) - const config = process.env.CI ? ['-p', 'tools/pptr.config'] : [] - return execa('mmdc', [...config, '-i', p, '-o', path.join(outDir, outFile)], { - preferLocal: true, - }) + return await mmdc.run(p, path.join(outDir, outFile)) } const runMmdAll = async () => { @@ -21,13 +17,15 @@ const runMmdAll = async () => { await Promise.all(paths.map(runMmd)) } -const runDot = (p) => { +const runDot = async (p) => { const outDir = path.dirname(p).replace('content/', 'static/_gen/diagrams/') const outFile = path.basename(p).replace('.dot', '.svg') fs.mkdirSync(outDir, { recursive: true }) - return execa('graphviz', ['-Tsvg', `-o${path.join(outDir, outFile)}`, p], { - preferLocal: true, - }) + + return await graphviz.renderGraphFromSource( + { name: p }, + { format: 'svg', name: path.join(outDir, outFile) } + ) } const runDotAll = async () => { @@ -43,7 +41,7 @@ const run = async () => { console.timeEnd('Processed *.{mmd,dot}') } -module.exports.configureWatcher = (watcher) => { +exports.configureWatcher = (watcher) => { watcher.on('all', async (_, p) => { const ext = path.extname(p) switch (ext) { diff --git a/tools/pptr.config b/tools/pptr.config deleted file mode 100644 index bc84d2dcc..000000000 --- a/tools/pptr.config +++ /dev/null @@ -1,4 +0,0 @@ -{ - "args": ["--no-sandbox"], - "executablePath": "google-chrome-stable" -}