Skip to content

Commit c33da03

Browse files
juliangruberjuliangruber
committed
move source files fn to own lib
1 parent 9530b10 commit c33da03

File tree

3 files changed

+175
-168
lines changed

3 files changed

+175
-168
lines changed

‎lib/modules.js

Lines changed: 2 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,19 @@
11
import os from 'node:os'
22
import assert from 'node:assert'
33
import { join } from 'node:path'
4-
import { mkdir, chmod, rm, readFile, writeFile, stat } from 'node:fs/promises'
5-
import { fetch, Headers } from 'undici'
4+
import { mkdir, chmod } from 'node:fs/promises'
5+
import { fetch } from 'undici'
66
import { pipeline } from 'node:stream/promises'
77
import unzip from 'unzip-stream'
88
import { createWriteStream } from 'node:fs'
99
import { moduleBinaries } from './paths.js'
10-
import * as Name from 'w3name'
11-
import { CarReader } from '@ipld/car'
12-
import { validateBlock } from '@web3-storage/car-block-validator'
13-
import { recursive as exporter } from 'ipfs-unixfs-exporter'
1410
import * as tar from 'tar'
15-
import { reportW3NameError } from './telemetry.js'
1611

1712
/** @typedef {import('unzip-stream').UnzipStreamEntry} UnzipStreamEntry */
1813

1914
const { GITHUB_TOKEN } = process.env
2015
const authorization = GITHUB_TOKEN ? `Bearer ${GITHUB_TOKEN}` : undefined
2116

22-
const gateways = [
23-
'w3s.link',
24-
'cf-ipfs.com',
25-
'dweb.link'
26-
]
27-
2817
export const getBinaryModuleExecutable = ({
2918
module,
3019
executable
@@ -104,157 +93,3 @@ export const installBinaryModule = async ({
10493
}
10594
console.log(`[${module}] ✓ ${outFile}`)
10695
}
107-
108-
async function getLatestCID (ipnsKey) {
109-
const name = Name.parse(ipnsKey)
110-
let revision
111-
try {
112-
revision = await Name.resolve(name)
113-
} catch (err) {
114-
reportW3NameError()
115-
// These errors aren't actionable
116-
err.reportToSentry = false
117-
throw err
118-
}
119-
// /ipfs/:cid
120-
return revision.value.split('/').pop()
121-
}
122-
123-
async function getLastSeenModuleCID ({ module, subnetVersionsDir }) {
124-
try {
125-
return await readFile(join(subnetVersionsDir, module), 'utf-8')
126-
} catch (err) {
127-
if (err.code !== 'ENOENT') {
128-
throw err
129-
}
130-
}
131-
return undefined
132-
}
133-
134-
async function setLastSeenModuleCID ({ module, cid, subnetVersionsDir }) {
135-
await mkdir(subnetVersionsDir, { recursive: true })
136-
await writeFile(join(subnetVersionsDir, module), cid)
137-
}
138-
139-
export async function updateSourceFiles ({
140-
module,
141-
ipnsKey,
142-
subnetVersionsDir,
143-
subnetSourcesDir,
144-
noCache
145-
}) {
146-
await mkdir(subnetSourcesDir, { recursive: true })
147-
const outDir = join(subnetSourcesDir, module)
148-
149-
const lastSeenCID = await getLastSeenModuleCID({ module, subnetVersionsDir })
150-
if (lastSeenCID !== undefined) {
151-
// Use `console.error` because with `--json` stdout needs to be JSON only
152-
console.error(`[${module}] ⇣ checking for updates`)
153-
}
154-
155-
const cid = await getLatestCID(ipnsKey)
156-
const isUpdate = lastSeenCID !== cid
157-
if (!isUpdate) {
158-
try {
159-
await stat(join(outDir, 'main.js'))
160-
console.error(`[${module}] ✓ no update available`)
161-
return false
162-
} catch (err) {
163-
console.error(`[${module}] Cannot find sources on disk`)
164-
}
165-
}
166-
167-
let res
168-
for (const gateway of gateways) {
169-
try {
170-
const url = `https://${cid}.ipfs.${gateway}?format=car`
171-
console.error(`[${module}] ⇣ downloading source files via ${url}`)
172-
const headers = new Headers()
173-
if (noCache) headers.append('Cache-Control', 'no-cache')
174-
res = await fetch(url, {
175-
signal: AbortSignal.timeout(10_000),
176-
headers
177-
})
178-
179-
if (res.status >= 300) {
180-
throw new Error(
181-
`[${module}] Cannot fetch ${module} archive for ${cid}: ${res.status}\n` +
182-
await res.text()
183-
)
184-
}
185-
186-
if (!res.body) {
187-
throw new Error(
188-
`[${module}] Cannot fetch ${module} archive for ${cid}: no response body`
189-
)
190-
}
191-
break
192-
} catch (err) {
193-
if (gateway === gateways[gateways.length - 1]) {
194-
throw new Error(
195-
`[${module}] Can't download module sources from any gateway`,
196-
{ cause: err }
197-
)
198-
} else {
199-
console.error(err)
200-
}
201-
}
202-
}
203-
204-
const tarExtractWarnings = []
205-
const tarExtractEntries = []
206-
try {
207-
const reader = await CarReader.fromIterable(res.body)
208-
const entries = exporter(cid, {
209-
async get (blockCid) {
210-
const block = await reader.get(blockCid)
211-
try {
212-
await validateBlock(block)
213-
} catch (err) {
214-
throw new Error(`Invalid block ${blockCid} of root ${cid}`, {
215-
cause: err
216-
})
217-
}
218-
return block.bytes
219-
}
220-
})
221-
const { value: entry } = await entries.next()
222-
assert(entry, `No entries in ${module} archive`)
223-
// Depending on size, entries might be packaged as `file` or `raw`
224-
// https://github.com/web3-storage/w3up/blob/e8bffe2ee0d3a59a977d2c4b7efe425699424e19/packages/upload-client/src/unixfs.js#L11
225-
if (entry.type === 'file' || entry.type === 'raw') {
226-
await mkdir(outDir, { recursive: true })
227-
// `{ strip: 1 }` tells tar to remove the top-level directory (e.g. `mod-peer-checker-v1.0.0`)
228-
await pipeline(
229-
/** @type {any} */(entry.content()),
230-
/** @type {any} */(tar.x({
231-
strip: 1,
232-
C: outDir,
233-
onwarn (code, message, data) {
234-
tarExtractWarnings.push({ code, message, data })
235-
},
236-
onReadEntry (entry) {
237-
tarExtractEntries.push(entry.path)
238-
}
239-
}))
240-
)
241-
await stat(join(outDir, 'main.js'))
242-
}
243-
} catch (err) {
244-
try {
245-
await rm(outDir, { recursive: true })
246-
} catch {
247-
if (err.code !== 'ENOENT') {
248-
throw err
249-
}
250-
}
251-
err.tarExtractWarnings = tarExtractWarnings
252-
err.tarExtractEntries = tarExtractEntries
253-
throw err
254-
}
255-
256-
await setLastSeenModuleCID({ module, cid, subnetVersionsDir })
257-
console.error(`[${module}] ✓ ${outDir}`)
258-
259-
return isUpdate
260-
}

‎lib/subnets.js

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import assert from 'node:assert'
2+
import { join } from 'node:path'
3+
import { mkdir, rm, readFile, writeFile, stat } from 'node:fs/promises'
4+
import { fetch, Headers } from 'undici'
5+
import { pipeline } from 'node:stream/promises'
6+
import * as Name from 'w3name'
7+
import { CarReader } from '@ipld/car'
8+
import { validateBlock } from '@web3-storage/car-block-validator'
9+
import { recursive as exporter } from 'ipfs-unixfs-exporter'
10+
import { reportW3NameError } from './telemetry.js'
11+
import * as tar from 'tar'
12+
13+
const gateways = [
14+
'w3s.link',
15+
'cf-ipfs.com',
16+
'dweb.link'
17+
]
18+
19+
async function getLatestCID (ipnsKey) {
20+
const name = Name.parse(ipnsKey)
21+
let revision
22+
try {
23+
revision = await Name.resolve(name)
24+
} catch (err) {
25+
reportW3NameError()
26+
// These errors aren't actionable
27+
err.reportToSentry = false
28+
throw err
29+
}
30+
// /ipfs/:cid
31+
return revision.value.split('/').pop()
32+
}
33+
34+
async function getLastSeenModuleCID ({ module, subnetVersionsDir }) {
35+
try {
36+
return await readFile(join(subnetVersionsDir, module), 'utf-8')
37+
} catch (err) {
38+
if (err.code !== 'ENOENT') {
39+
throw err
40+
}
41+
}
42+
return undefined
43+
}
44+
45+
async function setLastSeenModuleCID ({ module, cid, subnetVersionsDir }) {
46+
await mkdir(subnetVersionsDir, { recursive: true })
47+
await writeFile(join(subnetVersionsDir, module), cid)
48+
}
49+
50+
export async function updateSourceFiles ({
51+
module,
52+
ipnsKey,
53+
subnetVersionsDir,
54+
subnetSourcesDir,
55+
noCache
56+
}) {
57+
await mkdir(subnetSourcesDir, { recursive: true })
58+
const outDir = join(subnetSourcesDir, module)
59+
60+
const lastSeenCID = await getLastSeenModuleCID({ module, subnetVersionsDir })
61+
if (lastSeenCID !== undefined) {
62+
// Use `console.error` because with `--json` stdout needs to be JSON only
63+
console.error(`[${module}] ⇣ checking for updates`)
64+
}
65+
66+
const cid = await getLatestCID(ipnsKey)
67+
const isUpdate = lastSeenCID !== cid
68+
if (!isUpdate) {
69+
try {
70+
await stat(join(outDir, 'main.js'))
71+
console.error(`[${module}] ✓ no update available`)
72+
return false
73+
} catch (err) {
74+
console.error(`[${module}] Cannot find sources on disk`)
75+
}
76+
}
77+
78+
let res
79+
for (const gateway of gateways) {
80+
try {
81+
const url = `https://${cid}.ipfs.${gateway}?format=car`
82+
console.error(`[${module}] ⇣ downloading source files via ${url}`)
83+
const headers = new Headers()
84+
if (noCache) headers.append('Cache-Control', 'no-cache')
85+
res = await fetch(url, {
86+
signal: AbortSignal.timeout(10_000),
87+
headers
88+
})
89+
90+
if (res.status >= 300) {
91+
throw new Error(
92+
`[${module}] Cannot fetch ${module} archive for ${cid}: ${res.status}\n` +
93+
await res.text()
94+
)
95+
}
96+
97+
if (!res.body) {
98+
throw new Error(
99+
`[${module}] Cannot fetch ${module} archive for ${cid}: no response body`
100+
)
101+
}
102+
break
103+
} catch (err) {
104+
if (gateway === gateways[gateways.length - 1]) {
105+
throw new Error(
106+
`[${module}] Can't download module sources from any gateway`,
107+
{ cause: err }
108+
)
109+
} else {
110+
console.error(err)
111+
}
112+
}
113+
}
114+
115+
const tarExtractWarnings = []
116+
const tarExtractEntries = []
117+
try {
118+
const reader = await CarReader.fromIterable(res.body)
119+
const entries = exporter(cid, {
120+
async get (blockCid) {
121+
const block = await reader.get(blockCid)
122+
try {
123+
await validateBlock(block)
124+
} catch (err) {
125+
throw new Error(`Invalid block ${blockCid} of root ${cid}`, {
126+
cause: err
127+
})
128+
}
129+
return block.bytes
130+
}
131+
})
132+
const { value: entry } = await entries.next()
133+
assert(entry, `No entries in ${module} archive`)
134+
// Depending on size, entries might be packaged as `file` or `raw`
135+
// https://github.com/web3-storage/w3up/blob/e8bffe2ee0d3a59a977d2c4b7efe425699424e19/packages/upload-client/src/unixfs.js#L11
136+
if (entry.type === 'file' || entry.type === 'raw') {
137+
await mkdir(outDir, { recursive: true })
138+
// `{ strip: 1 }` tells tar to remove the top-level directory (e.g. `mod-peer-checker-v1.0.0`)
139+
await pipeline(
140+
/** @type {any} */(entry.content()),
141+
/** @type {any} */(tar.x({
142+
strip: 1,
143+
C: outDir,
144+
onwarn (code, message, data) {
145+
tarExtractWarnings.push({ code, message, data })
146+
},
147+
onReadEntry (entry) {
148+
tarExtractEntries.push(entry.path)
149+
}
150+
}))
151+
)
152+
await stat(join(outDir, 'main.js'))
153+
}
154+
} catch (err) {
155+
try {
156+
await rm(outDir, { recursive: true })
157+
} catch {
158+
if (err.code !== 'ENOENT') {
159+
throw err
160+
}
161+
}
162+
err.tarExtractWarnings = tarExtractWarnings
163+
err.tarExtractEntries = tarExtractEntries
164+
throw err
165+
}
166+
167+
await setLastSeenModuleCID({ module, cid, subnetVersionsDir })
168+
console.error(`[${module}] ✓ ${outDir}`)
169+
170+
return isUpdate
171+
}

‎lib/zinnia.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { execa } from 'execa'
22
import * as Sentry from '@sentry/node'
3-
import { installBinaryModule, updateSourceFiles, getBinaryModuleExecutable } from './modules.js'
3+
import { installBinaryModule, getBinaryModuleExecutable } from './modules.js'
4+
import { updateSourceFiles } from './subnets.js'
45
import os from 'node:os'
56
import pRetry from 'p-retry'
67
import timers from 'node:timers/promises'

0 commit comments

Comments
 (0)