diff --git a/.gitignore b/.gitignore index 952e1bb..a1cc359 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ +.idea +.vscode + node_modules dist __pycache__ *.egg-info *coverage* *.tar.gz -*.tgz \ No newline at end of file +*.tgz diff --git a/cli/commands/publish/upload.manifest.spec.ts b/cli/commands/publish/upload.manifest.spec.ts index 0c174f0..4764149 100644 --- a/cli/commands/publish/upload.manifest.spec.ts +++ b/cli/commands/publish/upload.manifest.spec.ts @@ -1,12 +1,13 @@ -import { ethers, Wallet } from "ethers"; -import { keccak256 } from "../../utils"; +import {ethers, Wallet} from "ethers"; +import {keccak256} from "../../utils"; import provideUploadManifest, { - ChainSettings, + DocumentationSetting, UploadManifest, } from "./upload.manifest"; describe("uploadManifest", () => { let uploadManifest: UploadManifest; + const mockSystemTime = new Date(); const mockFilesystem = { existsSync: jest.fn(), readFileSync: jest.fn(), @@ -19,7 +20,11 @@ describe("uploadManifest", () => { const mockLongDescription = "some long description"; const mockAgentId = "0xagentId"; const mockVersion = "0.1"; - const mockDocumentation = "README.md"; + const mockReadmeFilePath = "README.md"; + const mockDocumentationSettings: DocumentationSetting[] = [ + {title: 'General', filePath: 'General.md'}, + {title: 'API Guide', filePath: 'API.md'}, + ]; const mockRepository = "github.com/myrepository"; const mockLicenseUrl = "github.com/myrepository"; const mockPromoUrl = "github.com/myrepository"; @@ -60,9 +65,11 @@ describe("uploadManifest", () => { const resetMocks = () => { mockFilesystem.existsSync.mockReset(); mockFilesystem.statSync.mockReset(); + mockFilesystem.readFileSync.mockReset(); + mockAddToIpfs.mockReset(); }; - beforeAll(() => { + const setupUploadManifest = (props?: { documentationSettings?: DocumentationSetting[] }) => { uploadManifest = provideUploadManifest( mockFilesystem, mockAddToIpfs, @@ -72,7 +79,8 @@ describe("uploadManifest", () => { mockLongDescription, mockAgentId, mockVersion, - mockDocumentation, + mockReadmeFilePath, + props?.documentationSettings, mockRepository, mockLicenseUrl, mockPromoUrl, @@ -81,54 +89,144 @@ describe("uploadManifest", () => { mockExternal, mockChainSettings ); - }); + } beforeEach(() => { + jest.useFakeTimers().setSystemTime(mockSystemTime); resetMocks(); }); - it("throws error if documentation not found", async () => { + afterEach(() => { + jest.useRealTimers(); + }) + + it("throws error if readme file not found and documentation settings undefined", async () => { mockFilesystem.existsSync.mockReturnValueOnce(false); try { + setupUploadManifest(); await uploadManifest(mockImageRef, mockPrivateKey); } catch (e) { expect(e.message).toBe( - `documentation file ${mockDocumentation} not found` + `documentation file ${mockReadmeFilePath} not found` ); } expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(1); - expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockDocumentation); + expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockReadmeFilePath); }); - it("throws error if documentation file is empty", async () => { + it("throws error if readme file is empty and documentation settings undefined", async () => { mockFilesystem.existsSync.mockReturnValueOnce(true); - mockFilesystem.statSync.mockReturnValueOnce({ size: 0 }); + mockFilesystem.statSync.mockReturnValueOnce({size: 0}); try { + setupUploadManifest(); await uploadManifest(mockImageRef, mockPrivateKey); } catch (e) { expect(e.message).toBe( - `documentation file ${mockDocumentation} cannot be empty` + `documentation file ${mockReadmeFilePath} cannot be empty` ); } expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(1); - expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockDocumentation); + expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockReadmeFilePath); + expect(mockFilesystem.statSync).toHaveBeenCalledTimes(1); + expect(mockFilesystem.statSync).toHaveBeenCalledWith(mockReadmeFilePath); + }); + + it("throws error if one of documentation files not found", async () => { + const missingFile = 'API.md'; + + mockFilesystem.existsSync.mockImplementation((fileName: string) => fileName !== missingFile); + mockFilesystem.statSync.mockImplementation((fileName: string) => { + if (fileName === missingFile) throw new Error(`Cannot find ${fileName}`); + return {size: 1}; + }); + + try { + setupUploadManifest({documentationSettings: mockDocumentationSettings}); + await uploadManifest(mockImageRef, mockPrivateKey); + } catch (e) { + expect(e.message).toBe( + `documentation file ${missingFile} not found` + ); + } + + expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(2); expect(mockFilesystem.statSync).toHaveBeenCalledTimes(1); - expect(mockFilesystem.statSync).toHaveBeenCalledWith(mockDocumentation); + }); + + it('throws error if documentation settings defined with empty array', async () => { + try { + setupUploadManifest({documentationSettings: []}); + await uploadManifest(mockImageRef, mockPrivateKey); + } catch (e) { + expect(e.message).toBe( + `documentationSettings must be non-empty array` + ); + } + + expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(0); + expect(mockFilesystem.statSync).toHaveBeenCalledTimes(0); + }) + + it("throws error if one of documentation files is empty", async () => { + const emptyFile = 'API.md'; + + mockFilesystem.existsSync.mockReturnValue(true); + mockFilesystem.statSync.mockImplementation((fileName: string) => { + if (fileName === emptyFile) return {size: 0}; + return {size: 1}; + }); + + try { + setupUploadManifest({documentationSettings: mockDocumentationSettings}); + await uploadManifest(mockImageRef, mockPrivateKey); + } catch (e) { + expect(e.message).toBe( + `documentation file ${emptyFile} cannot be empty` + ); + } + + expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(2); + expect(mockFilesystem.statSync).toHaveBeenCalledTimes(2); + }); + + it("throws error if one of documentation settings has empty title", async () => { + mockFilesystem.existsSync.mockReturnValue(true); + mockFilesystem.statSync.mockReturnValue({size: 1}); + + try { + setupUploadManifest({ + documentationSettings: [{ + title: ' ', + filePath: 'FILE.md' + }] + }); + await uploadManifest(mockImageRef, mockPrivateKey); + } catch (e) { + expect(e.message).toBe( + `title in documentationSettings must be non-empty string` + ); + } + + expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(0); + expect(mockFilesystem.statSync).toHaveBeenCalledTimes(0); }); it("uploads signed manifest to ipfs and returns ipfs reference", async () => { - const systemTime = new Date(); - jest.useFakeTimers().setSystemTime(systemTime); + setupUploadManifest(); + mockFilesystem.existsSync.mockReturnValueOnce(true); - mockFilesystem.statSync.mockReturnValueOnce({ size: 1 }); - const mockDocumentationFile = JSON.stringify({ some: "documentation" }); + mockFilesystem.statSync.mockReturnValueOnce({size: 1}); + + const mockDocumentationFile = JSON.stringify({some: "documentation"}); mockFilesystem.readFileSync.mockReturnValueOnce(mockDocumentationFile); + const mockDocumentationRef = "docRef"; mockAddToIpfs.mockReturnValueOnce(mockDocumentationRef); + const mockManifest = { from: new Wallet(mockPrivateKey).address, name: mockAgentDisplayName, @@ -137,9 +235,12 @@ describe("uploadManifest", () => { agentId: mockAgentName, agentIdHash: mockAgentId, version: mockVersion, - timestamp: systemTime.toUTCString(), + timestamp: mockSystemTime.toUTCString(), imageReference: mockImageRef, - documentation: mockDocumentationRef, + documentation: JSON.stringify([{ + title: 'README', + ipfsUrl: mockDocumentationRef + }]), repository: mockRepository, licenseUrl: mockLicenseUrl, promoUrl: mockPromoUrl, @@ -155,12 +256,12 @@ describe("uploadManifest", () => { expect(manifestRef).toBe(mockManifestRef); expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(1); - expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockDocumentation); + expect(mockFilesystem.existsSync).toHaveBeenCalledWith(mockReadmeFilePath); expect(mockFilesystem.statSync).toHaveBeenCalledTimes(1); - expect(mockFilesystem.statSync).toHaveBeenCalledWith(mockDocumentation); + expect(mockFilesystem.statSync).toHaveBeenCalledWith(mockReadmeFilePath); expect(mockFilesystem.readFileSync).toHaveBeenCalledTimes(1); expect(mockFilesystem.readFileSync).toHaveBeenCalledWith( - mockDocumentation, + mockReadmeFilePath, "utf8" ); expect(mockAddToIpfs).toHaveBeenCalledTimes(2); @@ -171,8 +272,46 @@ describe("uploadManifest", () => { ); expect(mockAddToIpfs).toHaveBeenNthCalledWith( 2, - JSON.stringify({ manifest: mockManifest, signature }) + JSON.stringify({manifest: mockManifest, signature}) ); - jest.useRealTimers(); + }); + + it("uploads documentation settings correctly", async () => { + setupUploadManifest({ documentationSettings: mockDocumentationSettings }); + + // mocking file existence and content + mockFilesystem.existsSync.mockReturnValue(true); + mockFilesystem.statSync.mockReturnValue({ size: 1 }); + mockFilesystem.readFileSync.mockImplementation((fileName: string) => { + if (fileName === 'General.md') return 'General Content'; + if (fileName === 'API.md') return 'API Content'; + }); + + // mocking IPFS addition + const mockGeneralDocRef = "generalDocRef"; + const mockApiDocRef = "apiDocRef"; + mockAddToIpfs.mockImplementation((content: string) => { + if (content === 'General Content') return mockGeneralDocRef; + if (content === 'API Content') return mockApiDocRef; + }); + + const manifestRef = await uploadManifest(mockImageRef, mockPrivateKey); + + // asserting that files were read correctly + expect(mockFilesystem.existsSync).toHaveBeenCalledTimes(2); + expect(mockFilesystem.readFileSync).toHaveBeenCalledWith('General.md', 'utf8'); + expect(mockFilesystem.readFileSync).toHaveBeenCalledWith('API.md', 'utf8'); + + // asserting that files were added to IPFS correctly + expect(mockAddToIpfs).toHaveBeenCalledTimes(3); // 2 docs + 1 manifest + expect(mockAddToIpfs).toHaveBeenCalledWith('General Content'); + expect(mockAddToIpfs).toHaveBeenCalledWith('API Content'); + + // asserting manifest structure + const expectedDocumentation = JSON.stringify([ + { title: 'General', ipfsUrl: mockGeneralDocRef }, + { title: 'API Guide', ipfsUrl: mockApiDocRef }, + ]); + expect(JSON.parse(mockAddToIpfs.mock.calls[2][0]).manifest.documentation).toEqual(expectedDocumentation); }); }); diff --git a/cli/commands/publish/upload.manifest.ts b/cli/commands/publish/upload.manifest.ts index 1d3fade..1dae768 100644 --- a/cli/commands/publish/upload.manifest.ts +++ b/cli/commands/publish/upload.manifest.ts @@ -1,6 +1,12 @@ import fs from "fs" import { ethers, Wallet } from "ethers" -import { assertExists, assertIsNonEmptyString, assertIsValidChainSettings, keccak256 } from "../../utils" +import { + assertExists, + assertIsValidDocumentationSettings, + assertIsNonEmptyString, + assertIsValidChainSettings, + keccak256 +} from "../../utils" import { AddToIpfs } from "../../utils/add.to.ipfs" // uploads signed agent manifest to ipfs and returns ipfs reference @@ -13,6 +19,16 @@ export type ChainSetting = { export type ChainSettings = { [id: string]: ChainSetting } +export type DocumentationSetting = { + title: string; + filePath: string +}; + +export type DocumentationItem = { + title: string; + ipfsUrl: string +}; + type Manifest = { from: string, name: string, @@ -43,6 +59,7 @@ export default function provideUploadManifest( agentId: string, version: string, documentation: string, + documentationSettings: DocumentationSetting[] | undefined, repository: string, licenseUrl: string, promoUrl: string, @@ -57,22 +74,57 @@ export default function provideUploadManifest( assertIsNonEmptyString(description, 'description') assertIsNonEmptyString(agentId, 'agentId') assertIsNonEmptyString(version, 'version') - assertIsNonEmptyString(documentation, 'documentation') assertIsNonEmptyString(cliVersion, 'cliVersion') assertExists(chainIds, 'chainIds') assertIsValidChainSettings(chainSettings) + if(documentationSettings) { + assertIsValidDocumentationSettings(documentationSettings); + } else { + assertIsNonEmptyString(documentation, 'documentation') + } return async function uploadManifest(imageReference: string | undefined, privateKey: string) { - // upload documentation to ipfs - if (!filesystem.existsSync(documentation)) { - throw new Error(`documentation file ${documentation} not found`) + const assertDocumentationFile = (documentationFile: string) => { + if (!filesystem.existsSync(documentationFile)) { + throw new Error(`documentation file ${documentationFile} not found`) + } + if (!filesystem.statSync(documentationFile).size) { + throw new Error(`documentation file ${documentationFile} cannot be empty`) + } } - if (!filesystem.statSync(documentation).size) { - throw new Error(`documentation file ${documentation} cannot be empty`) + + if(documentationSettings) { + for(const item of documentationSettings) { + assertDocumentationFile(item.filePath) + } + } else { + assertDocumentationFile(documentation) + } + + // normalize to one format + const settings: DocumentationSetting[] = []; + if(documentationSettings) { + settings.push(...documentationSettings); + } else { + settings.push({ + title: 'README', + filePath: documentation + }) } + + // upload documentation to ipfs console.log('pushing agent documentation to IPFS...') - const documentationFile = filesystem.readFileSync(documentation, 'utf8') - const documentationReference = await addToIpfs(documentationFile) + + const items: DocumentationItem[] = []; + for(const setting of settings) { + const documentationFile = filesystem.readFileSync(setting.filePath, 'utf8') + const documentationReference = await addToIpfs(documentationFile) + + items.push({ + title: setting.title, + ipfsUrl: documentationReference + }) + } // create agent manifest const manifest: Manifest = { @@ -85,7 +137,7 @@ export default function provideUploadManifest( version, timestamp: new Date().toUTCString(), imageReference, - documentation: documentationReference, + documentation: JSON.stringify(items), repository, licenseUrl, promoUrl, @@ -96,7 +148,7 @@ export default function provideUploadManifest( } // sign agent manifest - const signingKey = new ethers.utils.SigningKey(privateKey) + const signingKey = new ethers.utils.SigningKey(privateKey) const signature = ethers.utils.joinSignature(signingKey.signDigest(keccak256(JSON.stringify(manifest)))) // upload signed manifest to ipfs @@ -126,4 +178,4 @@ function formatChainSettings(chainSettings?: ChainSettings): ChainSettings | und } } return formattedChainSettings -} \ No newline at end of file +} diff --git a/cli/di.container.ts b/cli/di.container.ts index a089753..508e0f5 100644 --- a/cli/di.container.ts +++ b/cli/di.container.ts @@ -160,6 +160,25 @@ export default function configureContainer(args: any = {}) { }).singleton(), version: asFunction((packageJson: any) => packageJson.version), documentation: asFunction((contextPath: string) => { return join(contextPath, 'README.md') }).singleton(), + documentationSettings: asFunction((packageJson: any, contextPath: string) => { + const { documentation } = packageJson + if(Array.isArray(documentation)) { + const items = []; + // join filePath with contextPath, if possible + for(const setting of documentation) { + if(typeof setting.filePath === 'string') { + items.push({ + ...setting, + filePath: join(contextPath, setting.filePath) + }) + } else { + items.push(setting) + } + } + return items + } + return undefined; + }).singleton(), repository: asFunction((packageJson: any) => { const repository = packageJson.repository if (typeof repository === 'string') { diff --git a/cli/utils/index.ts b/cli/utils/index.ts index 70be1a7..87e4191 100644 --- a/cli/utils/index.ts +++ b/cli/utils/index.ts @@ -59,11 +59,11 @@ export const assertFindings = (findings: Finding[]) => { export const assertIsValidChainSettings = (chainSettings?: any) => { if(!chainSettings) { - return + return } for(let key in chainSettings) { if(key == "default") { - continue + continue } if(isNaN(parseInt(key))) { throw new Error("keys in chainSettings must be numerical string or default") @@ -71,6 +71,33 @@ export const assertIsValidChainSettings = (chainSettings?: any) => { } } +export const assertIsValidDocumentationSettings = (documentationSettings: any) => { + if(!Array.isArray(documentationSettings)) { + throw new Error('documentationSettings must be array') + } + if(documentationSettings.length === 0) { + throw new Error("documentationSettings must be non-empty array") + } + if(documentationSettings.length > 10) { + throw new Error("documentationSettings must contain up to 10 items") + } + + for(const item of documentationSettings) { + if(typeof item.title !== 'string') { + throw new Error('title in documentationSettings must be string') + } + if(!item.title?.trim()) { + throw new Error('title in documentationSettings must be non-empty string'); + } + if(typeof item.filePath !== 'string') { + throw new Error('filePath in documentationSettings must be string') + } + if(!item.filePath?.trim()) { + throw new Error('filePath in documentationSettings must be non-empty string') + } + } +} + export const isValidTimeRange = (earliestTimestamp: Date, latestTimestamp: Date): boolean => { // If given a start range and end range return earliestTimestamp < latestTimestamp; @@ -121,9 +148,9 @@ export const createBlockEvent: CreateBlockEvent = (block: JsonRpcBlock, networkI // creates a Forta TransactionEvent from a json-rpc transaction receipt and block object export type CreateTransactionEvent = (transaction: JsonRpcTransaction, block: JsonRpcBlock, networkId: number, traces: Trace[], logs: JsonRpcLog[]) => TransactionEvent export const createTransactionEvent: CreateTransactionEvent = ( - transaction: JsonRpcTransaction, - block: JsonRpcBlock, - networkId: number, + transaction: JsonRpcTransaction, + block: JsonRpcBlock, + networkId: number, traces: Trace[] = [], logs: JsonRpcLog[] = [] ) => {