diff --git a/ci/build-upload.ts b/ci/build-upload.ts index d4ec9a5..bb34006 100644 --- a/ci/build-upload.ts +++ b/ci/build-upload.ts @@ -4,6 +4,7 @@ import { buildWithDockerAndValidate, checkPackagesValidity, parsePackageJson, + uploadImage, uploadTarballToS3, uploadTarballToSupabaseStorage, } from "./src/utils"; @@ -120,6 +121,10 @@ for (const buildResult of buildResults) { // continue; // } const filesize = fs.statSync(buildResult.tarballPath).size; + const demoImgPaths = buildResult.pkg.jarvis.demoImages + .map((p) => join(buildResult.extPath, p)) + .filter((p) => fs.existsSync); + const imgStoragePaths = await Promise.all(demoImgPaths.map((p) => uploadImage(p))); // file storage paths const supabasePath = await uploadTarballToSupabaseStorage( buildResult.tarballPath, @@ -143,6 +148,7 @@ for (const buildResult of buildResults) { packagejson: buildResult.pkg, size: filesize, tarball_path: supabasePath, + demo_images_paths: imgStoragePaths, }, ]); if (error) { diff --git a/ci/package.json b/ci/package.json index 40caf73..5c6bd46 100644 --- a/ci/package.json +++ b/ci/package.json @@ -18,6 +18,7 @@ "@aws-sdk/client-s3": "^3.583.0", "@supabase/supabase-js": "^2.43.4", "jarvis-api": "0.0.2-alpha.3", + "sharp": "^0.33.4", "zod": "^3.23.8" } } \ No newline at end of file diff --git a/ci/src/utils.ts b/ci/src/utils.ts index 5eaa511..d82ff52 100644 --- a/ci/src/utils.ts +++ b/ci/src/utils.ts @@ -12,16 +12,15 @@ import path, { join } from "path"; import { DOCKER_BUILD_ENTRYPOINT, REPO_ROOT } from "./constant"; import { spawn, exec } from "node:child_process"; import { supabase } from "./supabase"; +import sharp from "sharp"; + /** * Package Name can be scoped or not * Use regex to extract package name * @param packageName * @param version */ -export function computeTarballName( - packageName: string, - version: string, -): string { +export function computeTarballName(packageName: string, version: string): string { const scoped = packageName.startsWith("@"); if (scoped) { const [scope, name] = packageName.split("/"); @@ -32,9 +31,7 @@ export function computeTarballName( } export function parsePackageJson(pkgJsonPath: string) { - const parse = ExtPackageJson.safeParse( - JSON.parse(fs.readFileSync(pkgJsonPath, "utf8")), - ); + const parse = ExtPackageJson.safeParse(JSON.parse(fs.readFileSync(pkgJsonPath, "utf8"))); if (parse.error) { console.error(`Error parsing ${pkgJsonPath}: ${parse.error}`); process.exit(1); @@ -45,9 +42,7 @@ export function parsePackageJson(pkgJsonPath: string) { export function checkPackagesValidity(extPaths: string[]) { /* ------------------- make sure package.json is parseable ------------------ */ - const pkgs = extPaths.map((ext) => - parsePackageJson(join(ext, "package.json")), - ); + const pkgs = extPaths.map((ext) => parsePackageJson(join(ext, "package.json"))); /* --------------------- make sure identifier is unique --------------------- */ const identifiers = pkgs.map((pkg) => pkg.jarvis.identifier); @@ -55,20 +50,29 @@ export function checkPackagesValidity(extPaths: string[]) { if (identifiers.length !== uniqueIdentifiers.size) { console.error("Identifiers are not unique"); // find the duplicates - const duplicates = identifiers.filter( - (item, index) => identifiers.indexOf(item) !== index, - ); + const duplicates = identifiers.filter((item, index) => identifiers.indexOf(item) !== index); console.error("duplicates", duplicates); process.exit(1); } + + /* ----------------------- Check Demo Images Existence ---------------------- */ + for (const extPath of extPaths) { + const pkg = parsePackageJson(join(extPath, "package.json")); + for (const imgPath of pkg.jarvis.demoImages) { + const imgFullPath = join(extPath, imgPath); + if (!fs.existsSync(imgFullPath)) { + console.error(`Demo Image not found: ${imgFullPath} in ${extPath}`); + process.exit(1); + } + } + } + /* ------ make sure there is no tarball .tgz file in the each extension ----- */ for (const extPath of extPaths) { const files = fs.readdirSync(extPath); const tgzFiles = files.filter((file) => file.endsWith(".tgz")); if (tgzFiles.length > 0) { - console.error( - `Extension ${extPath} contains tarball files: ${tgzFiles.join(", ")}`, - ); + console.error(`Extension ${extPath} contains tarball files: ${tgzFiles.join(", ")}`); console.error( "If you are developing, run scripts/clean.sh to remove all .tgz file in the top level of each extension", ); @@ -77,14 +81,9 @@ export function checkPackagesValidity(extPaths: string[]) { } } -/** - * Compute SHA-1 checksum of a file - * @param filePath - * @returns - */ -export function computeShasum1(filePath: string): Promise { +export function computeFileHash(filePath: string, algorithm: string): Promise { return new Promise((resolve, reject) => { - const hash = crypto.createHash("sha1"); + const hash = crypto.createHash(algorithm); const stream = fs.createReadStream(filePath); stream.on("data", (data) => { @@ -102,6 +101,14 @@ export function computeShasum1(filePath: string): Promise { }); } +export function computeFileSha1(filePath: string): Promise { + return computeFileHash(filePath, "sha1"); +} + +export function computeFileSha512(filePath: string): Promise { + return computeFileHash(filePath, "sha512"); +} + /** * Docker is used to build each individual extension for safety * Packages could potentially modify other extensions if they share environment. @@ -152,9 +159,7 @@ export function buildWithDocker(extPath: string): Promise<{ } if (dataStr.includes("npm notice filename:")) { - const tarballFilename = dataStr.match( - /npm notice filename:\s+([^\s]+)/, - ); + const tarballFilename = dataStr.match(/npm notice filename:\s+([^\s]+)/); if (tarballFilename) { stderrTarballFilename = tarballFilename[1]; console.log("Parsed tarball:", stderrTarballFilename); @@ -172,10 +177,7 @@ export function buildWithDocker(extPath: string): Promise<{ }); subprocess.on("close", (code) => { console.log(`child process exited with code ${code}`); - if ( - stderrShasum.trim().length === 0 || - stderrTarballFilename.trim().length === 0 - ) { + if (stderrShasum.trim().length === 0 || stderrTarballFilename.trim().length === 0) { return reject("shasum or tarball filename not found"); } if (code !== 0) { @@ -201,9 +203,7 @@ export type BuildResult = { * @param extPath Extension Path * @returns */ -export function buildWithDockerAndValidate( - extPath: string, -): Promise { +export function buildWithDockerAndValidate(extPath: string): Promise { return buildWithDocker(extPath) .then((res) => { const parsedTarballPath = join(extPath, res.stderrTarballFilename); @@ -211,7 +211,7 @@ export function buildWithDockerAndValidate( console.error(`Tarball not found: ${parsedTarballPath}`); process.exit(1); } - return computeShasum1(parsedTarballPath).then((computedShasum) => { + return computeFileSha1(parsedTarballPath).then((computedShasum) => { if (computedShasum !== res.stderrShasum) { console.error( `Shasum mismatch: Computed(${computedShasum}) !== Output from docker(${res.stderrShasum})`, @@ -296,12 +296,10 @@ export async function uploadTarballToSupabaseStorage( const tarball = fs.readFileSync(tarballPath); console.log("uploading to supabase storage"); - const { data, error } = await supabase.storage - .from("extensions") - .upload(key, tarball, { - cacheControl: "3600", - upsert: true, - }); + const { data, error } = await supabase.storage.from("extensions").upload(key, tarball, { + cacheControl: "3600", + upsert: true, + }); if (error) { console.error("Failed to upload tarball to supabase storage"); console.error(error); @@ -310,3 +308,84 @@ export async function uploadTarballToSupabaseStorage( console.log("Tarball uploaded to supabase storage"); return data.path; } + +export function computeHash(buffer: Buffer, algorithm: "sha1" | "sha256" | "sha512") { + const hash = crypto.createHash(algorithm); + hash.update(buffer); + return hash.digest("hex"); +} + +export async function uploadImage(imagePath: string) { + // make sure imagePath exists and is a file + if (!fs.existsSync(imagePath)) { + console.error(`Image not found: ${imagePath}`); + process.exit(1); + } + const imageSize = fs.statSync(imagePath).size; + const img = + imageSize > 200 * 1024 + ? await sharp(imagePath) + .resize({ + height: 720, + fit: sharp.fit.inside, + withoutEnlargement: true, + }) + .jpeg() + .toBuffer() + : await sharp(imagePath).jpeg().toBuffer(); + const imgSha512 = computeHash(img, "sha512"); + /* ----------------------- Check if image exists in db ---------------------- */ + const dbRes = await supabase.from("ext_demo_images").select("*").eq("sha512", imgSha512); + const exists = dbRes.data && dbRes.data.length > 0; + if (exists) { + return dbRes.data[0].image_path; + } + + /* --------------------- Upload to supabase file storage -------------------- */ + const key = `ext-images/${imgSha512}.jpeg`; + + const { data, error } = await supabase.storage.from("extensions").upload(key, img, { + cacheControl: "3600", + upsert: true, + }); + if (error) { + console.error(error); + throw new Error("Failed to upload image to supabase storage."); + } + /* ------------------------------ Upload to S3 ------------------------------ */ + const s3Client = new S3Client({ + endpoint: z.string().parse(process.env.S3_ENDPOINT), + region: "auto", + credentials: { + accessKeyId: z.string().parse(process.env.S3_ACCESS_KEY_ID), + secretAccessKey: z.string().parse(process.env.S3_SECRET_ACCESS_KEY), + }, + }); + await s3Client + .send( + new PutObjectCommand({ + Bucket: "jarvis-extensions", + Key: key, + Body: img, + ContentType: "application/jpeg", + }), + ) + .then((res) => { + return key; + }) + .catch((err) => { + console.error("Failed to upload tarball"); + console.error(err); + }); + + /* ------------------------- Insert into database -------------------------- */ + const { data: insertData, error: insertError } = await supabase + .from("ext_demo_images") + .insert([{ sha512: imgSha512, image_path: data.path }]); + if (insertError) { + console.error(insertError); + throw new Error("Failed to insert image into database"); + } + + return data?.path; +} diff --git a/ci/supabase/types/supabase.ts b/ci/supabase/types/supabase.ts index 4c783ec..105e936 100644 --- a/ci/supabase/types/supabase.ts +++ b/ci/supabase/types/supabase.ts @@ -4,64 +4,88 @@ export type Json = | boolean | null | { [key: string]: Json | undefined } - | Json[]; + | Json[] export type Database = { public: { Tables: { + ext_demo_images: { + Row: { + created_at: string + id: number + image_path: string + sha512: string + } + Insert: { + created_at?: string + id?: number + image_path: string + sha512: string + } + Update: { + created_at?: string + id?: number + image_path?: string + sha512?: string + } + Relationships: [] + } extensions: { Row: { - created_at: string; - id: number; - identifier: string; - name: string; - packagejson: Json; - shasum: string; - size: number; - tarball_path: string; - version: string; - }; + created_at: string + demo_images_paths: string[] | null + id: number + identifier: string + name: string + packagejson: Json + shasum: string + size: number + tarball_path: string + version: string + } Insert: { - created_at?: string; - id?: number; - identifier: string; - name: string; - packagejson: Json; - shasum: string; - size: number; - tarball_path: string; - version: string; - }; + created_at?: string + demo_images_paths?: string[] | null + id?: number + identifier: string + name: string + packagejson: Json + shasum: string + size: number + tarball_path: string + version: string + } Update: { - created_at?: string; - id?: number; - identifier?: string; - name?: string; - packagejson?: Json; - shasum?: string; - size?: number; - tarball_path?: string; - version?: string; - }; - Relationships: []; - }; - }; + created_at?: string + demo_images_paths?: string[] | null + id?: number + identifier?: string + name?: string + packagejson?: Json + shasum?: string + size?: number + tarball_path?: string + version?: string + } + Relationships: [] + } + } Views: { - [_ in never]: never; - }; + [_ in never]: never + } Functions: { - [_ in never]: never; - }; + [_ in never]: never + } Enums: { - [_ in never]: never; - }; + [_ in never]: never + } CompositeTypes: { - [_ in never]: never; - }; - }; -}; + [_ in never]: never + } + } +} -type PublicSchema = Database[Extract]; +type PublicSchema = Database[Extract] export type Tables< PublicTableNameOrOptions extends @@ -74,7 +98,7 @@ export type Tables< > = PublicTableNameOrOptions extends { schema: keyof Database } ? (Database[PublicTableNameOrOptions["schema"]]["Tables"] & Database[PublicTableNameOrOptions["schema"]]["Views"])[TableName] extends { - Row: infer R; + Row: infer R } ? R : never @@ -82,11 +106,11 @@ export type Tables< PublicSchema["Views"]) ? (PublicSchema["Tables"] & PublicSchema["Views"])[PublicTableNameOrOptions] extends { - Row: infer R; + Row: infer R } ? R : never - : never; + : never export type TablesInsert< PublicTableNameOrOptions extends @@ -97,17 +121,17 @@ export type TablesInsert< : never = never, > = PublicTableNameOrOptions extends { schema: keyof Database } ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends { - Insert: infer I; + Insert: infer I } ? I : never : PublicTableNameOrOptions extends keyof PublicSchema["Tables"] ? PublicSchema["Tables"][PublicTableNameOrOptions] extends { - Insert: infer I; + Insert: infer I } ? I : never - : never; + : never export type TablesUpdate< PublicTableNameOrOptions extends @@ -118,17 +142,17 @@ export type TablesUpdate< : never = never, > = PublicTableNameOrOptions extends { schema: keyof Database } ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends { - Update: infer U; + Update: infer U } ? U : never : PublicTableNameOrOptions extends keyof PublicSchema["Tables"] ? PublicSchema["Tables"][PublicTableNameOrOptions] extends { - Update: infer U; + Update: infer U } ? U : never - : never; + : never export type Enums< PublicEnumNameOrOptions extends @@ -141,4 +165,4 @@ export type Enums< ? Database[PublicEnumNameOrOptions["schema"]]["Enums"][EnumName] : PublicEnumNameOrOptions extends keyof PublicSchema["Enums"] ? PublicSchema["Enums"][PublicEnumNameOrOptions] - : never; + : never diff --git a/extensions/download-twitter-video/demo/download-twitter.png b/extensions/download-twitter-video/demo/download-twitter.png new file mode 100644 index 0000000..037cf39 Binary files /dev/null and b/extensions/download-twitter-video/demo/download-twitter.png differ diff --git a/extensions/download-twitter-video/package.json b/extensions/download-twitter-video/package.json index 43e47c8..c3b4922 100644 --- a/extensions/download-twitter-video/package.json +++ b/extensions/download-twitter-video/package.json @@ -7,7 +7,9 @@ "type": "module", "jarvis": { "identifier": "tech.huakun.jarvis-twitter-download-video-ext", - "demoImages": [], + "demoImages": [ + "./demo/download-twitter.png" + ], "icon": { "type": "iconify", "icon": "skill-icons:twitter" diff --git a/extensions/ip-info/demo/ip-info.png b/extensions/ip-info/demo/ip-info.png new file mode 100644 index 0000000..fba05f6 Binary files /dev/null and b/extensions/ip-info/demo/ip-info.png differ diff --git a/extensions/ip-info/package.json b/extensions/ip-info/package.json index 2cf38a2..5ea73cd 100644 --- a/extensions/ip-info/package.json +++ b/extensions/ip-info/package.json @@ -26,7 +26,9 @@ }, "jarvis": { "identifier": "tech.huakun.jarvis-ext-ip-info", - "demoImages": [], + "demoImages": [ + "./demo/ip-info.png" + ], "icon": { "type": "iconify", "icon": "mdi:ip-network" diff --git a/extensions/jwt/demo/jwt-search.png b/extensions/jwt/demo/jwt-search.png new file mode 100644 index 0000000..6ee0b2e Binary files /dev/null and b/extensions/jwt/demo/jwt-search.png differ diff --git a/extensions/jwt/demo/jwt-view.png b/extensions/jwt/demo/jwt-view.png new file mode 100644 index 0000000..5361278 Binary files /dev/null and b/extensions/jwt/demo/jwt-view.png differ diff --git a/extensions/jwt/package.json b/extensions/jwt/package.json index 69e21d5..42b2422 100644 --- a/extensions/jwt/package.json +++ b/extensions/jwt/package.json @@ -7,7 +7,10 @@ "description": "JWT Inspector", "jarvis": { "identifier": "tech.huakun.jarvis-jwt-inspector", - "demoImages": [], + "demoImages": [ + "./demo/jwt-view.png", + "./demo/jwt-search.png" + ], "icon": { "type": "iconify", "icon": "logos:jwt-icon" diff --git a/extensions/qrcode/demo/gen-qrcode.png b/extensions/qrcode/demo/gen-qrcode.png new file mode 100644 index 0000000..2b076dd Binary files /dev/null and b/extensions/qrcode/demo/gen-qrcode.png differ diff --git a/extensions/qrcode/package.json b/extensions/qrcode/package.json index f10dc95..fec0775 100644 --- a/extensions/qrcode/package.json +++ b/extensions/qrcode/package.json @@ -7,7 +7,9 @@ "description": "Jarvis QRCode Extension", "jarvis": { "identifier": "tech.huakun.jarvis-qrcode-ext", - "demoImages": [], + "demoImages": [ + "./demo/gen-qrcode.png" + ], "icon": { "type": "iconify", "icon": "mingcute:qrcode-line" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9cf0dd3..bc028f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: jarvis-api: specifier: 0.0.2-alpha.3 version: 0.0.2-alpha.3(typescript@5.4.5) + sharp: + specifier: ^0.33.4 + version: 0.33.4 typescript: specifier: ^5.0.0 version: 5.4.5 @@ -6766,7 +6769,6 @@ packages: color-name: 1.1.4 simple-swizzle: 0.2.2 dev: false - optional: true /color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} @@ -6781,7 +6783,6 @@ packages: color-convert: 2.0.1 color-string: 1.9.1 dev: false - optional: true /colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} @@ -8466,7 +8467,6 @@ packages: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} requiresBuild: true dev: false - optional: true /is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} @@ -11675,7 +11675,6 @@ packages: '@img/sharp-win32-ia32': 0.33.4 '@img/sharp-win32-x64': 0.33.4 dev: false - optional: true /shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} @@ -11740,7 +11739,6 @@ packages: dependencies: is-arrayish: 0.3.2 dev: false - optional: true /sirv@2.0.4: resolution: {integrity: sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==}