simple progress tracking

Author: jordanverasamy

packages/store/src/cli/commands/store/execute.ts

@@ -1,47 +1,56 @@
+import {parseGraphQLOperation} from '../../services/graphql-parser.js'
+import {runBulkQuery} from '../../services/bulk-operations.js'
 import {Command, Flags} from '@oclif/core'
 import {globalFlags} from '@shopify/cli-kit/node/cli'
-import {readFile} from '@shopify/cli-kit/node/fs'
+import {readFile, writeFile} from '@shopify/cli-kit/node/fs'
 import {ensureAuthenticatedAdmin} from '@shopify/cli-kit/node/session'
 import {adminRequest} from '@shopify/cli-kit/node/api/admin'
-import {parseGraphQLOperation} from '../../services/graphql-parser.js'
-import {runBulkQuery} from '../../services/bulk-operations.js'
+import {outputInfo, outputSuccess} from '@shopify/cli-kit/node/output'
 
 export default class Execute extends Command {
   static summary = 'execute a graphql query or mutation on a store'
 
-  static description = 'executes a graphql query or mutation on the specified store, and writes the result to stdout or a file. supports bulk operations.'
+  static description =
+    'executes a graphql query or mutation on the specified store, and writes the result to stdout or a file. supports bulk operations.'
 
   static flags = {
     ...globalFlags,
     query: Flags.string({
       char: 'q',
       description: 'the graphql query or mutation, as a string',
       exclusive: ['query-file'],
+      env: 'SHOPIFY_FLAG_QUERY',
     }),
     'query-file': Flags.string({
       description: 'a file containing the graphql query or mutation',
       exclusive: ['query'],
+      env: 'SHOPIFY_FLAG_QUERY_FILE',
     }),
     store: Flags.string({
       char: 's',
       description: 'the myshopify.com domain of the store',
+      env: 'SHOPIFY_FLAG_STORE',
     }),
     variables: Flags.string({
       char: 'v',
       description: 'the values for graphql variables, in json format',
       multiple: true,
       exclusive: ['variable-file'],
+      env: 'SHOPIFY_FLAG_VARIABLES',
     }),
     'variable-file': Flags.string({
       description: 'a file containing graphql variables, in jsonl format',
       exclusive: ['variables'],
+      env: 'SHOPIFY_FLAG_VARIABLE_FILE',
     }),
     'output-file': Flags.string({
       description: 'the file name where results should be written',
+      env: 'SHOPIFY_FLAG_OUTPUT_FILE',
     }),
     'bulk-operation': Flags.boolean({
       description: 'execute as a bulk operation',
       default: false,
+      env: 'SHOPIFY_FLAG_BULK_OPERATION',
     }),
   }
 
@@ -63,15 +72,17 @@ export default class Execute extends Command {
       this.error('--store is required')
     }
 
-    let variables: Record<string, unknown> | undefined
+    let variables: {[key: string]: unknown} | undefined
 
     if (flags.variables && flags.variables.length > 0) {
       const variableString = flags.variables[0]
       if (variableString) {
         try {
           variables = JSON.parse(variableString)
+          /* eslint-disable-next-line no-catch-all/no-catch-all */
         } catch (error) {
-          this.error(`invalid json in --variables: ${error}`)
+          const message = error instanceof Error ? error.message : String(error)
+          this.error(`invalid json in --variables: ${message}`)
         }
       }
     } else if (flags['variable-file']) {
@@ -80,8 +91,10 @@ export default class Execute extends Command {
       if (firstLine) {
         try {
           variables = JSON.parse(firstLine)
+          /* eslint-disable-next-line no-catch-all/no-catch-all */
         } catch (error) {
-          this.error(`invalid json in --variable-file: ${error}`)
+          const message = error instanceof Error ? error.message : String(error)
+          this.error(`invalid json in --variable-file: ${message}`)
         }
       }
     }
@@ -92,9 +105,19 @@ export default class Execute extends Command {
       const operationType = parseGraphQLOperation(query)
 
       if (operationType === 'query') {
-        this.log('starting bulk query...')
-        const results = await runBulkQuery(query, adminSession)
-        this.log(results)
+        const result = await runBulkQuery(query, adminSession, (status, objectCount, rate, spinner) => {
+          const rateStr = rate > 0 ? ` • ${Math.round(rate)} obj/sec` : ''
+          process.stdout.write(`\r\x1b[K${status.toLowerCase()}: ${objectCount} objects${rateStr} ${spinner}`)
+        })
+
+        const outputFilePath = 'bulk-operation-results.jsonl'
+        await writeFile(outputFilePath, result.content)
+
+        this.log('\n')
+        outputSuccess(`wrote ${result.totalObjects} objects to ${outputFilePath}`)
+        outputInfo(
+          `completed in ${result.totalTimeSeconds.toFixed(1)}s (${Math.round(result.averageRate)} obj/sec average)`,
+        )
       } else {
         this.error('bulk mutations not yet implemented')
       }

packages/store/src/cli/services/bulk-operations.ts

@@ -2,7 +2,9 @@ import {AdminSession} from '@shopify/cli-kit/node/session'
 import {adminRequest} from '@shopify/cli-kit/node/api/admin'
 import {sleep} from '@shopify/cli-kit/node/system'
 import {fetch} from '@shopify/cli-kit/node/http'
+import {outputInfo} from '@shopify/cli-kit/node/output'
 
+/* eslint-disable @shopify/cli/no-inline-graphql */
 const BULK_QUERY_MUTATION = `
   mutation bulkOperationRunQuery($query: String!) {
     bulkOperationRunQuery(query: $query) {
@@ -29,6 +31,20 @@ const BULK_OPERATION_STATUS_QUERY = `
     }
   }
 `
+/* eslint-enable @shopify/cli/no-inline-graphql */
+
+interface BulkOperationStartResult {
+  bulkOperationRunQuery: {
+    bulkOperation: {
+      id: string
+      status: string
+    }
+    userErrors: {
+      field: string
+      message: string
+    }[]
+  }
+}
 
 interface BulkOperationStatus {
   currentBulkOperation: {
@@ -43,30 +59,72 @@ interface BulkOperationStatus {
 export async function runBulkQuery(
   query: string,
   session: AdminSession,
-): Promise<string> {
-  const startResult = await adminRequest<any>(BULK_QUERY_MUTATION, session, {query})
+  onProgress?: (status: string, objectCount: string, rate: number, spinner: string) => void,
+): Promise<{content: string; totalObjects: number; totalTimeSeconds: number; averageRate: number}> {
+  const startResult = await adminRequest<BulkOperationStartResult>(BULK_QUERY_MUTATION, session, {query})
 
   if (startResult.bulkOperationRunQuery.userErrors.length > 0) {
-    const errors = startResult.bulkOperationRunQuery.userErrors.map((e: any) => e.message).join(', ')
+    const errors = startResult.bulkOperationRunQuery.userErrors.map((err) => err.message).join(', ')
     throw new Error(`bulk operation failed: ${errors}`)
   }
 
+  outputInfo(`bulk operation started: ${startResult.bulkOperationRunQuery.bulkOperation.id}`)
+
+  const operationStartTime = Date.now()
+  let lastObjectCount = '0'
+  let lastUpdateTime = Date.now()
+  let rate = 0
+  const spinnerFrames = ['.', '..', '...']
+  let spinnerIndex = 0
+
+  /* eslint-disable no-await-in-loop */
   while (true) {
-    await sleep(1000)
+    await sleep(1)
 
     const statusResult = await adminRequest<BulkOperationStatus>(BULK_OPERATION_STATUS_QUERY, session)
     const operation = statusResult.currentBulkOperation
 
+    if (operation.status === 'CREATED' || operation.status === 'RUNNING') {
+      const currentCount = parseInt(operation.objectCount, 10)
+      const lastCount = parseInt(lastObjectCount, 10)
+      const timeDiff = (Date.now() - lastUpdateTime) / 1000
+
+      if (timeDiff > 0 && currentCount > lastCount) {
+        rate = (currentCount - lastCount) / timeDiff
+        lastUpdateTime = Date.now()
+      }
+
+      lastObjectCount = operation.objectCount
+
+      const spinner = spinnerFrames[spinnerIndex] ?? '...'
+      if (onProgress) {
+        onProgress(operation.status, operation.objectCount, rate, spinner)
+      }
+      spinnerIndex = (spinnerIndex + 1) % spinnerFrames.length
+    }
+
     if (operation.status === 'COMPLETED') {
       if (!operation.url) {
         throw new Error('bulk operation completed but no results url')
       }
+      const totalTimeSeconds = (Date.now() - operationStartTime) / 1000
+      const totalObjects = parseInt(operation.objectCount, 10)
+      const averageRate = totalObjects / totalTimeSeconds
+
       const response = await fetch(operation.url)
-      return await response.text()
+      const content = await response.text()
+
+      return {
+        content,
+        totalObjects,
+        totalTimeSeconds,
+        averageRate,
+      }
     }
 
     if (operation.status === 'FAILED') {
       throw new Error(`bulk operation failed: ${operation.errorCode}`)
     }
   }
+  /* eslint-enable no-await-in-loop */
 }

packages/store/src/cli/services/graphql-parser.ts

@@ -1,10 +1,8 @@
-import {parse, OperationDefinitionNode, OperationTypeNode} from 'graphql'
+import {parse, OperationTypeNode} from 'graphql'
 
 export function parseGraphQLOperation(document: string): OperationTypeNode {
   const ast = parse(document)
-  const operation = ast.definitions.find(
-    (def) => def.kind === 'OperationDefinition',
-  ) as OperationDefinitionNode | undefined
+  const operation = ast.definitions.find((def) => def.kind === 'OperationDefinition')
 
   if (!operation) {
     throw new Error('no operation found in graphql document')