Merge pull request #414 from snyk-tech-services/feat/bulk-import-targ…

…et-files

feat: bulk import target file batches

src/lib/api/poll-import/index.ts

@@ -78,7 +78,7 @@ export async function pollImportUrl(
 export async function pollImportUrls(
   requestManager: requestsManager,
   locationUrls: string[],
-): Promise<{ projects: Project[] }> {
+): Promise<{ projects: Project[]; failed: FailedProject[] }> {
   if (!locationUrls) {
     throw new Error(
       'Missing required parameters. Please ensure you have provided: locationUrls.',
@@ -130,5 +130,5 @@ export async function pollImportUrls(
 
   await logFailedProjects(allFailedProjects);
 
-  return { projects: projectsArray };
+  return { projects: projectsArray, failed: allFailedProjects };
 }

src/lib/types.ts

@@ -251,3 +251,9 @@ export type SyncTargetsConfig = {
   manifestTypes?: string[];
   exclusionGlobs?: string[];
 };
+
+export enum ProjectUpdateType {
+  BRANCH = 'branch',
+  DEACTIVATE = 'deactivate',
+  IMPORT = 'import',
+}

src/loggers/log-failed-projects.ts

@@ -9,6 +9,7 @@ const debug = debugLib('snyk:import-projects-script');
 
 export interface FailedProject extends Project {
   locationUrl: string;
+  userMessage?: string;
 }
 
 export async function logFailedProjects(

src/scripts/sync/import-target.ts

@@ -2,11 +2,13 @@ import type { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 import { defaultExclusionGlobs } from '../../common';
 import { importTarget, listIntegrations, pollImportUrls } from '../../lib';
+
 import type {
   Project,
-  SupportedIntegrationTypesUpdateProject,
   Target,
+  SupportedIntegrationTypesUpdateProject,
 } from '../../lib/types';
+import type { FailedProject } from '../../loggers/log-failed-projects';
 
 const debug = debugLib('snyk:import-single-target');
 
@@ -18,7 +20,7 @@ export async function importSingleTarget(
   filesToImport: string[] = [],
   excludeFolders?: string,
   loggingPath?: string,
-): Promise<{ projects: Project[] }> {
+): Promise<{ projects: Project[]; failed: FailedProject[] }> {
   const integrationsData = await listIntegrations(requestManager, orgId);
   const integrationId = integrationsData[integrationType];
   const files = filesToImport.map((f) => ({ path: f }));

src/scripts/sync/sync-projects-per-target.ts

@@ -10,17 +10,15 @@ import type {
   RepoMetaData,
   SyncTargetsConfig,
 } from '../../lib/types';
+import { ProjectUpdateType } from '../../lib/types';
 import { SupportedIntegrationTypesUpdateProject } from '../../lib/types';
 import { targetGenerators } from '../generate-imported-targets-from-snyk';
 import { deactivateProject, listProjects } from '../../lib';
 import pMap = require('p-map');
 import { cloneAndAnalyze } from './clone-and-analyze';
+import { importSingleTarget } from './import-target';
 const debug = debugLib('snyk:sync-projects-per-target');
 
-export enum ProjectUpdateType {
-  BRANCH = 'branch',
-  DEACTIVATE = 'deactivate',
-}
 export function getMetaDataGenerator(
   origin: SupportedIntegrationTypesUpdateProject,
 ): (target: Target, host?: string | undefined) => Promise<RepoMetaData> {
@@ -272,3 +270,73 @@ export async function bulkDeactivateProjects(
 
   return { updated, failed };
 }
+
+export async function bulkImportTargetFiles(
+  requestManager: requestsManager,
+  orgId: string,
+  files: string[] = [],
+  integrationType: SupportedIntegrationTypesUpdateProject,
+  target: Target,
+  dryRun = false,
+  concurrentFilesImport = 30,
+): Promise<{ created: ProjectUpdate[]; failed: ProjectUpdateFailure[] }> {
+  const created: ProjectUpdate[] = [];
+  const failed: ProjectUpdateFailure[] = [];
+
+  if (!files.length) {
+    return { created, failed };
+  }
+  debug(`Importing ${files.length} files`);
+
+  if (dryRun) {
+    files.map((f) =>
+      created.push({
+        projectPublicId: '',
+        type: ProjectUpdateType.IMPORT,
+        from: f,
+        to: `https://app.snyk.io/org/example-org-name/project/example-project-id-uuid`,
+        dryRun,
+      }),
+    );
+
+    return { created, failed: [] };
+  }
+
+  for (
+    let index = 0;
+    index < files.length;
+    index = index + concurrentFilesImport
+  ) {
+    const batch = files.slice(index, index + concurrentFilesImport);
+
+    const { projects, failed: failedProjects } = await importSingleTarget(
+      requestManager,
+      orgId,
+      integrationType,
+      target,
+      batch,
+    );
+    projects.map((p) => {
+      const projectId = p.projectUrl?.split('/').slice(-1)[0];
+      created.push({
+        projectPublicId: projectId,
+        type: ProjectUpdateType.IMPORT,
+        from: p.targetFile ?? '', // TODO: is there something more intuitive here?
+        to: p.projectUrl,
+        dryRun,
+      });
+    });
+    failedProjects.map((f) => {
+      failed.push({
+        projectPublicId: '',
+        type: ProjectUpdateType.IMPORT,
+        from: f.targetFile ?? '', // TODO: is there something more intuitive here?
+        to: f.projectUrl,
+        dryRun,
+        errorMessage:
+          f.userMessage ?? 'Failed to import project via Snyk Import API',
+      });
+    });
+  }
+  return { created, failed };
+}