Merge pull request #566 from justinlampley/targetRepo

Support Separate Targets Repo

graphql.schema.json

@@ -1055,6 +1055,18 @@
         "description": null,
         "fields": null,
         "inputFields": [
+          {
+            "name": "hardwareArtifactUrl",
+            "description": null,
+            "type": {
+              "kind": "SCALAR",
+              "name": "String",
+              "ofType": null
+            },
+            "defaultValue": null,
+            "isDeprecated": false,
+            "deprecationReason": null
+          },
           {
             "name": "owner",
             "description": null,

src/api/index.ts

@@ -138,10 +138,16 @@ export default class ApiServer {
         targetsLoader
       );
 
+    const targetStorageGitPath = path.join(
+      config.userDataPath,
+      'firmwares',
+      'binary-targets'
+    );
+
     const deviceDescriptionsLoader = new DeviceDescriptionsLoader(
       logger,
       config.PATH,
-      path.join(config.userDataPath, 'firmwares', 'binary-targets'),
+      targetStorageGitPath,
       path.join(config.userDataPath, 'firmwares', 'device-options')
     );
     const cloudBinariesCache = new CloudBinariesCache(
@@ -158,6 +164,7 @@ export default class ApiServer {
       firmwareBuilder,
       deviceDescriptionsLoader,
       cloudBinariesCache,
+      targetStorageGitPath,
       logger
     );
 

src/api/src/graphql/inputs/GitRepositoryInput.ts

@@ -17,17 +17,22 @@ export default class GitRepository {
   @Field()
   srcFolder: string;
 
+  @Field({ nullable: true })
+  hardwareArtifactUrl: string | null;
+
   constructor(
     url: string,
     owner: string,
     repositoryName: string,
     rawRepoUrl: string,
-    srcFolder: string
+    srcFolder: string,
+    hardwareArtifactUrl: string | null
   ) {
     this.url = url;
     this.owner = owner;
     this.repositoryName = repositoryName;
     this.rawRepoUrl = rawRepoUrl;
     this.srcFolder = srcFolder;
+    this.hardwareArtifactUrl = hardwareArtifactUrl;
   }
 }

src/api/src/library/AmazonS3/index.ts

@@ -0,0 +1,29 @@
+import fetch, { Headers } from 'node-fetch';
+import { writeFile } from 'fs/promises';
+import fs from 'fs';
+
+export default class AmazonS3 {
+  async downloadIfModified(url: string, outputFile: string): Promise<boolean> {
+    const headers = new Headers();
+    if (fs.existsSync(outputFile)) {
+      const stat = await fs.promises.stat(outputFile);
+      headers.append('If-Modified-Since', stat.mtime.toUTCString());
+    }
+
+    const response = await fetch(url, { headers });
+
+    if (response.status === 304) {
+      return false;
+    }
+
+    if (response.status === 200) {
+      const buffer = await response.buffer();
+      await writeFile(outputFile, buffer);
+      return true;
+    }
+
+    throw new Error(
+      `Error encountered while retrieving ${url}, ${response.status} - ${response.statusText}`
+    );
+  }
+}

src/api/src/services/BinaryFlashingStrategy/BinaryConfigurator/index.ts

@@ -36,12 +36,18 @@ export default class BinaryConfigurator {
   buildBinaryConfigFlags(
     outputDirectory: string,
     firmwareBinaryPath: string,
-    hardwareDefinitionsPath: string,
+    hardwareDefinitionsPath: string | null,
+    firmwareArtifactsDirPath: string,
     params: BuildFlashFirmwareParams
   ): string[][] {
     const flags: string[][] = [];
 
-    flags.push(['--dir', hardwareDefinitionsPath]);
+    if (hardwareDefinitionsPath && firmwareArtifactsDirPath) {
+      flags.push(['--dir', hardwareDefinitionsPath]);
+      flags.push(['--fdir', firmwareArtifactsDirPath]);
+    } else {
+      flags.push(['--dir', firmwareArtifactsDirPath]);
+    }
 
     const [manufacturer, subType, device, uploadMethod] =
       params.target.split('.');

src/api/src/services/BinaryFlashingStrategy/CloudBinariesCache/index.ts

@@ -1,9 +1,8 @@
-import fetch from 'node-fetch';
-import { writeFile } from 'fs/promises';
 import extractZip from 'extract-zip';
 import path from 'path';
 import mkdirp from 'mkdirp';
-import fs from 'fs';
+import { removeDirectoryContents } from '../../FlashingStrategyLocator/artefacts';
+import AmazonS3 from '../../../library/AmazonS3';
 
 export default class CloudBinariesCache {
   constructor(private baseURL: string, private firmwareCachePath: string) {}
@@ -17,28 +16,19 @@ export default class CloudBinariesCache {
     await mkdirp(workingDir);
     const firmwareCacheDir = path.join(workingDir, 'firmware');
 
-    // if we have firmware in our local cache already no need to download it the second time.
-    if (fs.existsSync(firmwareCacheDir)) {
-      return firmwareCacheDir;
-    }
-
     const cacheUrl = `${this.baseURL}/${repositoryName}/${commitHash}/firmware.zip`;
-    const response = await fetch(cacheUrl);
-    console.log(response);
-    if (response.status !== 200) {
-      throw new Error(
-        `cached build for ${repositoryName}/${commitHash} is not available`
-      );
-    }
-
     const outputZipFile = path.join(workingDir, 'firmware.zip');
-    const buffer = await response.buffer();
-    await writeFile(outputZipFile, buffer);
 
-    await extractZip(outputZipFile, {
-      dir: workingDir,
-    });
+    if (await new AmazonS3().downloadIfModified(cacheUrl, outputZipFile)) {
+      await extractZip(outputZipFile, {
+        dir: workingDir,
+      });
+    }
 
     return firmwareCacheDir;
   }
+
+  async clearCache() {
+    await removeDirectoryContents(this.firmwareCachePath);
+  }
 }

src/api/src/services/BinaryFlashingStrategy/DeviceDescriptionsLoader/index.ts

@@ -1,5 +1,8 @@
+import extractZip from 'extract-zip';
 import { Service } from 'typedi';
 import path from 'path';
+import mkdirp from 'mkdirp';
+import semver from 'semver';
 import FirmwareSource from '../../../models/enum/FirmwareSource';
 import TargetArgs from '../../../graphql/args/Target';
 import { LoggerService } from '../../../logger';
@@ -19,10 +22,12 @@ import TargetUserDefinesFactory from '../../../factories/TargetUserDefinesFactor
 import UserDefineKey from '../../../library/FirmwareBuilder/Enum/UserDefineKey';
 import PullRequest from '../../../models/PullRequest';
 import { removeDirectoryContents } from '../../FlashingStrategyLocator/artefacts';
+import AmazonS3 from '../../../library/AmazonS3';
 
 export interface GitRepository {
   url: string;
   srcFolder: string;
+  hardwareArtifactUrl: string | null;
 }
 
 export interface FirmwareVersion {
@@ -125,7 +130,17 @@ export default class DeviceDescriptionsLoader {
     );
     const devices: Device[] = [];
     Object.keys(data).forEach((id) => {
-      devices.push(this.configToDevice(id, data[id].category, data[id].config));
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      const { min_version } = data[id].config;
+      if (
+        !min_version ||
+        args.source !== FirmwareSource.GitTag ||
+        semver.gte(args.gitTag, min_version)
+      ) {
+        devices.push(
+          this.configToDevice(id, data[id].category, data[id].config)
+        );
+      }
     });
     return devices;
   }
@@ -149,6 +164,25 @@ export default class DeviceDescriptionsLoader {
       gitPath,
     });
 
+    if (gitRepository.hardwareArtifactUrl) {
+      const workingDir = path.join(gitRepositoryPath, 'hardware');
+      await mkdirp(workingDir);
+      const outputZipFile = path.join(workingDir, 'hardware.zip');
+
+      if (
+        await new AmazonS3().downloadIfModified(
+          gitRepository.hardwareArtifactUrl,
+          outputZipFile
+        )
+      ) {
+        await extractZip(outputZipFile, {
+          dir: workingDir,
+        });
+      }
+
+      return workingDir;
+    }
+
     const firmwareDownload = new GitFirmwareDownloader(
       {
         baseDirectory: gitRepositoryPath,
@@ -339,14 +373,14 @@ export default class DeviceDescriptionsLoader {
   async clearCache() {
     await this.targetsMutex.tryLockWithTimeout(60000);
     try {
-      return await removeDirectoryContents(this.targetStorageGitPath);
+      await removeDirectoryContents(this.targetStorageGitPath);
     } finally {
       this.targetsMutex.unlock();
     }
 
     await this.deviceOptionsMutex.tryLockWithTimeout(60000);
     try {
-      return await removeDirectoryContents(this.deviceOptionsGitPath);
+      await removeDirectoryContents(this.deviceOptionsGitPath);
     } finally {
       this.deviceOptionsMutex.unlock();
     }

src/api/src/services/BinaryFlashingStrategy/TargetsJSONLoader/index.ts

@@ -15,6 +15,7 @@ export interface DeviceDescription {
   overlay?: { [key: string]: string };
   frequency?: number;
   type?: string;
+  min_version?: string;
 }
 
 export type TargetsJSONRaw = {

src/api/src/services/BinaryFlashingStrategy/index.ts

@@ -62,6 +62,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
     private builder: FirmwareBuilder,
     private deviceDescriptionsLoader: DeviceDescriptionsLoader,
     private cloudBinariesCache: CloudBinariesCache,
+    private targetStorageGitPath: string,
     private logger: LoggerService
   ) {
     this.mutex = new Mutex();
@@ -473,6 +474,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
         {
           url: gitRepositoryUrl,
           srcFolder: gitRepositorySrcFolder,
+          hardwareArtifactUrl: gitRepository.hardwareArtifactUrl,
         }
       );
 
@@ -481,7 +483,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
       const workingDirectory = await this.createWorkingDirectory(params.target);
       const outputDirectory = await this.createWorkingDirectory(params.target);
       let firmwareBinFile = '';
-      let hardwareDescriptionsPath = firmwareSourcePath;
+      let firmwareDescriptionsPath = firmwareSourcePath;
       let flasherPath = path.join(
         firmwareSourcePath,
         'python',
@@ -510,13 +512,13 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
           if (fs.existsSync(flasherPyzPath)) {
             flasherPath = flasherPyzPath;
           }
-          hardwareDescriptionsPath = cacheLocation;
+          firmwareDescriptionsPath = cacheLocation;
           this.logger.log('paths', {
             cacheLocation,
             cachedBinaryPath,
             flasherPyzPath,
             sourceFirmwareBinaryPath: sourceFirmwareBinPath,
-            hardwareDescriptionsPath,
+            hardwareDescriptionsPath: firmwareDescriptionsPath,
           });
         } catch (e) {
           this.logger.log(
@@ -530,7 +532,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
       }
 
       // we were not able to find cloud binaries, so we will build them on the spot
-      if (hardwareDescriptionsPath === firmwareSourcePath) {
+      if (firmwareDescriptionsPath === firmwareSourcePath) {
         const target = this.getCompileTarget(config);
         sourceFirmwareBinPath = await this.compileBinary(
           target,
@@ -557,13 +559,24 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
         BuildFirmwareStep.BUILDING_FIRMWARE
       );
 
-      const flasherArgs: string[][] =
-        this.binaryConfigurator.buildBinaryConfigFlags(
+      let flasherArgs: string[][];
+      if (gitRepository.hardwareArtifactUrl) {
+        flasherArgs = this.binaryConfigurator.buildBinaryConfigFlags(
           outputDirectory,
           firmwareBinFile,
-          hardwareDescriptionsPath,
+          this.targetStorageGitPath,
+          firmwareDescriptionsPath,
           params
         );
+      } else {
+        flasherArgs = this.binaryConfigurator.buildBinaryConfigFlags(
+          outputDirectory,
+          firmwareBinFile,
+          null,
+          firmwareDescriptionsPath,
+          params
+        );
+      }
       const binaryConfiguratorResult = await this.binaryConfigurator.run(
         flasherPath,
         flasherArgs,
@@ -629,10 +642,11 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
 
   async clearFirmwareFiles(): Promise<void> {
     await this.deviceDescriptionsLoader.clearCache();
+    await this.cloudBinariesCache.clearCache();
 
     this.logger.log('BinaryConfigurator - clearFirmwareFiles', {
       firmwaresPath: this.firmwaresPath,
     });
-    return removeDirectoryContents(this.firmwaresPath);
+    await removeDirectoryContents(this.firmwaresPath);
   }
 }

src/api/src/services/FlashingStrategyLocator/artefacts.ts

@@ -110,7 +110,7 @@ export const removeDirectoryContents = async (firmwaresPath: string) => {
     return;
   }
   const files = await listFiles(firmwaresPath);
-  if (files.length > 4) {
+  if (files.length > 20) {
     throw new Error(`unexpected number of files to remove: ${files}`);
   }
   await Promise.all(files.map((item) => rmrf(item)));

src/api/src/services/PlatformioFlashingStrategy/index.ts

@@ -424,6 +424,7 @@ export default class PlatformioFlashingStrategyService
     this.logger.log('PlatformioConfigurator - clearFirmwareFiles', {
       firmwaresPath: this.firmwaresPath,
     });
+    await this.userDefinesBuilder.clearFiles();
     return removeDirectoryContents(this.firmwaresPath);
   }
 }

src/api/src/services/UserDefinesBuilder/index.ts

@@ -58,4 +58,8 @@ export default class UserDefinesBuilder {
       return compatibleKeys.indexOf(userDefine.key) > -1;
     });
   }
+
+  async clearFiles() {
+    await this.userDefinesLoader.clearFiles();
+  }
 }