only import files to poto-project once #27

.vscode/launch.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.4.6",
+  "version": "0.5.0",
   "configurations": [
     {
       "name": "dev-prepare-dataset1",

README.md

@@ -58,7 +58,7 @@ Install [node](https://node.org) latest version and run:
 ```bash
 npm install -g poto-siril
 poto -v
-# Should print `poto-siril 0.4.6`.
+# Should print `poto-siril 0.5.0`.
 
 # Make sure to have `siril` registered in your PATH.
 siril -v

package.json

@@ -1,6 +1,6 @@
 {
   "name": "poto-siril",
-  "version": "0.4.6",
+  "version": "0.5.0",
   "description": "Automatization around Siril (<https://siril.org/>) for deep sky astrophotography.",
   "bin": {
     "poto": "dist/src/poto-siril.js"

src/commands/prepare.ts

@@ -180,16 +180,16 @@
 
   const autorunFiles = fs.existsSync(autorunDirectory)
     ? getFitsFromDirectory({
         directory: autorunDirectory,
         projectDirectory,
       })
     : [];
 
   const planFiles = fs.existsSync(planDirectory)
     ? getFitsFromDirectory({
         directory: planDirectory,
         projectDirectory,
       })
     : [];
 
   if (autorunFiles.length === 0 && planFiles.length === 0) {
@@ -449,12 +449,12 @@
   for (const lightsFlatsMatch of lightsFlatsMatches) {
     const lights = lightsFlatsMatch.isManualMatch
       ? allLights.filter(
           light =>
             light.sequenceId === lightsFlatsMatch.lightSequenceId &&
             light.setName === lightsFlatsMatch.lightSetName,
         )
       : allLights.filter(
           light => light.setName === lightsFlatsMatch.lightSetName,
         );
     if (!lights) {
       throw new Error(
@@ -744,14 +744,16 @@
 
   fs.writeFileSync(potoJsonPath, JSON.stringify(potoProject, null, 2));
 
+  let alreadyImported: FileImageSpec[] = [];
+
   for (const layerSet of potoProject.layerSets) {
     for (const file of [
       ...layerSet.lights,
       ...layerSet.darks,
       ...layerSet.flats,
       ...layerSet.biases,
     ]) {
-      copyFileToProject(file);
+      alreadyImported = copyFileToProject(file, alreadyImported);
     }
   }
 };

src/commands/preprocess.generate-scripts.ts

@@ -62,7 +62,7 @@ const generateScriptForLightSet = (
     fs.mkdirSync(mastersDirectory, { recursive: false });
   }
 
-  const lightsdirs = [...new Set(set.lights.map(x => x.projectDirectory))];
+  const lightsdirs = [...new Set(set.lights.map(x => x.projectFileDirectory))];
   if (lightsdirs.length === 0) {
     throw new Error("No lights found for filter " + set.layerSetId);
   }
@@ -71,7 +71,7 @@ const generateScriptForLightSet = (
   }
   const lightsDir = path.resolve(lightsdirs[0]);
 
-  const flatsDirs = [...new Set(set.flats.map(x => x.projectDirectory))];
+  const flatsDirs = [...new Set(set.flats.map(x => x.projectFileDirectory))];
   if (flatsDirs.length === 0) {
     throw new Error("No flats found for filter " + set.layerSetId);
   }
@@ -80,7 +80,7 @@ const generateScriptForLightSet = (
   }
   const flatsDir = path.resolve(flatsDirs[0]);
 
-  const darksDirs = [...new Set(set.darks.map(x => x.projectDirectory))];
+  const darksDirs = [...new Set(set.darks.map(x => x.projectFileDirectory))];
   if (darksDirs.length === 0) {
     throw new Error("No darks found for filter " + set.layerSetId);
   }
@@ -89,7 +89,7 @@ const generateScriptForLightSet = (
   }
   const darksDir = path.resolve(darksDirs[0]);
 
-  const biasesDirs = [...new Set(set.biases.map(x => x.projectDirectory))];
+  const biasesDirs = [...new Set(set.biases.map(x => x.projectFileDirectory))];
   if (biasesDirs.length === 0) {
     throw new Error("No biases found for filter " + set.layerSetId);
   }

src/utils/const.ts

@@ -1,4 +1,4 @@
-export const POTO_VERSION = "poto-siril 0.4.6";
+export const POTO_VERSION = "poto-siril 0.5.0";
 
 export const POTO_JSON = "_poto_siril.json";
 

src/utils/tests/__snapshots__/utils.spec.ts.snap

@@ -10,11 +10,12 @@ exports[`utils getFileImageSpecFromFilename should match snapshot (ASIAIR plan t
   "fileName": "Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001.fit",
   "filter": "H",
   "gain": 100,
-  "projectDirectory": "project/bar/H/Light_120.0s_Bin1_H_gain100",
+  "projectFileDirectory": "project/bar/H/Light_120.0s_Bin1_H_gain100",
+  "projectFilePath": "project/bar/H/Light_120.0s_Bin1_H_gain100/Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001.fit",
   "sequenceId": "20240707-002348",
   "sequencePosition": 1,
   "setName": "Light_120.0s_Bin1_H_gain100",
-  "sourceDirectory": "input/bar",
+  "sourceFileDirectory": "input/bar",
   "sourceFilePath": "input/bar/Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001.fit",
   "temperature": -10,
   "type": "Light",
@@ -31,11 +32,12 @@ exports[`utils getFileImageSpecFromFilename should match snapshot (filter named
   "fileName": "Light_LDN 1093_120.0s_Bin1_filter h_gain100_20240707-002348_-10.0C_0001.fit",
   "filter": "filterh",
   "gain": 100,
-  "projectDirectory": "project/bar/filterh/Light_120.0s_Bin1_filterh_gain100",
+  "projectFileDirectory": "project/bar/filterh/Light_120.0s_Bin1_filterh_gain100",
+  "projectFilePath": "project/bar/filterh/Light_120.0s_Bin1_filterh_gain100/Light_LDN 1093_120.0s_Bin1_filter h_gain100_20240707-002348_-10.0C_0001.fit",
   "sequenceId": "20240707-002348",
   "sequencePosition": 1,
   "setName": "Light_120.0s_Bin1_filterh_gain100",
-  "sourceDirectory": "input/bar",
+  "sourceFileDirectory": "input/bar",
   "sourceFilePath": "input/bar/Light_LDN 1093_120.0s_Bin1_filter h_gain100_20240707-002348_-10.0C_0001.fit",
   "temperature": -10,
   "type": "Light",
@@ -52,11 +54,12 @@ exports[`utils getFileImageSpecFromFilename should match snapshot (no filter) 1`
   "fileName": "Light_LDN 1093_120.0s_Bin1_gain100_20240707-002348_-10.0C_0001.fit",
   "filter": null,
   "gain": 100,
-  "projectDirectory": "project/bar/Light_120.0s_Bin1_gain100",
+  "projectFileDirectory": "project/bar/Light_120.0s_Bin1_gain100",
+  "projectFilePath": "project/bar/Light_120.0s_Bin1_gain100/Light_LDN 1093_120.0s_Bin1_gain100_20240707-002348_-10.0C_0001.fit",
   "sequenceId": "20240707-002348",
   "sequencePosition": 1,
   "setName": "Light_120.0s_Bin1_gain100",
-  "sourceDirectory": "input/bar",
+  "sourceFileDirectory": "input/bar",
   "sourceFilePath": "input/bar/Light_LDN 1093_120.0s_Bin1_gain100_20240707-002348_-10.0C_0001.fit",
   "temperature": -10,
   "type": "Light",

src/utils/tests/utils.spec.ts

@@ -16,7 +16,7 @@ describe("utils", () => {
       const file = new fs.Dirent();
       file.name =
         "Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001.fit";
-      file.path = "input/bar";
+      file.parentPath = "input/bar";
       const projectDirectory = "project/bar";
       const previousFile = null;
 
@@ -29,7 +29,7 @@ describe("utils", () => {
       const file = new fs.Dirent();
       file.name =
         "Light_LDN 1093_120.0s_Bin1_filter h_gain100_20240707-002348_-10.0C_0001.fit";
-      file.path = "input/bar";
+      file.parentPath = "input/bar";
       const projectDirectory = "project/bar";
       const previousFile = null;
 
@@ -46,7 +46,7 @@ describe("utils", () => {
       const file = new fs.Dirent();
       file.name =
         "Light_LDN 1093_120.0s_Bin1_gain100_20240707-002348_-10.0C_0001.fit";
-      file.path = "input/bar";
+      file.parentPath = "input/bar";
       const projectDirectory = "project/bar";
       const previousFile = null;
 
@@ -66,7 +66,7 @@ describe("utils", () => {
   ])("should parse bulb in ms (%s)", data => {
     const file = new fs.Dirent();
     file.name = `Light_LDN 1093_${data.bulb}_Bin1_gain100_20240707-002348_-10.0C_0001.fit`;
-    file.path = "input/bar";
+    file.parentPath = "input/bar";
     const projectDirectory = "project/bar";
     const previousFile = null;
 
@@ -84,17 +84,17 @@ describe("utils", () => {
         Object.assign(new fs.Dirent(), {
           name: "Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001.fit",
           isFile: () => true,
-          path: "input/dir",
+          parentPath: "input/dir",
         }),
         Object.assign(new fs.Dirent(), {
           name: "Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002349_-10.0C_0002.fit",
           isFile: () => true,
-          path: "input/dir",
+          parentPath: "input/dir",
         }),
         Object.assign(new fs.Dirent(), {
           name: "Light_LDN 1093_120.0s_Bin1_H_gain100_20240707-002348_-10.0C_0001_thn.jpg",
           isFile: () => true,
-          path: "input/dir",
+          parentPath: "input/dir",
         }),
       ]);
 
@@ -116,16 +116,19 @@ describe("utils", () => {
   describe("copyFileToProject", () => {
     it("should copy a file to the project directory", () => {
       const file = {
-        projectDirectory: "project/dir",
         fileName: "file1.fit",
+        projectFileDirectory: "project/dir",
+        projectFilePath: "project/dir/file1.fit",
         sourceFilePath: "input/dir/file1.fit",
       } as FileImageSpec;
 
       jest.spyOn(fs, "existsSync").mockReturnValue(false);
       jest.spyOn(fs, "mkdirSync");
       jest.spyOn(fs, "copyFileSync").mockImplementation(() => {});
 
-      copyFileToProject(file);
+      let alreadyImported: FileImageSpec[] = [];
+
+      alreadyImported = copyFileToProject(file, alreadyImported);
 
       expect(fs.existsSync).toHaveBeenCalledWith("project/dir");
       expect(fs.mkdirSync).toHaveBeenCalledWith("project/dir", {
@@ -135,6 +138,16 @@ describe("utils", () => {
         "input/dir/file1.fit",
         "project/dir/file1.fit",
       );
+      expect(alreadyImported).toMatchInlineSnapshot(`
+[
+  {
+    "fileName": "file1.fit",
+    "projectFileDirectory": "project/dir",
+    "projectFilePath": "project/dir/file1.fit",
+    "sourceFilePath": "input/dir/file1.fit",
+  },
+]
+`);
     });
   });
 

src/utils/types.ts

@@ -47,18 +47,22 @@ export type FileImageSpec = ImageSpec & {
   extension: string;
 
   /**
-   * Path to the ASIAIR dump directory.
+   * Directory path from the source input directory.
    */
-  sourceDirectory: string;
+  sourceFileDirectory: string;
   /**
-   * Path to the file in the ASIAIR dump directory.
+   * File path file from the source input directory.
    */
   sourceFilePath: string;
 
   /**
    * Path to the project directory.
    */
-  projectDirectory: string;
+  projectFileDirectory: string;
+  /**
+   * File path in the project directory.
+   */
+  projectFilePath: string;
 
   /**
    * Recorded time.

src/utils/utils.ts

@@ -40,7 +40,7 @@ export const getFitsFromDirectory = ({
     if (
       file.name === ".DS_Store" ||
       file.name.startsWith("._") ||
-      file.path.includes("._")
+      file.parentPath.includes("._")
     ) {
       return;
     }
@@ -120,25 +120,25 @@ export const getFileImageSpecFromFilename = (
       fileName: fileFS.name,
       extension: match.groups.extension,
 
-      sourceDirectory: fileFS.path,
-      sourceFilePath: path.join(fileFS.path, fileFS.name),
-
-      projectDirectory,
+      sourceFileDirectory: fileFS.parentPath,
+      sourceFilePath: path.join(fileFS.parentPath, fileFS.name),
     } as FileImageSpec;
 
     file.setName = getSetName(file);
 
     if (["Light", "Flat"].includes(file.type)) {
       const directory = file.setName;
 
-      file.projectDirectory = file.filter
+      file.projectFileDirectory = file.filter
         ? path.join(projectDirectory, file.filter, directory)
         : path.join(projectDirectory, directory);
     } else {
       const directory = `${file.type}_${file.bulb}_${file.bin}_gain${file.gain}`;
-      file.projectDirectory = path.join(projectDirectory, "any", directory); // We ignore the filter for darks and biases.
+      file.projectFileDirectory = path.join(projectDirectory, "any", directory); // We ignore the filter for darks and biases.
     }
 
+    file.projectFilePath = path.join(file.projectFileDirectory, file.fileName);
+
     return file;
   } else {
     throw new Error(
@@ -200,15 +200,24 @@ const parseBulbString = (bulbString: string): number => {
   }
 };
 
-export const copyFileToProject = (file: FileImageSpec) => {
-  if (!fs.existsSync(file.projectDirectory)) {
-    fs.mkdirSync(file.projectDirectory, { recursive: true });
+export const copyFileToProject = (
+  file: FileImageSpec,
+  alreadyImported: FileImageSpec[],
+): FileImageSpec[] => {
+  if (!fs.existsSync(file.projectFileDirectory)) {
+    fs.mkdirSync(file.projectFileDirectory, { recursive: true });
   }
 
-  const targetFile = path.join(file.projectDirectory, file.fileName);
-  fs.copyFileSync(file.sourceFilePath, targetFile);
+  if (alreadyImported.find(f => f.fileName === file.fileName)) {
+    logger.debug(`- ${file.fileName} already imported.`);
+  } else {
+    fs.copyFileSync(file.sourceFilePath, file.projectFilePath);
+    alreadyImported.push(file);
+
+    logger.debug(`- ${file.fileName} imported.`);
+  }
 
-  logger.debug(`- ${file.fileName} dispatched.`);
+  return alreadyImported;
 };
 
 /**