Merge pull request #19 from garzj/full-migration

Full structured migration

.gitignore

@@ -6,6 +6,7 @@ error/
 
 # Build
 lib/
+build/
 
 # Env files
 .env

Dockerfile

@@ -0,0 +1,13 @@
+# we need some image and video processing tools too, so use Debian 12 "Bookworm" as the base so we can get them.
+FROM docker.io/node:bookworm
+
+# get our image and video processing dependencies
+RUN apt-get update && apt-get install -y ffmpeg exiftool
+
+COPY . /app
+
+WORKDIR /app
+# build the application
+RUN yarn && yarn build
+
+ENTRYPOINT [ "yarn", "start" ]

README.md

@@ -12,13 +12,109 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos
 
 ## Run this tool
 
+### Natively
+
+**Prerec**: Must have at least node 18 & yarn installed.
+
+#### Full structured migration
+
+If you wish to migrate an entire takeout folder (and keep the album directory structure):
+
 ```bash
 mkdir output error
 
-npx google-photos-migrate@latest '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000
+npx google-photos-migrate@latest full '/path/to/takeout' './output' './error' --timeout 60000
+```
+
+The folder names in the `output` and `error` directories will now correspond to the original album names.
+
+#### Flat migration
+
+If you wish to migrate your Google Photos folder into a flat directory (and don't care about albums):
+
+```bash
+mkdir output error
+
+npx google-photos-migrate@latest flat '/path/to/takeout/Google Photos' './output' './error' --timeout 60000
+```
+
+#### Optional flags (see `--help` for all details):
+
+```
+--timeout integer
+    Shorthand: -t integer
+    Meaning: Sets the timeout for exiftool, default is 30000 (30s)
+--force
+    Shorthand: -f
+    Meaning: Forces the migration and overwrites files in the target directory.
+```
+
+### Docker
+
+**Prerec:** You must have a working `docker` or `podman` install.
+
+A Dockerfile is also provided to make running this tool easier on most hosts. The image must be built manually (see below), no pre-built images are provided. Using it will by default use only software-based format conversion, hardware accelerated format conversion is beyond these instructions.
+
+Build the image once before you run it:
+
+```shell
+git clone https://github.com/garzj/google-photos-migrate
+cd google-photos-migrate
+
+# build the image
+docker build -f Dockerfile -t localhost/google-photos-migrate:latest .
+```
+
+To run the full migration:
+
+```shell
+mkdir output error
+docker run --rm -it -security-opt=label=disable \
+    -v $(readlink -e path/to/takeout):/takeout \
+    -v $(readlink -e ./output):/output \
+    -v $(readlink -e ./error):/error \
+   localhost/google-photos-migrate:latest \
+     fullMigrate '/takeout' '/output' '/error' --timeout=60000
+```
+
+To run the flat migration:
+
+```shell
+mkdir output error
+docker run --rm -it --security-opt=label=disable \
+    -v $(readlink -e path/to/takeout):/takeout \
+    -v $(readlink -e ./output):/output \
+    -v $(readlink -e ./error):/error \
+   localhost/google-photos-migrate:latest \
+     flat '/takeout/Google Fotos' '/output' '/error' --timeout=60000
+```
+
+All other options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container.
+
+For the overall help:
+
+```shell
+docker run --rm -it --security-opt=label=disable \
+   localhost/google-photos-migrate:latest \
+     --help
 ```
 
 ## Further steps
 
 - If you use Linux + Android, you might want to check out the scripts I used to locate duplicate media and keep the better versions in the [android-dups](./android-dups/) directory.
 - Use a tool like [Immich](https://github.com/immich-app/immich) and upload your photos
+
+## Development
+
+**Prerec**: Must have node 18 & yarn installed.
+
+To test the app:
+
+```bash
+git clone https://github.com/garzj/google-photos-migrate
+cd google-photos-migrate
+yarn
+yarn dev <subcommand>
+```
+
+The entrypoint of the cli is in `src/cli.ts` and library code should be exported from `src/index.ts`.

package.json

@@ -32,7 +32,12 @@
   "dependencies": {
     "cmd-ts": "^0.12.1",
     "exiftool-vendored": "^22.0.0",
-    "sanitize-filename": "^1.6.3"
+    "glob": "^10.3.3",
+    "sanitize-filename": "^1.6.3",
+    "ts-node": "^10.9.1"
+  },
+  "resolutions": {
+    "string-width-cjs": "5.1.1"
   },
   "keywords": [
     "google",

src/cli.ts

@@ -1,102 +1,14 @@
 #!/usr/bin/env node
 
-import { existsSync } from 'fs';
-import { command, run, string, positional, flag, number, option } from 'cmd-ts';
-import { migrateGoogleDirGen } from './media/migrate-google-dir';
-import { isEmptyDir } from './fs/is-empty-dir';
-import { ExifTool } from 'exiftool-vendored';
+import { subcommands, run } from 'cmd-ts';
+import { migrateFull } from './commands/migrate-full';
+import { migrateFlat } from './commands/migrate-flat';
 
-const app = command({
+const app = subcommands({
   name: 'google-photos-migrate',
-  args: {
-    googleDir: positional({
-      type: string,
-      displayName: 'google_dir',
-      description: 'The path to your "Google Photos" directory.',
-    }),
-    outputDir: positional({
-      type: string,
-      displayName: 'output_dir',
-      description: 'The path to your flat output directory.',
-    }),
-    errorDir: positional({
-      type: string,
-      displayName: 'error_dir',
-      description: 'Failed media will be saved here.',
-    }),
-    force: flag({
-      short: 'f',
-      long: 'force',
-      description:
-        "Forces the operation if the given directories aren't empty.",
-    }),
-    timeout: option({
-      type: number,
-      defaultValue: () => 30000,
-      short: 't',
-      long: 'timeout',
-      description:
-        'Sets the task timeout in milliseconds that will be passed to ExifTool.',
-    }),
-  },
-  handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => {
-    const errs: string[] = [];
-    if (!existsSync(googleDir)) {
-      errs.push('The specified google directory does not exist.');
-    }
-    if (!existsSync(outputDir)) {
-      errs.push('The specified output directory does not exist.');
-    }
-    if (!existsSync(errorDir)) {
-      errs.push('The specified error directory does not exist.');
-    }
-    if (errs.length !== 0) {
-      errs.forEach((e) => console.error(e));
-      process.exit(1);
-    }
-
-    if (!force && !(await isEmptyDir(outputDir))) {
-      errs.push(
-        'The output directory is not empty. Pass "-f" to force the operation.'
-      );
-    }
-    if (!force && !(await isEmptyDir(errorDir))) {
-      errs.push(
-        'The error directory is not empty. Pass "-f" to force the operation.'
-      );
-    }
-    if (await isEmptyDir(googleDir)) {
-      errs.push('The google directory is empty. Nothing to do.');
-    }
-    if (errs.length !== 0) {
-      errs.forEach((e) => console.error(e));
-      process.exit(1);
-    }
-
-    console.log(`Started migration.`);
-    const migGen = migrateGoogleDirGen({
-      googleDir,
-      outputDir,
-      errorDir,
-      warnLog: console.error,
-      exiftool: new ExifTool({ taskTimeoutMillis: timeout }),
-      endExifTool: true,
-    });
-
-    const counts = { err: 0, suc: 0 };
-    for await (const result of migGen) {
-      if (result instanceof Error) {
-        console.error(`Error: ${result}`);
-        counts.err++;
-        continue;
-      }
-
-      counts.suc++;
-    }
-
-    console.log(`Done! Processed ${counts.suc + counts.err} files.`);
-    console.log(`Files migrated: ${counts.suc}`);
-    console.log(`Files failed: ${counts.err}`);
+  cmds: {
+    full: migrateFull,
+    flat: migrateFlat,
   },
 });
 

src/commands/common.ts

@@ -0,0 +1,22 @@
+import { flag, number, option, positional, string } from 'cmd-ts';
+
+export const errorDirArg = positional({
+  type: string,
+  displayName: 'error_dir',
+  description: 'Failed media will be saved here.',
+});
+
+export const timeoutArg = option({
+  type: number,
+  defaultValue: () => 30000,
+  short: 't',
+  long: 'timeout',
+  description:
+    'Sets the task timeout in milliseconds that will be passed to ExifTool.',
+});
+
+export const forceArg = flag({
+  short: 'f',
+  long: 'force',
+  description: "Forces the operation if the given directories aren't empty.",
+});

src/commands/migrate-flat.ts

@@ -0,0 +1,86 @@
+import { command, string, positional } from 'cmd-ts';
+import { isEmptyDir } from '../fs/is-empty-dir';
+import { fileExists } from '../fs/file-exists';
+import { errorDirArg, forceArg, timeoutArg } from './common';
+import { migrateDirFlatGen } from '../dir/migrate-flat';
+import { ExifTool } from 'exiftool-vendored';
+import { MediaMigrationError } from '../media/MediaMigrationError';
+
+export const migrateFlat = command({
+  name: 'google-photos-migrate-flat',
+  args: {
+    inputDir: positional({
+      type: string,
+      displayName: 'input_dir',
+      description: 'The path to your "Google Photos" directory.',
+    }),
+    outputDir: positional({
+      type: string,
+      displayName: 'output_dir',
+      description: 'The path to your flat output directory.',
+    }),
+    errorDir: errorDirArg,
+    force: forceArg,
+    timeout: timeoutArg,
+  },
+  handler: async ({ inputDir, outputDir, errorDir, force, timeout }) => {
+    const errs: string[] = [];
+    const checkErrs = () => {
+      if (errs.length !== 0) {
+        errs.forEach((e) => console.error(e));
+        process.exit(1);
+      }
+    };
+
+    if (!(await fileExists(inputDir))) {
+      errs.push(`The specified google directory does not exist: ${inputDir}`);
+    }
+    if (!(await fileExists(outputDir))) {
+      errs.push(`The specified output directory does not exist: ${inputDir}`);
+    }
+    if (!(await fileExists(errorDir))) {
+      errs.push(`The specified error directory does not exist: ${inputDir}`);
+    }
+    checkErrs();
+
+    if (!force && !(await isEmptyDir(outputDir))) {
+      errs.push(
+        'The output directory is not empty. Pass "-f" to force the operation.'
+      );
+    }
+    if (!force && !(await isEmptyDir(errorDir))) {
+      errs.push(
+        'The error directory is not empty. Pass "-f" to force the operation.'
+      );
+    }
+    if (await isEmptyDir(inputDir)) {
+      errs.push(`Nothing to do, the source directory is empty: ${inputDir}`);
+    }
+    checkErrs();
+
+    console.log('Started migration.');
+    const migGen = migrateDirFlatGen({
+      inputDir,
+      outputDir,
+      errorDir,
+      log: console.log,
+      warnLog: console.error,
+      exiftool: new ExifTool({ taskTimeoutMillis: timeout }),
+      endExifTool: true,
+    });
+    const counts = { err: 0, suc: 0 };
+    for await (const result of migGen) {
+      if (result instanceof MediaMigrationError) {
+        console.error(`Error: ${result}`);
+        counts.err++;
+        continue;
+      } else {
+        counts.suc++;
+      }
+    }
+
+    console.log(`Done! Processed ${counts.suc + counts.err} files.`);
+    console.log(`Files migrated: ${counts.suc}`);
+    console.log(`Files failed: ${counts.err}`);
+  },
+});