From d2f95f71f4f5ff9e178c3eebc599d48b948ef19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Groven?= Date: Mon, 31 Jul 2023 10:36:34 +0200 Subject: [PATCH 01/50] Update suffix for french --- src/meta/find-meta-file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/find-meta-file.ts b/src/meta/find-meta-file.ts index 00508e6..ebef34c 100644 --- a/src/meta/find-meta-file.ts +++ b/src/meta/find-meta-file.ts @@ -50,7 +50,7 @@ export async function findMetaFile( }; let base = mediaPath.slice(0, mediaPath.length - ext.suffix.length); - base = base.replace(/-(edited|bearbeitet)$/i, ''); + base = base.replace(/-(edited|bearbeitet|modifié)$/i, ''); const potExts: string[] = []; From d69dffa096cff2a9c668771952d3d7373dbab493 Mon Sep 17 00:00:00 2001 From: covalent Date: Wed, 30 Aug 2023 16:46:44 -0400 Subject: [PATCH 02/50] added full auto migration functionality --- package.json | 3 +- src/cli.ts | 287 ++++++++++++++++++++++++++++++++++++++++++++++----- yarn.lock | 197 ++++++++++++++++++++++++++++++++++- 3 files changed, 456 insertions(+), 31 deletions(-) diff --git a/package.json b/package.json index 6cf1ac6..3e1c053 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "dotenv": "^16.1.4", "exiftool-vendored": "^22.0.0", "sanitize-filename": "^1.6.3", - "ts-node": "^10.9.1" + "ts-node": "^10.9.1", + "glob": "^10.13.3" } } diff --git a/src/cli.ts b/src/cli.ts index 4f10a2f..0630e21 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,11 +1,259 @@ -import { existsSync } from 'fs'; -import { command, run, string, positional, flag, number, option } from 'cmd-ts'; +import { existsSync, mkdirSync, rename } from 'fs'; +import { + command, + subcommands, + run, + string, + positional, + flag, + number, + option, +} from 'cmd-ts'; import { migrateGoogleDirGen } from './media/migrate-google-dir'; import { isEmptyDir } from './fs/is-empty-dir'; import { ExifTool } from 'exiftool-vendored'; +import { glob } from 'glob'; +import { basename } from 'path'; +import path = require('path'); -const app = command({ - name: 'google-photos-migrate', +const unzip = command({ + name: 'unzipper', + args: {}, + handler: async () => {}, +}); + +async function run_basic_migration( + googleDir: string, + outputDir: string, + errorDir: string, + exifTool: ExifTool +) { + console.log(`Started migration.`); + const migGen = migrateGoogleDirGen({ + googleDir, + outputDir, + errorDir, + warnLog: console.error, + exiftool: exifTool, + endExifTool: true, + }); + + const counts = { err: 0, suc: 0 }; + for await (const result of migGen) { + if (result instanceof Error) { + console.error(`Error: ${result}`); + counts.err++; + continue; + } + + counts.suc++; + } + + console.log(`Done! Processed ${counts.suc + counts.err} files.`); + console.log(`Files migrated: ${counts.suc}`); + console.log(`Files failed: ${counts.err}`); +} + +async function run_migrations_checked( + albumDir: string, + outDir: string, + errDir: string, + timeout: number, + exifTool: ExifTool, + check_errDir: boolean +) { + const errs: string[] = []; + if (!existsSync(albumDir)) { + errs.push('The specified google directory does not exist.'); + } + if (!existsSync(outDir)) { + errs.push('The specified output directory does not exist.'); + } + if (!existsSync(errDir)) { + errs.push('The specified error directory does not exist.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + await run_basic_migration(albumDir, outDir, errDir, exifTool); + + if (check_errDir && !(await isEmptyDir(errDir))) { + const errFiles: string[] = await glob(`${errDir}/*`); + for (let file of errFiles) { + exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + } + await run_migrations_checked( + albumDir, + outDir, + errDir, + timeout, + exifTool, + false + ); + } +} + +async function process_albums( + rootDir: string, + timeout: number, + exifTool: ExifTool +) { + const globStr: string = `${rootDir}/Albums/*/`; + const albums: string[] = await glob(globStr); + if (albums.length == 0) { + console.log(`WARN: No albums found at ${globStr}`); + } + for (let album of albums) { + console.log(`Processing album ${album}...`); + let albumName = basename(album); + let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; + let errDir = `${rootDir}/AlbumsError/${albumName}`; + await run_migrations_checked( + album, + outDir, + errDir, + timeout, + exifTool, + true + ); + } +} + +async function process_photos( + rootDir: string, + timeout: number, + exifTool: ExifTool +) { + // Also run the exif fix for the photos + console.log('Processing photos...'); + const albumDir = `${rootDir}/Photos`; + const outDir = `${rootDir}/PhotosProcessed`; + const errDir = `${rootDir}/PhotosError`; + + await run_migrations_checked( + albumDir, + outDir, + errDir, + timeout, + exifTool, + true + ); +} + +async function _restructure_if_needed(folders: string[], targetDir: string) { + if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { + console.log( + `${targetDir} exists and is non-empty, assuming no further restructing needed` + ); + return; + } + mkdirSync(targetDir); + if (folders.length == 0) { + console.log(`Warning: no folders were moved to ${targetDir}`); + } + for (let folder of folders) { + rename(folder, path.join(targetDir, folder), function (err) { + if (err) throw err; + console.log('Successfully moved!'); + }); + } + console.log(`Restructured ${folders.length} folders`); +} + +async function restructure_if_needed(rootDir: string) { + // before + // $rootdir/My Album 1 + // $rootdir/My Album 2 + // $rootdir/Photos from 2008 + + // after + // $rootdir/Albums/My Album 1 + // $rootdir/Albums/My Album 2 + // $rootdir/Photos/Photos from 2008 + + const photosDir: string = `${rootDir}/Photos`; + + // move the "Photos from $YEAR" directories to Photos/ + _restructure_if_needed(await glob(`${rootDir}/Photos from */`), photosDir); + + // move everythingg else to Albums/, so we end up with two top level folders + const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); + const photoSet: Set = new Set([photosDir]); + const everythingExceptPhotosDir: string[] = Array.from( + new Set([...fullSet].filter((x) => !photoSet.has(x))) + ); + _restructure_if_needed(everythingExceptPhotosDir, `${rootDir}/Albums`); +} + +async function run_full_migration( + rootDir: string, + timeout: number, + exifTool: ExifTool +) { + // at least in my takeout, the Takeout folder contains a subfolder + // Takeout/Google Foto + // rootdir refers to that subfolder + + rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); + await restructure_if_needed(rootDir); + await process_albums(rootDir, timeout, exifTool); + await process_photos(rootDir, timeout, exifTool); +} + +const full_migrate = command({ + name: 'google-photos-migrate-full', + args: { + takeoutDir: positional({ + type: string, + displayName: 'takeout_dir', + description: 'The path to your "Takeout" directory.', + }), + force: flag({ + short: 'f', + long: 'force', + description: + "Forces the operation if the given directories aren't empty.", + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: async ({ takeoutDir, force, timeout }) => { + const errs: string[] = []; + if (!existsSync(takeoutDir)) { + errs.push('The specified takeout directory does not exist.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + if (!force) { + errs.push( + 'The output directory is not empty. Pass "-f" to force the operation.' + ); + } + if (await isEmptyDir(takeoutDir)) { + errs.push('The google directory is empty. Nothing to do.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + run_full_migration(takeoutDir, timeout, exifTool); + }, +}); + +const folder_migrate = command({ + name: 'google-photos-migrate-folder', args: { googleDir: positional({ type: string, @@ -70,32 +318,15 @@ const app = command({ errs.forEach((e) => console.error(e)); process.exit(1); } + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - console.log(`Started migration.`); - const migGen = migrateGoogleDirGen({ - googleDir, - outputDir, - errorDir, - warnLog: console.error, - exiftool: new ExifTool({ taskTimeoutMillis: timeout }), - endExifTool: true, - }); - - const counts = { err: 0, suc: 0 }; - for await (const result of migGen) { - if (result instanceof Error) { - console.error(`Error: ${result}`); - counts.err++; - continue; - } - - counts.suc++; - } - - console.log(`Done! Processed ${counts.suc + counts.err} files.`); - console.log(`Files migrated: ${counts.suc}`); - console.log(`Files failed: ${counts.err}`); + await run_basic_migration(googleDir, outputDir, errorDir, exifTool); }, }); +const app = subcommands({ + name: 'google-photos-migrate', + cmds: { full_migrate, folder_migrate, unzip }, +}); + run(app, process.argv.slice(2)); diff --git a/yarn.lock b/yarn.lock index 0d09809..6d940c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,6 +9,18 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" + "@jridgewell/resolve-uri@^3.0.3": version "3.1.1" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" @@ -32,6 +44,11 @@ resolved "https://registry.yarnpkg.com/@photostructure/tz-lookup/-/tz-lookup-7.0.0.tgz#b079412e7fea59033e441c08fcdb37d7c3c05901" integrity sha512-pTRsZz7Sn4yAtItC7I4+0segDHosMyOtJgAXg+xvDOolT0Xz4IFWqBV33OMCWoaNd3oQb60wbWhLeCQgJCyZAA== +"@pkgjs/parseargs@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" + integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== + "@tsconfig/node10@^1.0.7": version "1.0.9" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" @@ -87,13 +104,23 @@ ansi-regex@^5.0.1: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== -ansi-styles@^4.1.0: +ansi-regex@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a" + integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA== + +ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: color-convert "^2.0.1" +ansi-styles@^6.1.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" + integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + anymatch@~3.1.2: version "3.1.3" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e" @@ -130,6 +157,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -197,6 +231,15 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cross-spawn@^7.0.0: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + debug@^4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" @@ -226,6 +269,21 @@ dynamic-dedupe@^0.3.0: dependencies: xtend "^4.0.0" +eastasianwidth@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +emoji-regex@^9.2.2: + version "9.2.2" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" + integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== + exiftool-vendored.exe@12.62.0: version "12.62.0" resolved "https://registry.yarnpkg.com/exiftool-vendored.exe/-/exiftool-vendored.exe-12.62.0.tgz#5a13b4bbfa0309f16fe30f71a10636e7fd73e95e" @@ -257,6 +315,14 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +foreground-child@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.1.1.tgz#1d173e776d75d2772fed08efe4a0de1ea1b12d0d" + integrity sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -279,6 +345,17 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" +glob@^10.13.3: + version "10.3.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.3.tgz#8360a4ffdd6ed90df84aa8d52f21f452e86a123b" + integrity sha512-92vPiMb/iqpmEgsOoIDvTjc50wf9CCCvMzsi6W0JLPeUKE8TWP1a73PgqSrqy7iAZxaSD1YdzU7QZR5LF51MJw== + dependencies: + foreground-child "^3.1.0" + jackspeak "^2.0.3" + minimatch "^9.0.1" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + path-scurry "^1.10.1" + glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -340,6 +417,11 @@ is-extglob@^2.1.1: resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + is-glob@^4.0.1, is-glob@~4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" @@ -352,6 +434,25 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + +jackspeak@^2.0.3: + version "2.3.1" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-2.3.1.tgz#ce2effa4c458e053640e61938865a5b5fae98456" + integrity sha512-4iSY3Bh1Htv+kLhiiZunUhQ+OYXIn0ze3ulq8JeWrFKmhPAJSySV2+kdtRh2pGcCeF0s6oR8Oc+pYZynJj4t8A== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + +"lru-cache@^9.1.1 || ^10.0.0": + version "10.0.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.0.1.tgz#0a3be479df549cca0e5d693ac402ff19537a6b7a" + integrity sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g== + luxon@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.3.0.tgz#d73ab5b5d2b49a461c47cedbc7e73309b4805b48" @@ -369,11 +470,23 @@ minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@^9.0.1: + version "9.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" + integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0": + version "7.0.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.3.tgz#05ea638da44e475037ed94d1c7efcc76a25e1974" + integrity sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg== + mkdirp@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -401,11 +514,24 @@ path-is-absolute@^1.0.0: resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== +path-scurry@^1.10.1: + version "1.10.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.10.1.tgz#9ba6bf5aa8500fe9fd67df4f0d9483b2b0bfc698" + integrity sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ== + dependencies: + lru-cache "^9.1.1 || ^10.0.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" @@ -441,6 +567,23 @@ sanitize-filename@^1.6.3: dependencies: truncate-utf8-bytes "^1.0.0" +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + source-map-support@^0.5.12: version "0.5.21" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" @@ -454,13 +597,38 @@ source-map@^0.6.0: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -strip-ansi@^6.0.0: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^5.0.1, string-width@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== + dependencies: + eastasianwidth "^0.2.0" + emoji-regex "^9.2.2" + strip-ansi "^7.0.1" + +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: ansi-regex "^5.0.1" +strip-ansi@^7.0.1: + version "7.1.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== + dependencies: + ansi-regex "^6.0.1" + strip-bom@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" @@ -562,6 +730,31 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== + dependencies: + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From a99b35b711d9b6f228cb3d1851342b7d7b39830b Mon Sep 17 00:00:00 2001 From: covalent Date: Wed, 30 Aug 2023 17:38:24 -0400 Subject: [PATCH 03/50] fixed borked glob install --- package.json | 4 ++-- yarn.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 3e1c053..457ec24 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "cmd-ts": "^0.12.1", "dotenv": "^16.1.4", "exiftool-vendored": "^22.0.0", + "glob": "^10.3.3", "sanitize-filename": "^1.6.3", - "ts-node": "^10.9.1", - "glob": "^10.13.3" + "ts-node": "^10.9.1" } } diff --git a/yarn.lock b/yarn.lock index 6d940c7..ebbdbc8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -345,7 +345,7 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@^10.13.3: +glob@^10.3.3: version "10.3.3" resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.3.tgz#8360a4ffdd6ed90df84aa8d52f21f452e86a123b" integrity sha512-92vPiMb/iqpmEgsOoIDvTjc50wf9CCCvMzsi6W0JLPeUKE8TWP1a73PgqSrqy7iAZxaSD1YdzU7QZR5LF51MJw== From ca18f94a2cd1943d00c039b506bfb164be048a57 Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 8 Sep 2023 11:06:43 -0400 Subject: [PATCH 04/50] fix improper set subtraction and general fixes --- src/cli.ts | 83 ++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 0630e21..74010e5 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, rename } from 'fs'; +import { cpSync, existsSync, mkdirSync } from 'fs'; import { command, subcommands, @@ -22,7 +22,7 @@ const unzip = command({ handler: async () => {}, }); -async function run_basic_migration( +async function runBasicMigration( googleDir: string, outputDir: string, errorDir: string, @@ -54,7 +54,7 @@ async function run_basic_migration( console.log(`Files failed: ${counts.err}`); } -async function run_migrations_checked( +async function runMigrationsChecked( albumDir: string, outDir: string, errDir: string, @@ -77,14 +77,14 @@ async function run_migrations_checked( process.exit(1); } - await run_basic_migration(albumDir, outDir, errDir, exifTool); + await runBasicMigration(albumDir, outDir, errDir, exifTool); if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); for (let file of errFiles) { - exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); } - await run_migrations_checked( + await runMigrationsChecked( albumDir, outDir, errDir, @@ -95,7 +95,7 @@ async function run_migrations_checked( } } -async function process_albums( +async function processAlbums( rootDir: string, timeout: number, exifTool: ExifTool @@ -110,7 +110,10 @@ async function process_albums( let albumName = basename(album); let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; let errDir = `${rootDir}/AlbumsError/${albumName}`; - await run_migrations_checked( + mkdirSync(album, {recursive: true}); + mkdirSync(outDir, {recursive: true}); + mkdirSync(errDir, {recursive: true}); + await runMigrationsChecked( album, outDir, errDir, @@ -121,7 +124,7 @@ async function process_albums( } } -async function process_photos( +async function processPhotos( rootDir: string, timeout: number, exifTool: ExifTool @@ -131,8 +134,11 @@ async function process_photos( const albumDir = `${rootDir}/Photos`; const outDir = `${rootDir}/PhotosProcessed`; const errDir = `${rootDir}/PhotosError`; + mkdirSync(albumDir, {recursive: true}); + mkdirSync(outDir, {recursive: true}); + mkdirSync(errDir, {recursive: true}); - await run_migrations_checked( + await runMigrationsChecked( albumDir, outDir, errDir, @@ -142,27 +148,21 @@ async function process_photos( ); } -async function _restructure_if_needed(folders: string[], targetDir: string) { - if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { - console.log( - `${targetDir} exists and is non-empty, assuming no further restructing needed` - ); +async function _restructureIfNeeded(folders: string[], targetDir: string) { + if (existsSync(targetDir) && ((await glob(`${targetDir}/*`)).length) > 0){ + console.log(`${targetDir} exists and is not empty. No restructuring needed.`); return; } - mkdirSync(targetDir); - if (folders.length == 0) { - console.log(`Warning: no folders were moved to ${targetDir}`); - } - for (let folder of folders) { - rename(folder, path.join(targetDir, folder), function (err) { - if (err) throw err; - console.log('Successfully moved!'); - }); + console.log(`Starting restructure of ${folders.length} directories`) + mkdirSync(targetDir, {recursive: true}); + for (let folder of folders){ + console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`) + cpSync(folder, `${targetDir}/${basename(folder)}`, {recursive: true}); } - console.log(`Restructured ${folders.length} folders`); + console.log(`Sucsessfully restructured ${folders.length} directories`) } -async function restructure_if_needed(rootDir: string) { +async function restructureIfNeeded(rootDir: string) { // before // $rootdir/My Album 1 // $rootdir/My Album 2 @@ -176,15 +176,16 @@ async function restructure_if_needed(rootDir: string) { const photosDir: string = `${rootDir}/Photos`; // move the "Photos from $YEAR" directories to Photos/ - _restructure_if_needed(await glob(`${rootDir}/Photos from */`), photosDir); - + _restructureIfNeeded(await glob(`${rootDir}/Photos from */`), photosDir); + // move everythingg else to Albums/, so we end up with two top level folders const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); - const photoSet: Set = new Set([photosDir]); + const photoSet: Set = new Set(await glob(`${rootDir}/Photos from */`)); + photoSet.add(`${rootDir}/Photos`); const everythingExceptPhotosDir: string[] = Array.from( new Set([...fullSet].filter((x) => !photoSet.has(x))) ); - _restructure_if_needed(everythingExceptPhotosDir, `${rootDir}/Albums`); + _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); } async function run_full_migration( @@ -197,12 +198,13 @@ async function run_full_migration( // rootdir refers to that subfolder rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); - await restructure_if_needed(rootDir); - await process_albums(rootDir, timeout, exifTool); - await process_photos(rootDir, timeout, exifTool); + await restructureIfNeeded(rootDir); + await processAlbums(rootDir, timeout, exifTool); + await processPhotos(rootDir, timeout, exifTool); + exifTool.end(); } -const full_migrate = command({ +const fullMigrate = command({ name: 'google-photos-migrate-full', args: { takeoutDir: positional({ @@ -234,12 +236,6 @@ const full_migrate = command({ errs.forEach((e) => console.error(e)); process.exit(1); } - - if (!force) { - errs.push( - 'The output directory is not empty. Pass "-f" to force the operation.' - ); - } if (await isEmptyDir(takeoutDir)) { errs.push('The google directory is empty. Nothing to do.'); } @@ -248,11 +244,12 @@ const full_migrate = command({ process.exit(1); } const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + run_full_migration(takeoutDir, timeout, exifTool); }, }); -const folder_migrate = command({ +const folderMigrate = command({ name: 'google-photos-migrate-folder', args: { googleDir: positional({ @@ -320,13 +317,13 @@ const folder_migrate = command({ } const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await run_basic_migration(googleDir, outputDir, errorDir, exifTool); + await runBasicMigration(googleDir, outputDir, errorDir, exifTool); }, }); const app = subcommands({ name: 'google-photos-migrate', - cmds: { full_migrate, folder_migrate, unzip }, + cmds: { fullMigrate, folderMigrate, unzip }, }); run(app, process.argv.slice(2)); From 7a87fbaea1cb332b09683ad26c168d401beed65e Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 8 Sep 2023 11:12:26 -0400 Subject: [PATCH 05/50] complete merge --- build/cli.js | 284 +++++++++++++++++++++++++++++ build/config/env.js | 7 + build/config/extensions.js | 40 ++++ build/fs/file-exists.js | 8 + build/fs/is-empty-dir.js | 15 ++ build/fs/walk-dir.js | 42 +++++ build/index.js | 8 + build/media/InvalidExtError.js | 10 + build/media/MediaFile.js | 2 + build/media/MediaFileExtension.js | 2 + build/media/MediaMigrationError.js | 10 + build/media/MetaType.js | 9 + build/media/NoMetaFileError.js | 10 + build/media/migrate-google-dir.js | 142 +++++++++++++++ build/media/save-to-dir.js | 60 ++++++ build/media/title-json-map.js | 69 +++++++ build/meta/GoogleMeta.js | 2 + build/meta/apply-meta-errors.js | 46 +++++ build/meta/apply-meta-file.js | 72 ++++++++ build/meta/find-meta-file.js | 71 ++++++++ build/meta/read-meta-title.js | 23 +++ build/ts.js | 7 + 22 files changed, 939 insertions(+) create mode 100644 build/cli.js create mode 100644 build/config/env.js create mode 100644 build/config/extensions.js create mode 100644 build/fs/file-exists.js create mode 100644 build/fs/is-empty-dir.js create mode 100644 build/fs/walk-dir.js create mode 100644 build/index.js create mode 100644 build/media/InvalidExtError.js create mode 100644 build/media/MediaFile.js create mode 100644 build/media/MediaFileExtension.js create mode 100644 build/media/MediaMigrationError.js create mode 100644 build/media/MetaType.js create mode 100644 build/media/NoMetaFileError.js create mode 100644 build/media/migrate-google-dir.js create mode 100644 build/media/save-to-dir.js create mode 100644 build/media/title-json-map.js create mode 100644 build/meta/GoogleMeta.js create mode 100644 build/meta/apply-meta-errors.js create mode 100644 build/meta/apply-meta-file.js create mode 100644 build/meta/find-meta-file.js create mode 100644 build/meta/read-meta-title.js create mode 100644 build/ts.js diff --git a/build/cli.js b/build/cli.js new file mode 100644 index 0000000..31378ff --- /dev/null +++ b/build/cli.js @@ -0,0 +1,284 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs_1 = require("fs"); +const cmd_ts_1 = require("cmd-ts"); +const migrate_google_dir_1 = require("./media/migrate-google-dir"); +const is_empty_dir_1 = require("./fs/is-empty-dir"); +const exiftool_vendored_1 = require("exiftool-vendored"); +const glob_1 = require("glob"); +const path_1 = require("path"); +const path = require("path"); +const unzip = (0, cmd_ts_1.command)({ + name: 'unzipper', + args: {}, + handler: () => __awaiter(void 0, void 0, void 0, function* () { }), +}); +function runBasicMigration(googleDir, outputDir, errorDir, exifTool) { + var _a, e_1, _b, _c; + return __awaiter(this, void 0, void 0, function* () { + console.log(`Started migration.`); + const migGen = (0, migrate_google_dir_1.migrateGoogleDirGen)({ + googleDir, + outputDir, + errorDir, + warnLog: console.error, + exiftool: exifTool, + endExifTool: true, + }); + const counts = { err: 0, suc: 0 }; + try { + for (var _d = true, migGen_1 = __asyncValues(migGen), migGen_1_1; migGen_1_1 = yield migGen_1.next(), _a = migGen_1_1.done, !_a; _d = true) { + _c = migGen_1_1.value; + _d = false; + const result = _c; + if (result instanceof Error) { + console.error(`Error: ${result}`); + counts.err++; + continue; + } + counts.suc++; + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (!_d && !_a && (_b = migGen_1.return)) yield _b.call(migGen_1); + } + finally { if (e_1) throw e_1.error; } + } + console.log(`Done! Processed ${counts.suc + counts.err} files.`); + console.log(`Files migrated: ${counts.suc}`); + console.log(`Files failed: ${counts.err}`); + }); +} +function runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, check_errDir) { + return __awaiter(this, void 0, void 0, function* () { + const errs = []; + if (!(0, fs_1.existsSync)(albumDir)) { + errs.push('The specified google directory does not exist.'); + } + if (!(0, fs_1.existsSync)(outDir)) { + errs.push('The specified output directory does not exist.'); + } + if (!(0, fs_1.existsSync)(errDir)) { + errs.push('The specified error directory does not exist.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + yield runBasicMigration(albumDir, outDir, errDir, exifTool); + if (check_errDir && !(yield (0, is_empty_dir_1.isEmptyDir)(errDir))) { + const errFiles = yield (0, glob_1.glob)(`${errDir}/*`); + for (let file of errFiles) { + yield exifTool.rewriteAllTags(file, path.join(albumDir, (0, path_1.basename)(file))); + } + yield runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, false); + } + }); +} +function processAlbums(rootDir, timeout, exifTool) { + return __awaiter(this, void 0, void 0, function* () { + const globStr = `${rootDir}/Albums/*/`; + const albums = yield (0, glob_1.glob)(globStr); + if (albums.length == 0) { + console.log(`WARN: No albums found at ${globStr}`); + } + for (let album of albums) { + console.log(`Processing album ${album}...`); + let albumName = (0, path_1.basename)(album); + let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; + let errDir = `${rootDir}/AlbumsError/${albumName}`; + (0, fs_1.mkdirSync)(album, { recursive: true }); + (0, fs_1.mkdirSync)(outDir, { recursive: true }); + (0, fs_1.mkdirSync)(errDir, { recursive: true }); + yield runMigrationsChecked(album, outDir, errDir, timeout, exifTool, true); + } + }); +} +function processPhotos(rootDir, timeout, exifTool) { + return __awaiter(this, void 0, void 0, function* () { + // Also run the exif fix for the photos + console.log('Processing photos...'); + const albumDir = `${rootDir}/Photos`; + const outDir = `${rootDir}/PhotosProcessed`; + const errDir = `${rootDir}/PhotosError`; + (0, fs_1.mkdirSync)(albumDir, { recursive: true }); + (0, fs_1.mkdirSync)(outDir, { recursive: true }); + (0, fs_1.mkdirSync)(errDir, { recursive: true }); + yield runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, true); + }); +} +function _restructureIfNeeded(folders, targetDir) { + return __awaiter(this, void 0, void 0, function* () { + if ((0, fs_1.existsSync)(targetDir) && ((yield (0, glob_1.glob)(`${targetDir}/*`)).length) > 0) { + console.log(`${targetDir} exists and is not empty. No restructuring needed.`); + return; + } + console.log(`Starting restructure of ${folders.length} directories`); + (0, fs_1.mkdirSync)(targetDir, { recursive: true }); + for (let folder of folders) { + console.log(`Copying ${folder} to ${targetDir}/${(0, path_1.basename)(folder)}`); + (0, fs_1.cpSync)(folder, `${targetDir}/${(0, path_1.basename)(folder)}`, { recursive: true }); + } + console.log(`Sucsessfully restructured ${folders.length} directories`); + }); +} +function restructureIfNeeded(rootDir) { + return __awaiter(this, void 0, void 0, function* () { + // before + // $rootdir/My Album 1 + // $rootdir/My Album 2 + // $rootdir/Photos from 2008 + // after + // $rootdir/Albums/My Album 1 + // $rootdir/Albums/My Album 2 + // $rootdir/Photos/Photos from 2008 + const photosDir = `${rootDir}/Photos`; + // move the "Photos from $YEAR" directories to Photos/ + _restructureIfNeeded(yield (0, glob_1.glob)(`${rootDir}/Photos from */`), photosDir); + // move everythingg else to Albums/, so we end up with two top level folders + const fullSet = new Set(yield (0, glob_1.glob)(`${rootDir}/*/`)); + const photoSet = new Set(yield (0, glob_1.glob)(`${rootDir}/Photos from */`)); + photoSet.add(`${rootDir}/Photos`); + const everythingExceptPhotosDir = Array.from(new Set([...fullSet].filter((x) => !photoSet.has(x)))); + _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); + }); +} +function run_full_migration(rootDir, timeout, exifTool) { + return __awaiter(this, void 0, void 0, function* () { + // at least in my takeout, the Takeout folder contains a subfolder + // Takeout/Google Foto + // rootdir refers to that subfolder + rootDir = (yield (0, glob_1.glob)(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); + yield restructureIfNeeded(rootDir); + yield processAlbums(rootDir, timeout, exifTool); + yield processPhotos(rootDir, timeout, exifTool); + exifTool.end(); + }); +} +const fullMigrate = (0, cmd_ts_1.command)({ + name: 'google-photos-migrate-full', + args: { + takeoutDir: (0, cmd_ts_1.positional)({ + type: cmd_ts_1.string, + displayName: 'takeout_dir', + description: 'The path to your "Takeout" directory.', + }), + force: (0, cmd_ts_1.flag)({ + short: 'f', + long: 'force', + description: "Forces the operation if the given directories aren't empty.", + }), + timeout: (0, cmd_ts_1.option)({ + type: cmd_ts_1.number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: ({ takeoutDir, force, timeout }) => __awaiter(void 0, void 0, void 0, function* () { + const errs = []; + if (!(0, fs_1.existsSync)(takeoutDir)) { + errs.push('The specified takeout directory does not exist.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + if (yield (0, is_empty_dir_1.isEmptyDir)(takeoutDir)) { + errs.push('The google directory is empty. Nothing to do.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + const exifTool = new exiftool_vendored_1.ExifTool({ taskTimeoutMillis: timeout }); + run_full_migration(takeoutDir, timeout, exifTool); + }), +}); +const folderMigrate = (0, cmd_ts_1.command)({ + name: 'google-photos-migrate-folder', + args: { + googleDir: (0, cmd_ts_1.positional)({ + type: cmd_ts_1.string, + displayName: 'google_dir', + description: 'The path to your "Google Photos" directory.', + }), + outputDir: (0, cmd_ts_1.positional)({ + type: cmd_ts_1.string, + displayName: 'output_dir', + description: 'The path to your flat output directory.', + }), + errorDir: (0, cmd_ts_1.positional)({ + type: cmd_ts_1.string, + displayName: 'error_dir', + description: 'Failed media will be saved here.', + }), + force: (0, cmd_ts_1.flag)({ + short: 'f', + long: 'force', + description: "Forces the operation if the given directories aren't empty.", + }), + timeout: (0, cmd_ts_1.option)({ + type: cmd_ts_1.number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: ({ googleDir, outputDir, errorDir, force, timeout }) => __awaiter(void 0, void 0, void 0, function* () { + const errs = []; + if (!(0, fs_1.existsSync)(googleDir)) { + errs.push('The specified google directory does not exist.'); + } + if (!(0, fs_1.existsSync)(outputDir)) { + errs.push('The specified output directory does not exist.'); + } + if (!(0, fs_1.existsSync)(errorDir)) { + errs.push('The specified error directory does not exist.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + if (!force && !(yield (0, is_empty_dir_1.isEmptyDir)(outputDir))) { + errs.push('The output directory is not empty. Pass "-f" to force the operation.'); + } + if (!force && !(yield (0, is_empty_dir_1.isEmptyDir)(errorDir))) { + errs.push('The error directory is not empty. Pass "-f" to force the operation.'); + } + if (yield (0, is_empty_dir_1.isEmptyDir)(googleDir)) { + errs.push('The google directory is empty. Nothing to do.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + const exifTool = new exiftool_vendored_1.ExifTool({ taskTimeoutMillis: timeout }); + yield runBasicMigration(googleDir, outputDir, errorDir, exifTool); + }), +}); +const app = (0, cmd_ts_1.subcommands)({ + name: 'google-photos-migrate', + cmds: { fullMigrate, folderMigrate, unzip }, +}); +(0, cmd_ts_1.run)(app, process.argv.slice(2)); diff --git a/build/config/env.js b/build/config/env.js new file mode 100644 index 0000000..518178e --- /dev/null +++ b/build/config/env.js @@ -0,0 +1,7 @@ +"use strict"; +var _a; +var _b; +Object.defineProperty(exports, "__esModule", { value: true }); +const dotenv = require("dotenv"); +dotenv.config(); +(_a = (_b = process.env).NODE_ENV) !== null && _a !== void 0 ? _a : (_b.NODE_ENV = 'production'); diff --git a/build/config/extensions.js b/build/config/extensions.js new file mode 100644 index 0000000..c53be5a --- /dev/null +++ b/build/config/extensions.js @@ -0,0 +1,40 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.supportedExtensions = void 0; +const MetaType_1 = require("../media/MetaType"); +exports.supportedExtensions = [ + { suffix: '.jpg', metaType: MetaType_1.MetaType.EXIF }, + { suffix: '.jpeg', metaType: MetaType_1.MetaType.EXIF }, + { suffix: '.png', metaType: MetaType_1.MetaType.EXIF }, + { suffix: '.raw', metaType: MetaType_1.MetaType.NONE }, + { suffix: '.ico', metaType: MetaType_1.MetaType.NONE }, + { suffix: '.tiff', metaType: MetaType_1.MetaType.EXIF }, + { suffix: '.webp', metaType: MetaType_1.MetaType.EXIF }, + { suffix: '.heic', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.heif', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.gif', metaType: MetaType_1.MetaType.NONE }, + { + suffix: '.mp4', + metaType: MetaType_1.MetaType.QUICKTIME, + aliases: ['.heic', '.jpg', '.jpeg'], // Live photos + }, + { + suffix: '.mov', + metaType: MetaType_1.MetaType.QUICKTIME, + aliases: ['.heic', '.jpg', '.jpeg'], // Live photos + }, + { suffix: '.qt', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.mov.qt', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.3gp', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.mp4v', metaType: MetaType_1.MetaType.QUICKTIME }, + { suffix: '.mkv', metaType: MetaType_1.MetaType.NONE }, + { suffix: '.wmv', metaType: MetaType_1.MetaType.NONE }, + { suffix: '.webm', metaType: MetaType_1.MetaType.NONE }, +].flatMap((e) => { + var _a; + const aliases = (_a = e.aliases) === null || _a === void 0 ? void 0 : _a.flatMap((s) => [s.toLowerCase(), s.toUpperCase()]); + return [ + Object.assign(Object.assign({}, e), { suffix: e.suffix.toLowerCase(), aliases }), + Object.assign(Object.assign({}, e), { suffix: e.suffix.toUpperCase(), aliases }), + ]; +}); diff --git a/build/fs/file-exists.js b/build/fs/file-exists.js new file mode 100644 index 0000000..28534ed --- /dev/null +++ b/build/fs/file-exists.js @@ -0,0 +1,8 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.fileExists = void 0; +const promises_1 = require("fs/promises"); +const fileExists = (path) => (0, promises_1.stat)(path) + .then(() => true) + .catch((e) => (e.code === 'ENOENT' ? false : Promise.reject(e))); +exports.fileExists = fileExists; diff --git a/build/fs/is-empty-dir.js b/build/fs/is-empty-dir.js new file mode 100644 index 0000000..92092ea --- /dev/null +++ b/build/fs/is-empty-dir.js @@ -0,0 +1,15 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.isEmptyDir = void 0; +const promises_1 = require("fs/promises"); +const isEmptyDir = (dir) => __awaiter(void 0, void 0, void 0, function* () { return (yield (0, promises_1.readdir)(dir)).length === 0; }); +exports.isEmptyDir = isEmptyDir; diff --git a/build/fs/walk-dir.js b/build/fs/walk-dir.js new file mode 100644 index 0000000..c1fdc8c --- /dev/null +++ b/build/fs/walk-dir.js @@ -0,0 +1,42 @@ +"use strict"; +var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +var __asyncDelegator = (this && this.__asyncDelegator) || function (o) { + var i, p; + return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i; + function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: false } : f ? f(v) : v; } : f; } +}; +var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var g = generator.apply(thisArg, _arguments || []), i, q = []; + return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; + function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } + function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } + function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } + function fulfill(value) { resume("next", value); } + function reject(value) { resume("throw", value); } + function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.walkDir = void 0; +const promises_1 = require("fs/promises"); +const path_1 = require("path"); +function walkDir(dir) { + return __asyncGenerator(this, arguments, function* walkDir_1() { + for (const dirent of yield __await((0, promises_1.readdir)(dir, { withFileTypes: true }))) { + if (dirent.isDirectory()) { + yield __await(yield* __asyncDelegator(__asyncValues(walkDir((0, path_1.resolve)(dir, dirent.name))))); + } + else { + yield yield __await((0, path_1.resolve)(dir, dirent.name)); + } + } + }); +} +exports.walkDir = walkDir; diff --git a/build/index.js b/build/index.js new file mode 100644 index 0000000..9589ac8 --- /dev/null +++ b/build/index.js @@ -0,0 +1,8 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.supportedExtensions = exports.migrateGoogleDir = void 0; +require("./config/env"); +const migrate_google_dir_1 = require("./media/migrate-google-dir"); +Object.defineProperty(exports, "migrateGoogleDir", { enumerable: true, get: function () { return migrate_google_dir_1.migrateGoogleDir; } }); +const extensions_1 = require("./config/extensions"); +Object.defineProperty(exports, "supportedExtensions", { enumerable: true, get: function () { return extensions_1.supportedExtensions; } }); diff --git a/build/media/InvalidExtError.js b/build/media/InvalidExtError.js new file mode 100644 index 0000000..f18d3a5 --- /dev/null +++ b/build/media/InvalidExtError.js @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.InvalidExtError = void 0; +const MediaMigrationError_1 = require("./MediaMigrationError"); +class InvalidExtError extends MediaMigrationError_1.MediaMigrationError { + toString() { + return `File has invalid extension: ${this.failedMedia.path}`; + } +} +exports.InvalidExtError = InvalidExtError; diff --git a/build/media/MediaFile.js b/build/media/MediaFile.js new file mode 100644 index 0000000..c8ad2e5 --- /dev/null +++ b/build/media/MediaFile.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/media/MediaFileExtension.js b/build/media/MediaFileExtension.js new file mode 100644 index 0000000..c8ad2e5 --- /dev/null +++ b/build/media/MediaFileExtension.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/media/MediaMigrationError.js b/build/media/MediaMigrationError.js new file mode 100644 index 0000000..25eacff --- /dev/null +++ b/build/media/MediaMigrationError.js @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.MediaMigrationError = void 0; +class MediaMigrationError extends Error { + constructor(failedMedia) { + super(); + this.failedMedia = failedMedia; + } +} +exports.MediaMigrationError = MediaMigrationError; diff --git a/build/media/MetaType.js b/build/media/MetaType.js new file mode 100644 index 0000000..d750457 --- /dev/null +++ b/build/media/MetaType.js @@ -0,0 +1,9 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.MetaType = void 0; +var MetaType; +(function (MetaType) { + MetaType["NONE"] = "NONE"; + MetaType["EXIF"] = "EXIF"; + MetaType["QUICKTIME"] = "QUICKTIME"; +})(MetaType || (exports.MetaType = MetaType = {})); diff --git a/build/media/NoMetaFileError.js b/build/media/NoMetaFileError.js new file mode 100644 index 0000000..6fdfb30 --- /dev/null +++ b/build/media/NoMetaFileError.js @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.NoMetaFileError = void 0; +const MediaMigrationError_1 = require("./MediaMigrationError"); +class NoMetaFileError extends MediaMigrationError_1.MediaMigrationError { + toString() { + return `Unable to locate meta file for: ${this.failedMedia.path}`; + } +} +exports.NoMetaFileError = NoMetaFileError; diff --git a/build/media/migrate-google-dir.js b/build/media/migrate-google-dir.js new file mode 100644 index 0000000..b3d240d --- /dev/null +++ b/build/media/migrate-google-dir.js @@ -0,0 +1,142 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } +var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var g = generator.apply(thisArg, _arguments || []), i, q = []; + return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; + function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } + function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } + function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } + function fulfill(value) { resume("next", value); } + function reject(value) { resume("throw", value); } + function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.migrateGoogleDirGen = exports.migrateGoogleDir = void 0; +const walk_dir_1 = require("../fs/walk-dir"); +const path_1 = require("path"); +const find_meta_file_1 = require("../meta/find-meta-file"); +const apply_meta_file_1 = require("../meta/apply-meta-file"); +const title_json_map_1 = require("./title-json-map"); +const extensions_1 = require("../config/extensions"); +const InvalidExtError_1 = require("./InvalidExtError"); +const NoMetaFileError_1 = require("./NoMetaFileError"); +const apply_meta_errors_1 = require("../meta/apply-meta-errors"); +const exiftool_vendored_1 = require("exiftool-vendored"); +const read_meta_title_1 = require("../meta/read-meta-title"); +const save_to_dir_1 = require("./save-to-dir"); +function migrateGoogleDir(args) { + return __asyncGenerator(this, arguments, function* migrateGoogleDir_1() { + var _a, e_1, _b, _c; + const wg = []; + try { + for (var _d = true, _e = __asyncValues(migrateGoogleDirGen(args)), _f; _f = yield __await(_e.next()), _a = _f.done, !_a; _d = true) { + _c = _f.value; + _d = false; + const result = _c; + wg.push(result); + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (!_d && !_a && (_b = _e.return)) yield __await(_b.call(_e)); + } + finally { if (e_1) throw e_1.error; } + } + return yield __await(yield __await(Promise.all(wg))); + }); +} +exports.migrateGoogleDir = migrateGoogleDir; +function migrateGoogleDirGen(args) { + var _a, _b, _c; + return __asyncGenerator(this, arguments, function* migrateGoogleDirGen_1() { + var _d, e_2, _e, _f; + const migCtx = Object.assign(Object.assign({ titleJsonMap: yield __await((0, title_json_map_1.indexJsonFiles)(args.googleDir)), migrationLocks: new Map() }, args), { exiftool: (_a = args.exiftool) !== null && _a !== void 0 ? _a : new exiftool_vendored_1.ExifTool(), endExifTool: (_b = args.endExifTool) !== null && _b !== void 0 ? _b : !args.exiftool, warnLog: (_c = args.warnLog) !== null && _c !== void 0 ? _c : (() => { }) }); + try { + for (var _g = true, _h = __asyncValues((0, walk_dir_1.walkDir)(args.googleDir)), _j; _j = yield __await(_h.next()), _d = _j.done, !_d; _g = true) { + _f = _j.value; + _g = false; + const mediaPath = _f; + if (mediaPath.endsWith('.json')) + continue; + yield yield __await(migrateMediaFile(mediaPath, migCtx)); + } + } + catch (e_2_1) { e_2 = { error: e_2_1 }; } + finally { + try { + if (!_g && !_d && (_e = _h.return)) yield __await(_e.call(_h)); + } + finally { if (e_2) throw e_2.error; } + } + migCtx.endExifTool && migCtx.exiftool.end(); + }); +} +exports.migrateGoogleDirGen = migrateGoogleDirGen; +function migrateMediaFile(originalPath, migCtx) { + return __awaiter(this, void 0, void 0, function* () { + const mediaFileInfo = { + originalPath, + path: originalPath, + }; + const ext = extensions_1.supportedExtensions.reduce((longestMatch, cur) => { + if (!originalPath.endsWith(cur.suffix)) + return longestMatch; + if (longestMatch === null) + return cur; + return cur.suffix.length > longestMatch.suffix.length + ? cur + : longestMatch; + }, null); + if (!ext) { + mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.errorDir, migCtx); + return new InvalidExtError_1.InvalidExtError(mediaFileInfo); + } + const jsonPath = yield (0, find_meta_file_1.findMetaFile)(originalPath, ext, migCtx); + if (!jsonPath) { + mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.errorDir, migCtx); + return new NoMetaFileError_1.NoMetaFileError(mediaFileInfo); + } + mediaFileInfo.jsonPath = jsonPath; + mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.outputDir, migCtx, false, yield (0, read_meta_title_1.readMetaTitle)(mediaFileInfo)); + const mediaFile = Object.assign(Object.assign({}, mediaFileInfo), { ext, + jsonPath }); + let err = yield (0, apply_meta_file_1.applyMetaFile)(mediaFile, migCtx); + if (!err) { + return mediaFile; + } + if (err instanceof apply_meta_errors_1.WrongExtensionError) { + const oldBase = (0, path_1.basename)(mediaFile.path); + const newBase = oldBase.slice(0, oldBase.length - err.expectedExt.length) + err.actualExt; + mediaFile.path = yield (0, save_to_dir_1.saveToDir)(mediaFile.path, migCtx.outputDir, migCtx, true, newBase); + migCtx.warnLog(`Renamed wrong extension ${err.expectedExt} to ${err.actualExt}: ${mediaFile.path}`); + err = yield (0, apply_meta_file_1.applyMetaFile)(mediaFile, migCtx); + if (!err) { + return mediaFile; + } + } + const savedPaths = yield Promise.all([ + (0, save_to_dir_1.saveToDir)(mediaFile.path, migCtx.errorDir, migCtx, true), + (0, save_to_dir_1.saveToDir)(mediaFile.jsonPath, migCtx.errorDir, migCtx), + ]); + mediaFile.path = savedPaths[0]; + return err; + }); +} diff --git a/build/media/save-to-dir.js b/build/media/save-to-dir.js new file mode 100644 index 0000000..598b011 --- /dev/null +++ b/build/media/save-to-dir.js @@ -0,0 +1,60 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.saveToDir = void 0; +const path_1 = require("path"); +const file_exists_1 = require("../fs/file-exists"); +const sanitize = require("sanitize-filename"); +const promises_1 = require("fs/promises"); +function _saveToDir(file, destDir, saveBase, move = false, duplicateIndex = 0) { + return __awaiter(this, void 0, void 0, function* () { + const saveDir = (0, path_1.resolve)(destDir, duplicateIndex > 0 ? `duplicates-${duplicateIndex}` : '.'); + yield (0, promises_1.mkdir)(saveDir, { recursive: true }); + const savePath = (0, path_1.resolve)(saveDir, saveBase); + const exists = yield (0, file_exists_1.fileExists)(savePath); + if (exists) { + return _saveToDir(file, destDir, saveBase, move, duplicateIndex + 1); + } + if (move) { + yield (0, promises_1.rename)(file, savePath); + } + else { + yield (0, promises_1.copyFile)(file, savePath); + } + return savePath; + }); +} +/** Copies or moves a file to dir, saves duplicates in subfolders and returns the new path. + * Atomic within this app, sanitizes filenames. + */ +function saveToDir(file, destDir, migCtx, move = false, saveBase) { + return __awaiter(this, void 0, void 0, function* () { + saveBase = saveBase !== null && saveBase !== void 0 ? saveBase : (0, path_1.basename)(file); + const sanitized = sanitize(saveBase, { replacement: '_' }); + if (saveBase != sanitized) { + migCtx.warnLog(`Sanitized file: ${file}` + '\nNew filename: ${sanitized}'); + } + const lcBase = saveBase.toLowerCase(); + let lock; + while ((lock = migCtx.migrationLocks.get(lcBase))) { + yield lock; + } + lock = _saveToDir(file, destDir, sanitized, move); + migCtx.migrationLocks.set(lcBase, lock); + try { + return yield lock; + } + finally { + migCtx.migrationLocks.delete(lcBase); + } + }); +} +exports.saveToDir = saveToDir; diff --git a/build/media/title-json-map.js b/build/media/title-json-map.js new file mode 100644 index 0000000..417007b --- /dev/null +++ b/build/media/title-json-map.js @@ -0,0 +1,69 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.indexJsonFiles = void 0; +const promises_1 = require("fs/promises"); +const walk_dir_1 = require("../fs/walk-dir"); +const path_1 = require("path"); +const MAX_BASE_LENGTH = 51; +function trimTitle(title) { } +function indexJsonFiles(googleDir) { + var _a, e_1, _b, _c; + var _d; + return __awaiter(this, void 0, void 0, function* () { + const titleJsonMap = new Map(); + try { + for (var _e = true, _f = __asyncValues((0, walk_dir_1.walkDir)(googleDir)), _g; _g = yield _f.next(), _a = _g.done, !_a; _e = true) { + _c = _g.value; + _e = false; + const jsonPath = _c; + if (!jsonPath.endsWith('.json')) + continue; + let title; + try { + const data = JSON.parse((yield (0, promises_1.readFile)(jsonPath)).toString()); + title = data.title; + } + catch (e) { } + if (typeof title !== 'string') + continue; + const potTitles = new Set(); + const ext = (0, path_1.extname)(title); + const woExt = title.slice(0, -ext.length); + const maxWoExt = MAX_BASE_LENGTH - ext.length; + potTitles.add(woExt.slice(0, maxWoExt) + ext); + potTitles.add((woExt + '-edited').slice(0, maxWoExt) + ext); + potTitles.add((woExt + '-bearbeitet').slice(0, maxWoExt) + ext); + for (const potTitle of potTitles) { + const jsonPaths = (_d = titleJsonMap.get(potTitle)) !== null && _d !== void 0 ? _d : []; + jsonPaths.push(jsonPath); + titleJsonMap.set(potTitle, jsonPaths); + } + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (!_e && !_a && (_b = _f.return)) yield _b.call(_f); + } + finally { if (e_1) throw e_1.error; } + } + return titleJsonMap; + }); +} +exports.indexJsonFiles = indexJsonFiles; diff --git a/build/meta/GoogleMeta.js b/build/meta/GoogleMeta.js new file mode 100644 index 0000000..c8ad2e5 --- /dev/null +++ b/build/meta/GoogleMeta.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/meta/apply-meta-errors.js b/build/meta/apply-meta-errors.js new file mode 100644 index 0000000..0a44e2e --- /dev/null +++ b/build/meta/apply-meta-errors.js @@ -0,0 +1,46 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.MissingMetaError = exports.WrongExtensionError = exports.ExifToolError = exports.ApplyMetaError = void 0; +const MediaMigrationError_1 = require("../media/MediaMigrationError"); +class ApplyMetaError extends MediaMigrationError_1.MediaMigrationError { + constructor(failedMedia) { + super(failedMedia); + } + toString() { + return `Failed to apply meta tags on file: ${this.failedMedia.path}`; + } +} +exports.ApplyMetaError = ApplyMetaError; +class ExifToolError extends ApplyMetaError { + constructor(failedMedia, reason) { + super(failedMedia); + this.reason = reason; + } + toString() { + return (`ExifTool failed to modify file: ${this.failedMedia.path}` + + `\nReason: ${this.reason.message}`); + } +} +exports.ExifToolError = ExifToolError; +class WrongExtensionError extends ApplyMetaError { + constructor(failedMedia, expectedExt, actualExt) { + super(failedMedia); + this.expectedExt = expectedExt; + this.actualExt = actualExt; + } + toString() { + return `File has wrong file extension ${this.actualExt}, should be ${this.expectedExt}: ${this.failedMedia.path}`; + } +} +exports.WrongExtensionError = WrongExtensionError; +class MissingMetaError extends ApplyMetaError { + constructor(failedMedia, keyName) { + super(failedMedia); + this.keyName = keyName; + } + toString() { + return (`Missing key ${this.keyName} from meta file: ${this.failedMedia.jsonPath}` + + `\nOriginal file: ${this.failedMedia.path}`); + } +} +exports.MissingMetaError = MissingMetaError; diff --git a/build/meta/apply-meta-file.js b/build/meta/apply-meta-file.js new file mode 100644 index 0000000..629215c --- /dev/null +++ b/build/meta/apply-meta-file.js @@ -0,0 +1,72 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.applyMetaFile = void 0; +const ts_1 = require("../ts"); +const MetaType_1 = require("../media/MetaType"); +const promises_1 = require("fs/promises"); +const apply_meta_errors_1 = require("./apply-meta-errors"); +function applyMetaFile(mediaFile, migCtx) { + var _a, _b, _c; + return __awaiter(this, void 0, void 0, function* () { + const metaJson = (yield (0, promises_1.readFile)(mediaFile.jsonPath)).toString(); + const meta = JSON.parse(metaJson); + const timeTakenTimestamp = (_a = meta === null || meta === void 0 ? void 0 : meta.photoTakenTime) === null || _a === void 0 ? void 0 : _a.timestamp; + if (timeTakenTimestamp === undefined) + return new apply_meta_errors_1.MissingMetaError(mediaFile, 'photoTakenTime'); + const timeTaken = new Date(parseInt(timeTakenTimestamp) * 1000); + // always UTC as per https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString + const timeTakenUTC = timeTaken.toISOString(); + const tags = {}; + switch (mediaFile.ext.metaType) { + case MetaType_1.MetaType.EXIF: + tags.SubSecDateTimeOriginal = timeTakenUTC; + tags.SubSecCreateDate = timeTakenUTC; + tags.SubSecModifyDate = timeTakenUTC; + break; + case MetaType_1.MetaType.QUICKTIME: + tags.DateTimeOriginal = timeTakenUTC; + tags.CreateDate = timeTakenUTC; + tags.ModifyDate = timeTakenUTC; + tags.TrackCreateDate = timeTakenUTC; + tags.TrackModifyDate = timeTakenUTC; + tags.MediaCreateDate = timeTakenUTC; + tags.MediaModifyDate = timeTakenUTC; + break; + case MetaType_1.MetaType.NONE: + break; + default: + (0, ts_1.exhaustiveCheck)(mediaFile.ext.metaType); + } + tags.FileModifyDate = timeTakenUTC; + try { + yield migCtx.exiftool.write(mediaFile.path, tags, [ + '-overwrite_original', + '-api', + 'quicktimeutc', + ]); + } + catch (e) { + if (e instanceof Error) { + const wrongExtMatch = e.message.match(/Not a valid (?\w+) \(looks more like a (?\w+)\)/); + const expected = (_b = wrongExtMatch === null || wrongExtMatch === void 0 ? void 0 : wrongExtMatch.groups) === null || _b === void 0 ? void 0 : _b['expected']; + const actual = (_c = wrongExtMatch === null || wrongExtMatch === void 0 ? void 0 : wrongExtMatch.groups) === null || _c === void 0 ? void 0 : _c['actual']; + if (expected !== undefined && actual !== undefined) { + return new apply_meta_errors_1.WrongExtensionError(mediaFile, `.${expected.toLowerCase()}`, `.${actual.toLowerCase()}`); + } + return new apply_meta_errors_1.ExifToolError(mediaFile, e); + } + return new apply_meta_errors_1.ExifToolError(mediaFile, new Error(`${e}`)); + } + return null; + }); +} +exports.applyMetaFile = applyMetaFile; diff --git a/build/meta/find-meta-file.js b/build/meta/find-meta-file.js new file mode 100644 index 0000000..0a11566 --- /dev/null +++ b/build/meta/find-meta-file.js @@ -0,0 +1,71 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.findMetaFile = void 0; +const path_1 = require("path"); +const file_exists_1 = require("../fs/file-exists"); +function findMetaFile(mediaPath, ext, migCtx) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + const title = (0, path_1.basename)(mediaPath); + // Most json files can be matched by indexed titles + const indexedJson = migCtx.titleJsonMap.get(title); + if (indexedJson) { + const sameFolder = indexedJson.filter((jsonPath) => (0, path_1.dirname)(jsonPath) === (0, path_1.dirname)(mediaPath)); + if (sameFolder.length === 1) { + return sameFolder[0]; + } + } + // Otherwise, try (from most to least significant) + const potPaths = new Set(); + const pushWithPotExt = (base, potExt) => { + var _a, _b; + const potBases = []; + // (.)?.json + potBases.push(`${base}${potExt}`); + // Stolen from https://github.com/mattwilson1024/google-photos-exif/blob/master/src/helpers/get-companion-json-path-for-media-file.ts + const nameCounterMatch = base.match(/(?.*)(?\(\d+\))$/); + const name = (_a = nameCounterMatch === null || nameCounterMatch === void 0 ? void 0 : nameCounterMatch.groups) === null || _a === void 0 ? void 0 : _a['name']; + const counter = (_b = nameCounterMatch === null || nameCounterMatch === void 0 ? void 0 : nameCounterMatch.groups) === null || _b === void 0 ? void 0 : _b['counter']; + if (name !== undefined && counter !== undefined) { + // (.)?(n).json + potBases.push(`${name}${potExt}${counter}`); + } + // (_n-?|_n?|_?)(.)?.json + if (base.endsWith('_n-') || base.endsWith('_n') || base.endsWith('_')) { + potBases.push(`${base.slice(0, -1)}${potExt}`); + } + for (const potBase of potBases) { + potPaths.add(`${potBase}.json`); + } + }; + let base = mediaPath.slice(0, mediaPath.length - ext.suffix.length); + base = base.replace(/-(edited|bearbeitet|modifié)$/i, ''); + const potExts = []; + // ..json + potExts.push(ext.suffix); + // ..json + potExts.push(...((_a = ext.aliases) !== null && _a !== void 0 ? _a : [])); + // .json + potExts.push(''); + for (const potExt of potExts) { + pushWithPotExt(base, potExt); + } + for (const potPath of potPaths) { + if (!(yield (0, file_exists_1.fileExists)(potPath))) { + continue; + } + return potPath; + } + return null; + }); +} +exports.findMetaFile = findMetaFile; diff --git a/build/meta/read-meta-title.js b/build/meta/read-meta-title.js new file mode 100644 index 0000000..6183a55 --- /dev/null +++ b/build/meta/read-meta-title.js @@ -0,0 +1,23 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.readMetaTitle = void 0; +const promises_1 = require("fs/promises"); +function readMetaTitle(mediaFileInfo) { + return __awaiter(this, void 0, void 0, function* () { + if (!mediaFileInfo.jsonPath) + return undefined; + const metaJson = (yield (0, promises_1.readFile)(mediaFileInfo.jsonPath)).toString(); + const meta = JSON.parse(metaJson); + return meta === null || meta === void 0 ? void 0 : meta.title; + }); +} +exports.readMetaTitle = readMetaTitle; diff --git a/build/ts.js b/build/ts.js new file mode 100644 index 0000000..2e3a04a --- /dev/null +++ b/build/ts.js @@ -0,0 +1,7 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.exhaustiveCheck = void 0; +const exhaustiveCheck = (_) => { + throw new Error('Exhaustive type check failed.'); +}; +exports.exhaustiveCheck = exhaustiveCheck; From 4a858b91bc7714cf9791b0430f7231e73f87f139 Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 8 Sep 2023 11:14:31 -0400 Subject: [PATCH 06/50] remove some camel case and remove exiftool for now --- src/cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index ed7050a..f679e5c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -84,7 +84,7 @@ async function runMigrationsChecked( if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); for (let file of errFiles) { - await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + // await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); } await runMigrationsChecked( albumDir, @@ -190,7 +190,7 @@ async function restructureIfNeeded(rootDir: string) { _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); } -async function run_full_migration( +async function runFullMigration( rootDir: string, timeout: number, exifTool: ExifTool @@ -247,7 +247,7 @@ const fullMigrate = command({ } const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - run_full_migration(takeoutDir, timeout, exifTool); + runFullMigration(takeoutDir, timeout, exifTool); }, }); From 8e92808c49334981e02ba2b4943ac894e8f76ee7 Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 8 Sep 2023 11:14:31 -0400 Subject: [PATCH 07/50] remove some camel case and remove exiftool for now --- src/cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index ed7050a..f679e5c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -84,7 +84,7 @@ async function runMigrationsChecked( if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); for (let file of errFiles) { - await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + // await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); } await runMigrationsChecked( albumDir, @@ -190,7 +190,7 @@ async function restructureIfNeeded(rootDir: string) { _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); } -async function run_full_migration( +async function runFullMigration( rootDir: string, timeout: number, exifTool: ExifTool @@ -247,7 +247,7 @@ const fullMigrate = command({ } const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - run_full_migration(takeoutDir, timeout, exifTool); + runFullMigration(takeoutDir, timeout, exifTool); }, }); From b0c51691be7b213204616d9f4cc16fa3b9ac8fc8 Mon Sep 17 00:00:00 2001 From: covalent Date: Sat, 9 Sep 2023 12:06:21 -0400 Subject: [PATCH 08/50] fix improper handling of exiftool singleton --- src/cli.ts | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index f679e5c..67a068f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -61,7 +61,6 @@ async function runMigrationsChecked( outDir: string, errDir: string, timeout: number, - exifTool: ExifTool, check_errDir: boolean ) { const errs: string[] = []; @@ -79,19 +78,21 @@ async function runMigrationsChecked( process.exit(1); } + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await runBasicMigration(albumDir, outDir, errDir, exifTool); if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { - // await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + await exifTool.rewriteAllTags(file, path.join(albumDir, `${basename(file)}-1`)); } + exifTool.end(); await runMigrationsChecked( albumDir, outDir, errDir, timeout, - exifTool, false ); } @@ -100,7 +101,6 @@ async function runMigrationsChecked( async function processAlbums( rootDir: string, timeout: number, - exifTool: ExifTool ) { const globStr: string = `${rootDir}/Albums/*/`; const albums: string[] = await glob(globStr); @@ -120,7 +120,6 @@ async function processAlbums( outDir, errDir, timeout, - exifTool, true ); } @@ -129,7 +128,6 @@ async function processAlbums( async function processPhotos( rootDir: string, timeout: number, - exifTool: ExifTool ) { // Also run the exif fix for the photos console.log('Processing photos...'); @@ -145,7 +143,6 @@ async function processPhotos( outDir, errDir, timeout, - exifTool, true ); } @@ -193,7 +190,6 @@ async function restructureIfNeeded(rootDir: string) { async function runFullMigration( rootDir: string, timeout: number, - exifTool: ExifTool ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto @@ -201,9 +197,8 @@ async function runFullMigration( rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); await restructureIfNeeded(rootDir); - await processAlbums(rootDir, timeout, exifTool); - await processPhotos(rootDir, timeout, exifTool); - exifTool.end(); + await processPhotos(rootDir, timeout); + await processAlbums(rootDir, timeout); } const fullMigrate = command({ @@ -245,9 +240,8 @@ const fullMigrate = command({ errs.forEach((e) => console.error(e)); process.exit(1); } - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - runFullMigration(takeoutDir, timeout, exifTool); + runFullMigration(takeoutDir, timeout); }, }); @@ -317,8 +311,8 @@ const folderMigrate = command({ errs.forEach((e) => console.error(e)); process.exit(1); } - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await runBasicMigration(googleDir, outputDir, errorDir, exifTool); }, }); From 390d13686a149b2a08c2d44b18df6599588a0f97 Mon Sep 17 00:00:00 2001 From: covalent Date: Sat, 9 Sep 2023 12:06:21 -0400 Subject: [PATCH 09/50] fix improper handling of exiftool singleton --- src/cli.ts | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index f679e5c..67a068f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -61,7 +61,6 @@ async function runMigrationsChecked( outDir: string, errDir: string, timeout: number, - exifTool: ExifTool, check_errDir: boolean ) { const errs: string[] = []; @@ -79,19 +78,21 @@ async function runMigrationsChecked( process.exit(1); } + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await runBasicMigration(albumDir, outDir, errDir, exifTool); if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { - // await exifTool.rewriteAllTags(file, path.join(albumDir, basename(file))); + await exifTool.rewriteAllTags(file, path.join(albumDir, `${basename(file)}-1`)); } + exifTool.end(); await runMigrationsChecked( albumDir, outDir, errDir, timeout, - exifTool, false ); } @@ -100,7 +101,6 @@ async function runMigrationsChecked( async function processAlbums( rootDir: string, timeout: number, - exifTool: ExifTool ) { const globStr: string = `${rootDir}/Albums/*/`; const albums: string[] = await glob(globStr); @@ -120,7 +120,6 @@ async function processAlbums( outDir, errDir, timeout, - exifTool, true ); } @@ -129,7 +128,6 @@ async function processAlbums( async function processPhotos( rootDir: string, timeout: number, - exifTool: ExifTool ) { // Also run the exif fix for the photos console.log('Processing photos...'); @@ -145,7 +143,6 @@ async function processPhotos( outDir, errDir, timeout, - exifTool, true ); } @@ -193,7 +190,6 @@ async function restructureIfNeeded(rootDir: string) { async function runFullMigration( rootDir: string, timeout: number, - exifTool: ExifTool ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto @@ -201,9 +197,8 @@ async function runFullMigration( rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); await restructureIfNeeded(rootDir); - await processAlbums(rootDir, timeout, exifTool); - await processPhotos(rootDir, timeout, exifTool); - exifTool.end(); + await processPhotos(rootDir, timeout); + await processAlbums(rootDir, timeout); } const fullMigrate = command({ @@ -245,9 +240,8 @@ const fullMigrate = command({ errs.forEach((e) => console.error(e)); process.exit(1); } - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - runFullMigration(takeoutDir, timeout, exifTool); + runFullMigration(takeoutDir, timeout); }, }); @@ -317,8 +311,8 @@ const folderMigrate = command({ errs.forEach((e) => console.error(e)); process.exit(1); } - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await runBasicMigration(googleDir, outputDir, errorDir, exifTool); }, }); From 9dc71a874ad742042efabdfc24998e63ac739663 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 10 Sep 2023 10:04:05 -0400 Subject: [PATCH 10/50] fix json issues and remove force for fullMigrate --- src/cli.ts | 64 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 67a068f..47e41fb 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -17,12 +17,8 @@ import { ExifTool } from 'exiftool-vendored'; import { glob } from 'glob'; import { basename } from 'path'; import path = require('path'); +import { inferLikelyOffsetMinutes } from 'exiftool-vendored/dist/Timezones'; -const unzip = command({ - name: 'unzipper', - args: {}, - handler: async () => {}, -}); async function runBasicMigration( googleDir: string, @@ -85,7 +81,12 @@ async function runMigrationsChecked( const errFiles: string[] = await glob(`${errDir}/*`); const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { - await exifTool.rewriteAllTags(file, path.join(albumDir, `${basename(file)}-1`)); + if (file.endsWith(".json")){ + console.log(`Cannot fix metadata for ${file} as .json is an unsupported file type.`); + continue; + } + console.log(`Rewriting all tags from ${file}, to ${path.join(albumDir, `cleaned-${basename(file)}`)}`); + await exifTool.rewriteAllTags(file, path.join(albumDir, `cleaned-${basename(file)}`)); } exifTool.end(); await runMigrationsChecked( @@ -201,6 +202,35 @@ async function runFullMigration( await processAlbums(rootDir, timeout); } +const rewriteAllTags = command({ + name: 'rewrite all tags for single file', + args: { + inFile: positional({ + type: string, + displayName: 'in_file', + description: 'The path to your input file.', + }), + outFile: positional({ + type: string, + displayName: 'out_file', + description: 'The path to your output location for the file.', + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: async ({inFile, outFile, timeout}) => { + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + await exifTool.rewriteAllTags(inFile, outFile); + exifTool.end(); + }, +}); + const fullMigrate = command({ name: 'google-photos-migrate-full', args: { @@ -209,12 +239,6 @@ const fullMigrate = command({ displayName: 'takeout_dir', description: 'The path to your "Takeout" directory.', }), - force: flag({ - short: 'f', - long: 'force', - description: - "Forces the operation if the given directories aren't empty.", - }), timeout: option({ type: number, defaultValue: () => 30000, @@ -224,7 +248,7 @@ const fullMigrate = command({ 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), }, - handler: async ({ takeoutDir, force, timeout }) => { + handler: async ({ takeoutDir, timeout }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { errs.push('The specified takeout directory does not exist.'); @@ -236,11 +260,21 @@ const fullMigrate = command({ if (await isEmptyDir(takeoutDir)) { errs.push('The google directory is empty. Nothing to do.'); } + if (!(await isEmptyDir(`${takeoutDir}/Photos`))) { + errs.push( + 'The Photos directory is not empty. Please delete it and try again.' + ); + } + if (!(await isEmptyDir(`${takeoutDir}/Albums`))) { + errs.push( + 'The Albums directory is not empty. Please delete it and try again.' + ); + } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); process.exit(1); } - + runFullMigration(takeoutDir, timeout); }, }); @@ -319,7 +353,7 @@ const folderMigrate = command({ const app = subcommands({ name: 'google-photos-migrate', - cmds: { fullMigrate, folderMigrate, unzip }, + cmds: { fullMigrate, folderMigrate, rewriteAllTags }, }); run(app, process.argv.slice(2)); From 5e4353805c1654a103d0a0a600d813353dfb1c01 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 10 Sep 2023 10:04:05 -0400 Subject: [PATCH 11/50] fix json issues and remove force for fullMigrate --- src/cli.ts | 64 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 67a068f..47e41fb 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -17,12 +17,8 @@ import { ExifTool } from 'exiftool-vendored'; import { glob } from 'glob'; import { basename } from 'path'; import path = require('path'); +import { inferLikelyOffsetMinutes } from 'exiftool-vendored/dist/Timezones'; -const unzip = command({ - name: 'unzipper', - args: {}, - handler: async () => {}, -}); async function runBasicMigration( googleDir: string, @@ -85,7 +81,12 @@ async function runMigrationsChecked( const errFiles: string[] = await glob(`${errDir}/*`); const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { - await exifTool.rewriteAllTags(file, path.join(albumDir, `${basename(file)}-1`)); + if (file.endsWith(".json")){ + console.log(`Cannot fix metadata for ${file} as .json is an unsupported file type.`); + continue; + } + console.log(`Rewriting all tags from ${file}, to ${path.join(albumDir, `cleaned-${basename(file)}`)}`); + await exifTool.rewriteAllTags(file, path.join(albumDir, `cleaned-${basename(file)}`)); } exifTool.end(); await runMigrationsChecked( @@ -201,6 +202,35 @@ async function runFullMigration( await processAlbums(rootDir, timeout); } +const rewriteAllTags = command({ + name: 'rewrite all tags for single file', + args: { + inFile: positional({ + type: string, + displayName: 'in_file', + description: 'The path to your input file.', + }), + outFile: positional({ + type: string, + displayName: 'out_file', + description: 'The path to your output location for the file.', + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: async ({inFile, outFile, timeout}) => { + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + await exifTool.rewriteAllTags(inFile, outFile); + exifTool.end(); + }, +}); + const fullMigrate = command({ name: 'google-photos-migrate-full', args: { @@ -209,12 +239,6 @@ const fullMigrate = command({ displayName: 'takeout_dir', description: 'The path to your "Takeout" directory.', }), - force: flag({ - short: 'f', - long: 'force', - description: - "Forces the operation if the given directories aren't empty.", - }), timeout: option({ type: number, defaultValue: () => 30000, @@ -224,7 +248,7 @@ const fullMigrate = command({ 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), }, - handler: async ({ takeoutDir, force, timeout }) => { + handler: async ({ takeoutDir, timeout }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { errs.push('The specified takeout directory does not exist.'); @@ -236,11 +260,21 @@ const fullMigrate = command({ if (await isEmptyDir(takeoutDir)) { errs.push('The google directory is empty. Nothing to do.'); } + if (!(await isEmptyDir(`${takeoutDir}/Photos`))) { + errs.push( + 'The Photos directory is not empty. Please delete it and try again.' + ); + } + if (!(await isEmptyDir(`${takeoutDir}/Albums`))) { + errs.push( + 'The Albums directory is not empty. Please delete it and try again.' + ); + } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); process.exit(1); } - + runFullMigration(takeoutDir, timeout); }, }); @@ -319,7 +353,7 @@ const folderMigrate = command({ const app = subcommands({ name: 'google-photos-migrate', - cmds: { fullMigrate, folderMigrate, unzip }, + cmds: { fullMigrate, folderMigrate, rewriteAllTags }, }); run(app, process.argv.slice(2)); From 50824ec374dd226f3158535048ef6993fd56b7d2 Mon Sep 17 00:00:00 2001 From: covalent Date: Mon, 11 Sep 2023 13:47:13 -0400 Subject: [PATCH 12/50] update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 52b7252..fc2d56c 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,24 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos ## Run this tool +If you wish to migrate a single folder from a Google Photos takeout file: + +```bash +mkdir output error + +npx google-photos-migrate@latest migrateFolder '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +``` + +If you wish to migrate an entire takeout folder: + ```bash mkdir output error -npx google-photos-migrate@latest '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest migrateFolder '/path/to/takeout/' --timeout 60000 ``` +The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. + ## Further steps - If you use Linux + Android, you might want to check out the scripts I used to locate duplicate media and keep the better versions in the [android-dups](./android-dups/) directory. From 40d9d874be22dcd7f9038365879d7eef15f36268 Mon Sep 17 00:00:00 2001 From: covalent Date: Mon, 11 Sep 2023 13:47:13 -0400 Subject: [PATCH 13/50] update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 52b7252..fc2d56c 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,24 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos ## Run this tool +If you wish to migrate a single folder from a Google Photos takeout file: + +```bash +mkdir output error + +npx google-photos-migrate@latest migrateFolder '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +``` + +If you wish to migrate an entire takeout folder: + ```bash mkdir output error -npx google-photos-migrate@latest '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest migrateFolder '/path/to/takeout/' --timeout 60000 ``` +The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. + ## Further steps - If you use Linux + Android, you might want to check out the scripts I used to locate duplicate media and keep the better versions in the [android-dups](./android-dups/) directory. From ff93a1dd6debbe9793bbb2c5cecc30d45c3198b3 Mon Sep 17 00:00:00 2001 From: covalent Date: Mon, 11 Sep 2023 19:08:15 -0400 Subject: [PATCH 14/50] update .gitingore to ignore build directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 27aaaf7..2244834 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ error/ # Build lib/ +build/ # Env files .env From 083fe602790400c271a68d9ac8573076fbc09f8f Mon Sep 17 00:00:00 2001 From: covalent Date: Mon, 11 Sep 2023 19:08:15 -0400 Subject: [PATCH 15/50] update .gitingore to ignore build directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 27aaaf7..2244834 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ error/ # Build lib/ +build/ # Env files .env From 1ff1c50ac35f3bbcf4e70c9bbcf20f9b17905ef8 Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 15 Sep 2023 17:51:06 -0400 Subject: [PATCH 16/50] fix readme & improve error logging --- README.md | 4 ++-- src/cli.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index fc2d56c..7175fa3 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If you wish to migrate a single folder from a Google Photos takeout file: ```bash mkdir output error -npx google-photos-migrate@latest migrateFolder '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest fullMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` If you wish to migrate an entire takeout folder: @@ -24,7 +24,7 @@ If you wish to migrate an entire takeout folder: ```bash mkdir output error -npx google-photos-migrate@latest migrateFolder '/path/to/takeout/' --timeout 60000 +npx google-photos-migrate@latest folderMigrate '/path/to/takeout/' --timeout 60000 ``` The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. diff --git a/src/cli.ts b/src/cli.ts index 47e41fb..1552f4c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -61,13 +61,13 @@ async function runMigrationsChecked( ) { const errs: string[] = []; if (!existsSync(albumDir)) { - errs.push('The specified google directory does not exist.'); + errs.push(`The specified google directory does not exist: ${albumDir}`); } if (!existsSync(outDir)) { - errs.push('The specified output directory does not exist.'); + errs.push(`The specified google directory does not exist: ${outDir}`); } if (!existsSync(errDir)) { - errs.push('The specified error directory does not exist.'); + errs.push(`The specified google directory does not exist: ${errDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); @@ -251,7 +251,7 @@ const fullMigrate = command({ handler: async ({ takeoutDir, timeout }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { - errs.push('The specified takeout directory does not exist.'); + errs.push(`The specified takeout directory does not exist: ${takeoutDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); @@ -315,13 +315,13 @@ const folderMigrate = command({ handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; if (!existsSync(googleDir)) { - errs.push('The specified google directory does not exist.'); + errs.push(`The specified google directory does not exist: ${googleDir}`); } if (!existsSync(outputDir)) { - errs.push('The specified output directory does not exist.'); + errs.push(`The specified output directory does not exist: ${googleDir}`); } if (!existsSync(errorDir)) { - errs.push('The specified error directory does not exist.'); + errs.push(`The specified error directory does not exist: ${googleDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); From 91f67cf63321d643493284d099163662e8fd25e4 Mon Sep 17 00:00:00 2001 From: covalent Date: Fri, 15 Sep 2023 18:13:45 -0400 Subject: [PATCH 17/50] remove build dir for good --- build/cli.js | 284 ----------------------------- build/config/env.js | 7 - build/config/extensions.js | 40 ---- build/fs/file-exists.js | 8 - build/fs/is-empty-dir.js | 15 -- build/fs/walk-dir.js | 42 ----- build/index.js | 8 - build/media/InvalidExtError.js | 10 - build/media/MediaFile.js | 2 - build/media/MediaFileExtension.js | 2 - build/media/MediaMigrationError.js | 10 - build/media/MetaType.js | 9 - build/media/NoMetaFileError.js | 10 - build/media/migrate-google-dir.js | 142 --------------- build/media/save-to-dir.js | 60 ------ build/media/title-json-map.js | 69 ------- build/meta/GoogleMeta.js | 2 - build/meta/apply-meta-errors.js | 46 ----- build/meta/apply-meta-file.js | 72 -------- build/meta/find-meta-file.js | 71 -------- build/meta/read-meta-title.js | 23 --- build/ts.js | 7 - 22 files changed, 939 deletions(-) delete mode 100644 build/cli.js delete mode 100644 build/config/env.js delete mode 100644 build/config/extensions.js delete mode 100644 build/fs/file-exists.js delete mode 100644 build/fs/is-empty-dir.js delete mode 100644 build/fs/walk-dir.js delete mode 100644 build/index.js delete mode 100644 build/media/InvalidExtError.js delete mode 100644 build/media/MediaFile.js delete mode 100644 build/media/MediaFileExtension.js delete mode 100644 build/media/MediaMigrationError.js delete mode 100644 build/media/MetaType.js delete mode 100644 build/media/NoMetaFileError.js delete mode 100644 build/media/migrate-google-dir.js delete mode 100644 build/media/save-to-dir.js delete mode 100644 build/media/title-json-map.js delete mode 100644 build/meta/GoogleMeta.js delete mode 100644 build/meta/apply-meta-errors.js delete mode 100644 build/meta/apply-meta-file.js delete mode 100644 build/meta/find-meta-file.js delete mode 100644 build/meta/read-meta-title.js delete mode 100644 build/ts.js diff --git a/build/cli.js b/build/cli.js deleted file mode 100644 index 31378ff..0000000 --- a/build/cli.js +++ /dev/null @@ -1,284 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -const fs_1 = require("fs"); -const cmd_ts_1 = require("cmd-ts"); -const migrate_google_dir_1 = require("./media/migrate-google-dir"); -const is_empty_dir_1 = require("./fs/is-empty-dir"); -const exiftool_vendored_1 = require("exiftool-vendored"); -const glob_1 = require("glob"); -const path_1 = require("path"); -const path = require("path"); -const unzip = (0, cmd_ts_1.command)({ - name: 'unzipper', - args: {}, - handler: () => __awaiter(void 0, void 0, void 0, function* () { }), -}); -function runBasicMigration(googleDir, outputDir, errorDir, exifTool) { - var _a, e_1, _b, _c; - return __awaiter(this, void 0, void 0, function* () { - console.log(`Started migration.`); - const migGen = (0, migrate_google_dir_1.migrateGoogleDirGen)({ - googleDir, - outputDir, - errorDir, - warnLog: console.error, - exiftool: exifTool, - endExifTool: true, - }); - const counts = { err: 0, suc: 0 }; - try { - for (var _d = true, migGen_1 = __asyncValues(migGen), migGen_1_1; migGen_1_1 = yield migGen_1.next(), _a = migGen_1_1.done, !_a; _d = true) { - _c = migGen_1_1.value; - _d = false; - const result = _c; - if (result instanceof Error) { - console.error(`Error: ${result}`); - counts.err++; - continue; - } - counts.suc++; - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (!_d && !_a && (_b = migGen_1.return)) yield _b.call(migGen_1); - } - finally { if (e_1) throw e_1.error; } - } - console.log(`Done! Processed ${counts.suc + counts.err} files.`); - console.log(`Files migrated: ${counts.suc}`); - console.log(`Files failed: ${counts.err}`); - }); -} -function runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, check_errDir) { - return __awaiter(this, void 0, void 0, function* () { - const errs = []; - if (!(0, fs_1.existsSync)(albumDir)) { - errs.push('The specified google directory does not exist.'); - } - if (!(0, fs_1.existsSync)(outDir)) { - errs.push('The specified output directory does not exist.'); - } - if (!(0, fs_1.existsSync)(errDir)) { - errs.push('The specified error directory does not exist.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - yield runBasicMigration(albumDir, outDir, errDir, exifTool); - if (check_errDir && !(yield (0, is_empty_dir_1.isEmptyDir)(errDir))) { - const errFiles = yield (0, glob_1.glob)(`${errDir}/*`); - for (let file of errFiles) { - yield exifTool.rewriteAllTags(file, path.join(albumDir, (0, path_1.basename)(file))); - } - yield runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, false); - } - }); -} -function processAlbums(rootDir, timeout, exifTool) { - return __awaiter(this, void 0, void 0, function* () { - const globStr = `${rootDir}/Albums/*/`; - const albums = yield (0, glob_1.glob)(globStr); - if (albums.length == 0) { - console.log(`WARN: No albums found at ${globStr}`); - } - for (let album of albums) { - console.log(`Processing album ${album}...`); - let albumName = (0, path_1.basename)(album); - let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; - let errDir = `${rootDir}/AlbumsError/${albumName}`; - (0, fs_1.mkdirSync)(album, { recursive: true }); - (0, fs_1.mkdirSync)(outDir, { recursive: true }); - (0, fs_1.mkdirSync)(errDir, { recursive: true }); - yield runMigrationsChecked(album, outDir, errDir, timeout, exifTool, true); - } - }); -} -function processPhotos(rootDir, timeout, exifTool) { - return __awaiter(this, void 0, void 0, function* () { - // Also run the exif fix for the photos - console.log('Processing photos...'); - const albumDir = `${rootDir}/Photos`; - const outDir = `${rootDir}/PhotosProcessed`; - const errDir = `${rootDir}/PhotosError`; - (0, fs_1.mkdirSync)(albumDir, { recursive: true }); - (0, fs_1.mkdirSync)(outDir, { recursive: true }); - (0, fs_1.mkdirSync)(errDir, { recursive: true }); - yield runMigrationsChecked(albumDir, outDir, errDir, timeout, exifTool, true); - }); -} -function _restructureIfNeeded(folders, targetDir) { - return __awaiter(this, void 0, void 0, function* () { - if ((0, fs_1.existsSync)(targetDir) && ((yield (0, glob_1.glob)(`${targetDir}/*`)).length) > 0) { - console.log(`${targetDir} exists and is not empty. No restructuring needed.`); - return; - } - console.log(`Starting restructure of ${folders.length} directories`); - (0, fs_1.mkdirSync)(targetDir, { recursive: true }); - for (let folder of folders) { - console.log(`Copying ${folder} to ${targetDir}/${(0, path_1.basename)(folder)}`); - (0, fs_1.cpSync)(folder, `${targetDir}/${(0, path_1.basename)(folder)}`, { recursive: true }); - } - console.log(`Sucsessfully restructured ${folders.length} directories`); - }); -} -function restructureIfNeeded(rootDir) { - return __awaiter(this, void 0, void 0, function* () { - // before - // $rootdir/My Album 1 - // $rootdir/My Album 2 - // $rootdir/Photos from 2008 - // after - // $rootdir/Albums/My Album 1 - // $rootdir/Albums/My Album 2 - // $rootdir/Photos/Photos from 2008 - const photosDir = `${rootDir}/Photos`; - // move the "Photos from $YEAR" directories to Photos/ - _restructureIfNeeded(yield (0, glob_1.glob)(`${rootDir}/Photos from */`), photosDir); - // move everythingg else to Albums/, so we end up with two top level folders - const fullSet = new Set(yield (0, glob_1.glob)(`${rootDir}/*/`)); - const photoSet = new Set(yield (0, glob_1.glob)(`${rootDir}/Photos from */`)); - photoSet.add(`${rootDir}/Photos`); - const everythingExceptPhotosDir = Array.from(new Set([...fullSet].filter((x) => !photoSet.has(x)))); - _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); - }); -} -function run_full_migration(rootDir, timeout, exifTool) { - return __awaiter(this, void 0, void 0, function* () { - // at least in my takeout, the Takeout folder contains a subfolder - // Takeout/Google Foto - // rootdir refers to that subfolder - rootDir = (yield (0, glob_1.glob)(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); - yield restructureIfNeeded(rootDir); - yield processAlbums(rootDir, timeout, exifTool); - yield processPhotos(rootDir, timeout, exifTool); - exifTool.end(); - }); -} -const fullMigrate = (0, cmd_ts_1.command)({ - name: 'google-photos-migrate-full', - args: { - takeoutDir: (0, cmd_ts_1.positional)({ - type: cmd_ts_1.string, - displayName: 'takeout_dir', - description: 'The path to your "Takeout" directory.', - }), - force: (0, cmd_ts_1.flag)({ - short: 'f', - long: 'force', - description: "Forces the operation if the given directories aren't empty.", - }), - timeout: (0, cmd_ts_1.option)({ - type: cmd_ts_1.number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - }, - handler: ({ takeoutDir, force, timeout }) => __awaiter(void 0, void 0, void 0, function* () { - const errs = []; - if (!(0, fs_1.existsSync)(takeoutDir)) { - errs.push('The specified takeout directory does not exist.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - if (yield (0, is_empty_dir_1.isEmptyDir)(takeoutDir)) { - errs.push('The google directory is empty. Nothing to do.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - const exifTool = new exiftool_vendored_1.ExifTool({ taskTimeoutMillis: timeout }); - run_full_migration(takeoutDir, timeout, exifTool); - }), -}); -const folderMigrate = (0, cmd_ts_1.command)({ - name: 'google-photos-migrate-folder', - args: { - googleDir: (0, cmd_ts_1.positional)({ - type: cmd_ts_1.string, - displayName: 'google_dir', - description: 'The path to your "Google Photos" directory.', - }), - outputDir: (0, cmd_ts_1.positional)({ - type: cmd_ts_1.string, - displayName: 'output_dir', - description: 'The path to your flat output directory.', - }), - errorDir: (0, cmd_ts_1.positional)({ - type: cmd_ts_1.string, - displayName: 'error_dir', - description: 'Failed media will be saved here.', - }), - force: (0, cmd_ts_1.flag)({ - short: 'f', - long: 'force', - description: "Forces the operation if the given directories aren't empty.", - }), - timeout: (0, cmd_ts_1.option)({ - type: cmd_ts_1.number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - }, - handler: ({ googleDir, outputDir, errorDir, force, timeout }) => __awaiter(void 0, void 0, void 0, function* () { - const errs = []; - if (!(0, fs_1.existsSync)(googleDir)) { - errs.push('The specified google directory does not exist.'); - } - if (!(0, fs_1.existsSync)(outputDir)) { - errs.push('The specified output directory does not exist.'); - } - if (!(0, fs_1.existsSync)(errorDir)) { - errs.push('The specified error directory does not exist.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - if (!force && !(yield (0, is_empty_dir_1.isEmptyDir)(outputDir))) { - errs.push('The output directory is not empty. Pass "-f" to force the operation.'); - } - if (!force && !(yield (0, is_empty_dir_1.isEmptyDir)(errorDir))) { - errs.push('The error directory is not empty. Pass "-f" to force the operation.'); - } - if (yield (0, is_empty_dir_1.isEmptyDir)(googleDir)) { - errs.push('The google directory is empty. Nothing to do.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - const exifTool = new exiftool_vendored_1.ExifTool({ taskTimeoutMillis: timeout }); - yield runBasicMigration(googleDir, outputDir, errorDir, exifTool); - }), -}); -const app = (0, cmd_ts_1.subcommands)({ - name: 'google-photos-migrate', - cmds: { fullMigrate, folderMigrate, unzip }, -}); -(0, cmd_ts_1.run)(app, process.argv.slice(2)); diff --git a/build/config/env.js b/build/config/env.js deleted file mode 100644 index 518178e..0000000 --- a/build/config/env.js +++ /dev/null @@ -1,7 +0,0 @@ -"use strict"; -var _a; -var _b; -Object.defineProperty(exports, "__esModule", { value: true }); -const dotenv = require("dotenv"); -dotenv.config(); -(_a = (_b = process.env).NODE_ENV) !== null && _a !== void 0 ? _a : (_b.NODE_ENV = 'production'); diff --git a/build/config/extensions.js b/build/config/extensions.js deleted file mode 100644 index c53be5a..0000000 --- a/build/config/extensions.js +++ /dev/null @@ -1,40 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.supportedExtensions = void 0; -const MetaType_1 = require("../media/MetaType"); -exports.supportedExtensions = [ - { suffix: '.jpg', metaType: MetaType_1.MetaType.EXIF }, - { suffix: '.jpeg', metaType: MetaType_1.MetaType.EXIF }, - { suffix: '.png', metaType: MetaType_1.MetaType.EXIF }, - { suffix: '.raw', metaType: MetaType_1.MetaType.NONE }, - { suffix: '.ico', metaType: MetaType_1.MetaType.NONE }, - { suffix: '.tiff', metaType: MetaType_1.MetaType.EXIF }, - { suffix: '.webp', metaType: MetaType_1.MetaType.EXIF }, - { suffix: '.heic', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.heif', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.gif', metaType: MetaType_1.MetaType.NONE }, - { - suffix: '.mp4', - metaType: MetaType_1.MetaType.QUICKTIME, - aliases: ['.heic', '.jpg', '.jpeg'], // Live photos - }, - { - suffix: '.mov', - metaType: MetaType_1.MetaType.QUICKTIME, - aliases: ['.heic', '.jpg', '.jpeg'], // Live photos - }, - { suffix: '.qt', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.mov.qt', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.3gp', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.mp4v', metaType: MetaType_1.MetaType.QUICKTIME }, - { suffix: '.mkv', metaType: MetaType_1.MetaType.NONE }, - { suffix: '.wmv', metaType: MetaType_1.MetaType.NONE }, - { suffix: '.webm', metaType: MetaType_1.MetaType.NONE }, -].flatMap((e) => { - var _a; - const aliases = (_a = e.aliases) === null || _a === void 0 ? void 0 : _a.flatMap((s) => [s.toLowerCase(), s.toUpperCase()]); - return [ - Object.assign(Object.assign({}, e), { suffix: e.suffix.toLowerCase(), aliases }), - Object.assign(Object.assign({}, e), { suffix: e.suffix.toUpperCase(), aliases }), - ]; -}); diff --git a/build/fs/file-exists.js b/build/fs/file-exists.js deleted file mode 100644 index 28534ed..0000000 --- a/build/fs/file-exists.js +++ /dev/null @@ -1,8 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.fileExists = void 0; -const promises_1 = require("fs/promises"); -const fileExists = (path) => (0, promises_1.stat)(path) - .then(() => true) - .catch((e) => (e.code === 'ENOENT' ? false : Promise.reject(e))); -exports.fileExists = fileExists; diff --git a/build/fs/is-empty-dir.js b/build/fs/is-empty-dir.js deleted file mode 100644 index 92092ea..0000000 --- a/build/fs/is-empty-dir.js +++ /dev/null @@ -1,15 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.isEmptyDir = void 0; -const promises_1 = require("fs/promises"); -const isEmptyDir = (dir) => __awaiter(void 0, void 0, void 0, function* () { return (yield (0, promises_1.readdir)(dir)).length === 0; }); -exports.isEmptyDir = isEmptyDir; diff --git a/build/fs/walk-dir.js b/build/fs/walk-dir.js deleted file mode 100644 index c1fdc8c..0000000 --- a/build/fs/walk-dir.js +++ /dev/null @@ -1,42 +0,0 @@ -"use strict"; -var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -var __asyncDelegator = (this && this.__asyncDelegator) || function (o) { - var i, p; - return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i; - function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: false } : f ? f(v) : v; } : f; } -}; -var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var g = generator.apply(thisArg, _arguments || []), i, q = []; - return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; - function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } - function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } - function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } - function fulfill(value) { resume("next", value); } - function reject(value) { resume("throw", value); } - function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.walkDir = void 0; -const promises_1 = require("fs/promises"); -const path_1 = require("path"); -function walkDir(dir) { - return __asyncGenerator(this, arguments, function* walkDir_1() { - for (const dirent of yield __await((0, promises_1.readdir)(dir, { withFileTypes: true }))) { - if (dirent.isDirectory()) { - yield __await(yield* __asyncDelegator(__asyncValues(walkDir((0, path_1.resolve)(dir, dirent.name))))); - } - else { - yield yield __await((0, path_1.resolve)(dir, dirent.name)); - } - } - }); -} -exports.walkDir = walkDir; diff --git a/build/index.js b/build/index.js deleted file mode 100644 index 9589ac8..0000000 --- a/build/index.js +++ /dev/null @@ -1,8 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.supportedExtensions = exports.migrateGoogleDir = void 0; -require("./config/env"); -const migrate_google_dir_1 = require("./media/migrate-google-dir"); -Object.defineProperty(exports, "migrateGoogleDir", { enumerable: true, get: function () { return migrate_google_dir_1.migrateGoogleDir; } }); -const extensions_1 = require("./config/extensions"); -Object.defineProperty(exports, "supportedExtensions", { enumerable: true, get: function () { return extensions_1.supportedExtensions; } }); diff --git a/build/media/InvalidExtError.js b/build/media/InvalidExtError.js deleted file mode 100644 index f18d3a5..0000000 --- a/build/media/InvalidExtError.js +++ /dev/null @@ -1,10 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.InvalidExtError = void 0; -const MediaMigrationError_1 = require("./MediaMigrationError"); -class InvalidExtError extends MediaMigrationError_1.MediaMigrationError { - toString() { - return `File has invalid extension: ${this.failedMedia.path}`; - } -} -exports.InvalidExtError = InvalidExtError; diff --git a/build/media/MediaFile.js b/build/media/MediaFile.js deleted file mode 100644 index c8ad2e5..0000000 --- a/build/media/MediaFile.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/media/MediaFileExtension.js b/build/media/MediaFileExtension.js deleted file mode 100644 index c8ad2e5..0000000 --- a/build/media/MediaFileExtension.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/media/MediaMigrationError.js b/build/media/MediaMigrationError.js deleted file mode 100644 index 25eacff..0000000 --- a/build/media/MediaMigrationError.js +++ /dev/null @@ -1,10 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.MediaMigrationError = void 0; -class MediaMigrationError extends Error { - constructor(failedMedia) { - super(); - this.failedMedia = failedMedia; - } -} -exports.MediaMigrationError = MediaMigrationError; diff --git a/build/media/MetaType.js b/build/media/MetaType.js deleted file mode 100644 index d750457..0000000 --- a/build/media/MetaType.js +++ /dev/null @@ -1,9 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.MetaType = void 0; -var MetaType; -(function (MetaType) { - MetaType["NONE"] = "NONE"; - MetaType["EXIF"] = "EXIF"; - MetaType["QUICKTIME"] = "QUICKTIME"; -})(MetaType || (exports.MetaType = MetaType = {})); diff --git a/build/media/NoMetaFileError.js b/build/media/NoMetaFileError.js deleted file mode 100644 index 6fdfb30..0000000 --- a/build/media/NoMetaFileError.js +++ /dev/null @@ -1,10 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.NoMetaFileError = void 0; -const MediaMigrationError_1 = require("./MediaMigrationError"); -class NoMetaFileError extends MediaMigrationError_1.MediaMigrationError { - toString() { - return `Unable to locate meta file for: ${this.failedMedia.path}`; - } -} -exports.NoMetaFileError = NoMetaFileError; diff --git a/build/media/migrate-google-dir.js b/build/media/migrate-google-dir.js deleted file mode 100644 index b3d240d..0000000 --- a/build/media/migrate-google-dir.js +++ /dev/null @@ -1,142 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } -var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var g = generator.apply(thisArg, _arguments || []), i, q = []; - return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; - function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } - function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } - function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } - function fulfill(value) { resume("next", value); } - function reject(value) { resume("throw", value); } - function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.migrateGoogleDirGen = exports.migrateGoogleDir = void 0; -const walk_dir_1 = require("../fs/walk-dir"); -const path_1 = require("path"); -const find_meta_file_1 = require("../meta/find-meta-file"); -const apply_meta_file_1 = require("../meta/apply-meta-file"); -const title_json_map_1 = require("./title-json-map"); -const extensions_1 = require("../config/extensions"); -const InvalidExtError_1 = require("./InvalidExtError"); -const NoMetaFileError_1 = require("./NoMetaFileError"); -const apply_meta_errors_1 = require("../meta/apply-meta-errors"); -const exiftool_vendored_1 = require("exiftool-vendored"); -const read_meta_title_1 = require("../meta/read-meta-title"); -const save_to_dir_1 = require("./save-to-dir"); -function migrateGoogleDir(args) { - return __asyncGenerator(this, arguments, function* migrateGoogleDir_1() { - var _a, e_1, _b, _c; - const wg = []; - try { - for (var _d = true, _e = __asyncValues(migrateGoogleDirGen(args)), _f; _f = yield __await(_e.next()), _a = _f.done, !_a; _d = true) { - _c = _f.value; - _d = false; - const result = _c; - wg.push(result); - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (!_d && !_a && (_b = _e.return)) yield __await(_b.call(_e)); - } - finally { if (e_1) throw e_1.error; } - } - return yield __await(yield __await(Promise.all(wg))); - }); -} -exports.migrateGoogleDir = migrateGoogleDir; -function migrateGoogleDirGen(args) { - var _a, _b, _c; - return __asyncGenerator(this, arguments, function* migrateGoogleDirGen_1() { - var _d, e_2, _e, _f; - const migCtx = Object.assign(Object.assign({ titleJsonMap: yield __await((0, title_json_map_1.indexJsonFiles)(args.googleDir)), migrationLocks: new Map() }, args), { exiftool: (_a = args.exiftool) !== null && _a !== void 0 ? _a : new exiftool_vendored_1.ExifTool(), endExifTool: (_b = args.endExifTool) !== null && _b !== void 0 ? _b : !args.exiftool, warnLog: (_c = args.warnLog) !== null && _c !== void 0 ? _c : (() => { }) }); - try { - for (var _g = true, _h = __asyncValues((0, walk_dir_1.walkDir)(args.googleDir)), _j; _j = yield __await(_h.next()), _d = _j.done, !_d; _g = true) { - _f = _j.value; - _g = false; - const mediaPath = _f; - if (mediaPath.endsWith('.json')) - continue; - yield yield __await(migrateMediaFile(mediaPath, migCtx)); - } - } - catch (e_2_1) { e_2 = { error: e_2_1 }; } - finally { - try { - if (!_g && !_d && (_e = _h.return)) yield __await(_e.call(_h)); - } - finally { if (e_2) throw e_2.error; } - } - migCtx.endExifTool && migCtx.exiftool.end(); - }); -} -exports.migrateGoogleDirGen = migrateGoogleDirGen; -function migrateMediaFile(originalPath, migCtx) { - return __awaiter(this, void 0, void 0, function* () { - const mediaFileInfo = { - originalPath, - path: originalPath, - }; - const ext = extensions_1.supportedExtensions.reduce((longestMatch, cur) => { - if (!originalPath.endsWith(cur.suffix)) - return longestMatch; - if (longestMatch === null) - return cur; - return cur.suffix.length > longestMatch.suffix.length - ? cur - : longestMatch; - }, null); - if (!ext) { - mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.errorDir, migCtx); - return new InvalidExtError_1.InvalidExtError(mediaFileInfo); - } - const jsonPath = yield (0, find_meta_file_1.findMetaFile)(originalPath, ext, migCtx); - if (!jsonPath) { - mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.errorDir, migCtx); - return new NoMetaFileError_1.NoMetaFileError(mediaFileInfo); - } - mediaFileInfo.jsonPath = jsonPath; - mediaFileInfo.path = yield (0, save_to_dir_1.saveToDir)(originalPath, migCtx.outputDir, migCtx, false, yield (0, read_meta_title_1.readMetaTitle)(mediaFileInfo)); - const mediaFile = Object.assign(Object.assign({}, mediaFileInfo), { ext, - jsonPath }); - let err = yield (0, apply_meta_file_1.applyMetaFile)(mediaFile, migCtx); - if (!err) { - return mediaFile; - } - if (err instanceof apply_meta_errors_1.WrongExtensionError) { - const oldBase = (0, path_1.basename)(mediaFile.path); - const newBase = oldBase.slice(0, oldBase.length - err.expectedExt.length) + err.actualExt; - mediaFile.path = yield (0, save_to_dir_1.saveToDir)(mediaFile.path, migCtx.outputDir, migCtx, true, newBase); - migCtx.warnLog(`Renamed wrong extension ${err.expectedExt} to ${err.actualExt}: ${mediaFile.path}`); - err = yield (0, apply_meta_file_1.applyMetaFile)(mediaFile, migCtx); - if (!err) { - return mediaFile; - } - } - const savedPaths = yield Promise.all([ - (0, save_to_dir_1.saveToDir)(mediaFile.path, migCtx.errorDir, migCtx, true), - (0, save_to_dir_1.saveToDir)(mediaFile.jsonPath, migCtx.errorDir, migCtx), - ]); - mediaFile.path = savedPaths[0]; - return err; - }); -} diff --git a/build/media/save-to-dir.js b/build/media/save-to-dir.js deleted file mode 100644 index 598b011..0000000 --- a/build/media/save-to-dir.js +++ /dev/null @@ -1,60 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.saveToDir = void 0; -const path_1 = require("path"); -const file_exists_1 = require("../fs/file-exists"); -const sanitize = require("sanitize-filename"); -const promises_1 = require("fs/promises"); -function _saveToDir(file, destDir, saveBase, move = false, duplicateIndex = 0) { - return __awaiter(this, void 0, void 0, function* () { - const saveDir = (0, path_1.resolve)(destDir, duplicateIndex > 0 ? `duplicates-${duplicateIndex}` : '.'); - yield (0, promises_1.mkdir)(saveDir, { recursive: true }); - const savePath = (0, path_1.resolve)(saveDir, saveBase); - const exists = yield (0, file_exists_1.fileExists)(savePath); - if (exists) { - return _saveToDir(file, destDir, saveBase, move, duplicateIndex + 1); - } - if (move) { - yield (0, promises_1.rename)(file, savePath); - } - else { - yield (0, promises_1.copyFile)(file, savePath); - } - return savePath; - }); -} -/** Copies or moves a file to dir, saves duplicates in subfolders and returns the new path. - * Atomic within this app, sanitizes filenames. - */ -function saveToDir(file, destDir, migCtx, move = false, saveBase) { - return __awaiter(this, void 0, void 0, function* () { - saveBase = saveBase !== null && saveBase !== void 0 ? saveBase : (0, path_1.basename)(file); - const sanitized = sanitize(saveBase, { replacement: '_' }); - if (saveBase != sanitized) { - migCtx.warnLog(`Sanitized file: ${file}` + '\nNew filename: ${sanitized}'); - } - const lcBase = saveBase.toLowerCase(); - let lock; - while ((lock = migCtx.migrationLocks.get(lcBase))) { - yield lock; - } - lock = _saveToDir(file, destDir, sanitized, move); - migCtx.migrationLocks.set(lcBase, lock); - try { - return yield lock; - } - finally { - migCtx.migrationLocks.delete(lcBase); - } - }); -} -exports.saveToDir = saveToDir; diff --git a/build/media/title-json-map.js b/build/media/title-json-map.js deleted file mode 100644 index 417007b..0000000 --- a/build/media/title-json-map.js +++ /dev/null @@ -1,69 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.indexJsonFiles = void 0; -const promises_1 = require("fs/promises"); -const walk_dir_1 = require("../fs/walk-dir"); -const path_1 = require("path"); -const MAX_BASE_LENGTH = 51; -function trimTitle(title) { } -function indexJsonFiles(googleDir) { - var _a, e_1, _b, _c; - var _d; - return __awaiter(this, void 0, void 0, function* () { - const titleJsonMap = new Map(); - try { - for (var _e = true, _f = __asyncValues((0, walk_dir_1.walkDir)(googleDir)), _g; _g = yield _f.next(), _a = _g.done, !_a; _e = true) { - _c = _g.value; - _e = false; - const jsonPath = _c; - if (!jsonPath.endsWith('.json')) - continue; - let title; - try { - const data = JSON.parse((yield (0, promises_1.readFile)(jsonPath)).toString()); - title = data.title; - } - catch (e) { } - if (typeof title !== 'string') - continue; - const potTitles = new Set(); - const ext = (0, path_1.extname)(title); - const woExt = title.slice(0, -ext.length); - const maxWoExt = MAX_BASE_LENGTH - ext.length; - potTitles.add(woExt.slice(0, maxWoExt) + ext); - potTitles.add((woExt + '-edited').slice(0, maxWoExt) + ext); - potTitles.add((woExt + '-bearbeitet').slice(0, maxWoExt) + ext); - for (const potTitle of potTitles) { - const jsonPaths = (_d = titleJsonMap.get(potTitle)) !== null && _d !== void 0 ? _d : []; - jsonPaths.push(jsonPath); - titleJsonMap.set(potTitle, jsonPaths); - } - } - } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (!_e && !_a && (_b = _f.return)) yield _b.call(_f); - } - finally { if (e_1) throw e_1.error; } - } - return titleJsonMap; - }); -} -exports.indexJsonFiles = indexJsonFiles; diff --git a/build/meta/GoogleMeta.js b/build/meta/GoogleMeta.js deleted file mode 100644 index c8ad2e5..0000000 --- a/build/meta/GoogleMeta.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/meta/apply-meta-errors.js b/build/meta/apply-meta-errors.js deleted file mode 100644 index 0a44e2e..0000000 --- a/build/meta/apply-meta-errors.js +++ /dev/null @@ -1,46 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.MissingMetaError = exports.WrongExtensionError = exports.ExifToolError = exports.ApplyMetaError = void 0; -const MediaMigrationError_1 = require("../media/MediaMigrationError"); -class ApplyMetaError extends MediaMigrationError_1.MediaMigrationError { - constructor(failedMedia) { - super(failedMedia); - } - toString() { - return `Failed to apply meta tags on file: ${this.failedMedia.path}`; - } -} -exports.ApplyMetaError = ApplyMetaError; -class ExifToolError extends ApplyMetaError { - constructor(failedMedia, reason) { - super(failedMedia); - this.reason = reason; - } - toString() { - return (`ExifTool failed to modify file: ${this.failedMedia.path}` + - `\nReason: ${this.reason.message}`); - } -} -exports.ExifToolError = ExifToolError; -class WrongExtensionError extends ApplyMetaError { - constructor(failedMedia, expectedExt, actualExt) { - super(failedMedia); - this.expectedExt = expectedExt; - this.actualExt = actualExt; - } - toString() { - return `File has wrong file extension ${this.actualExt}, should be ${this.expectedExt}: ${this.failedMedia.path}`; - } -} -exports.WrongExtensionError = WrongExtensionError; -class MissingMetaError extends ApplyMetaError { - constructor(failedMedia, keyName) { - super(failedMedia); - this.keyName = keyName; - } - toString() { - return (`Missing key ${this.keyName} from meta file: ${this.failedMedia.jsonPath}` + - `\nOriginal file: ${this.failedMedia.path}`); - } -} -exports.MissingMetaError = MissingMetaError; diff --git a/build/meta/apply-meta-file.js b/build/meta/apply-meta-file.js deleted file mode 100644 index 629215c..0000000 --- a/build/meta/apply-meta-file.js +++ /dev/null @@ -1,72 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.applyMetaFile = void 0; -const ts_1 = require("../ts"); -const MetaType_1 = require("../media/MetaType"); -const promises_1 = require("fs/promises"); -const apply_meta_errors_1 = require("./apply-meta-errors"); -function applyMetaFile(mediaFile, migCtx) { - var _a, _b, _c; - return __awaiter(this, void 0, void 0, function* () { - const metaJson = (yield (0, promises_1.readFile)(mediaFile.jsonPath)).toString(); - const meta = JSON.parse(metaJson); - const timeTakenTimestamp = (_a = meta === null || meta === void 0 ? void 0 : meta.photoTakenTime) === null || _a === void 0 ? void 0 : _a.timestamp; - if (timeTakenTimestamp === undefined) - return new apply_meta_errors_1.MissingMetaError(mediaFile, 'photoTakenTime'); - const timeTaken = new Date(parseInt(timeTakenTimestamp) * 1000); - // always UTC as per https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString - const timeTakenUTC = timeTaken.toISOString(); - const tags = {}; - switch (mediaFile.ext.metaType) { - case MetaType_1.MetaType.EXIF: - tags.SubSecDateTimeOriginal = timeTakenUTC; - tags.SubSecCreateDate = timeTakenUTC; - tags.SubSecModifyDate = timeTakenUTC; - break; - case MetaType_1.MetaType.QUICKTIME: - tags.DateTimeOriginal = timeTakenUTC; - tags.CreateDate = timeTakenUTC; - tags.ModifyDate = timeTakenUTC; - tags.TrackCreateDate = timeTakenUTC; - tags.TrackModifyDate = timeTakenUTC; - tags.MediaCreateDate = timeTakenUTC; - tags.MediaModifyDate = timeTakenUTC; - break; - case MetaType_1.MetaType.NONE: - break; - default: - (0, ts_1.exhaustiveCheck)(mediaFile.ext.metaType); - } - tags.FileModifyDate = timeTakenUTC; - try { - yield migCtx.exiftool.write(mediaFile.path, tags, [ - '-overwrite_original', - '-api', - 'quicktimeutc', - ]); - } - catch (e) { - if (e instanceof Error) { - const wrongExtMatch = e.message.match(/Not a valid (?\w+) \(looks more like a (?\w+)\)/); - const expected = (_b = wrongExtMatch === null || wrongExtMatch === void 0 ? void 0 : wrongExtMatch.groups) === null || _b === void 0 ? void 0 : _b['expected']; - const actual = (_c = wrongExtMatch === null || wrongExtMatch === void 0 ? void 0 : wrongExtMatch.groups) === null || _c === void 0 ? void 0 : _c['actual']; - if (expected !== undefined && actual !== undefined) { - return new apply_meta_errors_1.WrongExtensionError(mediaFile, `.${expected.toLowerCase()}`, `.${actual.toLowerCase()}`); - } - return new apply_meta_errors_1.ExifToolError(mediaFile, e); - } - return new apply_meta_errors_1.ExifToolError(mediaFile, new Error(`${e}`)); - } - return null; - }); -} -exports.applyMetaFile = applyMetaFile; diff --git a/build/meta/find-meta-file.js b/build/meta/find-meta-file.js deleted file mode 100644 index 0a11566..0000000 --- a/build/meta/find-meta-file.js +++ /dev/null @@ -1,71 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.findMetaFile = void 0; -const path_1 = require("path"); -const file_exists_1 = require("../fs/file-exists"); -function findMetaFile(mediaPath, ext, migCtx) { - var _a; - return __awaiter(this, void 0, void 0, function* () { - const title = (0, path_1.basename)(mediaPath); - // Most json files can be matched by indexed titles - const indexedJson = migCtx.titleJsonMap.get(title); - if (indexedJson) { - const sameFolder = indexedJson.filter((jsonPath) => (0, path_1.dirname)(jsonPath) === (0, path_1.dirname)(mediaPath)); - if (sameFolder.length === 1) { - return sameFolder[0]; - } - } - // Otherwise, try (from most to least significant) - const potPaths = new Set(); - const pushWithPotExt = (base, potExt) => { - var _a, _b; - const potBases = []; - // (.)?.json - potBases.push(`${base}${potExt}`); - // Stolen from https://github.com/mattwilson1024/google-photos-exif/blob/master/src/helpers/get-companion-json-path-for-media-file.ts - const nameCounterMatch = base.match(/(?.*)(?\(\d+\))$/); - const name = (_a = nameCounterMatch === null || nameCounterMatch === void 0 ? void 0 : nameCounterMatch.groups) === null || _a === void 0 ? void 0 : _a['name']; - const counter = (_b = nameCounterMatch === null || nameCounterMatch === void 0 ? void 0 : nameCounterMatch.groups) === null || _b === void 0 ? void 0 : _b['counter']; - if (name !== undefined && counter !== undefined) { - // (.)?(n).json - potBases.push(`${name}${potExt}${counter}`); - } - // (_n-?|_n?|_?)(.)?.json - if (base.endsWith('_n-') || base.endsWith('_n') || base.endsWith('_')) { - potBases.push(`${base.slice(0, -1)}${potExt}`); - } - for (const potBase of potBases) { - potPaths.add(`${potBase}.json`); - } - }; - let base = mediaPath.slice(0, mediaPath.length - ext.suffix.length); - base = base.replace(/-(edited|bearbeitet|modifié)$/i, ''); - const potExts = []; - // ..json - potExts.push(ext.suffix); - // ..json - potExts.push(...((_a = ext.aliases) !== null && _a !== void 0 ? _a : [])); - // .json - potExts.push(''); - for (const potExt of potExts) { - pushWithPotExt(base, potExt); - } - for (const potPath of potPaths) { - if (!(yield (0, file_exists_1.fileExists)(potPath))) { - continue; - } - return potPath; - } - return null; - }); -} -exports.findMetaFile = findMetaFile; diff --git a/build/meta/read-meta-title.js b/build/meta/read-meta-title.js deleted file mode 100644 index 6183a55..0000000 --- a/build/meta/read-meta-title.js +++ /dev/null @@ -1,23 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.readMetaTitle = void 0; -const promises_1 = require("fs/promises"); -function readMetaTitle(mediaFileInfo) { - return __awaiter(this, void 0, void 0, function* () { - if (!mediaFileInfo.jsonPath) - return undefined; - const metaJson = (yield (0, promises_1.readFile)(mediaFileInfo.jsonPath)).toString(); - const meta = JSON.parse(metaJson); - return meta === null || meta === void 0 ? void 0 : meta.title; - }); -} -exports.readMetaTitle = readMetaTitle; diff --git a/build/ts.js b/build/ts.js deleted file mode 100644 index 2e3a04a..0000000 --- a/build/ts.js +++ /dev/null @@ -1,7 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.exhaustiveCheck = void 0; -const exhaustiveCheck = (_) => { - throw new Error('Exhaustive type check failed.'); -}; -exports.exhaustiveCheck = exhaustiveCheck; From 595080443c2d6fb8f8e7388e553a7952bb718153 Mon Sep 17 00:00:00 2001 From: covalent Date: Sat, 16 Sep 2023 19:31:56 -0400 Subject: [PATCH 18/50] fix bugs, improve docs, add "Untitled" flag --- README.md | 30 ++++++++++++++++++++++++++++++ src/cli.ts | 42 ++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7175fa3..c7271f4 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,15 @@ mkdir output error npx google-photos-migrate@latest fullMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` +Optional flags for folder takeout: +``` +--timeout integer + Shorthand: -t integer + Meaning: Sets the timeout for exiftool, default is 30000 (30s) +--force + Shorthand: -f + Meaning: Forces the migration and overwrites files in the target directory. +``` If you wish to migrate an entire takeout folder: @@ -26,6 +35,15 @@ mkdir output error npx google-photos-migrate@latest folderMigrate '/path/to/takeout/' --timeout 60000 ``` +Optional flags for full takeout: +``` +--timeout integer + Shorthand: -t integer + Meaning: Sets the timeout for exiftool, default is 30000 (30s) +--untitled + Shorthand: -u + Meaning: Includes the largely superflous "Untitled" albums in the album migration. Even without this flag being passed, these images should already be included via the photos migration. +``` The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. @@ -57,3 +75,15 @@ Configured in [extensions.ts](./src/config/extensions.ts): - `.mkv` - `.wmv` - `.webm` + +## Development +**Prerec**: Must have node 18 & yarn installed. + +For basic deployment do the following: +```bash +git clone https://github.com/garzj/google-photos-migrate +yarn +yarn build +yarn start +``` +The entrypoint into the codebase is `src/cli.ts` \ No newline at end of file diff --git a/src/cli.ts b/src/cli.ts index 1552f4c..1ffc63c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,6 +9,7 @@ import { positional, flag, number, + boolean, option, } from 'cmd-ts'; import { migrateGoogleDirGen } from './media/migrate-google-dir'; @@ -61,13 +62,13 @@ async function runMigrationsChecked( ) { const errs: string[] = []; if (!existsSync(albumDir)) { - errs.push(`The specified google directory does not exist: ${albumDir}`); + errs.push(`The specified album directory does not exist: ${albumDir}`); } if (!existsSync(outDir)) { - errs.push(`The specified google directory does not exist: ${outDir}`); + errs.push(`The specified output directory does not exist: ${outDir}`); } if (!existsSync(errDir)) { - errs.push(`The specified google directory does not exist: ${errDir}`); + errs.push(`The specified error directory does not exist: ${errDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); @@ -148,7 +149,7 @@ async function processPhotos( ); } -async function _restructureIfNeeded(folders: string[], targetDir: string) { +async function _restructureIfNeeded(folders: string[], targetDir: string, untitled: boolean) { if (existsSync(targetDir) && ((await glob(`${targetDir}/*`)).length) > 0){ console.log(`${targetDir} exists and is not empty. No restructuring needed.`); return; @@ -156,13 +157,16 @@ async function _restructureIfNeeded(folders: string[], targetDir: string) { console.log(`Starting restructure of ${folders.length} directories`) mkdirSync(targetDir, {recursive: true}); for (let folder of folders){ + if (!untitled && basename(folder).includes("Untitled")){ + continue; // Skips untitled albums if flag is not passed, won't effect /Photos dir + } console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`) cpSync(folder, `${targetDir}/${basename(folder)}`, {recursive: true}); } console.log(`Sucsessfully restructured ${folders.length} directories`) } -async function restructureIfNeeded(rootDir: string) { +async function restructureIfNeeded(rootDir: string, untitled: boolean) { // before // $rootdir/My Album 1 // $rootdir/My Album 2 @@ -176,7 +180,7 @@ async function restructureIfNeeded(rootDir: string) { const photosDir: string = `${rootDir}/Photos`; // move the "Photos from $YEAR" directories to Photos/ - _restructureIfNeeded(await glob(`${rootDir}/Photos from */`), photosDir); + _restructureIfNeeded(await glob(`${rootDir}/Photos from */`), photosDir, untitled); // move everythingg else to Albums/, so we end up with two top level folders const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); @@ -185,19 +189,20 @@ async function restructureIfNeeded(rootDir: string) { const everythingExceptPhotosDir: string[] = Array.from( new Set([...fullSet].filter((x) => !photoSet.has(x))) ); - _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`); + _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`, untitled); } async function runFullMigration( rootDir: string, timeout: number, + untitled: boolean, ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); - await restructureIfNeeded(rootDir); + await restructureIfNeeded(rootDir, untitled); await processPhotos(rootDir, timeout); await processAlbums(rootDir, timeout); } @@ -247,25 +252,30 @@ const fullMigrate = command({ description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), + untitled: flag({ + type: boolean, + defaultValue: () => false, + short: 'u', + long: 'untitled', + description: + 'If passed, the untitled Album Directories will be processed. If not passed they will be ignored.', + }), }, - handler: async ({ takeoutDir, timeout }) => { + handler: async ({ takeoutDir, timeout, untitled }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { errs.push(`The specified takeout directory does not exist: ${takeoutDir}`); } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } if (await isEmptyDir(takeoutDir)) { errs.push('The google directory is empty. Nothing to do.'); } - if (!(await isEmptyDir(`${takeoutDir}/Photos`))) { + var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace(/\/+$/, ''); + if ((await existsSync(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Photos`))) { errs.push( 'The Photos directory is not empty. Please delete it and try again.' ); } - if (!(await isEmptyDir(`${takeoutDir}/Albums`))) { + if ((await existsSync(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Albums`))) { errs.push( 'The Albums directory is not empty. Please delete it and try again.' ); @@ -275,7 +285,7 @@ const fullMigrate = command({ process.exit(1); } - runFullMigration(takeoutDir, timeout); + runFullMigration(takeoutDir, timeout, untitled); }, }); From c7783e0d537a321818fda4309c58e5f87fbf63cc Mon Sep 17 00:00:00 2001 From: garzj Date: Sun, 17 Sep 2023 23:32:04 +0200 Subject: [PATCH 19/50] Format files --- README.md | 17 +++-- src/cli.ts | 133 ++++++++++++++++++---------------- src/meta/apply-meta-errors.ts | 5 +- 3 files changed, 88 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index c7271f4..d3e742d 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,14 @@ mkdir output error npx google-photos-migrate@latest fullMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` + Optional flags for folder takeout: + ``` ---timeout integer +--timeout integer Shorthand: -t integer Meaning: Sets the timeout for exiftool, default is 30000 (30s) ---force +--force Shorthand: -f Meaning: Forces the migration and overwrites files in the target directory. ``` @@ -35,9 +37,11 @@ mkdir output error npx google-photos-migrate@latest folderMigrate '/path/to/takeout/' --timeout 60000 ``` + Optional flags for full takeout: + ``` ---timeout integer +--timeout integer Shorthand: -t integer Meaning: Sets the timeout for exiftool, default is 30000 (30s) --untitled @@ -77,13 +81,16 @@ Configured in [extensions.ts](./src/config/extensions.ts): - `.webm` ## Development + **Prerec**: Must have node 18 & yarn installed. For basic deployment do the following: + ```bash git clone https://github.com/garzj/google-photos-migrate -yarn +yarn yarn build yarn start ``` -The entrypoint into the codebase is `src/cli.ts` \ No newline at end of file + +The entrypoint into the codebase is `src/cli.ts` diff --git a/src/cli.ts b/src/cli.ts index 1ffc63c..5fbc703 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -18,8 +18,6 @@ import { ExifTool } from 'exiftool-vendored'; import { glob } from 'glob'; import { basename } from 'path'; import path = require('path'); -import { inferLikelyOffsetMinutes } from 'exiftool-vendored/dist/Timezones'; - async function runBasicMigration( googleDir: string, @@ -82,28 +80,29 @@ async function runMigrationsChecked( const errFiles: string[] = await glob(`${errDir}/*`); const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { - if (file.endsWith(".json")){ - console.log(`Cannot fix metadata for ${file} as .json is an unsupported file type.`); + if (file.endsWith('.json')) { + console.log( + `Cannot fix metadata for ${file} as .json is an unsupported file type.` + ); continue; } - console.log(`Rewriting all tags from ${file}, to ${path.join(albumDir, `cleaned-${basename(file)}`)}`); - await exifTool.rewriteAllTags(file, path.join(albumDir, `cleaned-${basename(file)}`)); + console.log( + `Rewriting all tags from ${file}, to ${path.join( + albumDir, + `cleaned-${basename(file)}` + )}` + ); + await exifTool.rewriteAllTags( + file, + path.join(albumDir, `cleaned-${basename(file)}`) + ); } exifTool.end(); - await runMigrationsChecked( - albumDir, - outDir, - errDir, - timeout, - false - ); + await runMigrationsChecked(albumDir, outDir, errDir, timeout, false); } } -async function processAlbums( - rootDir: string, - timeout: number, -) { +async function processAlbums(rootDir: string, timeout: number) { const globStr: string = `${rootDir}/Albums/*/`; const albums: string[] = await glob(globStr); if (albums.length == 0) { @@ -114,56 +113,47 @@ async function processAlbums( let albumName = basename(album); let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; let errDir = `${rootDir}/AlbumsError/${albumName}`; - mkdirSync(album, {recursive: true}); - mkdirSync(outDir, {recursive: true}); - mkdirSync(errDir, {recursive: true}); - await runMigrationsChecked( - album, - outDir, - errDir, - timeout, - true - ); + mkdirSync(album, { recursive: true }); + mkdirSync(outDir, { recursive: true }); + mkdirSync(errDir, { recursive: true }); + await runMigrationsChecked(album, outDir, errDir, timeout, true); } } -async function processPhotos( - rootDir: string, - timeout: number, -) { +async function processPhotos(rootDir: string, timeout: number) { // Also run the exif fix for the photos console.log('Processing photos...'); const albumDir = `${rootDir}/Photos`; const outDir = `${rootDir}/PhotosProcessed`; const errDir = `${rootDir}/PhotosError`; - mkdirSync(albumDir, {recursive: true}); - mkdirSync(outDir, {recursive: true}); - mkdirSync(errDir, {recursive: true}); + mkdirSync(albumDir, { recursive: true }); + mkdirSync(outDir, { recursive: true }); + mkdirSync(errDir, { recursive: true }); - await runMigrationsChecked( - albumDir, - outDir, - errDir, - timeout, - true - ); + await runMigrationsChecked(albumDir, outDir, errDir, timeout, true); } -async function _restructureIfNeeded(folders: string[], targetDir: string, untitled: boolean) { - if (existsSync(targetDir) && ((await glob(`${targetDir}/*`)).length) > 0){ - console.log(`${targetDir} exists and is not empty. No restructuring needed.`); +async function _restructureIfNeeded( + folders: string[], + targetDir: string, + untitled: boolean +) { + if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { + console.log( + `${targetDir} exists and is not empty. No restructuring needed.` + ); return; } - console.log(`Starting restructure of ${folders.length} directories`) - mkdirSync(targetDir, {recursive: true}); - for (let folder of folders){ - if (!untitled && basename(folder).includes("Untitled")){ + console.log(`Starting restructure of ${folders.length} directories`); + mkdirSync(targetDir, { recursive: true }); + for (let folder of folders) { + if (!untitled && basename(folder).includes('Untitled')) { continue; // Skips untitled albums if flag is not passed, won't effect /Photos dir } - console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`) - cpSync(folder, `${targetDir}/${basename(folder)}`, {recursive: true}); + console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); + cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); } - console.log(`Sucsessfully restructured ${folders.length} directories`) + console.log(`Sucsessfully restructured ${folders.length} directories`); } async function restructureIfNeeded(rootDir: string, untitled: boolean) { @@ -180,22 +170,32 @@ async function restructureIfNeeded(rootDir: string, untitled: boolean) { const photosDir: string = `${rootDir}/Photos`; // move the "Photos from $YEAR" directories to Photos/ - _restructureIfNeeded(await glob(`${rootDir}/Photos from */`), photosDir, untitled); - + _restructureIfNeeded( + await glob(`${rootDir}/Photos from */`), + photosDir, + untitled + ); + // move everythingg else to Albums/, so we end up with two top level folders const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); - const photoSet: Set = new Set(await glob(`${rootDir}/Photos from */`)); + const photoSet: Set = new Set( + await glob(`${rootDir}/Photos from */`) + ); photoSet.add(`${rootDir}/Photos`); const everythingExceptPhotosDir: string[] = Array.from( new Set([...fullSet].filter((x) => !photoSet.has(x))) ); - _restructureIfNeeded(everythingExceptPhotosDir, `${rootDir}/Albums`, untitled); + _restructureIfNeeded( + everythingExceptPhotosDir, + `${rootDir}/Albums`, + untitled + ); } async function runFullMigration( rootDir: string, timeout: number, - untitled: boolean, + untitled: boolean ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto @@ -229,7 +229,7 @@ const rewriteAllTags = command({ 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), }, - handler: async ({inFile, outFile, timeout}) => { + handler: async ({ inFile, outFile, timeout }) => { const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await exifTool.rewriteAllTags(inFile, outFile); exifTool.end(); @@ -264,18 +264,29 @@ const fullMigrate = command({ handler: async ({ takeoutDir, timeout, untitled }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { - errs.push(`The specified takeout directory does not exist: ${takeoutDir}`); + errs.push( + `The specified takeout directory does not exist: ${takeoutDir}` + ); } if (await isEmptyDir(takeoutDir)) { errs.push('The google directory is empty. Nothing to do.'); } - var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace(/\/+$/, ''); - if ((await existsSync(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Photos`))) { + var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace( + /\/+$/, + '' + ); + if ( + (await existsSync(`${rootDir}/Photos`)) && + !(await isEmptyDir(`${rootDir}/Photos`)) + ) { errs.push( 'The Photos directory is not empty. Please delete it and try again.' ); } - if ((await existsSync(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Albums`))) { + if ( + (await existsSync(`${rootDir}/Photos`)) && + !(await isEmptyDir(`${rootDir}/Albums`)) + ) { errs.push( 'The Albums directory is not empty. Please delete it and try again.' ); diff --git a/src/meta/apply-meta-errors.ts b/src/meta/apply-meta-errors.ts index 59e0c92..4d89fd1 100644 --- a/src/meta/apply-meta-errors.ts +++ b/src/meta/apply-meta-errors.ts @@ -13,7 +13,10 @@ export class ApplyMetaError extends MediaMigrationError { } export class ExifToolError extends ApplyMetaError { - constructor(failedMedia: MediaFileInfo, public reason: Error) { + constructor( + failedMedia: MediaFileInfo, + public reason: Error + ) { super(failedMedia); } From 8a6454f75ea0dd75c564e60641050c0fb919f1e7 Mon Sep 17 00:00:00 2001 From: garzj Date: Mon, 18 Sep 2023 02:00:00 +0200 Subject: [PATCH 20/50] Refactor new code --- src/cli.ts | 375 +----------------- src/commands/migrate-folder.ts | 77 ++++ src/commands/migrate-full.ts | 77 ++++ src/commands/rewrite-all-tags.ts | 31 ++ src/dir/migrate-basic.ts | 34 ++ src/dir/migrate-checked.ts | 57 +++ src/dir/migrate-flat.ts | 51 +++ src/dir/migrate-full.ts | 19 + src/dir/process-albums.ts | 22 + src/dir/process-photos.ts | 15 + src/dir/restructure-if-needed.ts | 62 +++ src/index.ts | 2 +- ...te-google-dir.ts => migrate-media-file.ts} | 51 +-- 13 files changed, 452 insertions(+), 421 deletions(-) create mode 100644 src/commands/migrate-folder.ts create mode 100644 src/commands/migrate-full.ts create mode 100644 src/commands/rewrite-all-tags.ts create mode 100644 src/dir/migrate-basic.ts create mode 100644 src/dir/migrate-checked.ts create mode 100644 src/dir/migrate-flat.ts create mode 100644 src/dir/migrate-full.ts create mode 100644 src/dir/process-albums.ts create mode 100644 src/dir/process-photos.ts create mode 100644 src/dir/restructure-if-needed.ts rename src/media/{migrate-google-dir.ts => migrate-media-file.ts} (65%) diff --git a/src/cli.ts b/src/cli.ts index 5fbc703..bb5c0e9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,376 +1,9 @@ #!/usr/bin/env node -import { cpSync, existsSync, mkdirSync } from 'fs'; -import { - command, - subcommands, - run, - string, - positional, - flag, - number, - boolean, - option, -} from 'cmd-ts'; -import { migrateGoogleDirGen } from './media/migrate-google-dir'; -import { isEmptyDir } from './fs/is-empty-dir'; -import { ExifTool } from 'exiftool-vendored'; -import { glob } from 'glob'; -import { basename } from 'path'; -import path = require('path'); - -async function runBasicMigration( - googleDir: string, - outputDir: string, - errorDir: string, - exifTool: ExifTool -) { - console.log(`Started migration.`); - const migGen = migrateGoogleDirGen({ - googleDir, - outputDir, - errorDir, - warnLog: console.error, - exiftool: exifTool, - endExifTool: true, - }); - - const counts = { err: 0, suc: 0 }; - for await (const result of migGen) { - if (result instanceof Error) { - console.error(`Error: ${result}`); - counts.err++; - continue; - } - - counts.suc++; - } - - console.log(`Done! Processed ${counts.suc + counts.err} files.`); - console.log(`Files migrated: ${counts.suc}`); - console.log(`Files failed: ${counts.err}`); -} - -async function runMigrationsChecked( - albumDir: string, - outDir: string, - errDir: string, - timeout: number, - check_errDir: boolean -) { - const errs: string[] = []; - if (!existsSync(albumDir)) { - errs.push(`The specified album directory does not exist: ${albumDir}`); - } - if (!existsSync(outDir)) { - errs.push(`The specified output directory does not exist: ${outDir}`); - } - if (!existsSync(errDir)) { - errs.push(`The specified error directory does not exist: ${errDir}`); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await runBasicMigration(albumDir, outDir, errDir, exifTool); - - if (check_errDir && !(await isEmptyDir(errDir))) { - const errFiles: string[] = await glob(`${errDir}/*`); - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - for (let file of errFiles) { - if (file.endsWith('.json')) { - console.log( - `Cannot fix metadata for ${file} as .json is an unsupported file type.` - ); - continue; - } - console.log( - `Rewriting all tags from ${file}, to ${path.join( - albumDir, - `cleaned-${basename(file)}` - )}` - ); - await exifTool.rewriteAllTags( - file, - path.join(albumDir, `cleaned-${basename(file)}`) - ); - } - exifTool.end(); - await runMigrationsChecked(albumDir, outDir, errDir, timeout, false); - } -} - -async function processAlbums(rootDir: string, timeout: number) { - const globStr: string = `${rootDir}/Albums/*/`; - const albums: string[] = await glob(globStr); - if (albums.length == 0) { - console.log(`WARN: No albums found at ${globStr}`); - } - for (let album of albums) { - console.log(`Processing album ${album}...`); - let albumName = basename(album); - let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; - let errDir = `${rootDir}/AlbumsError/${albumName}`; - mkdirSync(album, { recursive: true }); - mkdirSync(outDir, { recursive: true }); - mkdirSync(errDir, { recursive: true }); - await runMigrationsChecked(album, outDir, errDir, timeout, true); - } -} - -async function processPhotos(rootDir: string, timeout: number) { - // Also run the exif fix for the photos - console.log('Processing photos...'); - const albumDir = `${rootDir}/Photos`; - const outDir = `${rootDir}/PhotosProcessed`; - const errDir = `${rootDir}/PhotosError`; - mkdirSync(albumDir, { recursive: true }); - mkdirSync(outDir, { recursive: true }); - mkdirSync(errDir, { recursive: true }); - - await runMigrationsChecked(albumDir, outDir, errDir, timeout, true); -} - -async function _restructureIfNeeded( - folders: string[], - targetDir: string, - untitled: boolean -) { - if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { - console.log( - `${targetDir} exists and is not empty. No restructuring needed.` - ); - return; - } - console.log(`Starting restructure of ${folders.length} directories`); - mkdirSync(targetDir, { recursive: true }); - for (let folder of folders) { - if (!untitled && basename(folder).includes('Untitled')) { - continue; // Skips untitled albums if flag is not passed, won't effect /Photos dir - } - console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); - cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); - } - console.log(`Sucsessfully restructured ${folders.length} directories`); -} - -async function restructureIfNeeded(rootDir: string, untitled: boolean) { - // before - // $rootdir/My Album 1 - // $rootdir/My Album 2 - // $rootdir/Photos from 2008 - - // after - // $rootdir/Albums/My Album 1 - // $rootdir/Albums/My Album 2 - // $rootdir/Photos/Photos from 2008 - - const photosDir: string = `${rootDir}/Photos`; - - // move the "Photos from $YEAR" directories to Photos/ - _restructureIfNeeded( - await glob(`${rootDir}/Photos from */`), - photosDir, - untitled - ); - - // move everythingg else to Albums/, so we end up with two top level folders - const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); - const photoSet: Set = new Set( - await glob(`${rootDir}/Photos from */`) - ); - photoSet.add(`${rootDir}/Photos`); - const everythingExceptPhotosDir: string[] = Array.from( - new Set([...fullSet].filter((x) => !photoSet.has(x))) - ); - _restructureIfNeeded( - everythingExceptPhotosDir, - `${rootDir}/Albums`, - untitled - ); -} - -async function runFullMigration( - rootDir: string, - timeout: number, - untitled: boolean -) { - // at least in my takeout, the Takeout folder contains a subfolder - // Takeout/Google Foto - // rootdir refers to that subfolder - - rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); - await restructureIfNeeded(rootDir, untitled); - await processPhotos(rootDir, timeout); - await processAlbums(rootDir, timeout); -} - -const rewriteAllTags = command({ - name: 'rewrite all tags for single file', - args: { - inFile: positional({ - type: string, - displayName: 'in_file', - description: 'The path to your input file.', - }), - outFile: positional({ - type: string, - displayName: 'out_file', - description: 'The path to your output location for the file.', - }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - }, - handler: async ({ inFile, outFile, timeout }) => { - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await exifTool.rewriteAllTags(inFile, outFile); - exifTool.end(); - }, -}); - -const fullMigrate = command({ - name: 'google-photos-migrate-full', - args: { - takeoutDir: positional({ - type: string, - displayName: 'takeout_dir', - description: 'The path to your "Takeout" directory.', - }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - untitled: flag({ - type: boolean, - defaultValue: () => false, - short: 'u', - long: 'untitled', - description: - 'If passed, the untitled Album Directories will be processed. If not passed they will be ignored.', - }), - }, - handler: async ({ takeoutDir, timeout, untitled }) => { - const errs: string[] = []; - if (!existsSync(takeoutDir)) { - errs.push( - `The specified takeout directory does not exist: ${takeoutDir}` - ); - } - if (await isEmptyDir(takeoutDir)) { - errs.push('The google directory is empty. Nothing to do.'); - } - var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace( - /\/+$/, - '' - ); - if ( - (await existsSync(`${rootDir}/Photos`)) && - !(await isEmptyDir(`${rootDir}/Photos`)) - ) { - errs.push( - 'The Photos directory is not empty. Please delete it and try again.' - ); - } - if ( - (await existsSync(`${rootDir}/Photos`)) && - !(await isEmptyDir(`${rootDir}/Albums`)) - ) { - errs.push( - 'The Albums directory is not empty. Please delete it and try again.' - ); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - - runFullMigration(takeoutDir, timeout, untitled); - }, -}); - -const folderMigrate = command({ - name: 'google-photos-migrate-folder', - args: { - googleDir: positional({ - type: string, - displayName: 'google_dir', - description: 'The path to your "Google Photos" directory.', - }), - outputDir: positional({ - type: string, - displayName: 'output_dir', - description: 'The path to your flat output directory.', - }), - errorDir: positional({ - type: string, - displayName: 'error_dir', - description: 'Failed media will be saved here.', - }), - force: flag({ - short: 'f', - long: 'force', - description: - "Forces the operation if the given directories aren't empty.", - }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - }, - handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { - const errs: string[] = []; - if (!existsSync(googleDir)) { - errs.push(`The specified google directory does not exist: ${googleDir}`); - } - if (!existsSync(outputDir)) { - errs.push(`The specified output directory does not exist: ${googleDir}`); - } - if (!existsSync(errorDir)) { - errs.push(`The specified error directory does not exist: ${googleDir}`); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - - if (!force && !(await isEmptyDir(outputDir))) { - errs.push( - 'The output directory is not empty. Pass "-f" to force the operation.' - ); - } - if (!force && !(await isEmptyDir(errorDir))) { - errs.push( - 'The error directory is not empty. Pass "-f" to force the operation.' - ); - } - if (await isEmptyDir(googleDir)) { - errs.push('The google directory is empty. Nothing to do.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } - - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await runBasicMigration(googleDir, outputDir, errorDir, exifTool); - }, -}); +import { subcommands, run } from 'cmd-ts'; +import { fullMigrate } from './commands/migrate-full'; +import { folderMigrate } from './commands/migrate-folder'; +import { rewriteAllTags } from './commands/rewrite-all-tags'; const app = subcommands({ name: 'google-photos-migrate', diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-folder.ts new file mode 100644 index 0000000..a6f75fd --- /dev/null +++ b/src/commands/migrate-folder.ts @@ -0,0 +1,77 @@ +import { command, string, positional, flag, number, option } from 'cmd-ts'; +import { existsSync } from 'fs'; +import { isEmptyDir } from '../fs/is-empty-dir'; +import { ExifTool } from 'exiftool-vendored'; +import { runBasicMigration } from '../dir/migrate-basic'; + +export const folderMigrate = command({ + name: 'google-photos-migrate-folder', + args: { + googleDir: positional({ + type: string, + displayName: 'google_dir', + description: 'The path to your "Google Photos" directory.', + }), + outputDir: positional({ + type: string, + displayName: 'output_dir', + description: 'The path to your flat output directory.', + }), + errorDir: positional({ + type: string, + displayName: 'error_dir', + description: 'Failed media will be saved here.', + }), + force: flag({ + short: 'f', + long: 'force', + description: + "Forces the operation if the given directories aren't empty.", + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { + const errs: string[] = []; + if (!existsSync(googleDir)) { + errs.push(`The specified google directory does not exist: ${googleDir}`); + } + if (!existsSync(outputDir)) { + errs.push(`The specified output directory does not exist: ${googleDir}`); + } + if (!existsSync(errorDir)) { + errs.push(`The specified error directory does not exist: ${googleDir}`); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + if (!force && !(await isEmptyDir(outputDir))) { + errs.push( + 'The output directory is not empty. Pass "-f" to force the operation.' + ); + } + if (!force && !(await isEmptyDir(errorDir))) { + errs.push( + 'The error directory is not empty. Pass "-f" to force the operation.' + ); + } + if (await isEmptyDir(googleDir)) { + errs.push('The google directory is empty. Nothing to do.'); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + await runBasicMigration(googleDir, outputDir, errorDir, exifTool); + }, +}); diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts new file mode 100644 index 0000000..03edaa0 --- /dev/null +++ b/src/commands/migrate-full.ts @@ -0,0 +1,77 @@ +import { + command, + string, + positional, + flag, + number, + option, + boolean, +} from 'cmd-ts'; +import { existsSync } from 'fs'; +import { isEmptyDir } from '../fs/is-empty-dir'; +import { glob } from 'glob'; +import { runFullMigration } from '../dir/migrate-full'; + +export const fullMigrate = command({ + name: 'google-photos-migrate-full', + args: { + takeoutDir: positional({ + type: string, + displayName: 'takeout_dir', + description: 'The path to your "Takeout" directory.', + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + untitled: flag({ + type: boolean, + defaultValue: () => false, + short: 'u', + long: 'untitled', + description: + 'If passed, the untitled Album Directories will be processed. If not passed they will be ignored.', + }), + }, + handler: async ({ takeoutDir, timeout, untitled }) => { + const errs: string[] = []; + if (!existsSync(takeoutDir)) { + errs.push( + `The specified takeout directory does not exist: ${takeoutDir}` + ); + } + if (await isEmptyDir(takeoutDir)) { + errs.push('The google directory is empty. Nothing to do.'); + } + var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace( + /\/+$/, + '' + ); + if ( + (await existsSync(`${rootDir}/Photos`)) && + !(await isEmptyDir(`${rootDir}/Photos`)) + ) { + errs.push( + 'The Photos directory is not empty. Please delete it and try again.' + ); + } + if ( + (await existsSync(`${rootDir}/Photos`)) && + !(await isEmptyDir(`${rootDir}/Albums`)) + ) { + errs.push( + 'The Albums directory is not empty. Please delete it and try again.' + ); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + runFullMigration(takeoutDir, timeout, untitled); + }, +}); diff --git a/src/commands/rewrite-all-tags.ts b/src/commands/rewrite-all-tags.ts new file mode 100644 index 0000000..85204c4 --- /dev/null +++ b/src/commands/rewrite-all-tags.ts @@ -0,0 +1,31 @@ +import { command, string, positional, number, option } from 'cmd-ts'; +import { ExifTool } from 'exiftool-vendored'; + +export const rewriteAllTags = command({ + name: 'rewrite all tags for single file', + args: { + inFile: positional({ + type: string, + displayName: 'in_file', + description: 'The path to your input file.', + }), + outFile: positional({ + type: string, + displayName: 'out_file', + description: 'The path to your output location for the file.', + }), + timeout: option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', + }), + }, + handler: async ({ inFile, outFile, timeout }) => { + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + await exifTool.rewriteAllTags(inFile, outFile); + exifTool.end(); + }, +}); diff --git a/src/dir/migrate-basic.ts b/src/dir/migrate-basic.ts new file mode 100644 index 0000000..3004014 --- /dev/null +++ b/src/dir/migrate-basic.ts @@ -0,0 +1,34 @@ +import { ExifTool } from 'exiftool-vendored'; +import { migrateGoogleDirGen } from './migrate-flat'; + +export async function runBasicMigration( + googleDir: string, + outputDir: string, + errorDir: string, + exifTool: ExifTool +) { + console.log(`Started migration.`); + const migGen = migrateGoogleDirGen({ + googleDir, + outputDir, + errorDir, + warnLog: console.error, + exiftool: exifTool, + endExifTool: true, + }); + + const counts = { err: 0, suc: 0 }; + for await (const result of migGen) { + if (result instanceof Error) { + console.error(`Error: ${result}`); + counts.err++; + continue; + } + + counts.suc++; + } + + console.log(`Done! Processed ${counts.suc + counts.err} files.`); + console.log(`Files migrated: ${counts.suc}`); + console.log(`Files failed: ${counts.err}`); +} diff --git a/src/dir/migrate-checked.ts b/src/dir/migrate-checked.ts new file mode 100644 index 0000000..0b4d4b6 --- /dev/null +++ b/src/dir/migrate-checked.ts @@ -0,0 +1,57 @@ +import { existsSync } from 'fs'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { ExifTool } from 'exiftool-vendored'; +import { runBasicMigration } from './migrate-basic'; +import { isEmptyDir } from '../fs/is-empty-dir'; + +export async function runMigrationsChecked( + albumDir: string, + outDir: string, + errDir: string, + timeout: number, + check_errDir: boolean +) { + const errs: string[] = []; + if (!existsSync(albumDir)) { + errs.push(`The specified album directory does not exist: ${albumDir}`); + } + if (!existsSync(outDir)) { + errs.push(`The specified output directory does not exist: ${outDir}`); + } + if (!existsSync(errDir)) { + errs.push(`The specified error directory does not exist: ${errDir}`); + } + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + await runBasicMigration(albumDir, outDir, errDir, exifTool); + + if (check_errDir && !(await isEmptyDir(errDir))) { + const errFiles: string[] = await glob(`${errDir}/*`); + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + for (let file of errFiles) { + if (file.endsWith('.json')) { + console.log( + `Cannot fix metadata for ${file} as .json is an unsupported file type.` + ); + continue; + } + console.log( + `Rewriting all tags from ${file}, to ${join( + albumDir, + `cleaned-${basename(file)}` + )}` + ); + await exifTool.rewriteAllTags( + file, + join(albumDir, `cleaned-${basename(file)}`) + ); + } + exifTool.end(); + await runMigrationsChecked(albumDir, outDir, errDir, timeout, false); + } +} diff --git a/src/dir/migrate-flat.ts b/src/dir/migrate-flat.ts new file mode 100644 index 0000000..5785877 --- /dev/null +++ b/src/dir/migrate-flat.ts @@ -0,0 +1,51 @@ +import { walkDir } from '../fs/walk-dir'; +import { MediaFile } from '../media/MediaFile'; +import { indexJsonFiles } from '../meta/index-meta-files'; +import { MediaMigrationError } from '../media/MediaMigrationError'; +import { ExifTool } from 'exiftool-vendored'; +import { migrateMediaFile } from '../media/migrate-media-file'; + +export type MigrationArgs = { + googleDir: string; + outputDir: string; + errorDir: string; + warnLog?: (msg: string) => void; + exiftool?: ExifTool; + endExifTool?: boolean; +}; + +export type MigrationContext = Required & { + titleJsonMap: Map; + migrationLocks: Map>; +}; + +export async function* migrateGoogleDir( + args: MigrationArgs +): AsyncGenerator { + const wg: (MediaFile | MediaMigrationError)[] = []; + for await (const result of migrateGoogleDirGen(args)) { + wg.push(result); + } + return await Promise.all(wg); +} + +export async function* migrateGoogleDirGen( + args: MigrationArgs +): AsyncGenerator { + const migCtx: MigrationContext = { + titleJsonMap: await indexJsonFiles(args.googleDir), + migrationLocks: new Map(), + ...args, + exiftool: args.exiftool ?? new ExifTool(), + endExifTool: args.endExifTool ?? !args.exiftool, + warnLog: args.warnLog ?? (() => {}), + }; + + for await (const mediaPath of walkDir(args.googleDir)) { + if (mediaPath.endsWith('.json')) continue; + + yield migrateMediaFile(mediaPath, migCtx); + } + + migCtx.endExifTool && migCtx.exiftool.end(); +} diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts new file mode 100644 index 0000000..1834049 --- /dev/null +++ b/src/dir/migrate-full.ts @@ -0,0 +1,19 @@ +import { glob } from 'glob'; +import { restructureIfNeeded } from './restructure-if-needed'; +import { processPhotos } from './process-photos'; +import { processAlbums } from './process-albums'; + +export async function runFullMigration( + rootDir: string, + timeout: number, + untitled: boolean +) { + // at least in my takeout, the Takeout folder contains a subfolder + // Takeout/Google Foto + // rootdir refers to that subfolder + + rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); + await restructureIfNeeded(rootDir, untitled); + await processPhotos(rootDir, timeout); + await processAlbums(rootDir, timeout); +} diff --git a/src/dir/process-albums.ts b/src/dir/process-albums.ts new file mode 100644 index 0000000..ad95875 --- /dev/null +++ b/src/dir/process-albums.ts @@ -0,0 +1,22 @@ +import { mkdirSync } from 'fs'; +import { glob } from 'glob'; +import { basename } from 'path'; +import { runMigrationsChecked } from './migrate-checked'; + +export async function processAlbums(rootDir: string, timeout: number) { + const globStr: string = `${rootDir}/Albums/*/`; + const albums: string[] = await glob(globStr); + if (albums.length == 0) { + console.log(`WARN: No albums found at ${globStr}`); + } + for (let album of albums) { + console.log(`Processing album ${album}...`); + let albumName = basename(album); + let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; + let errDir = `${rootDir}/AlbumsError/${albumName}`; + mkdirSync(album, { recursive: true }); + mkdirSync(outDir, { recursive: true }); + mkdirSync(errDir, { recursive: true }); + await runMigrationsChecked(album, outDir, errDir, timeout, true); + } +} diff --git a/src/dir/process-photos.ts b/src/dir/process-photos.ts new file mode 100644 index 0000000..f7112d9 --- /dev/null +++ b/src/dir/process-photos.ts @@ -0,0 +1,15 @@ +import { mkdirSync } from 'fs'; +import { runMigrationsChecked } from './migrate-checked'; + +export async function processPhotos(rootDir: string, timeout: number) { + // Also run the exif fix for the photos + console.log('Processing photos...'); + const albumDir = `${rootDir}/Photos`; + const outDir = `${rootDir}/PhotosProcessed`; + const errDir = `${rootDir}/PhotosError`; + mkdirSync(albumDir, { recursive: true }); + mkdirSync(outDir, { recursive: true }); + mkdirSync(errDir, { recursive: true }); + + await runMigrationsChecked(albumDir, outDir, errDir, timeout, true); +} diff --git a/src/dir/restructure-if-needed.ts b/src/dir/restructure-if-needed.ts new file mode 100644 index 0000000..ece4eb8 --- /dev/null +++ b/src/dir/restructure-if-needed.ts @@ -0,0 +1,62 @@ +import { cpSync, existsSync, mkdirSync } from 'fs'; +import { glob } from 'glob'; +import { basename } from 'path'; + +async function _restructureIfNeeded( + folders: string[], + targetDir: string, + untitled: boolean +) { + if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { + console.log( + `${targetDir} exists and is not empty. No restructuring needed.` + ); + return; + } + console.log(`Starting restructure of ${folders.length} directories`); + mkdirSync(targetDir, { recursive: true }); + for (let folder of folders) { + if (!untitled && basename(folder).includes('Untitled')) { + continue; // Skips untitled albums if flag is not passed, won't effect /Photos dir + } + console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); + cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); + } + console.log(`Sucsessfully restructured ${folders.length} directories`); +} + +export async function restructureIfNeeded(rootDir: string, untitled: boolean) { + // before + // $rootdir/My Album 1 + // $rootdir/My Album 2 + // $rootdir/Photos from 2008 + + // after + // $rootdir/Albums/My Album 1 + // $rootdir/Albums/My Album 2 + // $rootdir/Photos/Photos from 2008 + + const photosDir: string = `${rootDir}/Photos`; + + // move the "Photos from $YEAR" directories to Photos/ + _restructureIfNeeded( + await glob(`${rootDir}/Photos from */`), + photosDir, + untitled + ); + + // move everythingg else to Albums/, so we end up with two top level folders + const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); + const photoSet: Set = new Set( + await glob(`${rootDir}/Photos from */`) + ); + photoSet.add(`${rootDir}/Photos`); + const everythingExceptPhotosDir: string[] = Array.from( + new Set([...fullSet].filter((x) => !photoSet.has(x))) + ); + _restructureIfNeeded( + everythingExceptPhotosDir, + `${rootDir}/Albums`, + untitled + ); +} diff --git a/src/index.ts b/src/index.ts index 4b16d43..406e34d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import './config/env'; -import { migrateGoogleDir } from './media/migrate-google-dir'; +import { migrateGoogleDir } from './dir/migrate-flat'; import type { MediaFileExtension } from './media/MediaFileExtension'; import { supportedExtensions } from './config/extensions'; diff --git a/src/media/migrate-google-dir.ts b/src/media/migrate-media-file.ts similarity index 65% rename from src/media/migrate-google-dir.ts rename to src/media/migrate-media-file.ts index 0673b66..a9e27ce 100644 --- a/src/media/migrate-google-dir.ts +++ b/src/media/migrate-media-file.ts @@ -1,65 +1,18 @@ -import { walkDir } from '../fs/walk-dir'; import { basename } from 'path'; import { findMetaFile } from '../meta/find-meta-file'; import { MediaFileExtension } from './MediaFileExtension'; import { MediaFile, MediaFileInfo } from './MediaFile'; import { applyMetaFile } from '../meta/apply-meta-file'; -import { indexJsonFiles } from './title-json-map'; import { supportedExtensions } from '../config/extensions'; import { MediaMigrationError } from './MediaMigrationError'; import { InvalidExtError } from './InvalidExtError'; import { NoMetaFileError } from '../meta/NoMetaFileError'; import { WrongExtensionError } from '../meta/apply-meta-errors'; -import { ExifTool } from 'exiftool-vendored'; import { readMetaTitle } from '../meta/read-meta-title'; import { saveToDir } from './save-to-dir'; +import { MigrationContext } from '../dir/migrate-flat'; -export type MigrationArgs = { - googleDir: string; - outputDir: string; - errorDir: string; - warnLog?: (msg: string) => void; - exiftool?: ExifTool; - endExifTool?: boolean; -}; - -export type MigrationContext = Required & { - titleJsonMap: Map; - migrationLocks: Map>; -}; - -export async function* migrateGoogleDir( - args: MigrationArgs -): AsyncGenerator { - const wg: (MediaFile | MediaMigrationError)[] = []; - for await (const result of migrateGoogleDirGen(args)) { - wg.push(result); - } - return await Promise.all(wg); -} - -export async function* migrateGoogleDirGen( - args: MigrationArgs -): AsyncGenerator { - const migCtx: MigrationContext = { - titleJsonMap: await indexJsonFiles(args.googleDir), - migrationLocks: new Map(), - ...args, - exiftool: args.exiftool ?? new ExifTool(), - endExifTool: args.endExifTool ?? !args.exiftool, - warnLog: args.warnLog ?? (() => {}), - }; - - for await (const mediaPath of walkDir(args.googleDir)) { - if (mediaPath.endsWith('.json')) continue; - - yield migrateMediaFile(mediaPath, migCtx); - } - - migCtx.endExifTool && migCtx.exiftool.end(); -} - -async function migrateMediaFile( +export async function migrateMediaFile( originalPath: string, migCtx: MigrationContext ): Promise { From b885988ba1f9f89b366c4f4f69105eec24f08db2 Mon Sep 17 00:00:00 2001 From: garzj Date: Mon, 18 Sep 2023 02:02:07 +0200 Subject: [PATCH 21/50] Fix dangling import --- src/media/save-to-dir.ts | 2 +- src/meta/apply-meta-file.ts | 2 +- src/meta/find-meta-file.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/media/save-to-dir.ts b/src/media/save-to-dir.ts index 2ad65fb..6782fcf 100644 --- a/src/media/save-to-dir.ts +++ b/src/media/save-to-dir.ts @@ -2,7 +2,7 @@ import { basename, resolve } from 'path'; import { fileExists } from '../fs/file-exists'; import sanitize = require('sanitize-filename'); import { copyFile, mkdir, rename } from 'fs/promises'; -import { MigrationContext } from './migrate-google-dir'; +import { MigrationContext } from '../dir/migrate-flat'; async function _saveToDir( file: string, diff --git a/src/meta/apply-meta-file.ts b/src/meta/apply-meta-file.ts index 5827de5..6d5be4b 100644 --- a/src/meta/apply-meta-file.ts +++ b/src/meta/apply-meta-file.ts @@ -10,7 +10,7 @@ import { MissingMetaError, WrongExtensionError, } from './apply-meta-errors'; -import { MigrationContext } from '../media/migrate-google-dir'; +import { MigrationContext } from '../dir/migrate-flat'; export async function applyMetaFile( mediaFile: MediaFile, diff --git a/src/meta/find-meta-file.ts b/src/meta/find-meta-file.ts index b1b60e5..8e5f06a 100644 --- a/src/meta/find-meta-file.ts +++ b/src/meta/find-meta-file.ts @@ -1,5 +1,5 @@ import { basename, dirname } from 'path'; -import { MigrationContext } from '../media/migrate-google-dir'; +import { MigrationContext } from '../dir/migrate-flat'; import { MediaFileExtension } from '../media/MediaFileExtension'; import { fileExists } from '../fs/file-exists'; import { editedSuffices } from '../config/langs'; From 93f92697a5a29541b4078d9e7fd36aef07b559c0 Mon Sep 17 00:00:00 2001 From: covalent Date: Wed, 20 Sep 2023 12:24:37 -0400 Subject: [PATCH 22/50] fix exiftool singleton, top level funciton names, and how untitled dirs are handled --- src/commands/migrate-folder.ts | 4 ++-- src/commands/migrate-full.ts | 12 ++--------- ...ed.ts => migrate-entire-takeout-folder.ts} | 15 ++++++-------- src/dir/migrate-full.ts | 20 +++++++++++++------ ...rate-basic.ts => migrate-single-folder.ts} | 7 ++++--- src/dir/process-albums.ts | 7 ++++--- src/dir/process-photos.ts | 7 ++++--- src/dir/restructure-if-needed.ts | 17 ++++++++-------- 8 files changed, 45 insertions(+), 44 deletions(-) rename src/dir/{migrate-checked.ts => migrate-entire-takeout-folder.ts} (75%) rename src/dir/{migrate-basic.ts => migrate-single-folder.ts} (85%) diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-folder.ts index a6f75fd..e6bd55e 100644 --- a/src/commands/migrate-folder.ts +++ b/src/commands/migrate-folder.ts @@ -2,7 +2,7 @@ import { command, string, positional, flag, number, option } from 'cmd-ts'; import { existsSync } from 'fs'; import { isEmptyDir } from '../fs/is-empty-dir'; import { ExifTool } from 'exiftool-vendored'; -import { runBasicMigration } from '../dir/migrate-basic'; +import { migrateSingleFolder } from '../dir/migrate-single-folder'; export const folderMigrate = command({ name: 'google-photos-migrate-folder', @@ -72,6 +72,6 @@ export const folderMigrate = command({ } const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await runBasicMigration(googleDir, outputDir, errorDir, exifTool); + await migrateSingleFolder(googleDir, outputDir, errorDir, exifTool, true); }, }); diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 03edaa0..3d96df7 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -28,16 +28,8 @@ export const fullMigrate = command({ description: 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), - untitled: flag({ - type: boolean, - defaultValue: () => false, - short: 'u', - long: 'untitled', - description: - 'If passed, the untitled Album Directories will be processed. If not passed they will be ignored.', - }), }, - handler: async ({ takeoutDir, timeout, untitled }) => { + handler: async ({ takeoutDir, timeout }) => { const errs: string[] = []; if (!existsSync(takeoutDir)) { errs.push( @@ -72,6 +64,6 @@ export const fullMigrate = command({ process.exit(1); } - runFullMigration(takeoutDir, timeout, untitled); + runFullMigration(takeoutDir, timeout); }, }); diff --git a/src/dir/migrate-checked.ts b/src/dir/migrate-entire-takeout-folder.ts similarity index 75% rename from src/dir/migrate-checked.ts rename to src/dir/migrate-entire-takeout-folder.ts index 0b4d4b6..c0a37ab 100644 --- a/src/dir/migrate-checked.ts +++ b/src/dir/migrate-entire-takeout-folder.ts @@ -2,15 +2,15 @@ import { existsSync } from 'fs'; import { glob } from 'glob'; import { basename, join } from 'path'; import { ExifTool } from 'exiftool-vendored'; -import { runBasicMigration } from './migrate-basic'; +import { migrateSingleFolder } from './migrate-single-folder'; import { isEmptyDir } from '../fs/is-empty-dir'; -export async function runMigrationsChecked( +export async function migrateEntireTakoutFolder( albumDir: string, outDir: string, errDir: string, - timeout: number, - check_errDir: boolean + check_errDir: boolean, + exifTool: ExifTool ) { const errs: string[] = []; if (!existsSync(albumDir)) { @@ -27,12 +27,10 @@ export async function runMigrationsChecked( process.exit(1); } - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await runBasicMigration(albumDir, outDir, errDir, exifTool); + await migrateSingleFolder(albumDir, outDir, errDir, exifTool, false); if (check_errDir && !(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); for (let file of errFiles) { if (file.endsWith('.json')) { console.log( @@ -51,7 +49,6 @@ export async function runMigrationsChecked( join(albumDir, `cleaned-${basename(file)}`) ); } - exifTool.end(); - await runMigrationsChecked(albumDir, outDir, errDir, timeout, false); + await migrateEntireTakoutFolder(albumDir, outDir, errDir, false, exifTool); } } diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 1834049..e3fc39b 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -2,18 +2,26 @@ import { glob } from 'glob'; import { restructureIfNeeded } from './restructure-if-needed'; import { processPhotos } from './process-photos'; import { processAlbums } from './process-albums'; +import { existsSync } from 'fs'; +import { ExifTool } from 'exiftool-vendored'; export async function runFullMigration( rootDir: string, timeout: number, - untitled: boolean ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder - - rootDir = (await glob(`${rootDir}/Google*`))[0].replace(/\/+$/, ''); - await restructureIfNeeded(rootDir, untitled); - await processPhotos(rootDir, timeout); - await processAlbums(rootDir, timeout); + // Can add more language support here in the future + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + rootDir; + if (existsSync(`${rootDir}/Google Photos`)){ + rootDir = `${rootDir}/Google Photos`; + } else if (existsSync(`${rootDir}/Google Fotos`)){ + rootDir = `${rootDir}/Google Fotos` + } + await restructureIfNeeded(rootDir); + await processPhotos(rootDir, exifTool); + await processAlbums(rootDir, exifTool); + exifTool.end(); } diff --git a/src/dir/migrate-basic.ts b/src/dir/migrate-single-folder.ts similarity index 85% rename from src/dir/migrate-basic.ts rename to src/dir/migrate-single-folder.ts index 3004014..eda002a 100644 --- a/src/dir/migrate-basic.ts +++ b/src/dir/migrate-single-folder.ts @@ -1,11 +1,12 @@ import { ExifTool } from 'exiftool-vendored'; import { migrateGoogleDirGen } from './migrate-flat'; -export async function runBasicMigration( +export async function migrateSingleFolder( googleDir: string, outputDir: string, errorDir: string, - exifTool: ExifTool + exifTool: ExifTool, + endExifTool: boolean, ) { console.log(`Started migration.`); const migGen = migrateGoogleDirGen({ @@ -14,7 +15,7 @@ export async function runBasicMigration( errorDir, warnLog: console.error, exiftool: exifTool, - endExifTool: true, + endExifTool: endExifTool, }); const counts = { err: 0, suc: 0 }; diff --git a/src/dir/process-albums.ts b/src/dir/process-albums.ts index ad95875..de95b2e 100644 --- a/src/dir/process-albums.ts +++ b/src/dir/process-albums.ts @@ -1,9 +1,10 @@ import { mkdirSync } from 'fs'; import { glob } from 'glob'; import { basename } from 'path'; -import { runMigrationsChecked } from './migrate-checked'; +import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; +import { ExifTool } from 'exiftool-vendored'; -export async function processAlbums(rootDir: string, timeout: number) { +export async function processAlbums(rootDir: string, exifTool: ExifTool) { const globStr: string = `${rootDir}/Albums/*/`; const albums: string[] = await glob(globStr); if (albums.length == 0) { @@ -17,6 +18,6 @@ export async function processAlbums(rootDir: string, timeout: number) { mkdirSync(album, { recursive: true }); mkdirSync(outDir, { recursive: true }); mkdirSync(errDir, { recursive: true }); - await runMigrationsChecked(album, outDir, errDir, timeout, true); + await migrateEntireTakoutFolder(album, outDir, errDir, true, exifTool); } } diff --git a/src/dir/process-photos.ts b/src/dir/process-photos.ts index f7112d9..15a9acc 100644 --- a/src/dir/process-photos.ts +++ b/src/dir/process-photos.ts @@ -1,7 +1,8 @@ import { mkdirSync } from 'fs'; -import { runMigrationsChecked } from './migrate-checked'; +import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; +import { ExifTool } from 'exiftool-vendored'; -export async function processPhotos(rootDir: string, timeout: number) { +export async function processPhotos(rootDir: string, exifTool: ExifTool) { // Also run the exif fix for the photos console.log('Processing photos...'); const albumDir = `${rootDir}/Photos`; @@ -11,5 +12,5 @@ export async function processPhotos(rootDir: string, timeout: number) { mkdirSync(outDir, { recursive: true }); mkdirSync(errDir, { recursive: true }); - await runMigrationsChecked(albumDir, outDir, errDir, timeout, true); + await migrateEntireTakoutFolder(albumDir, outDir, errDir, true, exifTool); } diff --git a/src/dir/restructure-if-needed.ts b/src/dir/restructure-if-needed.ts index ece4eb8..8ae5b15 100644 --- a/src/dir/restructure-if-needed.ts +++ b/src/dir/restructure-if-needed.ts @@ -5,7 +5,6 @@ import { basename } from 'path'; async function _restructureIfNeeded( folders: string[], targetDir: string, - untitled: boolean ) { if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { console.log( @@ -16,16 +15,20 @@ async function _restructureIfNeeded( console.log(`Starting restructure of ${folders.length} directories`); mkdirSync(targetDir, { recursive: true }); for (let folder of folders) { - if (!untitled && basename(folder).includes('Untitled')) { - continue; // Skips untitled albums if flag is not passed, won't effect /Photos dir + // Moves all Untitled(x) directories to one large folder + if (basename(folder).includes('Untitled')) { + console.log(`Copying ${folder} to ${targetDir}/Untitled`); + cpSync(folder, `${targetDir}/Untitled`, { recursive: true }); + } else { + console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); + cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); } - console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); - cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); + } console.log(`Sucsessfully restructured ${folders.length} directories`); } -export async function restructureIfNeeded(rootDir: string, untitled: boolean) { +export async function restructureIfNeeded(rootDir: string) { // before // $rootdir/My Album 1 // $rootdir/My Album 2 @@ -42,7 +45,6 @@ export async function restructureIfNeeded(rootDir: string, untitled: boolean) { _restructureIfNeeded( await glob(`${rootDir}/Photos from */`), photosDir, - untitled ); // move everythingg else to Albums/, so we end up with two top level folders @@ -57,6 +59,5 @@ export async function restructureIfNeeded(rootDir: string, untitled: boolean) { _restructureIfNeeded( everythingExceptPhotosDir, `${rootDir}/Albums`, - untitled ); } From 93f8068ac1b886c81db4da82245a36fe151d5aea Mon Sep 17 00:00:00 2001 From: covalent Date: Wed, 20 Sep 2023 12:26:13 -0400 Subject: [PATCH 23/50] update README --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 7e763e6..1813fa5 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,6 @@ Optional flags for full takeout: --timeout integer Shorthand: -t integer Meaning: Sets the timeout for exiftool, default is 30000 (30s) ---untitled - Shorthand: -u - Meaning: Includes the largely superflous "Untitled" albums in the album migration. Even without this flag being passed, these images should already be included via the photos migration. ``` The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. From 9e51597074caebac0524abdf24f3b8b9dd20316e Mon Sep 17 00:00:00 2001 From: covalent Date: Wed, 20 Sep 2023 13:00:42 -0400 Subject: [PATCH 24/50] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1813fa5..ed52712 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If you wish to migrate a single folder from a Google Photos takeout file: ```bash mkdir output error -npx google-photos-migrate@latest fullMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest folderMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` Optional flags for folder takeout: @@ -35,7 +35,7 @@ If you wish to migrate an entire takeout folder: ```bash mkdir output error -npx google-photos-migrate@latest folderMigrate '/path/to/takeout/' --timeout 60000 +npx google-photos-migrate@latest fullMigrate '/path/to/takeout/' --timeout 60000 ``` Optional flags for full takeout: From ecdcc7d0220c9496e5474be0567521bedbab59d1 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 20 Sep 2023 13:24:58 -0400 Subject: [PATCH 25/50] Add Dockerfile. Add a Dockerfile that just wraps the code into a Debian 12 Bookworm node.js container and sets it as the entrypoint. Add instructions to the README on building the docker image manually, and running it as a one-shot container. Fix a mistake in the subcommands in the README. Add some additional detail to the fullMigrate command about what folders are created. --- Dockerfile | 13 ++++++++++ README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd88524 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +# we need some image and video processing tools too, so use Debian 12 "Bookworm" as the base so we can get them. +FROM docker.io/node:bookworm + +# get our image and video processing dependencies +RUN apt-get update && apt-get install -y ffmpeg exiftool + +COPY . /app + +WORKDIR /app +# build the application +RUN yarn && yarn build + +ENTRYPOINT [ "yarn", "start" ] diff --git a/README.md b/README.md index 7e763e6..155de7b 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,18 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos ## Run this tool +### Natively + +**Prerec**: Must have at least node 18 & yarn installed. + If you wish to migrate a single folder from a Google Photos takeout file: ```bash mkdir output error -npx google-photos-migrate@latest fullMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest folderMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` - -Optional flags for folder takeout: +Optional flags for folder takeout (see `--help` for all details): ``` --timeout integer @@ -35,10 +38,10 @@ If you wish to migrate an entire takeout folder: ```bash mkdir output error -npx google-photos-migrate@latest folderMigrate '/path/to/takeout/' --timeout 60000 +npx google-photos-migrate@latest fullMigrate '/path/to/takeout/' ``` -Optional flags for full takeout: +Optional flags for full takeout (see `--help` for all details): ``` --timeout integer @@ -51,6 +54,65 @@ Optional flags for full takeout: The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. +**WARNING:** The `fullMigrate` command non-destructively modifies your files, which results in 3 nearly complete copies of your Takeout folder by the time it completes successfully: the original, the intermediate metadata-modified files, and the final organized and de-duplicated files. Make sure you have sufficient disk space for this. + +Additional intermediate folders are created as part of this command and in the event of errors will need to be manually removed before retrying. All are under the `/path/to/takeout/Google Photos[Fotos]/` folder: +``` +Albums +AlbumsProcessed +Photos +PhotosError +PhotosProcessed +``` + +### Docker + +**Prerec:** You must have a working `docker` or `podman` install. + +A Dockerfile is also provided to make running this tool easier on most hosts. The image must be built manually (see below), no pre-built images are provided. Using it will by default use only software-based format conversion, hardware accelerated format conversion is beyond these instructions. + +**You must build the image yourself (see above), no public pre-built images are provided.** + +You must build the image once before you run it: +```shell +# get the source code +git clone https://github.com/garzj/google-photos-migrate +cd google-photos-migrate + +# Build the image. Must be run from within the source code folder. +docker build -f Dockerfile -t localhost/google-photos-migrate:latest . +``` + +To run `folderMigrate`, which requires providing multiple folders: +```shell +mkdir output error +docker run --rm -it --security-opt=label=disable \ + -v $(readlink -e path/to/takeout):/takeout \ + -v $(readlink -e ./output):/output \ + -v $(readlink -e ./error):/error \ + localhost/google-photos-migrate:latest \ + folderMigrate '/takeout/Google Fotos' '/output' '/error' --timeout=60000 +``` + +To run `fullMigrate`, which requires only the Takeout folder: +```shell +mkdir output error +docker run --rm -it -security-opt=label=disable \ + -v $(readlink -e path/to/takeout):/takeout \ + localhost/google-photos-migrate:latest \ + fullMigrate '/takeout' --timeout=60000 +``` + +All other commands and options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. + +For the overall help: +```shell +# no folders needed, so keep it simple +docker run --rm -it --security-opt=label=disable \ + localhost/google-photos-migrate:latest \ + --help +``` + ## Further steps - If you use Linux + Android, you might want to check out the scripts I used to locate duplicate media and keep the better versions in the [android-dups](./android-dups/) directory. From 22b249af1a7a9e7661b9ba35b6f4d328cdde55c9 Mon Sep 17 00:00:00 2001 From: garzj Date: Thu, 21 Sep 2023 13:06:02 +0200 Subject: [PATCH 26/50] Fix string-width-cjs resolution [fixes #12] --- package.json | 3 +++ yarn.lock | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 971c037..d0b31a9 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,9 @@ "sanitize-filename": "^1.6.3", "ts-node": "^10.9.1" }, + "resolutions": { + "string-width-cjs": "5.1.1" + }, "keywords": [ "google", "photos", diff --git a/yarn.lock b/yarn.lock index ebbdbc8..3f3fbba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -597,7 +597,13 @@ source-map@^0.6.0: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: +string-width-cjs@5.1.1, "string-width-cjs@npm:string-width@^4.2.0": + version "5.1.1" + resolved "https://registry.yarnpkg.com/string-width-cjs/-/string-width-cjs-5.1.1.tgz#9db7e00211027471870621778b089402cf9bdee3" + integrity sha512-N5/f0RNFtXPNDLtEvo8Y/+mH4xZWzIAnnqXPzCB4BBAQL8P3sHlz0dqDzTkGQoVsL56aCsORFFlyvA70k5yq3A== + +string-width@^4.1.0: + name string-width-cjs version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== From 36d1a8dc99a1d26a82d44affa6dda728eabcfade Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 24 Sep 2023 11:24:35 -0400 Subject: [PATCH 27/50] change all sync to async calls, remove middleman restructure --- src/commands/migrate-folder.ts | 9 ++- src/commands/migrate-full.ts | 29 +++++--- src/dir/migrate-entire-takeout-folder.ts | 8 +- src/dir/migrate-full.ts | 23 +++--- src/dir/process-albums.ts | 8 +- src/dir/process-photos.ts | 8 +- src/dir/restructure-and-process.ts | 95 ++++++++++++++++++++++++ src/dir/restructure-if-needed.ts | 63 ---------------- 8 files changed, 140 insertions(+), 103 deletions(-) create mode 100644 src/dir/restructure-and-process.ts delete mode 100644 src/dir/restructure-if-needed.ts diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-folder.ts index e6bd55e..aeb5562 100644 --- a/src/commands/migrate-folder.ts +++ b/src/commands/migrate-folder.ts @@ -1,8 +1,8 @@ import { command, string, positional, flag, number, option } from 'cmd-ts'; -import { existsSync } from 'fs'; import { isEmptyDir } from '../fs/is-empty-dir'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from '../dir/migrate-single-folder'; +import { fileExists } from '../fs/file-exists'; export const folderMigrate = command({ name: 'google-photos-migrate-folder', @@ -39,13 +39,14 @@ export const folderMigrate = command({ }, handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; - if (!existsSync(googleDir)) { + + if (!(await fileExists(googleDir))) { errs.push(`The specified google directory does not exist: ${googleDir}`); } - if (!existsSync(outputDir)) { + if (!(await fileExists(outputDir))) { errs.push(`The specified output directory does not exist: ${googleDir}`); } - if (!existsSync(errorDir)) { + if (!(await fileExists(errorDir))) { errs.push(`The specified error directory does not exist: ${googleDir}`); } if (errs.length !== 0) { diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 3d96df7..68402b2 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -7,19 +7,24 @@ import { option, boolean, } from 'cmd-ts'; -import { existsSync } from 'fs'; import { isEmptyDir } from '../fs/is-empty-dir'; import { glob } from 'glob'; import { runFullMigration } from '../dir/migrate-full'; +import { fileExists } from '../fs/file-exists'; export const fullMigrate = command({ name: 'google-photos-migrate-full', args: { - takeoutDir: positional({ + sourceDir: positional({ type: string, - displayName: 'takeout_dir', + displayName: 'source_dir', description: 'The path to your "Takeout" directory.', }), + targetDir: positional({ + type: string, + displayName: 'target_dir', + description: 'The path where you want the processed directories to go.', + }), timeout: option({ type: number, defaultValue: () => 30000, @@ -29,22 +34,22 @@ export const fullMigrate = command({ 'Sets the task timeout in milliseconds that will be passed to ExifTool.', }), }, - handler: async ({ takeoutDir, timeout }) => { + handler: async ({ sourceDir, targetDir, timeout }) => { const errs: string[] = []; - if (!existsSync(takeoutDir)) { + if (!(await fileExists(sourceDir))) { errs.push( - `The specified takeout directory does not exist: ${takeoutDir}` + `The specified takeout directory does not exist: ${sourceDir}` ); } - if (await isEmptyDir(takeoutDir)) { - errs.push('The google directory is empty. Nothing to do.'); + if (await isEmptyDir(sourceDir)) { + errs.push('The source directory is empty. Nothing to do.'); } - var rootDir: string = (await glob(`${takeoutDir}/Google*`))[0].replace( + const rootDir: string = (await glob(`${sourceDir}/Google*`))[0].replace( /\/+$/, '' ); if ( - (await existsSync(`${rootDir}/Photos`)) && + (await fileExists(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Photos`)) ) { errs.push( @@ -52,7 +57,7 @@ export const fullMigrate = command({ ); } if ( - (await existsSync(`${rootDir}/Photos`)) && + (await fileExists(`${rootDir}/Photos`)) && !(await isEmptyDir(`${rootDir}/Albums`)) ) { errs.push( @@ -64,6 +69,6 @@ export const fullMigrate = command({ process.exit(1); } - runFullMigration(takeoutDir, timeout); + await runFullMigration(sourceDir, targetDir, timeout); }, }); diff --git a/src/dir/migrate-entire-takeout-folder.ts b/src/dir/migrate-entire-takeout-folder.ts index c0a37ab..9fc6834 100644 --- a/src/dir/migrate-entire-takeout-folder.ts +++ b/src/dir/migrate-entire-takeout-folder.ts @@ -1,9 +1,9 @@ -import { existsSync } from 'fs'; import { glob } from 'glob'; import { basename, join } from 'path'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from './migrate-single-folder'; import { isEmptyDir } from '../fs/is-empty-dir'; +import { fileExists } from '../fs/file-exists'; export async function migrateEntireTakoutFolder( albumDir: string, @@ -13,13 +13,13 @@ export async function migrateEntireTakoutFolder( exifTool: ExifTool ) { const errs: string[] = []; - if (!existsSync(albumDir)) { + if (!(await fileExists(albumDir))) { errs.push(`The specified album directory does not exist: ${albumDir}`); } - if (!existsSync(outDir)) { + if (!(await fileExists(outDir))) { errs.push(`The specified output directory does not exist: ${outDir}`); } - if (!existsSync(errDir)) { + if (!(await fileExists(errDir))) { errs.push(`The specified error directory does not exist: ${errDir}`); } if (errs.length !== 0) { diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index e3fc39b..eaf8663 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,12 +1,13 @@ import { glob } from 'glob'; -import { restructureIfNeeded } from './restructure-if-needed'; +import { restructureAndProcess } from './restructure-and-process'; import { processPhotos } from './process-photos'; import { processAlbums } from './process-albums'; -import { existsSync } from 'fs'; import { ExifTool } from 'exiftool-vendored'; +import { fileExists } from '../fs/file-exists'; export async function runFullMigration( - rootDir: string, + sourceDirectory: string, + processedDirectory: string, timeout: number, ) { // at least in my takeout, the Takeout folder contains a subfolder @@ -14,14 +15,12 @@ export async function runFullMigration( // rootdir refers to that subfolder // Can add more language support here in the future const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - rootDir; - if (existsSync(`${rootDir}/Google Photos`)){ - rootDir = `${rootDir}/Google Photos`; - } else if (existsSync(`${rootDir}/Google Fotos`)){ - rootDir = `${rootDir}/Google Fotos` + let googlePhotosDir: string = ""; + if (await fileExists(`${sourceDirectory}/Google Photos`)){ + googlePhotosDir = `${sourceDirectory}/Google Photos`; + } else if (await fileExists(`${sourceDirectory}/Google Fotos`)){ + googlePhotosDir = `${sourceDirectory}/Google Fotos` } - await restructureIfNeeded(rootDir); - await processPhotos(rootDir, exifTool); - await processAlbums(rootDir, exifTool); - exifTool.end(); + await restructureAndProcess(googlePhotosDir, exifTool); + await exifTool.end(); } diff --git a/src/dir/process-albums.ts b/src/dir/process-albums.ts index de95b2e..f838d3e 100644 --- a/src/dir/process-albums.ts +++ b/src/dir/process-albums.ts @@ -1,8 +1,8 @@ -import { mkdirSync } from 'fs'; import { glob } from 'glob'; import { basename } from 'path'; import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; import { ExifTool } from 'exiftool-vendored'; +import { mkdir } from 'fs/promises'; export async function processAlbums(rootDir: string, exifTool: ExifTool) { const globStr: string = `${rootDir}/Albums/*/`; @@ -15,9 +15,9 @@ export async function processAlbums(rootDir: string, exifTool: ExifTool) { let albumName = basename(album); let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; let errDir = `${rootDir}/AlbumsError/${albumName}`; - mkdirSync(album, { recursive: true }); - mkdirSync(outDir, { recursive: true }); - mkdirSync(errDir, { recursive: true }); + await mkdir(album, { recursive: true }); + await mkdir(outDir, { recursive: true }); + await mkdir(errDir, { recursive: true }); await migrateEntireTakoutFolder(album, outDir, errDir, true, exifTool); } } diff --git a/src/dir/process-photos.ts b/src/dir/process-photos.ts index 15a9acc..4ccbd9c 100644 --- a/src/dir/process-photos.ts +++ b/src/dir/process-photos.ts @@ -1,4 +1,4 @@ -import { mkdirSync } from 'fs'; +import { mkdir } from 'fs/promises'; import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; import { ExifTool } from 'exiftool-vendored'; @@ -8,9 +8,9 @@ export async function processPhotos(rootDir: string, exifTool: ExifTool) { const albumDir = `${rootDir}/Photos`; const outDir = `${rootDir}/PhotosProcessed`; const errDir = `${rootDir}/PhotosError`; - mkdirSync(albumDir, { recursive: true }); - mkdirSync(outDir, { recursive: true }); - mkdirSync(errDir, { recursive: true }); + await mkdir(albumDir, { recursive: true }); + await mkdir(outDir, { recursive: true }); + await mkdir(errDir, { recursive: true }); await migrateEntireTakoutFolder(albumDir, outDir, errDir, true, exifTool); } diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts new file mode 100644 index 0000000..a87cf9c --- /dev/null +++ b/src/dir/restructure-and-process.ts @@ -0,0 +1,95 @@ +import { glob } from 'glob'; +import { basename, dirname } from 'path'; +import { fileExists } from '../fs/file-exists'; +import { mkdir, cp } from 'fs/promises'; +import { ExifTool } from 'exiftool-vendored'; +import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; + +async function _restructureAndProcess( + folders: string[], + targetDir: string, + processingType: boolean, // true for Albums, false for Photos + exifTool: ExifTool, +) { + if ((!await fileExists(targetDir)) && (await glob(`${targetDir}/*`)).length > 0) { + console.log( + `${targetDir} exists and is not empty. No restructuring needed.` + ); + return; + } + console.log(`Starting restructure of ${folders.length} directories`); + await mkdir(targetDir, { recursive: true }); + for (let folder of folders) { + // Moves all Untitled(x) directories to one large folder + // if (basename(folder).includes('Untitled')) { + // console.log(`Copying ${folder} to ${targetDir}/Untitled`); + // cp(folder, `${targetDir}/Untitled`); + // } else { + // console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); + // cp(folder, `${targetDir}/${basename(folder)}`); + // } + + if (processingType){ // true for Albums, false for Photos + console.log(`Processing album ${folder}...`); + let outDir = `${dirname(folder)}/AlbumsProcessed/${basename(folder)}`; + let errDir = `${dirname(folder)}/AlbumsError/${basename(folder)}`; + if (basename(folder).includes('Untitled(')){ + outDir = `${dirname(folder)}/AlbumsProcessed/Untitled`; + errDir = `${dirname(folder)}/AlbumsError/Untitled`; + } + await mkdir(outDir, { recursive: true }); + await mkdir(errDir, { recursive: true }); + await migrateEntireTakoutFolder(folder, outDir, errDir, true, exifTool); + }else{ + const outDir = `${dirname(folder)}/PhotosProcessed`; + const errDir = `${dirname(folder)}/PhotosError`; + await mkdir(outDir, { recursive: true }); + await mkdir(errDir, { recursive: true }); + await migrateEntireTakoutFolder(folder, outDir, errDir, true, exifTool); + } + + } + console.log(`Sucsessfully restructured ${folders.length} directories`); +} + +export async function restructureAndProcess(rootDir: string, exifTool: ExifTool) { + // before + // $rootdir/My Album 1 + // $rootdir/My Album 2 + // $rootdir/Photos from 2008 + + // after + // $rootdir/Albums/My Album 1 + // $rootdir/Albums/My Album 2 + // $rootdir/Photos/Photos from 2008 + + console.log('Processing photos...'); + + const photosDir: string = `${rootDir}/Photos`; + + // move the "Photos from $YEAR" directories to Photos/ + await _restructureAndProcess( + await glob(`${rootDir}/Photos from */`), + photosDir, + false, + exifTool + ); + + console.log('Processing albums...'); + + // move everythingg else to Albums/, so we end up with two top level folders + const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); + const photoSet: Set = new Set( + await glob(`${rootDir}/Photos from */`) + ); + photoSet.add(`${rootDir}/Photos`); + const everythingExceptPhotosDir: string[] = Array.from( + new Set([...fullSet].filter((x) => !photoSet.has(x))) + ); + await _restructureAndProcess( + everythingExceptPhotosDir, + `${rootDir}/Albums`, + true, + exifTool + ); +} diff --git a/src/dir/restructure-if-needed.ts b/src/dir/restructure-if-needed.ts deleted file mode 100644 index 8ae5b15..0000000 --- a/src/dir/restructure-if-needed.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { cpSync, existsSync, mkdirSync } from 'fs'; -import { glob } from 'glob'; -import { basename } from 'path'; - -async function _restructureIfNeeded( - folders: string[], - targetDir: string, -) { - if (existsSync(targetDir) && (await glob(`${targetDir}/*`)).length > 0) { - console.log( - `${targetDir} exists and is not empty. No restructuring needed.` - ); - return; - } - console.log(`Starting restructure of ${folders.length} directories`); - mkdirSync(targetDir, { recursive: true }); - for (let folder of folders) { - // Moves all Untitled(x) directories to one large folder - if (basename(folder).includes('Untitled')) { - console.log(`Copying ${folder} to ${targetDir}/Untitled`); - cpSync(folder, `${targetDir}/Untitled`, { recursive: true }); - } else { - console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); - cpSync(folder, `${targetDir}/${basename(folder)}`, { recursive: true }); - } - - } - console.log(`Sucsessfully restructured ${folders.length} directories`); -} - -export async function restructureIfNeeded(rootDir: string) { - // before - // $rootdir/My Album 1 - // $rootdir/My Album 2 - // $rootdir/Photos from 2008 - - // after - // $rootdir/Albums/My Album 1 - // $rootdir/Albums/My Album 2 - // $rootdir/Photos/Photos from 2008 - - const photosDir: string = `${rootDir}/Photos`; - - // move the "Photos from $YEAR" directories to Photos/ - _restructureIfNeeded( - await glob(`${rootDir}/Photos from */`), - photosDir, - ); - - // move everythingg else to Albums/, so we end up with two top level folders - const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); - const photoSet: Set = new Set( - await glob(`${rootDir}/Photos from */`) - ); - photoSet.add(`${rootDir}/Photos`); - const everythingExceptPhotosDir: string[] = Array.from( - new Set([...fullSet].filter((x) => !photoSet.has(x))) - ); - _restructureIfNeeded( - everythingExceptPhotosDir, - `${rootDir}/Albums`, - ); -} From cfdc4916bc2433ec22fccbbcc73954713881f5b5 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 24 Sep 2023 12:18:00 -0400 Subject: [PATCH 28/50] - allow specifiying target directory - improve fully rewriting exif pipeline --- src/commands/migrate-full.ts | 23 ++------- src/dir/check-error-dir.ts | 40 +++++++++++++++ src/dir/migrate-entire-takeout-folder.ts | 28 ++--------- src/dir/migrate-full.ts | 7 +-- src/dir/process-albums.ts | 23 --------- src/dir/process-photos.ts | 16 ------ src/dir/restructure-and-process.ts | 64 ++++++++++-------------- src/fs/is-dir.ts | 12 +++++ 8 files changed, 86 insertions(+), 127 deletions(-) create mode 100644 src/dir/check-error-dir.ts delete mode 100644 src/dir/process-albums.ts delete mode 100644 src/dir/process-photos.ts create mode 100644 src/fs/is-dir.ts diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 68402b2..84c52cd 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -42,27 +42,10 @@ export const fullMigrate = command({ ); } if (await isEmptyDir(sourceDir)) { - errs.push('The source directory is empty. Nothing to do.'); + errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); } - const rootDir: string = (await glob(`${sourceDir}/Google*`))[0].replace( - /\/+$/, - '' - ); - if ( - (await fileExists(`${rootDir}/Photos`)) && - !(await isEmptyDir(`${rootDir}/Photos`)) - ) { - errs.push( - 'The Photos directory is not empty. Please delete it and try again.' - ); - } - if ( - (await fileExists(`${rootDir}/Photos`)) && - !(await isEmptyDir(`${rootDir}/Albums`)) - ) { - errs.push( - 'The Albums directory is not empty. Please delete it and try again.' - ); + if ((await fileExists(targetDir)) && !(await isEmptyDir(targetDir))){ + errs.push(`The target directory is not empty, please delete it and try again: ${targetDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts new file mode 100644 index 0000000..b8552e6 --- /dev/null +++ b/src/dir/check-error-dir.ts @@ -0,0 +1,40 @@ +import { ExifTool } from 'exiftool-vendored'; +import { isEmptyDir } from '../fs/is-empty-dir'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { isDir } from '../fs/is-dir'; +import { fileExists } from '../fs/file-exists'; + +export async function checkErrorDir( + outDir: string, + errDir: string, + exifTool: ExifTool +) { + if (!(await isEmptyDir(errDir))) { + const errFiles: string[] = await glob(`${errDir}/*`); + for (let file of errFiles) { + if (file.endsWith('.json')) { + console.log( + `Cannot fix metadata for ${file} as .json is an unsupported file type.` + ); + continue; + } else if (await isDir(file)) { + console.log(`Cannot fix metadata for directory: ${file}`); + continue; + } else if (await fileExists(file)){ + console.log(`File already exists (you can ignore this): ${file}`); + continue; + } + console.log( + `Rewriting all tags from ${file}, to ${join( + outDir, + `cleaned-${basename(file)}` + )}` + ); + await exifTool.rewriteAllTags( + file, + join(outDir, `cleaned-${basename(file)}`) + ); + } + } +} diff --git a/src/dir/migrate-entire-takeout-folder.ts b/src/dir/migrate-entire-takeout-folder.ts index 9fc6834..7c0e629 100644 --- a/src/dir/migrate-entire-takeout-folder.ts +++ b/src/dir/migrate-entire-takeout-folder.ts @@ -4,12 +4,12 @@ import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from './migrate-single-folder'; import { isEmptyDir } from '../fs/is-empty-dir'; import { fileExists } from '../fs/file-exists'; +import { checkErrorDir } from './check-error-dir'; -export async function migrateEntireTakoutFolder( +export async function migrateSingleFolderAndCheckErrors( albumDir: string, outDir: string, errDir: string, - check_errDir: boolean, exifTool: ExifTool ) { const errs: string[] = []; @@ -28,27 +28,5 @@ export async function migrateEntireTakoutFolder( } await migrateSingleFolder(albumDir, outDir, errDir, exifTool, false); - - if (check_errDir && !(await isEmptyDir(errDir))) { - const errFiles: string[] = await glob(`${errDir}/*`); - for (let file of errFiles) { - if (file.endsWith('.json')) { - console.log( - `Cannot fix metadata for ${file} as .json is an unsupported file type.` - ); - continue; - } - console.log( - `Rewriting all tags from ${file}, to ${join( - albumDir, - `cleaned-${basename(file)}` - )}` - ); - await exifTool.rewriteAllTags( - file, - join(albumDir, `cleaned-${basename(file)}`) - ); - } - await migrateEntireTakoutFolder(albumDir, outDir, errDir, false, exifTool); - } + await checkErrorDir(outDir, errDir, exifTool); } diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index eaf8663..eb5c4a9 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,13 +1,10 @@ -import { glob } from 'glob'; import { restructureAndProcess } from './restructure-and-process'; -import { processPhotos } from './process-photos'; -import { processAlbums } from './process-albums'; import { ExifTool } from 'exiftool-vendored'; import { fileExists } from '../fs/file-exists'; export async function runFullMigration( sourceDirectory: string, - processedDirectory: string, + targetDirectory: string, timeout: number, ) { // at least in my takeout, the Takeout folder contains a subfolder @@ -21,6 +18,6 @@ export async function runFullMigration( } else if (await fileExists(`${sourceDirectory}/Google Fotos`)){ googlePhotosDir = `${sourceDirectory}/Google Fotos` } - await restructureAndProcess(googlePhotosDir, exifTool); + await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); await exifTool.end(); } diff --git a/src/dir/process-albums.ts b/src/dir/process-albums.ts deleted file mode 100644 index f838d3e..0000000 --- a/src/dir/process-albums.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { glob } from 'glob'; -import { basename } from 'path'; -import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; -import { ExifTool } from 'exiftool-vendored'; -import { mkdir } from 'fs/promises'; - -export async function processAlbums(rootDir: string, exifTool: ExifTool) { - const globStr: string = `${rootDir}/Albums/*/`; - const albums: string[] = await glob(globStr); - if (albums.length == 0) { - console.log(`WARN: No albums found at ${globStr}`); - } - for (let album of albums) { - console.log(`Processing album ${album}...`); - let albumName = basename(album); - let outDir = `${rootDir}/AlbumsProcessed/${albumName}`; - let errDir = `${rootDir}/AlbumsError/${albumName}`; - await mkdir(album, { recursive: true }); - await mkdir(outDir, { recursive: true }); - await mkdir(errDir, { recursive: true }); - await migrateEntireTakoutFolder(album, outDir, errDir, true, exifTool); - } -} diff --git a/src/dir/process-photos.ts b/src/dir/process-photos.ts deleted file mode 100644 index 4ccbd9c..0000000 --- a/src/dir/process-photos.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { mkdir } from 'fs/promises'; -import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; -import { ExifTool } from 'exiftool-vendored'; - -export async function processPhotos(rootDir: string, exifTool: ExifTool) { - // Also run the exif fix for the photos - console.log('Processing photos...'); - const albumDir = `${rootDir}/Photos`; - const outDir = `${rootDir}/PhotosProcessed`; - const errDir = `${rootDir}/PhotosError`; - await mkdir(albumDir, { recursive: true }); - await mkdir(outDir, { recursive: true }); - await mkdir(errDir, { recursive: true }); - - await migrateEntireTakoutFolder(albumDir, outDir, errDir, true, exifTool); -} diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index a87cf9c..ed3df09 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -3,56 +3,45 @@ import { basename, dirname } from 'path'; import { fileExists } from '../fs/file-exists'; import { mkdir, cp } from 'fs/promises'; import { ExifTool } from 'exiftool-vendored'; -import { migrateEntireTakoutFolder } from './migrate-entire-takeout-folder'; +import { migrateSingleFolderAndCheckErrors } from './migrate-entire-takeout-folder'; async function _restructureAndProcess( folders: string[], targetDir: string, processingType: boolean, // true for Albums, false for Photos - exifTool: ExifTool, + exifTool: ExifTool ) { - if ((!await fileExists(targetDir)) && (await glob(`${targetDir}/*`)).length > 0) { - console.log( - `${targetDir} exists and is not empty. No restructuring needed.` - ); - return; - } console.log(`Starting restructure of ${folders.length} directories`); await mkdir(targetDir, { recursive: true }); for (let folder of folders) { - // Moves all Untitled(x) directories to one large folder - // if (basename(folder).includes('Untitled')) { - // console.log(`Copying ${folder} to ${targetDir}/Untitled`); - // cp(folder, `${targetDir}/Untitled`); - // } else { - // console.log(`Copying ${folder} to ${targetDir}/${basename(folder)}`); - // cp(folder, `${targetDir}/${basename(folder)}`); - // } - - if (processingType){ // true for Albums, false for Photos + if (processingType) { + // true for Albums, false for Photos console.log(`Processing album ${folder}...`); - let outDir = `${dirname(folder)}/AlbumsProcessed/${basename(folder)}`; - let errDir = `${dirname(folder)}/AlbumsError/${basename(folder)}`; - if (basename(folder).includes('Untitled(')){ - outDir = `${dirname(folder)}/AlbumsProcessed/Untitled`; - errDir = `${dirname(folder)}/AlbumsError/Untitled`; + let outDir = `${targetDir}/AlbumsProcessed/${basename(folder)}`; + let errDir = `${targetDir}/AlbumsError/${basename(folder)}`; + if (basename(folder).includes('Untitled(')) { + outDir = `${targetDir}/AlbumsProcessed/Untitled`; + errDir = `${targetDir}/AlbumsError/Untitled`; } await mkdir(outDir, { recursive: true }); await mkdir(errDir, { recursive: true }); - await migrateEntireTakoutFolder(folder, outDir, errDir, true, exifTool); - }else{ - const outDir = `${dirname(folder)}/PhotosProcessed`; - const errDir = `${dirname(folder)}/PhotosError`; + await migrateSingleFolderAndCheckErrors(folder, outDir, errDir, exifTool); + } else { + const outDir = `${targetDir}/PhotosProcessed`; + const errDir = `${targetDir}/PhotosError`; await mkdir(outDir, { recursive: true }); await mkdir(errDir, { recursive: true }); - await migrateEntireTakoutFolder(folder, outDir, errDir, true, exifTool); + await migrateSingleFolderAndCheckErrors(folder, outDir, errDir, exifTool); } - } console.log(`Sucsessfully restructured ${folders.length} directories`); } -export async function restructureAndProcess(rootDir: string, exifTool: ExifTool) { +export async function restructureAndProcess( + sourceDir: string, + targetDir: string, + exifTool: ExifTool +) { // before // $rootdir/My Album 1 // $rootdir/My Album 2 @@ -64,13 +53,12 @@ export async function restructureAndProcess(rootDir: string, exifTool: ExifTool) // $rootdir/Photos/Photos from 2008 console.log('Processing photos...'); - - const photosDir: string = `${rootDir}/Photos`; + // move the "Photos from $YEAR" directories to Photos/ await _restructureAndProcess( - await glob(`${rootDir}/Photos from */`), - photosDir, + await glob(`${sourceDir}/Photos from */`), + targetDir, false, exifTool ); @@ -78,17 +66,17 @@ export async function restructureAndProcess(rootDir: string, exifTool: ExifTool) console.log('Processing albums...'); // move everythingg else to Albums/, so we end up with two top level folders - const fullSet: Set = new Set(await glob(`${rootDir}/*/`)); + const fullSet: Set = new Set(await glob(`${sourceDir}/*/`)); const photoSet: Set = new Set( - await glob(`${rootDir}/Photos from */`) + await glob(`${sourceDir}/Photos from */`) ); - photoSet.add(`${rootDir}/Photos`); + photoSet.add(`${sourceDir}/Photos`); const everythingExceptPhotosDir: string[] = Array.from( new Set([...fullSet].filter((x) => !photoSet.has(x))) ); await _restructureAndProcess( everythingExceptPhotosDir, - `${rootDir}/Albums`, + targetDir, true, exifTool ); diff --git a/src/fs/is-dir.ts b/src/fs/is-dir.ts new file mode 100644 index 0000000..c62e50b --- /dev/null +++ b/src/fs/is-dir.ts @@ -0,0 +1,12 @@ +import { lstat } from "fs/promises"; + + +export async function isDir(path: string) { + try { + var stat = await lstat(path); + return stat.isDirectory(); + } catch (e) { + // lstatSync throws an error if path doesn't exist + return false; + } +} \ No newline at end of file From 431908014ebcc3413ce0a6751795a42aaa433310 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 24 Sep 2023 12:23:24 -0400 Subject: [PATCH 29/50] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 381a454..8805795 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ If you wish to migrate an entire takeout folder: ```bash mkdir output error -npx google-photos-migrate@latest fullMigrate '/path/to/takeout/' --timeout 60000 +npx google-photos-migrate@latest fullMigrate '/path/to/takeout' '/path/to/target' --timeout 60000 ``` Optional flags for full takeout (see `--help` for all details): From 4615648f165742ff4a6475b621ea64f11e746a06 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 24 Sep 2023 13:48:11 -0400 Subject: [PATCH 30/50] check to make sure Google Photos(Fotos) directory exists --- src/dir/migrate-full.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index eb5c4a9..e1993d3 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -17,6 +17,9 @@ export async function runFullMigration( googlePhotosDir = `${sourceDirectory}/Google Photos`; } else if (await fileExists(`${sourceDirectory}/Google Fotos`)){ googlePhotosDir = `${sourceDirectory}/Google Fotos` + } else { + console.log("No Google Photos (Fotos) directory was found"); + process.exit(1); } await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); await exifTool.end(); From 32c36ce49c61673d5d35b1e78e97b367c991c3cc Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 24 Sep 2023 14:19:29 -0400 Subject: [PATCH 31/50] format files & move google photos locations to langs --- src/commands/migrate-full.ts | 4 ++-- src/config/langs.ts | 1 + src/dir/migrate-entire-takeout-folder.ts | 3 --- src/dir/migrate-flat.ts | 6 +++--- src/dir/migrate-full.ts | 21 ++++++++++++--------- src/dir/migrate-single-folder.ts | 6 +++--- src/dir/restructure-and-process.ts | 1 - src/fs/is-dir.ts | 19 +++++++++---------- src/index.ts | 5 +++-- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 84c52cd..37c3a54 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -9,7 +9,7 @@ import { } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; import { glob } from 'glob'; -import { runFullMigration } from '../dir/migrate-full'; +import { migrateFullDirectory } from '../dir/migrate-full'; import { fileExists } from '../fs/file-exists'; export const fullMigrate = command({ @@ -52,6 +52,6 @@ export const fullMigrate = command({ process.exit(1); } - await runFullMigration(sourceDir, targetDir, timeout); + await migrateFullDirectory(sourceDir, targetDir, timeout); }, }); diff --git a/src/config/langs.ts b/src/config/langs.ts index 51fe6ab..cfb07c4 100644 --- a/src/config/langs.ts +++ b/src/config/langs.ts @@ -1 +1,2 @@ export const editedSuffices = ['edited', 'bearbeitet', 'modifié']; +export const possiblePhotosLocations = ['Google Photos', 'Google Fotos']; diff --git a/src/dir/migrate-entire-takeout-folder.ts b/src/dir/migrate-entire-takeout-folder.ts index 7c0e629..f123c24 100644 --- a/src/dir/migrate-entire-takeout-folder.ts +++ b/src/dir/migrate-entire-takeout-folder.ts @@ -1,8 +1,5 @@ -import { glob } from 'glob'; -import { basename, join } from 'path'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from './migrate-single-folder'; -import { isEmptyDir } from '../fs/is-empty-dir'; import { fileExists } from '../fs/file-exists'; import { checkErrorDir } from './check-error-dir'; diff --git a/src/dir/migrate-flat.ts b/src/dir/migrate-flat.ts index 5785877..85641aa 100644 --- a/src/dir/migrate-flat.ts +++ b/src/dir/migrate-flat.ts @@ -19,17 +19,17 @@ export type MigrationContext = Required & { migrationLocks: Map>; }; -export async function* migrateGoogleDir( +export async function* migrateSingleDirectory( args: MigrationArgs ): AsyncGenerator { const wg: (MediaFile | MediaMigrationError)[] = []; - for await (const result of migrateGoogleDirGen(args)) { + for await (const result of migrateSingleDirectoryGen(args)) { wg.push(result); } return await Promise.all(wg); } -export async function* migrateGoogleDirGen( +export async function* migrateSingleDirectoryGen( args: MigrationArgs ): AsyncGenerator { const migCtx: MigrationContext = { diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index e1993d3..c9a9442 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,24 +1,27 @@ import { restructureAndProcess } from './restructure-and-process'; import { ExifTool } from 'exiftool-vendored'; import { fileExists } from '../fs/file-exists'; +import { possiblePhotosLocations } from '../config/langs'; -export async function runFullMigration( +export async function migrateFullDirectory( sourceDirectory: string, targetDirectory: string, - timeout: number, + timeout: number ) { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder // Can add more language support here in the future const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - let googlePhotosDir: string = ""; - if (await fileExists(`${sourceDirectory}/Google Photos`)){ - googlePhotosDir = `${sourceDirectory}/Google Photos`; - } else if (await fileExists(`${sourceDirectory}/Google Fotos`)){ - googlePhotosDir = `${sourceDirectory}/Google Fotos` - } else { - console.log("No Google Photos (Fotos) directory was found"); + let googlePhotosDir: string = ''; + for (let i of possiblePhotosLocations) { + if (await fileExists(`${sourceDirectory}/${i}`)) { + googlePhotosDir = `${sourceDirectory}/${i}`; + break; + } + } + if (googlePhotosDir == '') { + console.log('No Google Photos (Fotos) directory was found'); process.exit(1); } await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); diff --git a/src/dir/migrate-single-folder.ts b/src/dir/migrate-single-folder.ts index eda002a..1e92d76 100644 --- a/src/dir/migrate-single-folder.ts +++ b/src/dir/migrate-single-folder.ts @@ -1,15 +1,15 @@ import { ExifTool } from 'exiftool-vendored'; -import { migrateGoogleDirGen } from './migrate-flat'; +import { migrateSingleDirectory } from './migrate-flat'; export async function migrateSingleFolder( googleDir: string, outputDir: string, errorDir: string, exifTool: ExifTool, - endExifTool: boolean, + endExifTool: boolean ) { console.log(`Started migration.`); - const migGen = migrateGoogleDirGen({ + const migGen = migrateSingleDirectory({ googleDir, outputDir, errorDir, diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index ed3df09..fae85b0 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -1,6 +1,5 @@ import { glob } from 'glob'; import { basename, dirname } from 'path'; -import { fileExists } from '../fs/file-exists'; import { mkdir, cp } from 'fs/promises'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolderAndCheckErrors } from './migrate-entire-takeout-folder'; diff --git a/src/fs/is-dir.ts b/src/fs/is-dir.ts index c62e50b..bd98d4d 100644 --- a/src/fs/is-dir.ts +++ b/src/fs/is-dir.ts @@ -1,12 +1,11 @@ -import { lstat } from "fs/promises"; - +import { lstat } from 'fs/promises'; export async function isDir(path: string) { - try { - var stat = await lstat(path); - return stat.isDirectory(); - } catch (e) { - // lstatSync throws an error if path doesn't exist - return false; - } -} \ No newline at end of file + try { + var stat = await lstat(path); + return stat.isDirectory(); + } catch (e) { + // lstatSync throws an error if path doesn't exist + return false; + } +} diff --git a/src/index.ts b/src/index.ts index 406e34d..26ed77c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,9 @@ import './config/env'; -import { migrateGoogleDir } from './dir/migrate-flat'; +import { migrateSingleDirectory } from './dir/migrate-flat'; +import { migrateFullDirectory } from './dir/migrate-full'; import type { MediaFileExtension } from './media/MediaFileExtension'; import { supportedExtensions } from './config/extensions'; -export { migrateGoogleDir, supportedExtensions }; +export { migrateSingleDirectory, migrateFullDirectory, supportedExtensions }; export type { MediaFileExtension }; From c966ba139ff28f6e204f7cc134229d26b86f1363 Mon Sep 17 00:00:00 2001 From: covalent Date: Sat, 30 Sep 2023 17:57:47 -0400 Subject: [PATCH 32/50] - add better error handling - change fileExists to entityExists as it is used on dirs --- src/commands/migrate-folder.ts | 8 +++--- src/commands/migrate-full.ts | 6 ++-- src/dir/check-error-dir.ts | 4 +-- src/dir/migrate-entire-takeout-folder.ts | 10 +++---- src/dir/migrate-full.ts | 32 ++++++++++++--------- src/dir/migrate-single-folder.ts | 6 ++-- src/fs/{file-exists.ts => entity-exists.ts} | 2 +- src/media/save-to-dir.ts | 4 +-- src/meta/find-meta-file.ts | 4 +-- 9 files changed, 40 insertions(+), 36 deletions(-) rename src/fs/{file-exists.ts => entity-exists.ts} (75%) diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-folder.ts index aeb5562..f93e7bc 100644 --- a/src/commands/migrate-folder.ts +++ b/src/commands/migrate-folder.ts @@ -2,7 +2,7 @@ import { command, string, positional, flag, number, option } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from '../dir/migrate-single-folder'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; export const folderMigrate = command({ name: 'google-photos-migrate-folder', @@ -40,13 +40,13 @@ export const folderMigrate = command({ handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; - if (!(await fileExists(googleDir))) { + if (!(await entitiyExists(googleDir))) { errs.push(`The specified google directory does not exist: ${googleDir}`); } - if (!(await fileExists(outputDir))) { + if (!(await entitiyExists(outputDir))) { errs.push(`The specified output directory does not exist: ${googleDir}`); } - if (!(await fileExists(errorDir))) { + if (!(await entitiyExists(errorDir))) { errs.push(`The specified error directory does not exist: ${googleDir}`); } if (errs.length !== 0) { diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 37c3a54..d7933dd 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -10,7 +10,7 @@ import { import { isEmptyDir } from '../fs/is-empty-dir'; import { glob } from 'glob'; import { migrateFullDirectory } from '../dir/migrate-full'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; export const fullMigrate = command({ name: 'google-photos-migrate-full', @@ -36,7 +36,7 @@ export const fullMigrate = command({ }, handler: async ({ sourceDir, targetDir, timeout }) => { const errs: string[] = []; - if (!(await fileExists(sourceDir))) { + if (!(await entitiyExists(sourceDir))) { errs.push( `The specified takeout directory does not exist: ${sourceDir}` ); @@ -44,7 +44,7 @@ export const fullMigrate = command({ if (await isEmptyDir(sourceDir)) { errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); } - if ((await fileExists(targetDir)) && !(await isEmptyDir(targetDir))){ + if ((await entitiyExists(targetDir)) && !(await isEmptyDir(targetDir))){ errs.push(`The target directory is not empty, please delete it and try again: ${targetDir}`); } if (errs.length !== 0) { diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts index b8552e6..8ef7cee 100644 --- a/src/dir/check-error-dir.ts +++ b/src/dir/check-error-dir.ts @@ -3,7 +3,7 @@ import { isEmptyDir } from '../fs/is-empty-dir'; import { glob } from 'glob'; import { basename, join } from 'path'; import { isDir } from '../fs/is-dir'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; export async function checkErrorDir( outDir: string, @@ -21,7 +21,7 @@ export async function checkErrorDir( } else if (await isDir(file)) { console.log(`Cannot fix metadata for directory: ${file}`); continue; - } else if (await fileExists(file)){ + } else if (await entitiyExists(file)){ console.log(`File already exists (you can ignore this): ${file}`); continue; } diff --git a/src/dir/migrate-entire-takeout-folder.ts b/src/dir/migrate-entire-takeout-folder.ts index f123c24..4d9301d 100644 --- a/src/dir/migrate-entire-takeout-folder.ts +++ b/src/dir/migrate-entire-takeout-folder.ts @@ -1,6 +1,6 @@ import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from './migrate-single-folder'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; import { checkErrorDir } from './check-error-dir'; export async function migrateSingleFolderAndCheckErrors( @@ -10,18 +10,18 @@ export async function migrateSingleFolderAndCheckErrors( exifTool: ExifTool ) { const errs: string[] = []; - if (!(await fileExists(albumDir))) { + if (!(await entitiyExists(albumDir))) { errs.push(`The specified album directory does not exist: ${albumDir}`); } - if (!(await fileExists(outDir))) { + if (!(await entitiyExists(outDir))) { errs.push(`The specified output directory does not exist: ${outDir}`); } - if (!(await fileExists(errDir))) { + if (!(await entitiyExists(errDir))) { errs.push(`The specified error directory does not exist: ${errDir}`); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); - process.exit(1); + throw(Error(`Specified output directories don't exist: ${errs}`)); } await migrateSingleFolder(albumDir, outDir, errDir, exifTool, false); diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index c9a9442..5f8749f 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,29 +1,33 @@ import { restructureAndProcess } from './restructure-and-process'; import { ExifTool } from 'exiftool-vendored'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; import { possiblePhotosLocations } from '../config/langs'; export async function migrateFullDirectory( sourceDirectory: string, targetDirectory: string, timeout: number -) { +): Promise< Error | void > { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder // Can add more language support here in the future - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - let googlePhotosDir: string = ''; - for (let i of possiblePhotosLocations) { - if (await fileExists(`${sourceDirectory}/${i}`)) { - googlePhotosDir = `${sourceDirectory}/${i}`; - break; + try { + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); + let googlePhotosDir: string = ''; + for (let i of possiblePhotosLocations) { + if (await entitiyExists(`${sourceDirectory}/${i}`)) { + googlePhotosDir = `${sourceDirectory}/${i}`; + break; + } } + if (googlePhotosDir == '') { + return new Error('No Google Photos (Fotos) directory was found'); + } + await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); + await exifTool.end(); + } catch (e){ + return new Error(`Error while migrating: ${e}`); } - if (googlePhotosDir == '') { - console.log('No Google Photos (Fotos) directory was found'); - process.exit(1); - } - await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); - await exifTool.end(); + return; } diff --git a/src/dir/migrate-single-folder.ts b/src/dir/migrate-single-folder.ts index 1e92d76..9fc8db4 100644 --- a/src/dir/migrate-single-folder.ts +++ b/src/dir/migrate-single-folder.ts @@ -17,16 +17,16 @@ export async function migrateSingleFolder( exiftool: exifTool, endExifTool: endExifTool, }); - const counts = { err: 0, suc: 0 }; for await (const result of migGen) { + console.log(result); if (result instanceof Error) { console.error(`Error: ${result}`); counts.err++; continue; + } else { + counts.suc++; } - - counts.suc++; } console.log(`Done! Processed ${counts.suc + counts.err} files.`); diff --git a/src/fs/file-exists.ts b/src/fs/entity-exists.ts similarity index 75% rename from src/fs/file-exists.ts rename to src/fs/entity-exists.ts index e51c157..d84833a 100644 --- a/src/fs/file-exists.ts +++ b/src/fs/entity-exists.ts @@ -1,6 +1,6 @@ import { stat } from 'fs/promises'; -export const fileExists = (path: string) => +export const entitiyExists = (path: string) => stat(path) .then(() => true) .catch((e) => (e.code === 'ENOENT' ? false : Promise.reject(e))); diff --git a/src/media/save-to-dir.ts b/src/media/save-to-dir.ts index 6782fcf..75d2980 100644 --- a/src/media/save-to-dir.ts +++ b/src/media/save-to-dir.ts @@ -1,5 +1,5 @@ import { basename, resolve } from 'path'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; import sanitize = require('sanitize-filename'); import { copyFile, mkdir, rename } from 'fs/promises'; import { MigrationContext } from '../dir/migrate-flat'; @@ -18,7 +18,7 @@ async function _saveToDir( await mkdir(saveDir, { recursive: true }); const savePath = resolve(saveDir, saveBase); - const exists = await fileExists(savePath); + const exists = await entitiyExists(savePath); if (exists) { return _saveToDir(file, destDir, saveBase, move, duplicateIndex + 1); } diff --git a/src/meta/find-meta-file.ts b/src/meta/find-meta-file.ts index 8e5f06a..ddcbda9 100644 --- a/src/meta/find-meta-file.ts +++ b/src/meta/find-meta-file.ts @@ -1,7 +1,7 @@ import { basename, dirname } from 'path'; import { MigrationContext } from '../dir/migrate-flat'; import { MediaFileExtension } from '../media/MediaFileExtension'; -import { fileExists } from '../fs/file-exists'; +import { entitiyExists } from '../fs/entity-exists'; import { editedSuffices } from '../config/langs'; export async function findMetaFile( @@ -71,7 +71,7 @@ export async function findMetaFile( } for (const potPath of potPaths) { - if (!(await fileExists(potPath))) { + if (!(await entitiyExists(potPath))) { continue; } return potPath; From 2c1b527115617ce31ee2140fa96bf9814efe2eeb Mon Sep 17 00:00:00 2001 From: covalent Date: Sat, 30 Sep 2023 18:01:04 -0400 Subject: [PATCH 33/50] remove unnecesary print --- src/dir/migrate-single-folder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dir/migrate-single-folder.ts b/src/dir/migrate-single-folder.ts index 9fc8db4..39d1762 100644 --- a/src/dir/migrate-single-folder.ts +++ b/src/dir/migrate-single-folder.ts @@ -19,7 +19,6 @@ export async function migrateSingleFolder( }); const counts = { err: 0, suc: 0 }; for await (const result of migGen) { - console.log(result); if (result instanceof Error) { console.error(`Error: ${result}`); counts.err++; From 949c93ed575293d25b7a78569d474871f226ee98 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 1 Oct 2023 19:44:33 -0400 Subject: [PATCH 34/50] remove confusing log --- src/dir/check-error-dir.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts index 8ef7cee..de93ce3 100644 --- a/src/dir/check-error-dir.ts +++ b/src/dir/check-error-dir.ts @@ -14,9 +14,6 @@ export async function checkErrorDir( const errFiles: string[] = await glob(`${errDir}/*`); for (let file of errFiles) { if (file.endsWith('.json')) { - console.log( - `Cannot fix metadata for ${file} as .json is an unsupported file type.` - ); continue; } else if (await isDir(file)) { console.log(`Cannot fix metadata for directory: ${file}`); From 9155b997815e9380982ea6b114d14d3a79088e20 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 1 Oct 2023 20:08:59 -0400 Subject: [PATCH 35/50] improve error checking --- src/dir/check-error-dir.ts | 1 - src/dir/migrate-entire-takeout-folder.ts | 29 ---------------------- src/dir/migrate-full.ts | 8 +++--- src/dir/restructure-and-process.ts | 31 +++++++++++++++++++----- 4 files changed, 30 insertions(+), 39 deletions(-) delete mode 100644 src/dir/migrate-entire-takeout-folder.ts diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts index de93ce3..82b2b75 100644 --- a/src/dir/check-error-dir.ts +++ b/src/dir/check-error-dir.ts @@ -19,7 +19,6 @@ export async function checkErrorDir( console.log(`Cannot fix metadata for directory: ${file}`); continue; } else if (await entitiyExists(file)){ - console.log(`File already exists (you can ignore this): ${file}`); continue; } console.log( diff --git a/src/dir/migrate-entire-takeout-folder.ts b/src/dir/migrate-entire-takeout-folder.ts deleted file mode 100644 index 4d9301d..0000000 --- a/src/dir/migrate-entire-takeout-folder.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleFolder } from './migrate-single-folder'; -import { entitiyExists } from '../fs/entity-exists'; -import { checkErrorDir } from './check-error-dir'; - -export async function migrateSingleFolderAndCheckErrors( - albumDir: string, - outDir: string, - errDir: string, - exifTool: ExifTool -) { - const errs: string[] = []; - if (!(await entitiyExists(albumDir))) { - errs.push(`The specified album directory does not exist: ${albumDir}`); - } - if (!(await entitiyExists(outDir))) { - errs.push(`The specified output directory does not exist: ${outDir}`); - } - if (!(await entitiyExists(errDir))) { - errs.push(`The specified error directory does not exist: ${errDir}`); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - throw(Error(`Specified output directories don't exist: ${errs}`)); - } - - await migrateSingleFolder(albumDir, outDir, errDir, exifTool, false); - await checkErrorDir(outDir, errDir, exifTool); -} diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 5f8749f..8273abf 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,5 +1,5 @@ import { restructureAndProcess } from './restructure-and-process'; -import { ExifTool } from 'exiftool-vendored'; +import { ExifTool, exiftool } from 'exiftool-vendored'; import { entitiyExists } from '../fs/entity-exists'; import { possiblePhotosLocations } from '../config/langs'; @@ -12,8 +12,8 @@ export async function migrateFullDirectory( // Takeout/Google Foto // rootdir refers to that subfolder // Can add more language support here in the future + const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); try { - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); let googlePhotosDir: string = ''; for (let i of possiblePhotosLocations) { if (await entitiyExists(`${sourceDirectory}/${i}`)) { @@ -25,8 +25,10 @@ export async function migrateFullDirectory( return new Error('No Google Photos (Fotos) directory was found'); } await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); - await exifTool.end(); + exifTool.end(); } catch (e){ + exifTool.end(); + console.error(e); return new Error(`Error while migrating: ${e}`); } return; diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index fae85b0..e24074f 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -2,18 +2,20 @@ import { glob } from 'glob'; import { basename, dirname } from 'path'; import { mkdir, cp } from 'fs/promises'; import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleFolderAndCheckErrors } from './migrate-entire-takeout-folder'; +import { migrateSingleFolder } from './migrate-single-folder'; +import { checkErrorDir } from './check-error-dir'; +import path = require('path'); async function _restructureAndProcess( folders: string[], targetDir: string, - processingType: boolean, // true for Albums, false for Photos + processingAlbums: boolean, // true for Albums, false for Photos exifTool: ExifTool ) { console.log(`Starting restructure of ${folders.length} directories`); await mkdir(targetDir, { recursive: true }); for (let folder of folders) { - if (processingType) { + if (processingAlbums) { // true for Albums, false for Photos console.log(`Processing album ${folder}...`); let outDir = `${targetDir}/AlbumsProcessed/${basename(folder)}`; @@ -24,15 +26,33 @@ async function _restructureAndProcess( } await mkdir(outDir, { recursive: true }); await mkdir(errDir, { recursive: true }); - await migrateSingleFolderAndCheckErrors(folder, outDir, errDir, exifTool); + await migrateSingleFolder(folder, outDir, errDir, exifTool, false); } else { const outDir = `${targetDir}/PhotosProcessed`; const errDir = `${targetDir}/PhotosError`; await mkdir(outDir, { recursive: true }); await mkdir(errDir, { recursive: true }); - await migrateSingleFolderAndCheckErrors(folder, outDir, errDir, exifTool); + await migrateSingleFolder(folder, outDir, errDir, exifTool, false); } } + + // check for errors + if (!processingAlbums) { + const outDir = `${targetDir}/PhotosProcessed`; + const errDir = `${targetDir}/PhotosError`; + await checkErrorDir(outDir, errDir, exifTool); + } else { + const outDir = `${targetDir}/AlbumsProcessed`; + const errDir = `${targetDir}/AlbumsError`; + const errAlbumDirs = await glob(errDir); + for (let dir of errAlbumDirs) { + if (dir == errDir) { + continue; + } + await checkErrorDir(dir, path.join(outDir, basename(dir)), exifTool); + } + } + console.log(`Sucsessfully restructured ${folders.length} directories`); } @@ -53,7 +73,6 @@ export async function restructureAndProcess( console.log('Processing photos...'); - // move the "Photos from $YEAR" directories to Photos/ await _restructureAndProcess( await glob(`${sourceDir}/Photos from */`), From 1f632d70e4082ec2941efc577fb5bfd1e5efc4b0 Mon Sep 17 00:00:00 2001 From: covalent Date: Sun, 1 Oct 2023 20:10:57 -0400 Subject: [PATCH 36/50] update comment for accuracy --- src/dir/restructure-and-process.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index e24074f..e23f3fe 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -62,14 +62,14 @@ export async function restructureAndProcess( exifTool: ExifTool ) { // before - // $rootdir/My Album 1 - // $rootdir/My Album 2 - // $rootdir/Photos from 2008 + // $rootdir/My Album 1/* + // $rootdir/My Album 2/* + // $rootdir/Photos from 2008/* // after - // $rootdir/Albums/My Album 1 - // $rootdir/Albums/My Album 2 - // $rootdir/Photos/Photos from 2008 + // $rootdir/AlbumsProcessed/My Album 1/* + // $rootdir/AlbumsProcessed/My Album 2/* + // $rootdir/PhotosProcessed/* console.log('Processing photos...'); From b7f8ec41c48a3df53d8f304c252dacbbebfff6b4 Mon Sep 17 00:00:00 2001 From: garzj Date: Thu, 5 Oct 2023 13:09:06 +0200 Subject: [PATCH 37/50] Fix full migration display bug --- src/dir/migrate-flat.ts | 4 ++-- src/dir/migrate-single-folder.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dir/migrate-flat.ts b/src/dir/migrate-flat.ts index 85641aa..5ba7c57 100644 --- a/src/dir/migrate-flat.ts +++ b/src/dir/migrate-flat.ts @@ -19,9 +19,9 @@ export type MigrationContext = Required & { migrationLocks: Map>; }; -export async function* migrateSingleDirectory( +export async function migrateSingleDirectory( args: MigrationArgs -): AsyncGenerator { +): Promise<(MediaFile | MediaMigrationError)[]> { const wg: (MediaFile | MediaMigrationError)[] = []; for await (const result of migrateSingleDirectoryGen(args)) { wg.push(result); diff --git a/src/dir/migrate-single-folder.ts b/src/dir/migrate-single-folder.ts index 39d1762..2a6f9d1 100644 --- a/src/dir/migrate-single-folder.ts +++ b/src/dir/migrate-single-folder.ts @@ -1,5 +1,5 @@ import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleDirectory } from './migrate-flat'; +import { migrateSingleDirectoryGen } from './migrate-flat'; export async function migrateSingleFolder( googleDir: string, @@ -9,7 +9,7 @@ export async function migrateSingleFolder( endExifTool: boolean ) { console.log(`Started migration.`); - const migGen = migrateSingleDirectory({ + const migGen = migrateSingleDirectoryGen({ googleDir, outputDir, errorDir, From 548a2a319528a96ae650a428599ee77347e26a12 Mon Sep 17 00:00:00 2001 From: garzj Date: Thu, 5 Oct 2023 13:51:10 +0200 Subject: [PATCH 38/50] Format and fix eslint issues --- src/commands/migrate-full.ts | 10 +++++----- src/dir/check-error-dir.ts | 6 +++--- src/dir/migrate-full.ts | 6 +++--- src/dir/restructure-and-process.ts | 4 ++-- src/fs/is-dir.ts | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index d7933dd..379e6a9 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -37,15 +37,15 @@ export const fullMigrate = command({ handler: async ({ sourceDir, targetDir, timeout }) => { const errs: string[] = []; if (!(await entitiyExists(sourceDir))) { - errs.push( - `The specified takeout directory does not exist: ${sourceDir}` - ); + errs.push(`The specified takeout directory does not exist: ${sourceDir}`); } if (await isEmptyDir(sourceDir)) { errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); } - if ((await entitiyExists(targetDir)) && !(await isEmptyDir(targetDir))){ - errs.push(`The target directory is not empty, please delete it and try again: ${targetDir}`); + if ((await entitiyExists(targetDir)) && !(await isEmptyDir(targetDir))) { + errs.push( + `The target directory is not empty, please delete it and try again: ${targetDir}` + ); } if (errs.length !== 0) { errs.forEach((e) => console.error(e)); diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts index 82b2b75..f8072a5 100644 --- a/src/dir/check-error-dir.ts +++ b/src/dir/check-error-dir.ts @@ -12,13 +12,13 @@ export async function checkErrorDir( ) { if (!(await isEmptyDir(errDir))) { const errFiles: string[] = await glob(`${errDir}/*`); - for (let file of errFiles) { + for (const file of errFiles) { if (file.endsWith('.json')) { continue; - } else if (await isDir(file)) { + } else if (await isDir(file)) { console.log(`Cannot fix metadata for directory: ${file}`); continue; - } else if (await entitiyExists(file)){ + } else if (await entitiyExists(file)) { continue; } console.log( diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 8273abf..da3a52c 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -7,7 +7,7 @@ export async function migrateFullDirectory( sourceDirectory: string, targetDirectory: string, timeout: number -): Promise< Error | void > { +): Promise { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder @@ -15,7 +15,7 @@ export async function migrateFullDirectory( const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); try { let googlePhotosDir: string = ''; - for (let i of possiblePhotosLocations) { + for (const i of possiblePhotosLocations) { if (await entitiyExists(`${sourceDirectory}/${i}`)) { googlePhotosDir = `${sourceDirectory}/${i}`; break; @@ -26,7 +26,7 @@ export async function migrateFullDirectory( } await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); exifTool.end(); - } catch (e){ + } catch (e) { exifTool.end(); console.error(e); return new Error(`Error while migrating: ${e}`); diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index e23f3fe..8f6a25c 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -14,7 +14,7 @@ async function _restructureAndProcess( ) { console.log(`Starting restructure of ${folders.length} directories`); await mkdir(targetDir, { recursive: true }); - for (let folder of folders) { + for (const folder of folders) { if (processingAlbums) { // true for Albums, false for Photos console.log(`Processing album ${folder}...`); @@ -45,7 +45,7 @@ async function _restructureAndProcess( const outDir = `${targetDir}/AlbumsProcessed`; const errDir = `${targetDir}/AlbumsError`; const errAlbumDirs = await glob(errDir); - for (let dir of errAlbumDirs) { + for (const dir of errAlbumDirs) { if (dir == errDir) { continue; } diff --git a/src/fs/is-dir.ts b/src/fs/is-dir.ts index bd98d4d..8e8f0fb 100644 --- a/src/fs/is-dir.ts +++ b/src/fs/is-dir.ts @@ -2,7 +2,7 @@ import { lstat } from 'fs/promises'; export async function isDir(path: string) { try { - var stat = await lstat(path); + const stat = await lstat(path); return stat.isDirectory(); } catch (e) { // lstatSync throws an error if path doesn't exist From a9e705bceb1efcd879bbd7a6e6dcc39ef3d87977 Mon Sep 17 00:00:00 2001 From: Covalent <58121030+lukehmcc@users.noreply.github.com> Date: Thu, 5 Oct 2023 22:49:22 -0400 Subject: [PATCH 39/50] Update README.md --- README.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index dd99605..014ce29 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos **Prerec**: Must have at least node 18 & yarn installed. -If you wish to migrate a single folder from a Google Photos takeout file: +If you wish to migrate a single folder from a Google Photos takeout file (or flatten the entire Takout folder into a single output with no album hierarchy): ```bash mkdir output error @@ -34,7 +34,7 @@ Optional flags for folder takeout (see `--help` for all details): Meaning: Forces the migration and overwrites files in the target directory. ``` -If you wish to migrate an entire takeout folder: +If you wish to migrate an entire takeout folder (and keep the album directory structure): ```bash mkdir output error @@ -50,19 +50,17 @@ Optional flags for full takeout (see `--help` for all details): Meaning: Sets the timeout for exiftool, default is 30000 (30s) ``` -The processed folders will be automatically put in `/path/to/takeout/Google Photos[Fotos]/PhotosProcessed` & `/path/to/takeout/Google Photos[Fotos]/AlbumsProcessed`. +In the target directory, four sub-directories are created: -**WARNING:** The `fullMigrate` command non-destructively modifies your files, which results in 3 nearly complete copies of your Takeout folder by the time it completes successfully: the original, the intermediate metadata-modified files, and the final organized and de-duplicated files. Make sure you have sufficient disk space for this. - -Additional intermediate folders are created as part of this command and in the event of errors will need to be manually removed before retrying. All are under the `/path/to/takeout/Google Photos[Fotos]/` folder: ``` -Albums -AlbumsProcessed -Photos -PhotosError PhotosProcessed +PhotosError +AlbumsProcessed +AlbumsError ``` +If all goes well you can ignore the error directories and just use the output in the *Processed dirs. + ### Docker **Prerec:** You must have a working `docker` or `podman` install. From 41175c9adf85746d5550007a470d5ddafed4179c Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 09:04:52 +0200 Subject: [PATCH 40/50] Format README.md --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 014ce29..80d2185 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos ## Run this tool -### Natively +### Natively **Prerec**: Must have at least node 18 & yarn installed. @@ -23,6 +23,7 @@ mkdir output error npx google-photos-migrate@latest folderMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` + Optional flags for folder takeout (see `--help` for all details): ``` @@ -59,17 +60,18 @@ AlbumsProcessed AlbumsError ``` -If all goes well you can ignore the error directories and just use the output in the *Processed dirs. +If all goes well you can ignore the error directories and just use the output in the \*Processed dirs. ### Docker **Prerec:** You must have a working `docker` or `podman` install. -A Dockerfile is also provided to make running this tool easier on most hosts. The image must be built manually (see below), no pre-built images are provided. Using it will by default use only software-based format conversion, hardware accelerated format conversion is beyond these instructions. +A Dockerfile is also provided to make running this tool easier on most hosts. The image must be built manually (see below), no pre-built images are provided. Using it will by default use only software-based format conversion, hardware accelerated format conversion is beyond these instructions. **You must build the image yourself (see above), no public pre-built images are provided.** You must build the image once before you run it: + ```shell # get the source code git clone https://github.com/garzj/google-photos-migrate @@ -80,6 +82,7 @@ docker build -f Dockerfile -t localhost/google-photos-migrate:latest . ``` To run `folderMigrate`, which requires providing multiple folders: + ```shell mkdir output error docker run --rm -it --security-opt=label=disable \ @@ -91,6 +94,7 @@ docker run --rm -it --security-opt=label=disable \ ``` To run `fullMigrate`, which requires only the Takeout folder: + ```shell mkdir output error docker run --rm -it -security-opt=label=disable \ @@ -102,6 +106,7 @@ docker run --rm -it -security-opt=label=disable \ All other commands and options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. For the overall help: + ```shell # no folders needed, so keep it simple docker run --rm -it --security-opt=label=disable \ From da29b313de931851fb9b7094f1298c68d5f81f47 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 09:19:02 +0200 Subject: [PATCH 41/50] Update README.md --- README.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 80d2185..2f16c07 100644 --- a/README.md +++ b/README.md @@ -68,16 +68,13 @@ If all goes well you can ignore the error directories and just use the output in A Dockerfile is also provided to make running this tool easier on most hosts. The image must be built manually (see below), no pre-built images are provided. Using it will by default use only software-based format conversion, hardware accelerated format conversion is beyond these instructions. -**You must build the image yourself (see above), no public pre-built images are provided.** - -You must build the image once before you run it: +Build the image once before you run it: ```shell -# get the source code git clone https://github.com/garzj/google-photos-migrate cd google-photos-migrate -# Build the image. Must be run from within the source code folder. +# build the image docker build -f Dockerfile -t localhost/google-photos-migrate:latest . ``` @@ -96,11 +93,12 @@ docker run --rm -it --security-opt=label=disable \ To run `fullMigrate`, which requires only the Takeout folder: ```shell -mkdir output error +mkdir output docker run --rm -it -security-opt=label=disable \ -v $(readlink -e path/to/takeout):/takeout \ + -v $(readlink -e ./output):/output \ localhost/google-photos-migrate:latest \ - fullMigrate '/takeout' --timeout=60000 + fullMigrate '/takeout' '/output' --timeout=60000 ``` All other commands and options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. @@ -108,7 +106,6 @@ All other commands and options are also available. The only difference from runn For the overall help: ```shell -# no folders needed, so keep it simple docker run --rm -it --security-opt=label=disable \ localhost/google-photos-migrate:latest \ --help @@ -123,13 +120,13 @@ docker run --rm -it --security-opt=label=disable \ **Prerec**: Must have node 18 & yarn installed. -For basic deployment do the following: +To test the app: ```bash git clone https://github.com/garzj/google-photos-migrate +cd google-photos-migrate yarn -yarn build -yarn start +yarn dev ``` -The entrypoint into the codebase is `src/cli.ts` +The entrypoint of the cli is in `src/cli.ts` and library code should be exported from `src/index.ts`. From 7b6850e065d252fbeffbead8a7bc0b5b414e3485 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 10:41:43 +0200 Subject: [PATCH 42/50] Add force argument to full migration --- README.md | 32 ++++++++++------------- src/commands/common.ts | 16 ++++++++++++ src/commands/migrate-folder.ts | 37 ++++++++++---------------- src/commands/migrate-full.ts | 47 +++++++++++++++------------------- 4 files changed, 64 insertions(+), 68 deletions(-) create mode 100644 src/commands/common.ts diff --git a/README.md b/README.md index 2f16c07..a55ec4e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos **Prerec**: Must have at least node 18 & yarn installed. +#### Flat migration + If you wish to migrate a single folder from a Google Photos takeout file (or flatten the entire Takout folder into a single output with no album hierarchy): ```bash @@ -24,16 +26,7 @@ mkdir output error npx google-photos-migrate@latest folderMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 ``` -Optional flags for folder takeout (see `--help` for all details): - -``` ---timeout integer - Shorthand: -t integer - Meaning: Sets the timeout for exiftool, default is 30000 (30s) ---force - Shorthand: -f - Meaning: Forces the migration and overwrites files in the target directory. -``` +#### Structured migration If you wish to migrate an entire takeout folder (and keep the album directory structure): @@ -43,14 +36,6 @@ mkdir output error npx google-photos-migrate@latest fullMigrate '/path/to/takeout' '/path/to/target' --timeout 60000 ``` -Optional flags for full takeout (see `--help` for all details): - -``` ---timeout integer - Shorthand: -t integer - Meaning: Sets the timeout for exiftool, default is 30000 (30s) -``` - In the target directory, four sub-directories are created: ``` @@ -62,6 +47,17 @@ AlbumsError If all goes well you can ignore the error directories and just use the output in the \*Processed dirs. +#### Optional flags (see `--help` for all details): + +``` +--timeout integer + Shorthand: -t integer + Meaning: Sets the timeout for exiftool, default is 30000 (30s) +--force + Shorthand: -f + Meaning: Forces the migration and overwrites files in the target directory. +``` + ### Docker **Prerec:** You must have a working `docker` or `podman` install. diff --git a/src/commands/common.ts b/src/commands/common.ts new file mode 100644 index 0000000..9f0a576 --- /dev/null +++ b/src/commands/common.ts @@ -0,0 +1,16 @@ +import { flag, number, option } from 'cmd-ts'; + +export const timeoutArg = option({ + type: number, + defaultValue: () => 30000, + short: 't', + long: 'timeout', + description: + 'Sets the task timeout in milliseconds that will be passed to ExifTool.', +}); + +export const forceArg = flag({ + short: 'f', + long: 'force', + description: "Forces the operation if the given directories aren't empty.", +}); diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-folder.ts index f93e7bc..56c1173 100644 --- a/src/commands/migrate-folder.ts +++ b/src/commands/migrate-folder.ts @@ -1,8 +1,9 @@ -import { command, string, positional, flag, number, option } from 'cmd-ts'; +import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; import { ExifTool } from 'exiftool-vendored'; import { migrateSingleFolder } from '../dir/migrate-single-folder'; import { entitiyExists } from '../fs/entity-exists'; +import { forceArg, timeoutArg } from './common'; export const folderMigrate = command({ name: 'google-photos-migrate-folder', @@ -22,23 +23,17 @@ export const folderMigrate = command({ displayName: 'error_dir', description: 'Failed media will be saved here.', }), - force: flag({ - short: 'f', - long: 'force', - description: - "Forces the operation if the given directories aren't empty.", - }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), + force: forceArg, + timeout: timeoutArg, }, handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; + const checkErrs = () => { + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + }; if (!(await entitiyExists(googleDir))) { errs.push(`The specified google directory does not exist: ${googleDir}`); @@ -49,10 +44,7 @@ export const folderMigrate = command({ if (!(await entitiyExists(errorDir))) { errs.push(`The specified error directory does not exist: ${googleDir}`); } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); - } + checkErrs(); if (!force && !(await isEmptyDir(outputDir))) { errs.push( @@ -65,12 +57,9 @@ export const folderMigrate = command({ ); } if (await isEmptyDir(googleDir)) { - errs.push('The google directory is empty. Nothing to do.'); - } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); + errs.push(`Nothing to do, the source directory is empty: ${googleDir}`); } + checkErrs(); const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); await migrateSingleFolder(googleDir, outputDir, errorDir, exifTool, true); diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 379e6a9..ce6a05e 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -1,16 +1,8 @@ -import { - command, - string, - positional, - flag, - number, - option, - boolean, -} from 'cmd-ts'; +import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; -import { glob } from 'glob'; import { migrateFullDirectory } from '../dir/migrate-full'; import { entitiyExists } from '../fs/entity-exists'; +import { forceArg, timeoutArg } from './common'; export const fullMigrate = command({ name: 'google-photos-migrate-full', @@ -25,32 +17,35 @@ export const fullMigrate = command({ displayName: 'target_dir', description: 'The path where you want the processed directories to go.', }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), + force: forceArg, + timeout: timeoutArg, }, - handler: async ({ sourceDir, targetDir, timeout }) => { + handler: async ({ sourceDir, targetDir, force, timeout }) => { const errs: string[] = []; + const checkErrs = () => { + if (errs.length !== 0) { + errs.forEach((e) => console.error(e)); + process.exit(1); + } + }; + if (!(await entitiyExists(sourceDir))) { errs.push(`The specified takeout directory does not exist: ${sourceDir}`); } - if (await isEmptyDir(sourceDir)) { - errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); + if (!(await entitiyExists(targetDir))) { + errs.push(`The specified target directory does not exist: ${targetDir}`); } - if ((await entitiyExists(targetDir)) && !(await isEmptyDir(targetDir))) { + checkErrs(); + + if (!force && !(await isEmptyDir(targetDir))) { errs.push( - `The target directory is not empty, please delete it and try again: ${targetDir}` + `The target directory is not empty. Pass "-f" to force the operation.` ); } - if (errs.length !== 0) { - errs.forEach((e) => console.error(e)); - process.exit(1); + if (await isEmptyDir(sourceDir)) { + errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); } + checkErrs(); await migrateFullDirectory(sourceDir, targetDir, timeout); }, From 9d7b660a9b37f477d636fc6cdd1b08b8d09e1187 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 22:08:25 +0200 Subject: [PATCH 43/50] Refactor subcmds and share their migration context --- README.md | 30 ++--- src/cli.ts | 10 +- src/commands/common.ts | 8 +- .../{migrate-folder.ts => migrate-flat.ts} | 61 ++++++---- src/commands/migrate-full.ts | 64 ++++++++--- src/dir/DirMigrationError.ts | 15 +++ src/dir/migrate-flat.ts | 39 ++----- src/dir/migrate-full.ts | 49 ++++---- src/dir/migrate-single-folder.ts | 34 ------ src/dir/migration-args.ts | 25 +++++ src/dir/restructure-and-process.ts | 106 ++++++------------ src/index.ts | 15 ++- src/ts.ts | 14 +++ 13 files changed, 250 insertions(+), 220 deletions(-) rename src/commands/{migrate-folder.ts => migrate-flat.ts} (53%) create mode 100644 src/dir/DirMigrationError.ts delete mode 100644 src/dir/migrate-single-folder.ts create mode 100644 src/dir/migration-args.ts diff --git a/README.md b/README.md index a55ec4e..fb44a7e 100644 --- a/README.md +++ b/README.md @@ -23,29 +23,20 @@ If you wish to migrate a single folder from a Google Photos takeout file (or fla ```bash mkdir output error -npx google-photos-migrate@latest folderMigrate '/path/to/takeout/Google Fotos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest flat '/path/to/takeout/Google Photos' './output' './error' --timeout 60000 ``` -#### Structured migration +#### Full structured migration If you wish to migrate an entire takeout folder (and keep the album directory structure): ```bash mkdir output error -npx google-photos-migrate@latest fullMigrate '/path/to/takeout' '/path/to/target' --timeout 60000 +npx google-photos-migrate@latest full '/path/to/takeout' './output' './error' --timeout 60000 ``` -In the target directory, four sub-directories are created: - -``` -PhotosProcessed -PhotosError -AlbumsProcessed -AlbumsError -``` - -If all goes well you can ignore the error directories and just use the output in the \*Processed dirs. +The folder names in the `output` and `error` directories will now correspond to the original album names. #### Optional flags (see `--help` for all details): @@ -74,7 +65,7 @@ cd google-photos-migrate docker build -f Dockerfile -t localhost/google-photos-migrate:latest . ``` -To run `folderMigrate`, which requires providing multiple folders: +To run the flat migration: ```shell mkdir output error @@ -83,21 +74,22 @@ docker run --rm -it --security-opt=label=disable \ -v $(readlink -e ./output):/output \ -v $(readlink -e ./error):/error \ localhost/google-photos-migrate:latest \ - folderMigrate '/takeout/Google Fotos' '/output' '/error' --timeout=60000 + flat '/takeout/Google Fotos' '/output' '/error' --timeout=60000 ``` -To run `fullMigrate`, which requires only the Takeout folder: +To run the full migration: ```shell -mkdir output +mkdir output error docker run --rm -it -security-opt=label=disable \ -v $(readlink -e path/to/takeout):/takeout \ -v $(readlink -e ./output):/output \ + -v $(readlink -e ./error):/error \ localhost/google-photos-migrate:latest \ - fullMigrate '/takeout' '/output' --timeout=60000 + fullMigrate '/takeout' '/output' '/error' --timeout=60000 ``` -All other commands and options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. +All other options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. For the overall help: diff --git a/src/cli.ts b/src/cli.ts index bb5c0e9..3e6a38d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,13 +1,17 @@ #!/usr/bin/env node import { subcommands, run } from 'cmd-ts'; -import { fullMigrate } from './commands/migrate-full'; -import { folderMigrate } from './commands/migrate-folder'; +import { migrateFull } from './commands/migrate-full'; +import { migrateFlat } from './commands/migrate-flat'; import { rewriteAllTags } from './commands/rewrite-all-tags'; const app = subcommands({ name: 'google-photos-migrate', - cmds: { fullMigrate, folderMigrate, rewriteAllTags }, + cmds: { + full: migrateFull, + flat: migrateFlat, + rewriteAllTags, + }, }); run(app, process.argv.slice(2)); diff --git a/src/commands/common.ts b/src/commands/common.ts index 9f0a576..ad2106f 100644 --- a/src/commands/common.ts +++ b/src/commands/common.ts @@ -1,4 +1,10 @@ -import { flag, number, option } from 'cmd-ts'; +import { flag, number, option, positional, string } from 'cmd-ts'; + +export const errorDirArg = positional({ + type: string, + displayName: 'error_dir', + description: 'Failed media will be saved here.', +}); export const timeoutArg = option({ type: number, diff --git a/src/commands/migrate-folder.ts b/src/commands/migrate-flat.ts similarity index 53% rename from src/commands/migrate-folder.ts rename to src/commands/migrate-flat.ts index 56c1173..fcf3812 100644 --- a/src/commands/migrate-folder.ts +++ b/src/commands/migrate-flat.ts @@ -1,16 +1,17 @@ import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; -import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleFolder } from '../dir/migrate-single-folder'; import { entitiyExists } from '../fs/entity-exists'; -import { forceArg, timeoutArg } from './common'; +import { errorDirArg, forceArg, timeoutArg } from './common'; +import { migrateDirFlatGen } from '../dir/migrate-flat'; +import { ExifTool } from 'exiftool-vendored'; +import { MediaMigrationError } from '../media/MediaMigrationError'; -export const folderMigrate = command({ - name: 'google-photos-migrate-folder', +export const migrateFlat = command({ + name: 'google-photos-migrate-flat', args: { - googleDir: positional({ + inputDir: positional({ type: string, - displayName: 'google_dir', + displayName: 'input_dir', description: 'The path to your "Google Photos" directory.', }), outputDir: positional({ @@ -18,15 +19,11 @@ export const folderMigrate = command({ displayName: 'output_dir', description: 'The path to your flat output directory.', }), - errorDir: positional({ - type: string, - displayName: 'error_dir', - description: 'Failed media will be saved here.', - }), + errorDir: errorDirArg, force: forceArg, timeout: timeoutArg, }, - handler: async ({ googleDir, outputDir, errorDir, force, timeout }) => { + handler: async ({ inputDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; const checkErrs = () => { if (errs.length !== 0) { @@ -35,14 +32,14 @@ export const folderMigrate = command({ } }; - if (!(await entitiyExists(googleDir))) { - errs.push(`The specified google directory does not exist: ${googleDir}`); + if (!(await entitiyExists(inputDir))) { + errs.push(`The specified google directory does not exist: ${inputDir}`); } if (!(await entitiyExists(outputDir))) { - errs.push(`The specified output directory does not exist: ${googleDir}`); + errs.push(`The specified output directory does not exist: ${inputDir}`); } if (!(await entitiyExists(errorDir))) { - errs.push(`The specified error directory does not exist: ${googleDir}`); + errs.push(`The specified error directory does not exist: ${inputDir}`); } checkErrs(); @@ -56,12 +53,34 @@ export const folderMigrate = command({ 'The error directory is not empty. Pass "-f" to force the operation.' ); } - if (await isEmptyDir(googleDir)) { - errs.push(`Nothing to do, the source directory is empty: ${googleDir}`); + if (await isEmptyDir(inputDir)) { + errs.push(`Nothing to do, the source directory is empty: ${inputDir}`); } checkErrs(); - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await migrateSingleFolder(googleDir, outputDir, errorDir, exifTool, true); + console.log('Started migration.'); + const migGen = migrateDirFlatGen({ + inputDir, + outputDir, + errorDir, + log: console.log, + warnLog: console.error, + exiftool: new ExifTool({ taskTimeoutMillis: timeout }), + endExifTool: true, + }); + const counts = { err: 0, suc: 0 }; + for await (const result of migGen) { + if (result instanceof MediaMigrationError) { + console.error(`Error: ${result}`); + counts.err++; + continue; + } else { + counts.suc++; + } + } + + console.log(`Done! Processed ${counts.suc + counts.err} files.`); + console.log(`Files migrated: ${counts.suc}`); + console.log(`Files failed: ${counts.err}`); }, }); diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index ce6a05e..31c6c7d 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -1,26 +1,30 @@ import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; -import { migrateFullDirectory } from '../dir/migrate-full'; import { entitiyExists } from '../fs/entity-exists'; -import { forceArg, timeoutArg } from './common'; +import { errorDirArg, forceArg, timeoutArg } from './common'; +import { migrateDirFullGen } from '..'; +import { ExifTool } from 'exiftool-vendored'; +import { MediaMigrationError } from '../media/MediaMigrationError'; +import { DirMigrationError } from '../dir/DirMigrationError'; -export const fullMigrate = command({ +export const migrateFull = command({ name: 'google-photos-migrate-full', args: { - sourceDir: positional({ + inputDir: positional({ type: string, - displayName: 'source_dir', + displayName: 'input_dir', description: 'The path to your "Takeout" directory.', }), - targetDir: positional({ + outputDir: positional({ type: string, - displayName: 'target_dir', + displayName: 'output_dir', description: 'The path where you want the processed directories to go.', }), + errorDir: errorDirArg, force: forceArg, timeout: timeoutArg, }, - handler: async ({ sourceDir, targetDir, force, timeout }) => { + handler: async ({ inputDir, outputDir, errorDir, force, timeout }) => { const errs: string[] = []; const checkErrs = () => { if (errs.length !== 0) { @@ -29,24 +33,52 @@ export const fullMigrate = command({ } }; - if (!(await entitiyExists(sourceDir))) { - errs.push(`The specified takeout directory does not exist: ${sourceDir}`); + if (!(await entitiyExists(inputDir))) { + errs.push(`The specified takeout directory does not exist: ${inputDir}`); } - if (!(await entitiyExists(targetDir))) { - errs.push(`The specified target directory does not exist: ${targetDir}`); + if (!(await entitiyExists(outputDir))) { + errs.push(`The specified target directory does not exist: ${outputDir}`); } checkErrs(); - if (!force && !(await isEmptyDir(targetDir))) { + if (!force && !(await isEmptyDir(outputDir))) { errs.push( `The target directory is not empty. Pass "-f" to force the operation.` ); } - if (await isEmptyDir(sourceDir)) { - errs.push(`Nothing to do, the source directory is empty: ${sourceDir}`); + if (await isEmptyDir(inputDir)) { + errs.push(`Nothing to do, the source directory is empty: ${inputDir}`); } checkErrs(); - await migrateFullDirectory(sourceDir, targetDir, timeout); + console.log('Started migration.'); + const migGen = migrateDirFullGen({ + inputDir: inputDir, + errorDir, + outputDir, + log: console.log, + warnLog: console.error, + exiftool: new ExifTool({ taskTimeoutMillis: timeout }), + endExifTool: true, + }); + const counts = { err: 0, suc: 0 }; + for await (const result of migGen) { + if (result instanceof DirMigrationError) { + console.error(`Error: ${result}`); + process.exit(1); + } + + if (result instanceof MediaMigrationError) { + console.error(`Error: ${result}`); + counts.err++; + continue; + } + + counts.suc++; + } + + console.log(`Done! Processed ${counts.suc + counts.err} files.`); + console.log(`Files migrated: ${counts.suc}`); + console.log(`Files failed: ${counts.err}`); }, }); diff --git a/src/dir/DirMigrationError.ts b/src/dir/DirMigrationError.ts new file mode 100644 index 0000000..b17a622 --- /dev/null +++ b/src/dir/DirMigrationError.ts @@ -0,0 +1,15 @@ +export class DirMigrationError extends Error { + constructor(public folder: string) { + super(); + } +} + +export class NoPhotosDirError extends DirMigrationError { + constructor(public folder: string) { + super(folder); + } + + toString() { + return `Failed to find Google Photos directory in folder: ${this.folder}`; + } +} diff --git a/src/dir/migrate-flat.ts b/src/dir/migrate-flat.ts index 5ba7c57..9b0d0ca 100644 --- a/src/dir/migrate-flat.ts +++ b/src/dir/migrate-flat.ts @@ -1,47 +1,28 @@ import { walkDir } from '../fs/walk-dir'; import { MediaFile } from '../media/MediaFile'; +import { migrateMediaFile } from '../media/migrate-media-file'; +import { MigrationArgs, migrationArgsDefaults } from './migration-args'; +import { asyncGenToAsync } from '../ts'; import { indexJsonFiles } from '../meta/index-meta-files'; import { MediaMigrationError } from '../media/MediaMigrationError'; -import { ExifTool } from 'exiftool-vendored'; -import { migrateMediaFile } from '../media/migrate-media-file'; - -export type MigrationArgs = { - googleDir: string; - outputDir: string; - errorDir: string; - warnLog?: (msg: string) => void; - exiftool?: ExifTool; - endExifTool?: boolean; -}; export type MigrationContext = Required & { titleJsonMap: Map; - migrationLocks: Map>; }; -export async function migrateSingleDirectory( - args: MigrationArgs -): Promise<(MediaFile | MediaMigrationError)[]> { - const wg: (MediaFile | MediaMigrationError)[] = []; - for await (const result of migrateSingleDirectoryGen(args)) { - wg.push(result); - } - return await Promise.all(wg); -} +export const migrateDirFlat = asyncGenToAsync(migrateDirFlatGen); -export async function* migrateSingleDirectoryGen( - args: MigrationArgs +export async function* migrateDirFlatGen( + _args: MigrationArgs ): AsyncGenerator { + const args = await migrationArgsDefaults(_args); const migCtx: MigrationContext = { - titleJsonMap: await indexJsonFiles(args.googleDir), - migrationLocks: new Map(), ...args, - exiftool: args.exiftool ?? new ExifTool(), - endExifTool: args.endExifTool ?? !args.exiftool, - warnLog: args.warnLog ?? (() => {}), + titleJsonMap: await indexJsonFiles(args.inputDir), + endExifTool: false, }; - for await (const mediaPath of walkDir(args.googleDir)) { + for await (const mediaPath of walkDir(args.inputDir)) { if (mediaPath.endsWith('.json')) continue; yield migrateMediaFile(mediaPath, migCtx); diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index da3a52c..64afe08 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,35 +1,36 @@ import { restructureAndProcess } from './restructure-and-process'; -import { ExifTool, exiftool } from 'exiftool-vendored'; import { entitiyExists } from '../fs/entity-exists'; import { possiblePhotosLocations } from '../config/langs'; +import { asyncGenToAsync } from '../ts'; +import { MediaFile } from '../media/MediaFile'; +import { MigrationArgs, migrationArgsDefaults } from './migration-args'; +import { MediaMigrationError } from '../media/MediaMigrationError'; +import { DirMigrationError, NoPhotosDirError } from './DirMigrationError'; + +export type FullMigrationContext = Required; + +export const migrateDirFull = asyncGenToAsync(migrateDirFullGen); + +export async function* migrateDirFullGen( + args: MigrationArgs +): AsyncGenerator { + const migCtx: FullMigrationContext = await migrationArgsDefaults(args); -export async function migrateFullDirectory( - sourceDirectory: string, - targetDirectory: string, - timeout: number -): Promise { // at least in my takeout, the Takeout folder contains a subfolder // Takeout/Google Foto // rootdir refers to that subfolder // Can add more language support here in the future - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - try { - let googlePhotosDir: string = ''; - for (const i of possiblePhotosLocations) { - if (await entitiyExists(`${sourceDirectory}/${i}`)) { - googlePhotosDir = `${sourceDirectory}/${i}`; - break; - } + let googlePhotosDir: string = ''; + for (const i of possiblePhotosLocations) { + if (await entitiyExists(`${migCtx.inputDir}/${i}`)) { + googlePhotosDir = `${migCtx.inputDir}/${i}`; + break; } - if (googlePhotosDir == '') { - return new Error('No Google Photos (Fotos) directory was found'); - } - await restructureAndProcess(googlePhotosDir, targetDirectory, exifTool); - exifTool.end(); - } catch (e) { - exifTool.end(); - console.error(e); - return new Error(`Error while migrating: ${e}`); } - return; + if (googlePhotosDir == '') { + yield new NoPhotosDirError(migCtx.inputDir); + return; + } + + yield* restructureAndProcess(googlePhotosDir, migCtx); } diff --git a/src/dir/migrate-single-folder.ts b/src/dir/migrate-single-folder.ts deleted file mode 100644 index 2a6f9d1..0000000 --- a/src/dir/migrate-single-folder.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleDirectoryGen } from './migrate-flat'; - -export async function migrateSingleFolder( - googleDir: string, - outputDir: string, - errorDir: string, - exifTool: ExifTool, - endExifTool: boolean -) { - console.log(`Started migration.`); - const migGen = migrateSingleDirectoryGen({ - googleDir, - outputDir, - errorDir, - warnLog: console.error, - exiftool: exifTool, - endExifTool: endExifTool, - }); - const counts = { err: 0, suc: 0 }; - for await (const result of migGen) { - if (result instanceof Error) { - console.error(`Error: ${result}`); - counts.err++; - continue; - } else { - counts.suc++; - } - } - - console.log(`Done! Processed ${counts.suc + counts.err} files.`); - console.log(`Files migrated: ${counts.suc}`); - console.log(`Files failed: ${counts.err}`); -} diff --git a/src/dir/migration-args.ts b/src/dir/migration-args.ts new file mode 100644 index 0000000..fd42af4 --- /dev/null +++ b/src/dir/migration-args.ts @@ -0,0 +1,25 @@ +import { ExifTool } from 'exiftool-vendored'; + +export type MigrationArgs = { + inputDir: string; + outputDir: string; + errorDir: string; + log?: (msg: string) => void; + warnLog?: (msg: string) => void; + exiftool?: ExifTool; + endExifTool?: boolean; + migrationLocks?: Map>; +}; + +export async function migrationArgsDefaults( + args: MigrationArgs +): Promise> { + return { + ...args, + migrationLocks: args.migrationLocks ?? new Map(), + exiftool: args.exiftool ?? new ExifTool(), + endExifTool: args.endExifTool ?? !args.exiftool, + log: args.log ?? (() => {}), + warnLog: args.warnLog ?? (() => {}), + }; +} diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index 8f6a25c..c0b7bb4 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -1,65 +1,46 @@ import { glob } from 'glob'; -import { basename, dirname } from 'path'; -import { mkdir, cp } from 'fs/promises'; -import { ExifTool } from 'exiftool-vendored'; -import { migrateSingleFolder } from './migrate-single-folder'; +import { basename } from 'path'; +import { mkdir } from 'fs/promises'; import { checkErrorDir } from './check-error-dir'; -import path = require('path'); +import { migrateDirFlatGen } from './migrate-flat'; +import { FullMigrationContext } from './migrate-full'; -async function _restructureAndProcess( +async function* _restructureAndProcess( folders: string[], - targetDir: string, processingAlbums: boolean, // true for Albums, false for Photos - exifTool: ExifTool + migCtx: FullMigrationContext ) { - console.log(`Starting restructure of ${folders.length} directories`); - await mkdir(targetDir, { recursive: true }); + migCtx.log(`Starting restructure of ${folders.length} directories.`); + for (const folder of folders) { - if (processingAlbums) { - // true for Albums, false for Photos - console.log(`Processing album ${folder}...`); - let outDir = `${targetDir}/AlbumsProcessed/${basename(folder)}`; - let errDir = `${targetDir}/AlbumsError/${basename(folder)}`; - if (basename(folder).includes('Untitled(')) { - outDir = `${targetDir}/AlbumsProcessed/Untitled`; - errDir = `${targetDir}/AlbumsError/Untitled`; - } - await mkdir(outDir, { recursive: true }); - await mkdir(errDir, { recursive: true }); - await migrateSingleFolder(folder, outDir, errDir, exifTool, false); - } else { - const outDir = `${targetDir}/PhotosProcessed`; - const errDir = `${targetDir}/PhotosError`; - await mkdir(outDir, { recursive: true }); - await mkdir(errDir, { recursive: true }); - await migrateSingleFolder(folder, outDir, errDir, exifTool, false); - } - } + processingAlbums && migCtx.log(`Processing album ${folder}...`); - // check for errors - if (!processingAlbums) { - const outDir = `${targetDir}/PhotosProcessed`; - const errDir = `${targetDir}/PhotosError`; - await checkErrorDir(outDir, errDir, exifTool); - } else { - const outDir = `${targetDir}/AlbumsProcessed`; - const errDir = `${targetDir}/AlbumsError`; - const errAlbumDirs = await glob(errDir); - for (const dir of errAlbumDirs) { - if (dir == errDir) { - continue; - } - await checkErrorDir(dir, path.join(outDir, basename(dir)), exifTool); + let albumName = processingAlbums ? basename(folder) : 'Photos'; + if (albumName.startsWith('Untitled(')) { + albumName = 'Untitled'; } + + const outDir = `${migCtx.outputDir}/${albumName}`; + const errDir = `${migCtx.errorDir}/${albumName}`; + + await mkdir(outDir, { recursive: true }); + await mkdir(errDir, { recursive: true }); + yield* migrateDirFlatGen({ + ...migCtx, + inputDir: folder, + outputDir: outDir, + errorDir: errDir, + }); + + await checkErrorDir(outDir, errDir, migCtx.exiftool); } console.log(`Sucsessfully restructured ${folders.length} directories`); } -export async function restructureAndProcess( +export async function* restructureAndProcess( sourceDir: string, - targetDir: string, - exifTool: ExifTool + migCtx: FullMigrationContext ) { // before // $rootdir/My Album 1/* @@ -71,31 +52,18 @@ export async function restructureAndProcess( // $rootdir/AlbumsProcessed/My Album 2/* // $rootdir/PhotosProcessed/* - console.log('Processing photos...'); - // move the "Photos from $YEAR" directories to Photos/ - await _restructureAndProcess( - await glob(`${sourceDir}/Photos from */`), - targetDir, - false, - exifTool + migCtx.log('Processing photos...'); + const photoSet = new Set( + await glob([`${sourceDir}/Photos`, `${sourceDir}/Photos from */`]) ); - - console.log('Processing albums...'); + yield* _restructureAndProcess([...photoSet], false, migCtx); // move everythingg else to Albums/, so we end up with two top level folders + migCtx.log('Processing albums...'); const fullSet: Set = new Set(await glob(`${sourceDir}/*/`)); - const photoSet: Set = new Set( - await glob(`${sourceDir}/Photos from */`) - ); - photoSet.add(`${sourceDir}/Photos`); - const everythingExceptPhotosDir: string[] = Array.from( - new Set([...fullSet].filter((x) => !photoSet.has(x))) - ); - await _restructureAndProcess( - everythingExceptPhotosDir, - targetDir, - true, - exifTool - ); + const everythingExceptPhotosDir = [ + ...new Set([...fullSet].filter((x) => !photoSet.has(x))), + ]; + yield* _restructureAndProcess(everythingExceptPhotosDir, true, migCtx); } diff --git a/src/index.ts b/src/index.ts index 26ed77c..af8d387 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,16 @@ import './config/env'; -import { migrateSingleDirectory } from './dir/migrate-flat'; -import { migrateFullDirectory } from './dir/migrate-full'; +import { migrateDirFlat, migrateDirFlatGen } from './dir/migrate-flat'; +import { migrateDirFull, migrateDirFullGen } from './dir/migrate-full'; import type { MediaFileExtension } from './media/MediaFileExtension'; import { supportedExtensions } from './config/extensions'; +import type { MigrationArgs } from './dir/migration-args'; -export { migrateSingleDirectory, migrateFullDirectory, supportedExtensions }; -export type { MediaFileExtension }; +export { + migrateDirFlat, + migrateDirFlatGen, + migrateDirFull, + migrateDirFullGen, + supportedExtensions, +}; +export type { MigrationArgs, MediaFileExtension }; diff --git a/src/ts.ts b/src/ts.ts index a12364d..e161119 100644 --- a/src/ts.ts +++ b/src/ts.ts @@ -1,3 +1,17 @@ export const exhaustiveCheck = (_: never): never => { throw new Error('Exhaustive type check failed.'); }; + +export function asyncGenToAsync< + T extends (...args: P) => AsyncGenerator, + P extends any[] = Parameters, + R = ReturnType extends AsyncGenerator ? R : never, +>(f: T) { + return async (...args: P) => { + const wg: R[] = []; + for await (const result of f(...args)) { + wg.push(result); + } + return await Promise.all(wg); + }; +} From 5183d8f980bce5d542899ada7b1d4ad1de6bbe6a Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 22:25:16 +0200 Subject: [PATCH 44/50] Add language support for Untitled albums --- src/config/langs.ts | 3 ++- src/dir/migrate-full.ts | 4 ++-- src/dir/restructure-and-process.ts | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/config/langs.ts b/src/config/langs.ts index cfb07c4..7877762 100644 --- a/src/config/langs.ts +++ b/src/config/langs.ts @@ -1,2 +1,3 @@ export const editedSuffices = ['edited', 'bearbeitet', 'modifié']; -export const possiblePhotosLocations = ['Google Photos', 'Google Fotos']; +export const photosLocations = ['Google Photos', 'Google Fotos']; +export const untitledNames = ['Untitled', 'Unbenannt']; diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 64afe08..df10a67 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,6 +1,6 @@ import { restructureAndProcess } from './restructure-and-process'; import { entitiyExists } from '../fs/entity-exists'; -import { possiblePhotosLocations } from '../config/langs'; +import { photosLocations } from '../config/langs'; import { asyncGenToAsync } from '../ts'; import { MediaFile } from '../media/MediaFile'; import { MigrationArgs, migrationArgsDefaults } from './migration-args'; @@ -21,7 +21,7 @@ export async function* migrateDirFullGen( // rootdir refers to that subfolder // Can add more language support here in the future let googlePhotosDir: string = ''; - for (const i of possiblePhotosLocations) { + for (const i of photosLocations) { if (await entitiyExists(`${migCtx.inputDir}/${i}`)) { googlePhotosDir = `${migCtx.inputDir}/${i}`; break; diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index c0b7bb4..ed5dbcc 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -4,6 +4,7 @@ import { mkdir } from 'fs/promises'; import { checkErrorDir } from './check-error-dir'; import { migrateDirFlatGen } from './migrate-flat'; import { FullMigrationContext } from './migrate-full'; +import { untitledNames } from '../config/langs'; async function* _restructureAndProcess( folders: string[], @@ -16,8 +17,10 @@ async function* _restructureAndProcess( processingAlbums && migCtx.log(`Processing album ${folder}...`); let albumName = processingAlbums ? basename(folder) : 'Photos'; - if (albumName.startsWith('Untitled(')) { - albumName = 'Untitled'; + for (const untitledName of untitledNames) { + if (albumName.startsWith(`${untitledName}(`)) { + albumName = untitledName; + } } const outDir = `${migCtx.outputDir}/${albumName}`; From 73e9ab83308e8a1da2e5390cec73107d58d37fd3 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 22:42:59 +0200 Subject: [PATCH 45/50] End exiftool properly on full migrations --- src/dir/migrate-full.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index df10a67..69934a2 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -33,4 +33,6 @@ export async function* migrateDirFullGen( } yield* restructureAndProcess(googlePhotosDir, migCtx); + + migCtx.endExifTool && migCtx.exiftool.end(); } From 2efaa0467ac8c04c1598c38c8950091d29db84d9 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 22:58:10 +0200 Subject: [PATCH 46/50] Allow both takeout or photos dir as arg for full --- src/config/langs.ts | 4 ++-- src/dir/migrate-full.ts | 13 +++++++++---- src/dir/restructure-and-process.ts | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/config/langs.ts b/src/config/langs.ts index 7877762..b875d38 100644 --- a/src/config/langs.ts +++ b/src/config/langs.ts @@ -1,3 +1,3 @@ export const editedSuffices = ['edited', 'bearbeitet', 'modifié']; -export const photosLocations = ['Google Photos', 'Google Fotos']; -export const untitledNames = ['Untitled', 'Unbenannt']; +export const photosDirs = ['Google Photos', 'Google Fotos']; +export const untitledDirs = ['Untitled', 'Unbenannt']; diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 69934a2..479a4c5 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,11 +1,12 @@ import { restructureAndProcess } from './restructure-and-process'; import { entitiyExists } from '../fs/entity-exists'; -import { photosLocations } from '../config/langs'; +import { photosDirs } from '../config/langs'; import { asyncGenToAsync } from '../ts'; import { MediaFile } from '../media/MediaFile'; import { MigrationArgs, migrationArgsDefaults } from './migration-args'; import { MediaMigrationError } from '../media/MediaMigrationError'; import { DirMigrationError, NoPhotosDirError } from './DirMigrationError'; +import { basename } from 'path'; export type FullMigrationContext = Required; @@ -21,9 +22,13 @@ export async function* migrateDirFullGen( // rootdir refers to that subfolder // Can add more language support here in the future let googlePhotosDir: string = ''; - for (const i of photosLocations) { - if (await entitiyExists(`${migCtx.inputDir}/${i}`)) { - googlePhotosDir = `${migCtx.inputDir}/${i}`; + for (const photosDir of photosDirs) { + if (await entitiyExists(`${migCtx.inputDir}/${photosDir}`)) { + googlePhotosDir = `${migCtx.inputDir}/${photosDir}`; + break; + } + if (basename(migCtx.inputDir) === photosDir) { + googlePhotosDir = migCtx.inputDir; break; } } diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index ed5dbcc..5c42b51 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -4,7 +4,7 @@ import { mkdir } from 'fs/promises'; import { checkErrorDir } from './check-error-dir'; import { migrateDirFlatGen } from './migrate-flat'; import { FullMigrationContext } from './migrate-full'; -import { untitledNames } from '../config/langs'; +import { untitledDirs } from '../config/langs'; async function* _restructureAndProcess( folders: string[], @@ -17,7 +17,7 @@ async function* _restructureAndProcess( processingAlbums && migCtx.log(`Processing album ${folder}...`); let albumName = processingAlbums ? basename(folder) : 'Photos'; - for (const untitledName of untitledNames) { + for (const untitledName of untitledDirs) { if (albumName.startsWith(`${untitledName}(`)) { albumName = untitledName; } From 4988200e9e4e8d1f17d6d220923b25c8d25a8c13 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 22:59:31 +0200 Subject: [PATCH 47/50] Remove rewriteAllTags subcmd --- src/cli.ts | 2 -- src/commands/rewrite-all-tags.ts | 31 ------------------------------- 2 files changed, 33 deletions(-) delete mode 100644 src/commands/rewrite-all-tags.ts diff --git a/src/cli.ts b/src/cli.ts index 3e6a38d..85b01b4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,14 +3,12 @@ import { subcommands, run } from 'cmd-ts'; import { migrateFull } from './commands/migrate-full'; import { migrateFlat } from './commands/migrate-flat'; -import { rewriteAllTags } from './commands/rewrite-all-tags'; const app = subcommands({ name: 'google-photos-migrate', cmds: { full: migrateFull, flat: migrateFlat, - rewriteAllTags, }, }); diff --git a/src/commands/rewrite-all-tags.ts b/src/commands/rewrite-all-tags.ts deleted file mode 100644 index 85204c4..0000000 --- a/src/commands/rewrite-all-tags.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { command, string, positional, number, option } from 'cmd-ts'; -import { ExifTool } from 'exiftool-vendored'; - -export const rewriteAllTags = command({ - name: 'rewrite all tags for single file', - args: { - inFile: positional({ - type: string, - displayName: 'in_file', - description: 'The path to your input file.', - }), - outFile: positional({ - type: string, - displayName: 'out_file', - description: 'The path to your output location for the file.', - }), - timeout: option({ - type: number, - defaultValue: () => 30000, - short: 't', - long: 'timeout', - description: - 'Sets the task timeout in milliseconds that will be passed to ExifTool.', - }), - }, - handler: async ({ inFile, outFile, timeout }) => { - const exifTool = new ExifTool({ taskTimeoutMillis: timeout }); - await exifTool.rewriteAllTags(inFile, outFile); - exifTool.end(); - }, -}); From 4138e74c406790b0f7b43829138a0a1936ddb3b6 Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 23:09:38 +0200 Subject: [PATCH 48/50] Rename helper to fileExists --- src/commands/migrate-flat.ts | 8 ++++---- src/commands/migrate-full.ts | 6 +++--- src/dir/check-error-dir.ts | 4 ++-- src/dir/migrate-full.ts | 4 ++-- src/fs/{entity-exists.ts => file-exists.ts} | 2 +- src/media/save-to-dir.ts | 4 ++-- src/meta/find-meta-file.ts | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) rename src/fs/{entity-exists.ts => file-exists.ts} (75%) diff --git a/src/commands/migrate-flat.ts b/src/commands/migrate-flat.ts index fcf3812..c0070c9 100644 --- a/src/commands/migrate-flat.ts +++ b/src/commands/migrate-flat.ts @@ -1,6 +1,6 @@ import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; import { errorDirArg, forceArg, timeoutArg } from './common'; import { migrateDirFlatGen } from '../dir/migrate-flat'; import { ExifTool } from 'exiftool-vendored'; @@ -32,13 +32,13 @@ export const migrateFlat = command({ } }; - if (!(await entitiyExists(inputDir))) { + if (!(await fileExists(inputDir))) { errs.push(`The specified google directory does not exist: ${inputDir}`); } - if (!(await entitiyExists(outputDir))) { + if (!(await fileExists(outputDir))) { errs.push(`The specified output directory does not exist: ${inputDir}`); } - if (!(await entitiyExists(errorDir))) { + if (!(await fileExists(errorDir))) { errs.push(`The specified error directory does not exist: ${inputDir}`); } checkErrs(); diff --git a/src/commands/migrate-full.ts b/src/commands/migrate-full.ts index 31c6c7d..281366d 100644 --- a/src/commands/migrate-full.ts +++ b/src/commands/migrate-full.ts @@ -1,6 +1,6 @@ import { command, string, positional } from 'cmd-ts'; import { isEmptyDir } from '../fs/is-empty-dir'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; import { errorDirArg, forceArg, timeoutArg } from './common'; import { migrateDirFullGen } from '..'; import { ExifTool } from 'exiftool-vendored'; @@ -33,10 +33,10 @@ export const migrateFull = command({ } }; - if (!(await entitiyExists(inputDir))) { + if (!(await fileExists(inputDir))) { errs.push(`The specified takeout directory does not exist: ${inputDir}`); } - if (!(await entitiyExists(outputDir))) { + if (!(await fileExists(outputDir))) { errs.push(`The specified target directory does not exist: ${outputDir}`); } checkErrs(); diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts index f8072a5..836b8b5 100644 --- a/src/dir/check-error-dir.ts +++ b/src/dir/check-error-dir.ts @@ -3,7 +3,7 @@ import { isEmptyDir } from '../fs/is-empty-dir'; import { glob } from 'glob'; import { basename, join } from 'path'; import { isDir } from '../fs/is-dir'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; export async function checkErrorDir( outDir: string, @@ -18,7 +18,7 @@ export async function checkErrorDir( } else if (await isDir(file)) { console.log(`Cannot fix metadata for directory: ${file}`); continue; - } else if (await entitiyExists(file)) { + } else if (await fileExists(file)) { continue; } console.log( diff --git a/src/dir/migrate-full.ts b/src/dir/migrate-full.ts index 479a4c5..589cd5c 100644 --- a/src/dir/migrate-full.ts +++ b/src/dir/migrate-full.ts @@ -1,5 +1,5 @@ import { restructureAndProcess } from './restructure-and-process'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; import { photosDirs } from '../config/langs'; import { asyncGenToAsync } from '../ts'; import { MediaFile } from '../media/MediaFile'; @@ -23,7 +23,7 @@ export async function* migrateDirFullGen( // Can add more language support here in the future let googlePhotosDir: string = ''; for (const photosDir of photosDirs) { - if (await entitiyExists(`${migCtx.inputDir}/${photosDir}`)) { + if (await fileExists(`${migCtx.inputDir}/${photosDir}`)) { googlePhotosDir = `${migCtx.inputDir}/${photosDir}`; break; } diff --git a/src/fs/entity-exists.ts b/src/fs/file-exists.ts similarity index 75% rename from src/fs/entity-exists.ts rename to src/fs/file-exists.ts index d84833a..e51c157 100644 --- a/src/fs/entity-exists.ts +++ b/src/fs/file-exists.ts @@ -1,6 +1,6 @@ import { stat } from 'fs/promises'; -export const entitiyExists = (path: string) => +export const fileExists = (path: string) => stat(path) .then(() => true) .catch((e) => (e.code === 'ENOENT' ? false : Promise.reject(e))); diff --git a/src/media/save-to-dir.ts b/src/media/save-to-dir.ts index 75d2980..6782fcf 100644 --- a/src/media/save-to-dir.ts +++ b/src/media/save-to-dir.ts @@ -1,5 +1,5 @@ import { basename, resolve } from 'path'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; import sanitize = require('sanitize-filename'); import { copyFile, mkdir, rename } from 'fs/promises'; import { MigrationContext } from '../dir/migrate-flat'; @@ -18,7 +18,7 @@ async function _saveToDir( await mkdir(saveDir, { recursive: true }); const savePath = resolve(saveDir, saveBase); - const exists = await entitiyExists(savePath); + const exists = await fileExists(savePath); if (exists) { return _saveToDir(file, destDir, saveBase, move, duplicateIndex + 1); } diff --git a/src/meta/find-meta-file.ts b/src/meta/find-meta-file.ts index ddcbda9..8e5f06a 100644 --- a/src/meta/find-meta-file.ts +++ b/src/meta/find-meta-file.ts @@ -1,7 +1,7 @@ import { basename, dirname } from 'path'; import { MigrationContext } from '../dir/migrate-flat'; import { MediaFileExtension } from '../media/MediaFileExtension'; -import { entitiyExists } from '../fs/entity-exists'; +import { fileExists } from '../fs/file-exists'; import { editedSuffices } from '../config/langs'; export async function findMetaFile( @@ -71,7 +71,7 @@ export async function findMetaFile( } for (const potPath of potPaths) { - if (!(await entitiyExists(potPath))) { + if (!(await fileExists(potPath))) { continue; } return potPath; From ebe43b4a54e9cac9db4f4e88aefee8620a21e53a Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 23:15:10 +0200 Subject: [PATCH 49/50] Update README.md --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fb44a7e..f3233f3 100644 --- a/README.md +++ b/README.md @@ -16,28 +16,28 @@ A tool like [google-photos-exif](https://github.com/mattwilson1024/google-photos **Prerec**: Must have at least node 18 & yarn installed. -#### Flat migration +#### Full structured migration -If you wish to migrate a single folder from a Google Photos takeout file (or flatten the entire Takout folder into a single output with no album hierarchy): +If you wish to migrate an entire takeout folder (and keep the album directory structure): ```bash mkdir output error -npx google-photos-migrate@latest flat '/path/to/takeout/Google Photos' './output' './error' --timeout 60000 +npx google-photos-migrate@latest full '/path/to/takeout' './output' './error' --timeout 60000 ``` -#### Full structured migration +The folder names in the `output` and `error` directories will now correspond to the original album names. -If you wish to migrate an entire takeout folder (and keep the album directory structure): +#### Flat migration + +If you wish to migrate your Google Photos folder into a flat directory (and don't care about albums): ```bash mkdir output error -npx google-photos-migrate@latest full '/path/to/takeout' './output' './error' --timeout 60000 +npx google-photos-migrate@latest flat '/path/to/takeout/Google Photos' './output' './error' --timeout 60000 ``` -The folder names in the `output` and `error` directories will now correspond to the original album names. - #### Optional flags (see `--help` for all details): ``` @@ -65,28 +65,28 @@ cd google-photos-migrate docker build -f Dockerfile -t localhost/google-photos-migrate:latest . ``` -To run the flat migration: +To run the full migration: ```shell mkdir output error -docker run --rm -it --security-opt=label=disable \ +docker run --rm -it -security-opt=label=disable \ -v $(readlink -e path/to/takeout):/takeout \ -v $(readlink -e ./output):/output \ -v $(readlink -e ./error):/error \ localhost/google-photos-migrate:latest \ - flat '/takeout/Google Fotos' '/output' '/error' --timeout=60000 + fullMigrate '/takeout' '/output' '/error' --timeout=60000 ``` -To run the full migration: +To run the flat migration: ```shell mkdir output error -docker run --rm -it -security-opt=label=disable \ +docker run --rm -it --security-opt=label=disable \ -v $(readlink -e path/to/takeout):/takeout \ -v $(readlink -e ./output):/output \ -v $(readlink -e ./error):/error \ localhost/google-photos-migrate:latest \ - fullMigrate '/takeout' '/output' '/error' --timeout=60000 + flat '/takeout/Google Fotos' '/output' '/error' --timeout=60000 ``` All other options are also available. The only difference from running it natively is the lack of (possible) hardware acceleration, and the need to explicitly add any folders the command will need to reference as host-mounts for the container. From b5a081cd5b6b715c3b7558b754d3cee536ec709c Mon Sep 17 00:00:00 2001 From: garzj Date: Fri, 6 Oct 2023 23:44:47 +0200 Subject: [PATCH 50/50] Remove error dir check --- src/dir/check-error-dir.ts | 36 ------------------------------ src/dir/restructure-and-process.ts | 3 --- 2 files changed, 39 deletions(-) delete mode 100644 src/dir/check-error-dir.ts diff --git a/src/dir/check-error-dir.ts b/src/dir/check-error-dir.ts deleted file mode 100644 index 836b8b5..0000000 --- a/src/dir/check-error-dir.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { ExifTool } from 'exiftool-vendored'; -import { isEmptyDir } from '../fs/is-empty-dir'; -import { glob } from 'glob'; -import { basename, join } from 'path'; -import { isDir } from '../fs/is-dir'; -import { fileExists } from '../fs/file-exists'; - -export async function checkErrorDir( - outDir: string, - errDir: string, - exifTool: ExifTool -) { - if (!(await isEmptyDir(errDir))) { - const errFiles: string[] = await glob(`${errDir}/*`); - for (const file of errFiles) { - if (file.endsWith('.json')) { - continue; - } else if (await isDir(file)) { - console.log(`Cannot fix metadata for directory: ${file}`); - continue; - } else if (await fileExists(file)) { - continue; - } - console.log( - `Rewriting all tags from ${file}, to ${join( - outDir, - `cleaned-${basename(file)}` - )}` - ); - await exifTool.rewriteAllTags( - file, - join(outDir, `cleaned-${basename(file)}`) - ); - } - } -} diff --git a/src/dir/restructure-and-process.ts b/src/dir/restructure-and-process.ts index 5c42b51..a70bd85 100644 --- a/src/dir/restructure-and-process.ts +++ b/src/dir/restructure-and-process.ts @@ -1,7 +1,6 @@ import { glob } from 'glob'; import { basename } from 'path'; import { mkdir } from 'fs/promises'; -import { checkErrorDir } from './check-error-dir'; import { migrateDirFlatGen } from './migrate-flat'; import { FullMigrationContext } from './migrate-full'; import { untitledDirs } from '../config/langs'; @@ -34,8 +33,6 @@ async function* _restructureAndProcess( outputDir: outDir, errorDir: errDir, }); - - await checkErrorDir(outDir, errDir, migCtx.exiftool); } console.log(`Sucsessfully restructured ${folders.length} directories`);