From 359ac0232928543f9faed4fc324ef1d189be3a7a Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 12:18:56 +1300 Subject: [PATCH 1/9] fix: convert all .js imports to .ts --- src/__test__/eai_againRetry.test.ts | 2 +- src/__test__/fqdn.test.ts | 2 +- src/__test__/fs.test.ts | 2 +- .../__test__/create-pr-handler.test.ts | 10 +- .../__test__/create-pr.test.ts | 2 +- src/commands/basemaps-github/create-pr.ts | 14 +- .../basemaps-github/make.cog.github.ts | 10 +- .../__test__/create-mapsheet.test.ts | 2 +- .../basemaps-mapsheet/create-mapsheet.ts | 8 +- src/commands/common.ts | 8 +- src/commands/copy/__test__/copy.test.ts | 2 +- src/commands/copy/copy-worker.ts | 14 +- src/commands/copy/copy.ts | 10 +- .../__test__/create-manifest.test.ts | 2 +- .../create-manifest/create-manifest.ts | 10 +- .../__test__/generate.path.test.ts | 6 +- src/commands/generate-path/__test__/sample.ts | 2 +- src/commands/generate-path/path.generate.ts | 8 +- src/commands/group/__test__/group.test.ts | 2 +- src/commands/group/group.ts | 8 +- src/commands/index.ts | 34 ++--- .../lds-fetch-layer/lds.fetch.layer.ts | 6 +- src/commands/list/list.ts | 8 +- .../__test__/mapsheet.coverage.test.ts | 4 +- .../mapsheet-coverage/mapsheet.coverage.ts | 16 +- src/commands/pretty-print/pretty.print.ts | 10 +- .../__test__/stac.catalog.test.ts | 2 +- src/commands/stac-catalog/stac.catalog.ts | 8 +- .../__test__/stac.link.test.ts | 2 +- .../stac-github-import/stac.github.import.ts | 12 +- src/commands/stac-setup/__test__/sample.ts | 2 +- .../stac-setup/__test__/stac.setup.test.ts | 8 +- src/commands/stac-setup/stac.setup.ts | 12 +- .../stac-sync/__test__/stac.sync.test.ts | 4 +- src/commands/stac-sync/stac.sync.ts | 12 +- .../__test__/stac.validate.test.ts | 2 +- src/commands/stac-validate/stac.validate.ts | 12 +- .../__test__/tileindex.validate.data.ts | 2 +- .../__test__/tileindex.validate.test.ts | 14 +- .../tileindex-validate/tileindex.validate.ts | 20 +-- src/fs.register.ts | 4 +- src/index.ts | 4 +- src/readme/readme.generate.ts | 6 +- src/utils/__test__/argo.test.ts | 2 +- src/utils/__test__/band.test.ts | 4 +- src/utils/__test__/chunk.test.ts | 2 +- src/utils/__test__/config.test.ts | 2 +- src/utils/__test__/date.test.ts | 2 +- src/utils/__test__/filelist.test.ts | 4 +- src/utils/__test__/filter.test.ts | 2 +- src/utils/__test__/geotiff.test.ts | 4 +- src/utils/__test__/hash.test.ts | 2 +- src/utils/__test__/mapsheet.test.ts | 4 +- src/utils/__test__/slugify.test.ts | 2 +- src/utils/chunk.ts | 4 +- src/utils/filelist.ts | 2 +- src/utils/geotiff.ts | 2 +- src/utils/github.ts | 4 +- src/utils/mapsheet.chatham.islands.ts | 144 ++++++++++++++++++ 59 files changed, 327 insertions(+), 187 deletions(-) create mode 100644 src/utils/mapsheet.chatham.islands.ts diff --git a/src/__test__/eai_againRetry.test.ts b/src/__test__/eai_againRetry.test.ts index 3f1decac4..74f7ba793 100644 --- a/src/__test__/eai_againRetry.test.ts +++ b/src/__test__/eai_againRetry.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, it } from 'node:test'; import { BuildHandler, MetadataBearer } from '@smithy/types'; -import { eaiAgainBuilder } from '../fs.register.js'; +import { eaiAgainBuilder } from "../fs.register.ts"; let callCount = 0; diff --git a/src/__test__/fqdn.test.ts b/src/__test__/fqdn.test.ts index 64e7f3ff7..ee3c80a37 100644 --- a/src/__test__/fqdn.test.ts +++ b/src/__test__/fqdn.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import { FinalizeHandler, MetadataBearer } from '@smithy/types'; -import { fqdn } from '../fs.register.js'; +import { fqdn } from "../fs.register.ts"; describe('fqdnMiddleware', () => { const fakeNext: FinalizeHandler = () => { diff --git a/src/__test__/fs.test.ts b/src/__test__/fs.test.ts index 8d1c3d675..f8d8eb7a1 100644 --- a/src/__test__/fs.test.ts +++ b/src/__test__/fs.test.ts @@ -8,7 +8,7 @@ import { FsMemory } from '@chunkd/source-memory'; import { InitializeMiddleware, MetadataBearer } from '@smithy/types'; import assert from 'assert'; -import { registerFileSystem } from '../fs.register.js'; +import { registerFileSystem } from "../fs.register.ts"; export class HttpError extends Error { statusCode: number; diff --git a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts index 2b97626ab..9fe6852f2 100644 --- a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts @@ -1,15 +1,15 @@ import assert from 'node:assert'; import { afterEach, it } from 'node:test'; -import { TileSetType } from '@basemaps/config/build/config/tile.set.js'; +import { TileSetType } from "@basemaps/config/build/config/tile.set.ts"; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { StacVersion } from 'stac-ts'; -import { anySlug } from '../../../utils/__test__/slugify.test.js'; -import { GithubApi } from '../../../utils/github.js'; -import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from '../create-pr.js'; -import { Category } from '../make.cog.github.js'; +import { anySlug } from "../../../utils/__test__/slugify.test.ts"; +import { GithubApi } from "../../../utils/github.ts"; +import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from "../create-pr.ts"; +import { Category } from "../make.cog.github.ts"; const originalEnv = Object.assign({}, process.env); diff --git a/src/commands/basemaps-github/__test__/create-pr.test.ts b/src/commands/basemaps-github/__test__/create-pr.test.ts index b8f76525f..8a35e744a 100644 --- a/src/commands/basemaps-github/__test__/create-pr.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { parseTargetUrl, targetInfo } from '../create-pr.js'; +import { parseTargetUrl, targetInfo } from "../create-pr.ts"; describe('parseTargetUrl', () => { it('Should parse the correct target vector Urls', async () => { diff --git a/src/commands/basemaps-github/create-pr.ts b/src/commands/basemaps-github/create-pr.ts index d338b0545..7eaf3e144 100644 --- a/src/commands/basemaps-github/create-pr.ts +++ b/src/commands/basemaps-github/create-pr.ts @@ -5,21 +5,17 @@ import { fsa } from '@chunkd/fs'; import { boolean, command, flag, oneOf, option, optional, string } from 'cmd-ts'; import { StacCollection } from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { registerCli, verbose } from '../common.js'; -import { Category, MakeCogGithub } from './make.cog.github.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { registerCli, verbose } from '../common.ts'; +import { Category, MakeCogGithub } from './make.cog.github.ts'; export const ValidTargetBuckets: Set = new Set(['linz-basemaps', 'linz-basemaps-staging']); export const ValidSourceBuckets: Set = new Set(['nz-imagery', 'linz-imagery', 'nz-elevation']); export const LinzBasemapsSourceCollectionRel = 'linz_basemaps:source_collection'; -export enum ConfigType { - Raster = 'raster', - Vector = 'vector', - Elevation = 'elevation', -} +export type ConfigType = 'raster' | 'vector' | 'elevation'; export function assertValidBucket(bucket: string, validBuckets: Set): void { // Validate the target information diff --git a/src/commands/basemaps-github/make.cog.github.ts b/src/commands/basemaps-github/make.cog.github.ts index 48f848bb8..cfedc0fae 100644 --- a/src/commands/basemaps-github/make.cog.github.ts +++ b/src/commands/basemaps-github/make.cog.github.ts @@ -5,13 +5,13 @@ import { ConfigTileSetRaster, ConfigTileSetVector, TileSetType, -} from '@basemaps/config/build/config/tile.set.js'; +} from "@basemaps/config/build/config/tile.set.ts"; import { fsa } from '@chunkd/fs'; -import { logger } from '../../log.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { GithubApi } from '../../utils/github.js'; -import { prettyPrint } from '../pretty-print/pretty.print.js'; +import { logger } from "../../log.ts"; +import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; +import { GithubApi } from "../../utils/github.ts"; +import { prettyPrint } from "../pretty-print/pretty.print.ts"; export enum Category { Urban = 'Urban Aerial Photos', diff --git a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts index e50ca54af..517bf0fda 100644 --- a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts +++ b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts @@ -4,7 +4,7 @@ import { describe, it } from 'node:test'; import { ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; import { FeatureCollection } from 'geojson'; -import { createMapSheet } from '../create-mapsheet.js'; +import { createMapSheet } from "../create-mapsheet.ts"; describe('copyFiles', () => { const rest: FeatureCollection = { diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index 66d168707..31f4bb519 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -2,14 +2,14 @@ import { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } fro import { Bounds, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; -import * as fgb from 'flatgeobuf/lib/mjs/geojson.js'; +import * as fgb from "flatgeobuf/lib/mjs/geojson.ts"; import { FeatureCollection, MultiPolygon } from 'geojson'; import { promisify } from 'util'; import { gunzip } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { registerCli, verbose } from "../common.ts"; const gunzipProm = promisify(gunzip); diff --git a/src/commands/common.ts b/src/commands/common.ts index 513d55b87..006b9d4f8 100644 --- a/src/commands/common.ts +++ b/src/commands/common.ts @@ -4,10 +4,10 @@ import { boolean, flag, option, optional, string, Type } from 'cmd-ts'; import pLimit from 'p-limit'; import { fileURLToPath, pathToFileURL } from 'url'; -import { CliInfo } from '../cli.info.js'; -import { registerFileSystem } from '../fs.register.js'; -import { logger, registerLogger } from '../log.js'; -import { isArgo } from '../utils/argo.js'; +import { CliInfo } from "../cli.info.ts"; +import { registerFileSystem } from "../fs.register.ts"; +import { logger, registerLogger } from "../log.ts"; +import { isArgo } from "../utils/argo.ts"; export const config = option({ long: 'config', diff --git a/src/commands/copy/__test__/copy.test.ts b/src/commands/copy/__test__/copy.test.ts index de3ca47dd..c11042811 100644 --- a/src/commands/copy/__test__/copy.test.ts +++ b/src/commands/copy/__test__/copy.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { worker } from '../copy-worker.js'; +import { worker } from "../copy-worker.ts"; describe('copyFiles', () => { const memory = new FsMemory(); diff --git a/src/commands/copy/copy-worker.ts b/src/commands/copy/copy-worker.ts index 6160614ed..19ce23eb5 100644 --- a/src/commands/copy/copy-worker.ts +++ b/src/commands/copy/copy-worker.ts @@ -5,13 +5,13 @@ import { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; import { WorkerRpc } from '@wtrpc/core'; -import { logger } from '../../log.js'; -import { ConcurrentQueue } from '../../utils/concurrent.queue.js'; -import { HashKey, hashStream } from '../../utils/hash.js'; -import { HashTransform } from '../../utils/hash.stream.js'; -import { registerCli } from '../common.js'; -import { isTiff } from '../tileindex-validate/tileindex.validate.js'; -import { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.js'; +import { logger } from "../../log.ts"; +import { ConcurrentQueue } from "../../utils/concurrent.queue.ts"; +import { HashKey, hashStream } from "../../utils/hash.ts"; +import { HashTransform } from "../../utils/hash.stream.ts"; +import { registerCli } from "../common.ts"; +import { isTiff } from "../tileindex-validate/tileindex.validate.ts"; +import { CopyContract, CopyContractArgs, CopyStats } from "./copy-rpc.ts"; const Q = new ConcurrentQueue(10); diff --git a/src/commands/copy/copy.ts b/src/commands/copy/copy.ts index cae1181c1..9ac333461 100644 --- a/src/commands/copy/copy.ts +++ b/src/commands/copy/copy.ts @@ -4,11 +4,11 @@ import { boolean, command, flag, number, option, restPositionals, string } from import { performance } from 'perf_hooks'; import * as z from 'zod'; -import { CliInfo } from '../../cli.info.js'; -import { logger, logId } from '../../log.js'; -import { ActionCopy } from '../../utils/actions.js'; -import { config, registerCli, verbose } from '../common.js'; -import { CopyContract } from './copy-rpc.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger, logId } from "../../log.ts"; +import { ActionCopy } from "../../utils/actions.ts"; +import { config, registerCli, verbose } from "../common.ts"; +import { CopyContract } from "./copy-rpc.ts"; const CopyValidator = z.object({ source: z.string(), target: z.string() }); const CopyManifest = z.array(CopyValidator); diff --git a/src/commands/create-manifest/__test__/create-manifest.test.ts b/src/commands/create-manifest/__test__/create-manifest.test.ts index 26f9a2b7d..fb8dc726a 100644 --- a/src/commands/create-manifest/__test__/create-manifest.test.ts +++ b/src/commands/create-manifest/__test__/create-manifest.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createManifest, validatePaths } from '../create-manifest.js'; +import { createManifest, validatePaths } from "../create-manifest.ts"; describe('createManifest', () => { beforeEach(() => { diff --git a/src/commands/create-manifest/create-manifest.ts b/src/commands/create-manifest/create-manifest.ts index 91a325dad..4e31cdf5b 100644 --- a/src/commands/create-manifest/create-manifest.ts +++ b/src/commands/create-manifest/create-manifest.ts @@ -4,11 +4,11 @@ import { createHash } from 'crypto'; import path from 'path'; import { gzipSync } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { getActionLocation } from '../../utils/action.storage.js'; -import { ActionCopy } from '../../utils/actions.js'; -import { FileFilter, getFiles } from '../../utils/chunk.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { getActionLocation } from "../../utils/action.storage.ts"; +import { ActionCopy } from "../../utils/actions.ts"; +import { FileFilter, getFiles } from "../../utils/chunk.ts"; +import { config, registerCli, verbose } from "../common.ts"; export const commandCreateManifest = command({ name: 'create-manifest', diff --git a/src/commands/generate-path/__test__/generate.path.test.ts b/src/commands/generate-path/__test__/generate.path.test.ts index 9780145fa..6ed923d6c 100644 --- a/src/commands/generate-path/__test__/generate.path.test.ts +++ b/src/commands/generate-path/__test__/generate.path.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { SampleCollection } from '../../generate-path/__test__/sample.js'; -import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.js'; -import { extractEpsg, extractGsd, generatePath, PathMetadata } from '../path.generate.js'; +import { SampleCollection } from "../../generate-path/__test__/sample.ts"; +import { FakeCogTiff } from "../../tileindex-validate/__test__/tileindex.validate.data.ts"; +import { extractEpsg, extractGsd, generatePath, PathMetadata } from "../path.generate.ts"; describe('GeneratePathImagery', () => { it('Should match - urban aerial from slug', () => { diff --git a/src/commands/generate-path/__test__/sample.ts b/src/commands/generate-path/__test__/sample.ts index 486a7a246..8a976736a 100644 --- a/src/commands/generate-path/__test__/sample.ts +++ b/src/commands/generate-path/__test__/sample.ts @@ -1,6 +1,6 @@ import { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.js'; +import { StacCollectionLinz } from "../../../utils/metadata.ts"; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index 3332400f1..17f3f3d47 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -4,10 +4,10 @@ import { Tiff } from '@cogeotiff/core'; import { command, option, positional, string } from 'cmd-ts'; import { StacCollection, StacItem } from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.js'; -import { config, createTiff, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { GeospatialDataCategories, StacCollectionLinz } from "../../utils/metadata.ts"; +import { config, createTiff, registerCli, verbose } from "../common.ts"; export interface PathMetadata { targetBucketName: string; diff --git a/src/commands/group/__test__/group.test.ts b/src/commands/group/__test__/group.test.ts index 112564b90..1f8334a83 100644 --- a/src/commands/group/__test__/group.test.ts +++ b/src/commands/group/__test__/group.test.ts @@ -4,7 +4,7 @@ import { before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { commandGroup, groupItems } from '../group.js'; +import { commandGroup, groupItems } from "../group.ts"; describe('groupItems', () => { it('should group items', () => { diff --git a/src/commands/group/group.ts b/src/commands/group/group.ts index b20c70c0d..af53df0e4 100644 --- a/src/commands/group/group.ts +++ b/src/commands/group/group.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { config, forceOutput, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { isArgo } from "../../utils/argo.ts"; +import { config, forceOutput, registerCli, verbose } from "../common.ts"; /** Chunk an array into a group size * @example diff --git a/src/commands/index.ts b/src/commands/index.ts index 42b70b354..5f8f2fa06 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,22 +1,22 @@ import { subcommands } from 'cmd-ts'; -import { CliInfo } from '../cli.info.js'; -import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; -import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { commandCopy } from './copy/copy.js'; -import { commandCreateManifest } from './create-manifest/create-manifest.js'; -import { commandGeneratePath } from './generate-path/path.generate.js'; -import { commandGroup } from './group/group.js'; -import { commandLdsFetch } from './lds-fetch-layer/lds.fetch.layer.js'; -import { commandList } from './list/list.js'; -import { commandMapSheetCoverage } from './mapsheet-coverage/mapsheet.coverage.js'; -import { commandPrettyPrint } from './pretty-print/pretty.print.js'; -import { commandStacCatalog } from './stac-catalog/stac.catalog.js'; -import { commandStacGithubImport } from './stac-github-import/stac.github.import.js'; -import { commandStacSetup } from './stac-setup/stac.setup.js'; -import { commandStacSync } from './stac-sync/stac.sync.js'; -import { commandStacValidate } from './stac-validate/stac.validate.js'; -import { commandTileIndexValidate } from './tileindex-validate/tileindex.validate.js'; +import { CliInfo } from "../cli.info.ts"; +import { basemapsCreatePullRequest } from "./basemaps-github/create-pr.ts"; +import { basemapsCreateMapSheet } from "./basemaps-mapsheet/create-mapsheet.ts"; +import { commandCopy } from "./copy/copy.ts"; +import { commandCreateManifest } from "./create-manifest/create-manifest.ts"; +import { commandGeneratePath } from "./generate-path/path.generate.ts"; +import { commandGroup } from "./group/group.ts"; +import { commandLdsFetch } from "./lds-fetch-layer/lds.fetch.layer.ts"; +import { commandList } from "./list/list.ts"; +import { commandMapSheetCoverage } from "./mapsheet-coverage/mapsheet.coverage.ts"; +import { commandPrettyPrint } from "./pretty-print/pretty.print.ts"; +import { commandStacCatalog } from "./stac-catalog/stac.catalog.ts"; +import { commandStacGithubImport } from "./stac-github-import/stac.github.import.ts"; +import { commandStacSetup } from "./stac-setup/stac.setup.ts"; +import { commandStacSync } from "./stac-sync/stac.sync.ts"; +import { commandStacValidate } from "./stac-validate/stac.validate.ts"; +import { commandTileIndexValidate } from "./tileindex-validate/tileindex.validate.ts"; export const AllCommands = { copy: commandCopy, diff --git a/src/commands/lds-fetch-layer/lds.fetch.layer.ts b/src/commands/lds-fetch-layer/lds.fetch.layer.ts index 1956dc6f7..ffadd1553 100644 --- a/src/commands/lds-fetch-layer/lds.fetch.layer.ts +++ b/src/commands/lds-fetch-layer/lds.fetch.layer.ts @@ -3,9 +3,9 @@ import { command, option, restPositionals, string } from 'cmd-ts'; import * as stac from 'stac-ts'; import { createGunzip } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { config, registerCli, verbose } from "../common.ts"; function getTargetPath(source: string, path: string): string { if (path.startsWith('./')) return fsa.join(source, path.slice(2)); diff --git a/src/commands/list/list.ts b/src/commands/list/list.ts index ef28bc216..9a8f6a6e5 100644 --- a/src/commands/list/list.ts +++ b/src/commands/list/list.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getFiles } from '../../utils/chunk.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { getFiles } from "../../utils/chunk.ts"; +import { config, registerCli, verbose } from "../common.ts"; export const CommandListArgs = { config, diff --git a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts index 2ae260b5b..ebb96552a 100644 --- a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts +++ b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts @@ -7,8 +7,8 @@ import { FsMemory } from '@chunkd/source-memory'; import { featuresToMultiPolygon } from '@linzjs/geojson'; import { StacCollection } from 'stac-ts'; -import { MapSheet } from '../../../utils/mapsheet.js'; -import { commandMapSheetCoverage, isLargeRegion } from '../mapsheet.coverage.js'; +import { MapSheet } from "../../../utils/mapsheet.ts"; +import { commandMapSheetCoverage, isLargeRegion } from "../mapsheet.coverage.ts"; // convert a collection of map sheets into a multipolygon function mapSheetToGeoJson(...sheetCodes: string[]): GeoJSON.Feature { diff --git a/src/commands/mapsheet-coverage/mapsheet.coverage.ts b/src/commands/mapsheet-coverage/mapsheet.coverage.ts index 2eda7e362..45e036831 100644 --- a/src/commands/mapsheet-coverage/mapsheet.coverage.ts +++ b/src/commands/mapsheet-coverage/mapsheet.coverage.ts @@ -7,17 +7,17 @@ import { Area, truncate } from '@linzjs/geojson'; import buffer from '@turf/buffer'; import { command, number, option, optional, string } from 'cmd-ts'; import pLimit from 'p-limit'; -import { basename } from 'path/posix'; +import { basename } from "path/posix.ts"; import pc from 'polygon-clipping'; import { StacCollection, StacItem } from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getPacificAucklandYearMonthDay } from '../../utils/date.js'; -import { createFileList } from '../../utils/filelist.js'; -import { hashStream } from '../../utils/hash.js'; -import { MapSheet } from '../../utils/mapsheet.js'; -import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { getPacificAucklandYearMonthDay } from "../../utils/date.ts"; +import { createFileList } from "../../utils/filelist.ts"; +import { hashStream } from "../../utils/hash.ts"; +import { MapSheet } from "../../utils/mapsheet.ts"; +import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from "../common.ts"; /** Datasets to skip */ const Skip = new Set([ diff --git a/src/commands/pretty-print/pretty.print.ts b/src/commands/pretty-print/pretty.print.ts index 77ade3a07..14ba0c934 100644 --- a/src/commands/pretty-print/pretty.print.ts +++ b/src/commands/pretty-print/pretty.print.ts @@ -3,11 +3,11 @@ import { command, option, optional, positional, string } from 'cmd-ts'; import { basename } from 'path'; import prettier from 'prettier'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getFiles } from '../../utils/chunk.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { getFiles } from "../../utils/chunk.ts"; +import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; +import { config, registerCli, verbose } from "../common.ts"; function isJson(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/commands/stac-catalog/__test__/stac.catalog.test.ts b/src/commands/stac-catalog/__test__/stac.catalog.test.ts index 8a9a02fca..82769c4c9 100644 --- a/src/commands/stac-catalog/__test__/stac.catalog.test.ts +++ b/src/commands/stac-catalog/__test__/stac.catalog.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createLinks, makeRelative } from '../stac.catalog.js'; +import { createLinks, makeRelative } from "../stac.catalog.ts"; describe('stacCatalog', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-catalog/stac.catalog.ts b/src/commands/stac-catalog/stac.catalog.ts index d5098bf8b..068d034ee 100644 --- a/src/commands/stac-catalog/stac.catalog.ts +++ b/src/commands/stac-catalog/stac.catalog.ts @@ -3,10 +3,10 @@ import { command, option, positional, string } from 'cmd-ts'; import { isAbsolute } from 'path'; import * as st from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { hashBuffer } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { hashBuffer } from "../../utils/hash.ts"; +import { config, registerCli, verbose } from "../common.ts"; /** is a path a URL */ export function isUrl(path: string): boolean { diff --git a/src/commands/stac-github-import/__test__/stac.link.test.ts b/src/commands/stac-github-import/__test__/stac.link.test.ts index 2b0604fc3..9aa9a6be2 100644 --- a/src/commands/stac-github-import/__test__/stac.link.test.ts +++ b/src/commands/stac-github-import/__test__/stac.link.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { sortLinks } from '../stac.github.import.js'; +import { sortLinks } from "../stac.github.import.ts"; function shuffle(array: T[]): T[] { let currentIndex = array.length; diff --git a/src/commands/stac-github-import/stac.github.import.ts b/src/commands/stac-github-import/stac.github.import.ts index 6426abcff..b15f70e95 100644 --- a/src/commands/stac-github-import/stac.github.import.ts +++ b/src/commands/stac-github-import/stac.github.import.ts @@ -2,12 +2,12 @@ import { fsa } from '@chunkd/fs'; import { command, oneOf, option, string, Type } from 'cmd-ts'; import * as st from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { GithubApi } from '../../utils/github.js'; -import { config, registerCli, verbose } from '../common.js'; -import { prettyPrint } from '../pretty-print/pretty.print.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; +import { GithubApi } from "../../utils/github.ts"; +import { config, registerCli, verbose } from "../common.ts"; +import { prettyPrint } from "../pretty-print/pretty.print.ts"; const Url: Type = { async from(str) { diff --git a/src/commands/stac-setup/__test__/sample.ts b/src/commands/stac-setup/__test__/sample.ts index 999adbfa2..511eba333 100644 --- a/src/commands/stac-setup/__test__/sample.ts +++ b/src/commands/stac-setup/__test__/sample.ts @@ -1,6 +1,6 @@ import { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.js'; +import { StacCollectionLinz } from "../../../utils/metadata.ts"; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/stac-setup/__test__/stac.setup.test.ts b/src/commands/stac-setup/__test__/stac.setup.test.ts index 94c36d1a7..75217a597 100644 --- a/src/commands/stac-setup/__test__/stac.setup.test.ts +++ b/src/commands/stac-setup/__test__/stac.setup.test.ts @@ -4,10 +4,10 @@ import { afterEach, before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { GeospatialDataCategory } from '../../../utils/metadata.js'; -import { MeterAsString } from '../../common.js'; -import { commandStacSetup, formatDate, slugFromMetadata, SlugMetadata } from '../stac.setup.js'; -import { SampleCollection } from './sample.js'; +import { GeospatialDataCategory } from "../../../utils/metadata.ts"; +import { MeterAsString } from "../../common.ts"; +import { commandStacSetup, formatDate, slugFromMetadata, SlugMetadata } from "../stac.setup.ts"; +import { SampleCollection } from "./sample.ts"; describe('stac-setup', () => { const mem = new FsMemory(); diff --git a/src/commands/stac-setup/stac.setup.ts b/src/commands/stac-setup/stac.setup.ts index e9656665b..10f16c15d 100644 --- a/src/commands/stac-setup/stac.setup.ts +++ b/src/commands/stac-setup/stac.setup.ts @@ -1,13 +1,13 @@ import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; import ulid from 'ulid'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { GeospatialDataCategory, StacCollectionLinz } from '../../utils/metadata.js'; -import { slugify } from '../../utils/slugify.js'; -import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { GeospatialDataCategory, StacCollectionLinz } from "../../utils/metadata.ts"; +import { slugify } from "../../utils/slugify.ts"; +import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from "../common.ts"; export interface SlugMetadata { geospatialCategory: GeospatialDataCategory; diff --git a/src/commands/stac-sync/__test__/stac.sync.test.ts b/src/commands/stac-sync/__test__/stac.sync.test.ts index 8e985f5ab..3a796602f 100644 --- a/src/commands/stac-sync/__test__/stac.sync.test.ts +++ b/src/commands/stac-sync/__test__/stac.sync.test.ts @@ -4,8 +4,8 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { hashBuffer, HashKey } from '../../../utils/hash.js'; -import { synchroniseFiles } from '../stac.sync.js'; +import { hashBuffer, HashKey } from "../../../utils/hash.ts"; +import { synchroniseFiles } from "../stac.sync.ts"; describe('stacSync', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-sync/stac.sync.ts b/src/commands/stac-sync/stac.sync.ts index fe1f4bfe5..97753f50b 100644 --- a/src/commands/stac-sync/stac.sync.ts +++ b/src/commands/stac-sync/stac.sync.ts @@ -2,12 +2,12 @@ import { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; import { command, positional, string, Type } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { md } from '../../readme/markdown.js'; -import { annotateExample } from '../../readme/readme.example.js'; -import { hashBuffer, HashKey } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { md } from "../../readme/markdown.ts"; +import { annotateExample } from "../../readme/readme.example.ts"; +import { hashBuffer, HashKey } from "../../utils/hash.ts"; +import { config, registerCli, verbose } from "../common.ts"; const S3Path: Type = { async from(str) { diff --git a/src/commands/stac-validate/__test__/stac.validate.test.ts b/src/commands/stac-validate/__test__/stac.validate.test.ts index a86468348..6c067e1ca 100644 --- a/src/commands/stac-validate/__test__/stac.validate.test.ts +++ b/src/commands/stac-validate/__test__/stac.validate.test.ts @@ -13,7 +13,7 @@ import { validateAssets, validateLinks, validateStacChecksum, -} from '../stac.validate.js'; +} from "../stac.validate.ts"; describe('stacValidate', function () { it('listLocation', () => { diff --git a/src/commands/stac-validate/stac.validate.ts b/src/commands/stac-validate/stac.validate.ts index ae189fa06..c88180361 100644 --- a/src/commands/stac-validate/stac.validate.ts +++ b/src/commands/stac-validate/stac.validate.ts @@ -1,17 +1,17 @@ import { fsa } from '@chunkd/fs'; import Ajv, { DefinedError, ValidateFunction } from 'ajv'; -import { fastFormats } from 'ajv-formats/dist/formats.js'; +import { fastFormats } from "ajv-formats/dist/formats.ts"; import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; import { createHash } from 'crypto'; import { dirname, join } from 'path'; import { performance } from 'perf_hooks'; import * as st from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { ConcurrentQueue } from '../../utils/concurrent.queue.js'; -import { hashStream, Sha256Prefix } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { ConcurrentQueue } from "../../utils/concurrent.queue.ts"; +import { hashStream, Sha256Prefix } from "../../utils/hash.ts"; +import { config, registerCli, verbose } from "../common.ts"; export const commandStacValidate = command({ name: 'stac-validate', diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts index 115fc751a..aeed5afe8 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts @@ -1,6 +1,6 @@ import { SampleFormat, Size, Source, Tiff, TiffImage, TiffTag } from '@cogeotiff/core'; -import { MapSheet } from '../../../utils/mapsheet.js'; +import { MapSheet } from "../../../utils/mapsheet.ts"; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts index c0fcc31ae..d97138a9d 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts @@ -7,11 +7,11 @@ import { FsMemory } from '@chunkd/source-memory'; import { BBox } from '@linzjs/geojson'; import { FeatureCollection } from 'geojson'; -import { logger } from '../../../log.js'; -import { MapSheetData } from '../../../utils/__test__/mapsheet.data.js'; -import { FileListEntry } from '../../../utils/filelist.js'; -import { GridSize, MapSheet } from '../../../utils/mapsheet.js'; -import { createTiff } from '../../common.js'; +import { logger } from "../../../log.ts"; +import { MapSheetData } from "../../../utils/__test__/mapsheet.data.ts"; +import { FileListEntry } from "../../../utils/filelist.ts"; +import { GridSize, MapSheet } from "../../../utils/mapsheet.ts"; +import { createTiff } from "../../common.ts"; import { commandTileIndexValidate, extractTiffLocations, @@ -22,8 +22,8 @@ import { TiffLoader, validate8BitsTiff, validatePreset, -} from '../tileindex.validate.js'; -import { FakeCogTiff } from './tileindex.validate.data.js'; +} from "../tileindex.validate.ts"; +import { FakeCogTiff } from "./tileindex.validate.data.ts"; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/tileindex.validate.ts b/src/commands/tileindex-validate/tileindex.validate.ts index 20330d6e8..8b5671c66 100644 --- a/src/commands/tileindex-validate/tileindex.validate.ts +++ b/src/commands/tileindex-validate/tileindex.validate.ts @@ -4,16 +4,16 @@ import { Size, Tiff, TiffTag } from '@cogeotiff/core'; import { BBox } from '@linzjs/geojson'; import { boolean, command, flag, number, option, optional, restPositionals, string, Type } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { extractBandInformation } from '../../utils/band.js'; -import { FileFilter, getFiles } from '../../utils/chunk.js'; -import { createFileList } from '../../utils/filelist.js'; -import { findBoundingBox } from '../../utils/geotiff.js'; -import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.js'; -import { config, createTiff, forceOutput, registerCli, verbose } from '../common.js'; -import { CommandListArgs } from '../list/list.js'; +import { CliInfo } from "../../cli.info.ts"; +import { logger } from "../../log.ts"; +import { isArgo } from "../../utils/argo.ts"; +import { extractBandInformation } from "../../utils/band.ts"; +import { FileFilter, getFiles } from "../../utils/chunk.ts"; +import { createFileList } from "../../utils/filelist.ts"; +import { findBoundingBox } from "../../utils/geotiff.ts"; +import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from "../../utils/mapsheet.ts"; +import { config, createTiff, forceOutput, registerCli, verbose } from "../common.ts"; +import { CommandListArgs } from "../list/list.ts"; export function isTiff(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/fs.register.ts b/src/fs.register.ts index 492b6f667..1e08500d7 100644 --- a/src/fs.register.ts +++ b/src/fs.register.ts @@ -1,4 +1,4 @@ -import { setTimeout } from 'node:timers/promises'; +import { setTimeout } from "node:timers/promises.ts"; import { S3Client } from '@aws-sdk/client-s3'; import { fsa } from '@chunkd/fs'; @@ -6,7 +6,7 @@ import { FsAwsS3 } from '@chunkd/source-aws'; import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3'; import { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; -import { logger } from './log.js'; +import { logger } from "./log.ts"; /** Check to see if hostname exists inside of a object */ function hasHostName(x: unknown): x is { hostname: string } { diff --git a/src/index.ts b/src/index.ts index ca42249b4..c427613c5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,8 +3,8 @@ process.env['AWS_NODEJS_CONNECTION_REUSE_ENABLED'] = '1'; import { run } from 'cmd-ts'; -import { cmd } from './commands/index.js'; -import { logger } from './log.js'; +import { cmd } from "./commands/index.ts"; +import { logger } from "./log.ts"; const startTime = performance.now(); run(cmd, process.argv.slice(2)) diff --git a/src/readme/readme.generate.ts b/src/readme/readme.generate.ts index 7f7fb39a0..16005fa69 100644 --- a/src/readme/readme.generate.ts +++ b/src/readme/readme.generate.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; -import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.js'; +import type { HelpTopic, ProvidesHelp } from "cmd-ts/dist/cjs/helpdoc.ts"; import { writeFileSync } from 'fs'; import * as prettier from 'prettier'; -import { AllCommands } from '../commands/index.js'; -import { commandHasExample, ExampleSymbol } from './readme.example.js'; +import { AllCommands } from "../commands/index.ts"; +import { commandHasExample, ExampleSymbol } from "./readme.example.ts"; const AnsiRemove = /\u001b\[.*?m/g; function hasHelp(f: unknown): f is ProvidesHelp { diff --git a/src/utils/__test__/argo.test.ts b/src/utils/__test__/argo.test.ts index 07910a55e..7d3f985db 100644 --- a/src/utils/__test__/argo.test.ts +++ b/src/utils/__test__/argo.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { beforeEach, describe, it } from 'node:test'; -import { getActionLocation } from '../action.storage.js'; +import { getActionLocation } from "../action.storage.ts"; describe('argoLocation', () => { beforeEach(() => { diff --git a/src/utils/__test__/band.test.ts b/src/utils/__test__/band.test.ts index b861b8b17..42351d2f0 100644 --- a/src/utils/__test__/band.test.ts +++ b/src/utils/__test__/band.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { createTiff } from '../../commands/common.js'; -import { extractBandInformation } from '../band.js'; +import { createTiff } from "../../commands/common.ts"; +import { extractBandInformation } from "../band.ts"; describe('extractBandInformation', () => { it('should extract basic band information (8-bit)', async () => { diff --git a/src/utils/__test__/chunk.test.ts b/src/utils/__test__/chunk.test.ts index cf53c2fa4..3aa3b4aca 100644 --- a/src/utils/__test__/chunk.test.ts +++ b/src/utils/__test__/chunk.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { getFiles, splitPaths } from '../chunk.js'; +import { getFiles, splitPaths } from "../chunk.ts"; describe('splitPaths', () => { it('should split on ;', () => { diff --git a/src/utils/__test__/config.test.ts b/src/utils/__test__/config.test.ts index b5060b82f..8bbf991d6 100644 --- a/src/utils/__test__/config.test.ts +++ b/src/utils/__test__/config.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import prettier from 'prettier'; -import { DEFAULT_PRETTIER_FORMAT } from '../config.js'; +import { DEFAULT_PRETTIER_FORMAT } from "../config.ts"; describe('DefaultPrettierFormat', () => { it('should be the same prettier format with @linzjs/style', async () => { diff --git a/src/utils/__test__/date.test.ts b/src/utils/__test__/date.test.ts index aa2535c14..6e9efe5a4 100644 --- a/src/utils/__test__/date.test.ts +++ b/src/utils/__test__/date.test.ts @@ -2,7 +2,7 @@ import { describe, it } from 'node:test'; import assert from 'assert'; -import { getPacificAucklandYearMonthDay } from '../date.js'; +import { getPacificAucklandYearMonthDay } from "../date.ts"; describe('getPacificAucklandYearMonthDay', () => { it('should format as yyyy-mm-dd', () => { diff --git a/src/utils/__test__/filelist.test.ts b/src/utils/__test__/filelist.test.ts index 346b74f51..252619ea3 100644 --- a/src/utils/__test__/filelist.test.ts +++ b/src/utils/__test__/filelist.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { TiffLocation } from '../../commands/tileindex-validate/tileindex.validate.js'; -import { createFileList, FileListEntry } from '../filelist.js'; +import { TiffLocation } from "../../commands/tileindex-validate/tileindex.validate.ts"; +import { createFileList, FileListEntry } from "../filelist.ts"; describe('createFileList', () => { const locationTestOne: TiffLocation = { bands: [], bbox: [0, 0, 0, 0], tileNames: [], source: 'input1' }; diff --git a/src/utils/__test__/filter.test.ts b/src/utils/__test__/filter.test.ts index fb379b9aa..9970eb69a 100644 --- a/src/utils/__test__/filter.test.ts +++ b/src/utils/__test__/filter.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; -import { asyncFilter } from '../chunk.js'; +import { asyncFilter } from "../chunk.ts"; describe('AsyncFilter', () => { function makeGenerator(list: string[]): () => AsyncGenerator<{ path: string }> { diff --git a/src/utils/__test__/geotiff.test.ts b/src/utils/__test__/geotiff.test.ts index 1d6a5dedd..9eaeb9de8 100644 --- a/src/utils/__test__/geotiff.test.ts +++ b/src/utils/__test__/geotiff.test.ts @@ -5,8 +5,8 @@ import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; import { Source, Tiff, TiffImage } from '@cogeotiff/core'; -import { createTiff } from '../../commands/common.js'; -import { findBoundingBox, parseTfw, PixelIsPoint } from '../geotiff.js'; +import { createTiff } from "../../commands/common.ts"; +import { findBoundingBox, parseTfw, PixelIsPoint } from "../geotiff.ts"; describe('geotiff', () => { describe('parseTfw', () => { diff --git a/src/utils/__test__/hash.test.ts b/src/utils/__test__/hash.test.ts index 755f0e8fd..28d92fc5e 100644 --- a/src/utils/__test__/hash.test.ts +++ b/src/utils/__test__/hash.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { Readable } from 'node:stream'; import { describe, it } from 'node:test'; -import { hashBuffer, hashStream } from '../hash.js'; +import { hashBuffer, hashStream } from "../hash.ts"; describe('hashBuffer', () => { it('should return the expecting digest', () => { diff --git a/src/utils/__test__/mapsheet.test.ts b/src/utils/__test__/mapsheet.test.ts index c07264887..b4872ef6c 100644 --- a/src/utils/__test__/mapsheet.test.ts +++ b/src/utils/__test__/mapsheet.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { MapSheet, MapTileIndex, SheetRanges } from '../mapsheet.js'; -import { MapSheetData } from './mapsheet.data.js'; +import { MapSheet, MapTileIndex, SheetRanges } from "../mapsheet.ts"; +import { MapSheetData } from "./mapsheet.data.ts"; describe('MapSheets', () => { it('should extract MapTileIndex from 1:500 tile filename', () => { diff --git a/src/utils/__test__/slugify.test.ts b/src/utils/__test__/slugify.test.ts index bb52fa4e1..dda166481 100644 --- a/src/utils/__test__/slugify.test.ts +++ b/src/utils/__test__/slugify.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { slugify } from '../slugify.js'; +import { slugify } from "../slugify.ts"; const slugChars = 'abcdefghijklmnopqrstuvwxyz0123456789_.-'; diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts index 56eb8806d..1201249c2 100644 --- a/src/utils/chunk.ts +++ b/src/utils/chunk.ts @@ -1,7 +1,7 @@ import { fsa } from '@chunkd/fs'; -import { parseSize } from '../commands/common.js'; -import { logger } from '../log.js'; +import { parseSize } from "../commands/common.ts"; +import { logger } from "../log.ts"; export interface FileSizeInfo { path: string; diff --git a/src/utils/filelist.ts b/src/utils/filelist.ts index 1ae97fbf1..0aff66000 100644 --- a/src/utils/filelist.ts +++ b/src/utils/filelist.ts @@ -1,4 +1,4 @@ -import { TiffLocation } from '../commands/tileindex-validate/tileindex.validate.js'; +import { TiffLocation } from "../commands/tileindex-validate/tileindex.validate.ts"; export type FileListEntry = { output: string; diff --git a/src/utils/geotiff.ts b/src/utils/geotiff.ts index d0fc044d8..89afe4086 100644 --- a/src/utils/geotiff.ts +++ b/src/utils/geotiff.ts @@ -1,7 +1,7 @@ import { fsa } from '@chunkd/fs'; import { RasterTypeKey, Tiff, TiffTagGeo } from '@cogeotiff/core'; -import { urlToString } from '../commands/common.js'; +import { urlToString } from "../commands/common.ts"; /** * Attempt to parse a tiff world file diff --git a/src/utils/github.ts b/src/utils/github.ts index 1e4d38318..f341141a4 100644 --- a/src/utils/github.ts +++ b/src/utils/github.ts @@ -1,8 +1,8 @@ import { Octokit } from '@octokit/core'; import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'; -import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; +import { Api } from "@octokit/plugin-rest-endpoint-methods/dist-types/types.ts"; -import { logger } from '../log.js'; +import { logger } from "../log.ts"; export interface Blob { path: string; diff --git a/src/utils/mapsheet.chatham.islands.ts b/src/utils/mapsheet.chatham.islands.ts new file mode 100644 index 000000000..beb168800 --- /dev/null +++ b/src/utils/mapsheet.chatham.islands.ts @@ -0,0 +1,144 @@ +// import { Bounds, MapSheetRegex, MapTileIndex } from './mapsheet.js'; + +// export const CiSheetRegex = /(?CI\d{2})(_(?\d+)_(?\d+))?/; + +// export interface MapSheetCi { +// name: string; +// x: number; +// y: number; +// } + +// export const MapSheetChathamIslands = { +// /** Height of Topo 1:50k map sheets (meters) */ +// height: 36_000, +// /** Width of Topo 1:50k map sheets (meters) */ +// width: 24_000, + +// gridSizeMax: 50_000, +// scale: 50_000, + +// /** +// * Map sheets are laid out in a pattern resembling: +// * +// * ``` +// * [CI01] [CI02] [CI03] +// * [CI04] [CI05] +// * [CI06] +// * ``` +// */ +// origins: [ +// // First row +// { name: 'CI01', x: 3458000, y: 5176000 }, +// { name: 'CI02', x: 3482000, y: 5176000 }, +// { name: 'CI03', x: 3506000, y: 5176000 }, + +// // Second row +// { name: 'CI04', x: 3482000, y: 5140000 }, +// { name: 'CI05', x: 3506000, y: 5140000 }, + +// // Third row +// { name: 'CI06', x: 3506000, y: 5104000 }, +// ], + +// getMapTileIndex(str: string): MapTileIndex | null { +// const match = str.match(CiSheetRegex); +// if (match == null) return null; + +// const sheetCode = match?.groups?.['sheetCode']; +// if (sheetCode == null) return null; + +// const gridSize = Number(match?.groups?.['gridSize'] ?? this.gridSizeMax); +// const out: MapTileIndex = { +// mapSheet: sheetCode, +// gridSize: gridSize, +// x: -1, +// y: -1, +// name: match[0], +// origin: { x: 0, y: 0 }, +// width: 0, +// height: 0, +// bbox: [0, 0, 0, 0], +// }; +// const mapSheetOffset = this.offset(sheetCode); + +// if (out.gridSize === this.gridSizeMax) { +// out.y = mapSheetOffset.y; +// out.x = mapSheetOffset.x; +// out.origin = mapSheetOffset; +// out.width = this.width; +// out.height = this.height; +// // As in CITM negative Y goes north, the minY is actually the bottom right point +// out.bbox = [out.origin.x, out.origin.y - out.height, out.origin.x + out.width, out.origin.y]; +// return out; +// } + +// // 1:500 has X/Y is 3 digits not 2 +// if (out.gridSize === 500) { +// out.y = Number(match?.groups?.['tileId']?.slice(0, 3)); +// out.x = Number(match?.groups?.['tileId']?.slice(3)); +// } else { +// out.y = Number(match?.groups?.['tileId']?.slice(0, 2)); +// out.x = Number(match?.groups?.['tileId']?.slice(2)); +// } +// if (isNaN(out.gridSize) || isNaN(out.x) || isNaN(out.y)) return null; + +// const tileOffset = this.tileSize(out.gridSize, out.x, out.y); +// out.origin.x = mapSheetOffset.x + tileOffset.x; +// out.origin.y = mapSheetOffset.y - tileOffset.y; +// out.width = tileOffset.width; +// out.height = tileOffset.height; +// // As in NZTM negative Y goes north, the minY is actually the bottom right point +// out.bbox = [out.origin.x, out.origin.y - tileOffset.height, out.origin.x + tileOffset.width, out.origin.y]; + +// return out; +// }, +// tileSize(gridSize: number, x: number, y: number): Bounds { +// const scale = gridSize / this.gridSizeMax; +// const offsetX = this.width * scale; +// const offsetY = this.height * scale; +// return { x: (x - 1) * offsetX, y: (y - 1) * offsetY, width: offsetX, height: offsetY }; +// }, +// offset(sheet: string): { x: number; y: number } { +// for (const o of this.origins) if (sheet == o.name) return { x: o.x, y: o.y }; +// throw new Error('Sheet not found'); +// }, + +// getMapSheet(x: number, y: number): MapSheetCi { +// for (const o of MapSheetChathamIslands.origins) { +// if (o.x < x && x < o.x + this.width) { +// if (o.y > y && y > o.y - this.height) { +// return o; +// } +// } +// } + +// throw new Error(`Unable to find mapsheet for point: ${x},${y}`); +// }, + +// getTileName(x: number, y: number, gridSize: number): string { +// const ms = this.getMapSheet(x, y); + +// if (gridSize === MapSheetChathamIslands.gridSizeMax) return ms.name; + +// const tilesPerMapSheet = Math.floor(MapSheetChathamIslands.gridSizeMax / gridSize); +// const tileWidth = Math.floor(MapSheetChathamIslands.width / tilesPerMapSheet); +// const tileHeight = Math.floor(MapSheetChathamIslands.height / tilesPerMapSheet); + +// const nbDigits = gridSize === 500 ? 3 : 2; + +// const tileX = Math.round(Math.floor((x - ms.x) / tileWidth + 1)); +// const tileY = Math.round(Math.floor((ms.y - y) / tileHeight + 1)); +// const tileId = `${`${tileY}`.padStart(nbDigits, '0')}${`${tileX}`.padStart(nbDigits, '0')}`; +// return `${ms.name}_${gridSize}_${tileId}`; +// }, +// }; + +// /** +// * 3518000.19287 +// 5085999.80449 +// */ + +// // const pt = [3522800.00752, 5085999.99359, 3525200.00752, 5089599.99359] as const; + +// console.log(MapSheetChathamIslands.getTileName(3518000.19287, 5085999.80449, 5000)); +// console.log(MapSheetChathamIslands.getMapTileIndex('SN8066_CI06_5000_0606.tfw')); From 4fe0c12381adcfa894f1678900e3d70a18a24f00 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 12:21:40 +1300 Subject: [PATCH 2/9] fix: replace imports correctly --- Dockerfile | 8 ++++---- package-lock.json | 11 +++++----- package.json | 2 +- .../__test__/create-pr-handler.test.ts | 12 +++++------ src/commands/basemaps-github/create-pr.ts | 2 +- .../basemaps-mapsheet/create-mapsheet.ts | 10 +++++----- src/commands/generate-path/__test__/sample.ts | 4 ++-- src/commands/generate-path/path.generate.ts | 10 +++++----- .../lds-fetch-layer/lds.fetch.layer.ts | 8 ++++---- .../__test__/mapsheet.coverage.test.ts | 6 +++--- .../mapsheet-coverage/mapsheet.coverage.ts | 20 +++++++++---------- src/commands/stac-catalog/stac.catalog.ts | 10 +++++----- .../stac-github-import/stac.github.import.ts | 16 +++++++-------- src/commands/stac-setup/__test__/sample.ts | 4 ++-- .../__test__/stac.validate.test.ts | 4 ++-- src/commands/stac-validate/stac.validate.ts | 14 ++++++------- 16 files changed, 71 insertions(+), 70 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ce559342..c68c3d5bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 +FROM node:23-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 RUN apt-get update && apt-get install openssh-client git -y @@ -16,9 +16,9 @@ ENV NODE_ENV production ADD package.json package-lock.json /app/ RUN npm install --omit=dev -ADD build/src /app/ +ADD src /app/ # Cache of copy of the STAC JSON schemas by triggering a validation run -RUN node /app/index.js stac-validate https://nz-imagery.s3-ap-southeast-2.amazonaws.com/new-zealand/new-zealand_2020-2021_10m/rgb/2193/collection.json +RUN node /app/index.ts stac-validate https://nz-imagery.s3-ap-southeast-2.amazonaws.com/new-zealand/new-zealand_2020-2021_10m/rgb/2193/collection.json -ENTRYPOINT ["node", "/app/index.js"] +ENTRYPOINT ["node", "/app/index.ts"] diff --git a/package-lock.json b/package-lock.json index 3d8668b0b..24c2caee5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,7 @@ "@linzjs/style": "^5.4.0", "@types/node": "^20.12.10", "@types/prettier": "^3.0.0", - "stac-ts": "^1.0.3" + "stac-ts": "^1.0.4" }, "engines": { "node": "^20.13.1" @@ -5987,10 +5987,11 @@ } }, "node_modules/stac-ts": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stac-ts/-/stac-ts-1.0.3.tgz", - "integrity": "sha512-QYqE6CFQ5XuWds3/Pi/07HGvWLMMQSesTcrylWF/Hp/ZjvBsSdDf0uokvHsz1iA/6JyZuvekfFlElRz4rsQR+w==", - "dev": true + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/stac-ts/-/stac-ts-1.0.4.tgz", + "integrity": "sha512-5n09H4bmBfrjIkjcDs60joV9zEX55GU29WTCNB++YzRawUZ1cCDG7a/M7bDDyR7VcQz9nguRUnRCz/JWdsFuBQ==", + "dev": true, + "license": "MIT" }, "node_modules/stream-browserify": { "version": "3.0.0", diff --git a/package.json b/package.json index 6acb0b1ef..1b4009a0c 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "@linzjs/style": "^5.4.0", "@types/node": "^20.12.10", "@types/prettier": "^3.0.0", - "stac-ts": "^1.0.3" + "stac-ts": "^1.0.4" }, "scripts": { "build": "tsc", diff --git a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts index 9fe6852f2..dd7c6178c 100644 --- a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts @@ -1,15 +1,15 @@ import assert from 'node:assert'; import { afterEach, it } from 'node:test'; -import { TileSetType } from "@basemaps/config/build/config/tile.set.ts"; +import { TileSetType } from '@basemaps/config/build/config/tile.set.ts'; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { StacVersion } from 'stac-ts'; +import type { StacVersion } from 'stac-ts'; -import { anySlug } from "../../../utils/__test__/slugify.test.ts"; -import { GithubApi } from "../../../utils/github.ts"; -import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from "../create-pr.ts"; -import { Category } from "../make.cog.github.ts"; +import { anySlug } from '../../../utils/__test__/slugify.test.ts'; +import { GithubApi } from '../../../utils/github.ts'; +import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from '../create-pr.ts'; +import { Category } from '../make.cog.github.ts'; const originalEnv = Object.assign({}, process.env); diff --git a/src/commands/basemaps-github/create-pr.ts b/src/commands/basemaps-github/create-pr.ts index 7eaf3e144..01f914e52 100644 --- a/src/commands/basemaps-github/create-pr.ts +++ b/src/commands/basemaps-github/create-pr.ts @@ -3,7 +3,7 @@ import { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; import { Epsg, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { boolean, command, flag, oneOf, option, optional, string } from 'cmd-ts'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index 31f4bb519..a308aa075 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -2,14 +2,14 @@ import { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } fro import { Bounds, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; -import * as fgb from "flatgeobuf/lib/mjs/geojson.ts"; -import { FeatureCollection, MultiPolygon } from 'geojson'; +import * as fgb from 'flatgeobuf/lib/mjs/geojson.js'; +import type { FeatureCollection, MultiPolygon } from 'geojson'; import { promisify } from 'util'; import { gunzip } from 'zlib'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { registerCli, verbose } from '../common.ts'; const gunzipProm = promisify(gunzip); diff --git a/src/commands/generate-path/__test__/sample.ts b/src/commands/generate-path/__test__/sample.ts index 8a976736a..813e488c3 100644 --- a/src/commands/generate-path/__test__/sample.ts +++ b/src/commands/generate-path/__test__/sample.ts @@ -1,6 +1,6 @@ -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from "../../../utils/metadata.ts"; +import { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index 17f3f3d47..3c312897f 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -2,12 +2,12 @@ import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { Tiff } from '@cogeotiff/core'; import { command, option, positional, string } from 'cmd-ts'; -import { StacCollection, StacItem } from 'stac-ts'; +import type { StacCollection, StacItem } from 'stac-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { GeospatialDataCategories, StacCollectionLinz } from "../../utils/metadata.ts"; -import { config, createTiff, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.ts'; +import { config, createTiff, registerCli, verbose } from '../common.ts'; export interface PathMetadata { targetBucketName: string; diff --git a/src/commands/lds-fetch-layer/lds.fetch.layer.ts b/src/commands/lds-fetch-layer/lds.fetch.layer.ts index ffadd1553..cd7b89632 100644 --- a/src/commands/lds-fetch-layer/lds.fetch.layer.ts +++ b/src/commands/lds-fetch-layer/lds.fetch.layer.ts @@ -1,11 +1,11 @@ import { fsa } from '@chunkd/fs'; import { command, option, restPositionals, string } from 'cmd-ts'; -import * as stac from 'stac-ts'; +import type * as stac from 'stac-ts'; import { createGunzip } from 'zlib'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { config, registerCli, verbose } from '../common.ts'; function getTargetPath(source: string, path: string): string { if (path.startsWith('./')) return fsa.join(source, path.slice(2)); diff --git a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts index ebb96552a..fb23e9ed8 100644 --- a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts +++ b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts @@ -5,10 +5,10 @@ import { Nztm2000QuadTms, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; import { featuresToMultiPolygon } from '@linzjs/geojson'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { MapSheet } from "../../../utils/mapsheet.ts"; -import { commandMapSheetCoverage, isLargeRegion } from "../mapsheet.coverage.ts"; +import { MapSheet } from '../../../utils/mapsheet.ts'; +import { commandMapSheetCoverage, isLargeRegion } from '../mapsheet.coverage.ts'; // convert a collection of map sheets into a multipolygon function mapSheetToGeoJson(...sheetCodes: string[]): GeoJSON.Feature { diff --git a/src/commands/mapsheet-coverage/mapsheet.coverage.ts b/src/commands/mapsheet-coverage/mapsheet.coverage.ts index 45e036831..581db3a58 100644 --- a/src/commands/mapsheet-coverage/mapsheet.coverage.ts +++ b/src/commands/mapsheet-coverage/mapsheet.coverage.ts @@ -1,3 +1,4 @@ +import { basename } from 'node:path/posix'; import { gzipSync } from 'node:zlib'; import { ConfigTileSetRaster } from '@basemaps/config'; @@ -7,17 +8,16 @@ import { Area, truncate } from '@linzjs/geojson'; import buffer from '@turf/buffer'; import { command, number, option, optional, string } from 'cmd-ts'; import pLimit from 'p-limit'; -import { basename } from "path/posix.ts"; import pc from 'polygon-clipping'; -import { StacCollection, StacItem } from 'stac-ts'; - -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { getPacificAucklandYearMonthDay } from "../../utils/date.ts"; -import { createFileList } from "../../utils/filelist.ts"; -import { hashStream } from "../../utils/hash.ts"; -import { MapSheet } from "../../utils/mapsheet.ts"; -import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from "../common.ts"; +import type { StacCollection, StacItem } from 'stac-ts'; + +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getPacificAucklandYearMonthDay } from '../../utils/date.ts'; +import { createFileList } from '../../utils/filelist.ts'; +import { hashStream } from '../../utils/hash.ts'; +import { MapSheet } from '../../utils/mapsheet.ts'; +import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from '../common.ts'; /** Datasets to skip */ const Skip = new Set([ diff --git a/src/commands/stac-catalog/stac.catalog.ts b/src/commands/stac-catalog/stac.catalog.ts index 068d034ee..bc496de4b 100644 --- a/src/commands/stac-catalog/stac.catalog.ts +++ b/src/commands/stac-catalog/stac.catalog.ts @@ -1,12 +1,12 @@ import { fsa } from '@chunkd/fs'; import { command, option, positional, string } from 'cmd-ts'; import { isAbsolute } from 'path'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { hashBuffer } from "../../utils/hash.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { hashBuffer } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; /** is a path a URL */ export function isUrl(path: string): boolean { diff --git a/src/commands/stac-github-import/stac.github.import.ts b/src/commands/stac-github-import/stac.github.import.ts index b15f70e95..7da852d58 100644 --- a/src/commands/stac-github-import/stac.github.import.ts +++ b/src/commands/stac-github-import/stac.github.import.ts @@ -1,13 +1,13 @@ import { fsa } from '@chunkd/fs'; import { command, oneOf, option, string, Type } from 'cmd-ts'; -import * as st from 'stac-ts'; - -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; -import { GithubApi } from "../../utils/github.ts"; -import { config, registerCli, verbose } from "../common.ts"; -import { prettyPrint } from "../pretty-print/pretty.print.ts"; +import type * as st from 'stac-ts'; + +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { GithubApi } from '../../utils/github.ts'; +import { config, registerCli, verbose } from '../common.ts'; +import { prettyPrint } from '../pretty-print/pretty.print.ts'; const Url: Type = { async from(str) { diff --git a/src/commands/stac-setup/__test__/sample.ts b/src/commands/stac-setup/__test__/sample.ts index 511eba333..fcae3c9c8 100644 --- a/src/commands/stac-setup/__test__/sample.ts +++ b/src/commands/stac-setup/__test__/sample.ts @@ -1,6 +1,6 @@ -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from "../../../utils/metadata.ts"; +import { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/stac-validate/__test__/stac.validate.test.ts b/src/commands/stac-validate/__test__/stac.validate.test.ts index 6c067e1ca..20908e8bb 100644 --- a/src/commands/stac-validate/__test__/stac.validate.test.ts +++ b/src/commands/stac-validate/__test__/stac.validate.test.ts @@ -3,7 +3,7 @@ import { before, beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; import { getStacSchemaUrl, @@ -13,7 +13,7 @@ import { validateAssets, validateLinks, validateStacChecksum, -} from "../stac.validate.ts"; +} from '../stac.validate.ts'; describe('stacValidate', function () { it('listLocation', () => { diff --git a/src/commands/stac-validate/stac.validate.ts b/src/commands/stac-validate/stac.validate.ts index c88180361..741cc357b 100644 --- a/src/commands/stac-validate/stac.validate.ts +++ b/src/commands/stac-validate/stac.validate.ts @@ -1,17 +1,17 @@ import { fsa } from '@chunkd/fs'; import Ajv, { DefinedError, ValidateFunction } from 'ajv'; -import { fastFormats } from "ajv-formats/dist/formats.ts"; +import { fastFormats } from 'ajv-formats/dist/formats.js'; import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; import { createHash } from 'crypto'; import { dirname, join } from 'path'; import { performance } from 'perf_hooks'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { ConcurrentQueue } from "../../utils/concurrent.queue.ts"; -import { hashStream, Sha256Prefix } from "../../utils/hash.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { ConcurrentQueue } from '../../utils/concurrent.queue.ts'; +import { hashStream, Sha256Prefix } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const commandStacValidate = command({ name: 'stac-validate', From 278a7193a135759e63b62dd9f35cdb5ed0a244f3 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 12:31:21 +1300 Subject: [PATCH 3/9] refactor: move to import type --- .eslintrc.cjs | 4 +- src/commands/basemaps-github/create-pr.ts | 4 +- .../basemaps-github/make.cog.github.ts | 55 +++++++++---------- .../basemaps-mapsheet/create-mapsheet.ts | 2 +- src/commands/common.ts | 11 ++-- src/commands/copy/copy.ts | 10 ++-- .../create-manifest/create-manifest.ts | 10 ++-- src/commands/generate-path/path.generate.ts | 3 +- .../tileindex-validate/tileindex.validate.ts | 22 ++++---- src/fs.register.ts | 6 +- src/utils/github.ts | 4 +- 11 files changed, 67 insertions(+), 64 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 328fcd0ac..70982d67a 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -7,6 +7,8 @@ testOverrides.rules['@typescript-eslint/no-floating-promises'] = 'off'; // Async functions are required in a few places without awaits eg `async foo() { throw Bar }` const tsOverrides = cfg.overrides.find((ovr) => ovr.files.find((f) => f.endsWith('*.ts'))); -tsOverrides.rules['@typescript-eslint/require-await'] = 'off' +tsOverrides.rules['@typescript-eslint/require-await'] = 'off'; + +tsOverrides.rules['@typescript-eslint/consistent-type-imports'] = 'error'; module.exports = cfg; diff --git a/src/commands/basemaps-github/create-pr.ts b/src/commands/basemaps-github/create-pr.ts index 01f914e52..dd0630909 100644 --- a/src/commands/basemaps-github/create-pr.ts +++ b/src/commands/basemaps-github/create-pr.ts @@ -1,5 +1,5 @@ import { standardizeLayerName } from '@basemaps/config'; -import { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; +import type { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; import { Epsg, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { boolean, command, flag, oneOf, option, optional, string } from 'cmd-ts'; @@ -8,7 +8,7 @@ import type { StacCollection } from 'stac-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; import { registerCli, verbose } from '../common.ts'; -import { Category, MakeCogGithub } from './make.cog.github.ts'; +import { MakeCogGithub } from './make.cog.github.ts'; export const ValidTargetBuckets: Set = new Set(['linz-basemaps', 'linz-basemaps-staging']); export const ValidSourceBuckets: Set = new Set(['nz-imagery', 'linz-imagery', 'nz-elevation']); diff --git a/src/commands/basemaps-github/make.cog.github.ts b/src/commands/basemaps-github/make.cog.github.ts index cfedc0fae..9b792b349 100644 --- a/src/commands/basemaps-github/make.cog.github.ts +++ b/src/commands/basemaps-github/make.cog.github.ts @@ -1,27 +1,26 @@ import { DefaultColorRampOutput, DefaultTerrainRgbOutput, standardizeLayerName } from '@basemaps/config'; -import { +import type { ConfigLayer, ConfigTileSet, ConfigTileSetRaster, ConfigTileSetVector, TileSetType, -} from "@basemaps/config/build/config/tile.set.ts"; +} from '@basemaps/config/build/config/tile.set.js'; import { fsa } from '@chunkd/fs'; -import { logger } from "../../log.ts"; -import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; -import { GithubApi } from "../../utils/github.ts"; -import { prettyPrint } from "../pretty-print/pretty.print.ts"; +import { logger } from '../../log.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { GithubApi } from '../../utils/github.ts'; +import { prettyPrint } from '../pretty-print/pretty.print.ts'; -export enum Category { - Urban = 'Urban Aerial Photos', - Rural = 'Rural Aerial Photos', - Satellite = 'Satellite Imagery', - Elevation = 'Elevation', - Event = 'Event', - Scanned = 'Scanned Aerial Imagery', - Other = 'New Aerial Photos', -} +export type Category = + | 'Urban Aerial Photos' + | 'Rural Aerial Photos' + | 'Satellite Imagery' + | 'Elevation' + | 'Event' + | 'Scanned Aerial Imagery' + | 'New Aerial Photos'; export interface CategorySetting { minZoom?: number; @@ -29,13 +28,13 @@ export interface CategorySetting { } export const DefaultCategorySetting: Record = { - [Category.Urban]: { minZoom: 14 }, - [Category.Rural]: { minZoom: 13 }, - [Category.Satellite]: { minZoom: 5 }, - [Category.Elevation]: { minZoom: 9 }, - [Category.Scanned]: { minZoom: 0, maxZoom: 32 }, - [Category.Other]: {}, - [Category.Event]: {}, + 'Urban Aerial Photos': { minZoom: 14 }, + ['Rural Aerial Photos']: { minZoom: 13 }, + ['Satellite Imagery']: { minZoom: 5 }, + ['Elevation']: { minZoom: 9 }, + ['Scanned Aerial Imagery']: { minZoom: 0, maxZoom: 32 }, + ['New Aerial Photos']: {}, + ['Event']: {}, }; const botEmail = 'basemaps@linz.govt.nz'; @@ -245,16 +244,16 @@ export class MakeCogGithub { this.setDefaultConfig(layer, category); // Set layer zoom level and add to latest order - if (category === Category.Rural) { + if (category === 'Rural Aerial Photos') { for (let i = 0; i < tileSet.layers.length; i++) { // Add new layer above the first Urban - if (tileSet.layers[i]?.category === Category.Urban) { + if (tileSet.layers[i]?.category === 'Urban Aerial Photos') { // Find first valid Urban and insert new record above that. tileSet.layers.splice(i, 0, layer); break; } } - } else if (category === Category.Other) { + } else if (category === 'New Aerial Photos') { // Add new layer at the bottom tileSet.layers.push(layer); } else { @@ -296,19 +295,19 @@ export class MakeCogGithub { async prepareElevationTileSetConfig(layer: ConfigLayer, tileSet?: ConfigTileSetRaster): Promise { if (tileSet == null) { // Prepare individual elevation tileset config - const targetLayer = { ...layer, Category: Category.Elevation, minZoom: 0, maxZoom: 32 }; + const targetLayer = { ...layer, Category: 'Elevation', minZoom: 0, maxZoom: 32 }; return { type: TileSetType.Raster, id: `ts_${layer.name}`, name: layer.name, title: layer.title, - category: Category.Elevation, + category: 'Elevation', layers: [targetLayer], outputs: [DefaultTerrainRgbOutput, DefaultColorRampOutput], }; } - this.setDefaultConfig(layer, Category.Elevation); + this.setDefaultConfig(layer, 'Elevation'); // Remove elevation layer.category if already exists if (layer.category) delete layer.category; diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index a308aa075..eba7cb3af 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -1,4 +1,4 @@ -import { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; +import type { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; import { Bounds, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; diff --git a/src/commands/common.ts b/src/commands/common.ts index 006b9d4f8..54d1bce05 100644 --- a/src/commands/common.ts +++ b/src/commands/common.ts @@ -1,13 +1,14 @@ import { fsa } from '@chunkd/fs'; import { Tiff } from '@cogeotiff/core'; -import { boolean, flag, option, optional, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { boolean, flag, option, optional, string } from 'cmd-ts'; import pLimit from 'p-limit'; import { fileURLToPath, pathToFileURL } from 'url'; -import { CliInfo } from "../cli.info.ts"; -import { registerFileSystem } from "../fs.register.ts"; -import { logger, registerLogger } from "../log.ts"; -import { isArgo } from "../utils/argo.ts"; +import { CliInfo } from '../cli.info.ts'; +import { registerFileSystem } from '../fs.register.ts'; +import { logger, registerLogger } from '../log.ts'; +import { isArgo } from '../utils/argo.ts'; export const config = option({ long: 'config', diff --git a/src/commands/copy/copy.ts b/src/commands/copy/copy.ts index 9ac333461..73e2e50d6 100644 --- a/src/commands/copy/copy.ts +++ b/src/commands/copy/copy.ts @@ -4,11 +4,11 @@ import { boolean, command, flag, number, option, restPositionals, string } from import { performance } from 'perf_hooks'; import * as z from 'zod'; -import { CliInfo } from "../../cli.info.ts"; -import { logger, logId } from "../../log.ts"; -import { ActionCopy } from "../../utils/actions.ts"; -import { config, registerCli, verbose } from "../common.ts"; -import { CopyContract } from "./copy-rpc.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger, logId } from '../../log.ts'; +import type { ActionCopy } from '../../utils/actions.ts'; +import { config, registerCli, verbose } from '../common.ts'; +import type { CopyContract } from './copy-rpc.ts'; const CopyValidator = z.object({ source: z.string(), target: z.string() }); const CopyManifest = z.array(CopyValidator); diff --git a/src/commands/create-manifest/create-manifest.ts b/src/commands/create-manifest/create-manifest.ts index 4e31cdf5b..e23bf1c32 100644 --- a/src/commands/create-manifest/create-manifest.ts +++ b/src/commands/create-manifest/create-manifest.ts @@ -4,11 +4,11 @@ import { createHash } from 'crypto'; import path from 'path'; import { gzipSync } from 'zlib'; -import { CliInfo } from "../../cli.info.ts"; -import { getActionLocation } from "../../utils/action.storage.ts"; -import { ActionCopy } from "../../utils/actions.ts"; -import { FileFilter, getFiles } from "../../utils/chunk.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { getActionLocation } from '../../utils/action.storage.ts'; +import type { ActionCopy } from '../../utils/actions.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const commandCreateManifest = command({ name: 'create-manifest', diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index 3c312897f..ecc6827e3 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -6,7 +6,8 @@ import type { StacCollection, StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; -import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.ts'; +import type { StacCollectionLinz } from '../../utils/metadata.ts'; +import { GeospatialDataCategories } from '../../utils/metadata.ts'; import { config, createTiff, registerCli, verbose } from '../common.ts'; export interface PathMetadata { diff --git a/src/commands/tileindex-validate/tileindex.validate.ts b/src/commands/tileindex-validate/tileindex.validate.ts index 8b5671c66..58ffc9906 100644 --- a/src/commands/tileindex-validate/tileindex.validate.ts +++ b/src/commands/tileindex-validate/tileindex.validate.ts @@ -1,19 +1,19 @@ import { Bounds, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { Size, Tiff, TiffTag } from '@cogeotiff/core'; -import { BBox } from '@linzjs/geojson'; +import type { BBox } from '@linzjs/geojson'; import { boolean, command, flag, number, option, optional, restPositionals, string, Type } from 'cmd-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { isArgo } from "../../utils/argo.ts"; -import { extractBandInformation } from "../../utils/band.ts"; -import { FileFilter, getFiles } from "../../utils/chunk.ts"; -import { createFileList } from "../../utils/filelist.ts"; -import { findBoundingBox } from "../../utils/geotiff.ts"; -import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from "../../utils/mapsheet.ts"; -import { config, createTiff, forceOutput, registerCli, verbose } from "../common.ts"; -import { CommandListArgs } from "../list/list.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { isArgo } from '../../utils/argo.ts'; +import { extractBandInformation } from '../../utils/band.ts'; +import { FileFilter, getFiles } from '../../utils/chunk.ts'; +import { createFileList } from '../../utils/filelist.ts'; +import { findBoundingBox } from '../../utils/geotiff.ts'; +import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.ts'; +import { config, createTiff, forceOutput, registerCli, verbose } from '../common.ts'; +import { CommandListArgs } from '../list/list.ts'; export function isTiff(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/fs.register.ts b/src/fs.register.ts index 1e08500d7..ceab27d84 100644 --- a/src/fs.register.ts +++ b/src/fs.register.ts @@ -1,12 +1,12 @@ -import { setTimeout } from "node:timers/promises.ts"; +import { setTimeout } from 'node:timers/promises'; import { S3Client } from '@aws-sdk/client-s3'; import { fsa } from '@chunkd/fs'; import { FsAwsS3 } from '@chunkd/source-aws'; import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3'; -import { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; +import type { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; -import { logger } from "./log.ts"; +import { logger } from './log.ts'; /** Check to see if hostname exists inside of a object */ function hasHostName(x: unknown): x is { hostname: string } { diff --git a/src/utils/github.ts b/src/utils/github.ts index f341141a4..e697fff4e 100644 --- a/src/utils/github.ts +++ b/src/utils/github.ts @@ -1,8 +1,8 @@ import { Octokit } from '@octokit/core'; import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'; -import { Api } from "@octokit/plugin-rest-endpoint-methods/dist-types/types.ts"; +import type { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; -import { logger } from "../log.ts"; +import { logger } from '../log.ts'; export interface Blob { path: string; From e5ec704685a5e554baca601d07b952cc21af115d Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 12:45:03 +1300 Subject: [PATCH 4/9] refactor: cleanup imports again --- src/__test__/eai_againRetry.test.ts | 4 +-- src/__test__/fqdn.test.ts | 4 +-- src/__test__/fs.test.ts | 11 +++--- .../__test__/create-pr-handler.test.ts | 9 +++-- .../__test__/create-pr.test.ts | 3 +- src/commands/basemaps-github/create-pr.ts | 32 +++++++++-------- .../basemaps-github/make.cog.github.ts | 29 +++++++++++----- .../__test__/create-mapsheet.test.ts | 7 ++-- .../basemaps-mapsheet/create-mapsheet.ts | 3 +- src/commands/copy/__test__/copy.test.ts | 2 +- src/commands/copy/copy-worker.ts | 16 ++++----- .../__test__/create-manifest.test.ts | 2 +- .../create-manifest/create-manifest.ts | 1 + .../__test__/generate.path.test.ts | 7 ++-- src/commands/generate-path/__test__/sample.ts | 2 +- src/commands/generate-path/path.generate.ts | 2 +- src/commands/group/__test__/group.test.ts | 2 +- src/commands/group/group.ts | 8 ++--- src/commands/index.ts | 34 +++++++++---------- src/commands/list/list.ts | 8 ++--- .../mapsheet-coverage/mapsheet.coverage.ts | 2 +- src/commands/pretty-print/pretty.print.ts | 10 +++--- .../__test__/stac.catalog.test.ts | 2 +- .../__test__/stac.link.test.ts | 2 +- .../stac-github-import/stac.github.import.ts | 3 +- src/commands/stac-setup/__test__/sample.ts | 2 +- .../stac-setup/__test__/stac.setup.test.ts | 9 ++--- src/commands/stac-setup/stac.setup.ts | 10 +++--- .../stac-sync/__test__/stac.sync.test.ts | 4 +-- src/commands/stac-sync/stac.sync.ts | 17 +++++----- src/commands/stac-validate/stac.validate.ts | 3 +- .../__test__/tileindex.validate.data.ts | 5 +-- .../__test__/tileindex.validate.test.ts | 21 ++++++------ .../tileindex-validate/tileindex.validate.ts | 12 ++++--- src/fs.register.ts | 3 +- src/index.ts | 4 +-- src/readme/readme.generate.ts | 6 ++-- src/utils/__test__/argo.test.ts | 2 +- src/utils/__test__/band.test.ts | 4 +-- src/utils/__test__/chunk.test.ts | 2 +- src/utils/__test__/config.test.ts | 2 +- src/utils/__test__/date.test.ts | 2 +- src/utils/__test__/filelist.test.ts | 5 +-- src/utils/__test__/filter.test.ts | 2 +- src/utils/__test__/geotiff.test.ts | 6 ++-- src/utils/__test__/hash.test.ts | 2 +- src/utils/__test__/mapsheet.test.ts | 5 +-- src/utils/__test__/slugify.test.ts | 2 +- src/utils/band.ts | 3 +- src/utils/chunk.ts | 4 +-- src/utils/concurrent.queue.ts | 3 +- src/utils/config.ts | 2 +- src/utils/filelist.ts | 2 +- src/utils/geotiff.ts | 5 +-- src/utils/hash.stream.ts | 6 ++-- src/utils/hash.ts | 5 +-- tsconfig.json | 4 ++- 57 files changed, 204 insertions(+), 165 deletions(-) diff --git a/src/__test__/eai_againRetry.test.ts b/src/__test__/eai_againRetry.test.ts index 74f7ba793..0842065dd 100644 --- a/src/__test__/eai_againRetry.test.ts +++ b/src/__test__/eai_againRetry.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { beforeEach, describe, it } from 'node:test'; -import { BuildHandler, MetadataBearer } from '@smithy/types'; +import type { BuildHandler, MetadataBearer } from '@smithy/types'; -import { eaiAgainBuilder } from "../fs.register.ts"; +import { eaiAgainBuilder } from '../fs.register.ts'; let callCount = 0; diff --git a/src/__test__/fqdn.test.ts b/src/__test__/fqdn.test.ts index ee3c80a37..feee5268f 100644 --- a/src/__test__/fqdn.test.ts +++ b/src/__test__/fqdn.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { FinalizeHandler, MetadataBearer } from '@smithy/types'; +import type { FinalizeHandler, MetadataBearer } from '@smithy/types'; -import { fqdn } from "../fs.register.ts"; +import { fqdn } from '../fs.register.ts'; describe('fqdnMiddleware', () => { const fakeNext: FinalizeHandler = () => { diff --git a/src/__test__/fs.test.ts b/src/__test__/fs.test.ts index f8d8eb7a1..39f7854e2 100644 --- a/src/__test__/fs.test.ts +++ b/src/__test__/fs.test.ts @@ -1,14 +1,15 @@ import { after, before, beforeEach, describe, it } from 'node:test'; import { CompositeError } from '@chunkd/core'; -import { FileSystemAbstraction, fsa } from '@chunkd/fs'; -import { FsAwsS3 } from '@chunkd/source-aws'; -import { S3LikeV3 } from '@chunkd/source-aws-v3'; +import type { FileSystemAbstraction } from '@chunkd/fs'; +import { fsa } from '@chunkd/fs'; +import type { FsAwsS3 } from '@chunkd/source-aws'; +import type { S3LikeV3 } from '@chunkd/source-aws-v3'; import { FsMemory } from '@chunkd/source-memory'; -import { InitializeMiddleware, MetadataBearer } from '@smithy/types'; +import type { InitializeMiddleware, MetadataBearer } from '@smithy/types'; import assert from 'assert'; -import { registerFileSystem } from "../fs.register.ts"; +import { registerFileSystem } from '../fs.register.ts'; export class HttpError extends Error { statusCode: number; diff --git a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts index dd7c6178c..307ceaa58 100644 --- a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts @@ -1,15 +1,14 @@ import assert from 'node:assert'; import { afterEach, it } from 'node:test'; -import { TileSetType } from '@basemaps/config/build/config/tile.set.ts'; +import { TileSetType } from '@basemaps/config/build/config/tile.set.js'; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import type { StacVersion } from 'stac-ts'; import { anySlug } from '../../../utils/__test__/slugify.test.ts'; import { GithubApi } from '../../../utils/github.ts'; -import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from '../create-pr.ts'; -import { Category } from '../make.cog.github.ts'; +import { basemapsCreatePullRequest, LinzBasemapsSourceCollectionRel } from '../create-pr.ts'; const originalEnv = Object.assign({}, process.env); @@ -62,8 +61,8 @@ await it('basemapsCreatePullRequest.handler should handle S3 target', async (t) target: targetUrlsString, repository: 'any-owner/any-repository', verbose: false, - category: Category.Satellite, - configType: ConfigType.Raster, + category: 'Satellite Imagery', + configType: 'raster', individual: false, vector: false, ticket: 'any ticket', diff --git a/src/commands/basemaps-github/__test__/create-pr.test.ts b/src/commands/basemaps-github/__test__/create-pr.test.ts index 8a35e744a..2aaf58c4f 100644 --- a/src/commands/basemaps-github/__test__/create-pr.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { parseTargetUrl, targetInfo } from "../create-pr.ts"; +import type { targetInfo } from '../create-pr.ts'; +import { parseTargetUrl } from '../create-pr.ts'; describe('parseTargetUrl', () => { it('Should parse the correct target vector Urls', async () => { diff --git a/src/commands/basemaps-github/create-pr.ts b/src/commands/basemaps-github/create-pr.ts index dd0630909..c2e9b7ee6 100644 --- a/src/commands/basemaps-github/create-pr.ts +++ b/src/commands/basemaps-github/create-pr.ts @@ -1,6 +1,7 @@ import { standardizeLayerName } from '@basemaps/config'; import type { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; -import { Epsg, EpsgCode } from '@basemaps/geo'; +import type { EpsgCode } from '@basemaps/geo'; +import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { boolean, command, flag, oneOf, option, optional, string } from 'cmd-ts'; import type { StacCollection } from 'stac-ts'; @@ -8,14 +9,15 @@ import type { StacCollection } from 'stac-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; import { registerCli, verbose } from '../common.ts'; -import { MakeCogGithub } from './make.cog.github.ts'; +import { Categories, MakeCogGithub } from './make.cog.github.ts'; export const ValidTargetBuckets: Set = new Set(['linz-basemaps', 'linz-basemaps-staging']); export const ValidSourceBuckets: Set = new Set(['nz-imagery', 'linz-imagery', 'nz-elevation']); export const LinzBasemapsSourceCollectionRel = 'linz_basemaps:source_collection'; -export type ConfigType = 'raster' | 'vector' | 'elevation'; +export const ConfigTypes = ['raster', 'elevation', 'vector'] as const; +export type ConfigType = (typeof ConfigTypes)[number]; export function assertValidBucket(bucket: string, validBuckets: Set): void { // Validate the target information @@ -198,9 +200,9 @@ export const CommandCreatePRArgs = { description: 'New layers locations as array of strings import into basemaps-config', }), category: option({ - type: optional(oneOf(Object.values(Category))), + type: optional(oneOf(Object.values(Categories))), long: 'category', - description: [...Object.values(Category)].join(', '), + description: [...Object.values(Categories)].join(', '), }), repository: option({ type: string, @@ -210,10 +212,10 @@ export const CommandCreatePRArgs = { defaultValueIsSerializable: true, }), configType: option({ - type: optional(oneOf(Object.values(ConfigType))), + type: optional(oneOf(ConfigTypes)), long: 'config-type', - description: `Basemaps config file type, includes ${[...Object.values(ConfigType)].join(', ')}`, - defaultValue: () => ConfigType.Raster, + description: `Basemaps config file type, includes ${ConfigTypes.join(', ')}`, + defaultValue: () => 'raster' as const, }), individual: flag({ type: boolean, @@ -242,7 +244,7 @@ export const basemapsCreatePullRequest = command({ async handler(args) { registerCli(this, args); const target = args.target; - const category = args.category ?? Category.Other; + const category = args.category ?? Categories.Other; let targets: string[]; try { targets = JSON.parse(target) as string[]; @@ -252,15 +254,15 @@ export const basemapsCreatePullRequest = command({ const layer: ConfigLayer = { name: '', title: '' }; let region; - const configType = args.vector ? ConfigType.Vector : args.configType; - if (configType === ConfigType.Vector) { + const configType = args.vector ? 'vector' : args.configType; + if (configType === 'vector') { for (const target of targets) { const info = await parseVectorTargetInfo(target); layer.name = info.name; layer.title = info.title; layer[info.epsg] = target; } - } else if (configType === ConfigType.Elevation) { + } else if (configType === 'elevation') { for (const target of targets) { const info = await parseElevationTargetInfo(target, args.individual); layer.name = info.name; @@ -283,11 +285,11 @@ export const basemapsCreatePullRequest = command({ if (layer.name === '' || layer.title === '') throw new Error('Failed to find the imagery name or title.'); const git = new MakeCogGithub(layer.name, args.repository, args.ticket); - if (configType === ConfigType.Vector) { + if (configType === 'vector') { await git.updateVectorTileSet(layer.name, layer, args.individual); - } else if (configType === ConfigType.Raster) { + } else if (configType === 'raster') { await git.updateRasterTileSet(layer.name, layer, category, args.individual, region); - } else if (configType === ConfigType.Elevation) { + } else if (configType === 'elevation') { await git.updateElevationTileSet(layer.name, layer, args.individual, region); } else throw new Error(`Invalid Config File target: ${configType}`); }, diff --git a/src/commands/basemaps-github/make.cog.github.ts b/src/commands/basemaps-github/make.cog.github.ts index 9b792b349..ce7a938c4 100644 --- a/src/commands/basemaps-github/make.cog.github.ts +++ b/src/commands/basemaps-github/make.cog.github.ts @@ -4,8 +4,8 @@ import type { ConfigTileSet, ConfigTileSetRaster, ConfigTileSetVector, - TileSetType, } from '@basemaps/config/build/config/tile.set.js'; +import { TileSetType } from '@basemaps/config/build/config/tile.set.js'; import { fsa } from '@chunkd/fs'; import { logger } from '../../log.ts'; @@ -13,14 +13,25 @@ import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; import { GithubApi } from '../../utils/github.ts'; import { prettyPrint } from '../pretty-print/pretty.print.ts'; -export type Category = - | 'Urban Aerial Photos' - | 'Rural Aerial Photos' - | 'Satellite Imagery' - | 'Elevation' - | 'Event' - | 'Scanned Aerial Imagery' - | 'New Aerial Photos'; +export const Categories = { + Urban: 'Urban Aerial Photos', + Rural: 'Rural Aerial Photos', + Satellite: 'Satellite Imagery', + Elevation: 'Elevation', + Event: 'Event', + Scanned: 'Scanned Aerial Imagery', + Other: 'New Aerial Photos', +} as const; + +export type Category = (typeof Categories)[keyof typeof Categories]; + +// | 'Urban Aerial Photos' +// | 'Rural Aerial Photos' +// | 'Satellite Imagery' +// | 'Elevation' +// | 'Event' +// | 'Scanned Aerial Imagery' +// | 'New Aerial Photos'; export interface CategorySetting { minZoom?: number; diff --git a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts index 517bf0fda..666878b87 100644 --- a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts +++ b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts @@ -1,10 +1,11 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; -import { FeatureCollection } from 'geojson'; +import type { ConfigImagery, ConfigTileSet } from '@basemaps/config'; +import { ConfigProviderMemory } from '@basemaps/config'; +import type { FeatureCollection } from 'geojson'; -import { createMapSheet } from "../create-mapsheet.ts"; +import { createMapSheet } from '../create-mapsheet.ts'; describe('copyFiles', () => { const rest: FeatureCollection = { diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index eba7cb3af..fde04678e 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -1,4 +1,5 @@ -import type { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; +import type { ConfigBundled, ConfigImagery, ConfigTileSet } from '@basemaps/config'; +import { ConfigProviderMemory } from '@basemaps/config'; import { Bounds, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; diff --git a/src/commands/copy/__test__/copy.test.ts b/src/commands/copy/__test__/copy.test.ts index c11042811..9770e531b 100644 --- a/src/commands/copy/__test__/copy.test.ts +++ b/src/commands/copy/__test__/copy.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { worker } from "../copy-worker.ts"; +import { worker } from '../copy-worker.ts'; describe('copyFiles', () => { const memory = new FsMemory(); diff --git a/src/commands/copy/copy-worker.ts b/src/commands/copy/copy-worker.ts index 19ce23eb5..3600f1f3a 100644 --- a/src/commands/copy/copy-worker.ts +++ b/src/commands/copy/copy-worker.ts @@ -1,17 +1,17 @@ import { performance } from 'node:perf_hooks'; import { parentPort, threadId } from 'node:worker_threads'; -import { FileInfo } from '@chunkd/core'; +import type { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; import { WorkerRpc } from '@wtrpc/core'; -import { logger } from "../../log.ts"; -import { ConcurrentQueue } from "../../utils/concurrent.queue.ts"; -import { HashKey, hashStream } from "../../utils/hash.ts"; -import { HashTransform } from "../../utils/hash.stream.ts"; -import { registerCli } from "../common.ts"; -import { isTiff } from "../tileindex-validate/tileindex.validate.ts"; -import { CopyContract, CopyContractArgs, CopyStats } from "./copy-rpc.ts"; +import { logger } from '../../log.ts'; +import { ConcurrentQueue } from '../../utils/concurrent.queue.ts'; +import { HashTransform } from '../../utils/hash.stream.ts'; +import { HashKey, hashStream } from '../../utils/hash.ts'; +import { registerCli } from '../common.ts'; +import { isTiff } from '../tileindex-validate/tileindex.validate.ts'; +import type { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.ts'; const Q = new ConcurrentQueue(10); diff --git a/src/commands/create-manifest/__test__/create-manifest.test.ts b/src/commands/create-manifest/__test__/create-manifest.test.ts index fb8dc726a..d864b76cd 100644 --- a/src/commands/create-manifest/__test__/create-manifest.test.ts +++ b/src/commands/create-manifest/__test__/create-manifest.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createManifest, validatePaths } from "../create-manifest.ts"; +import { createManifest, validatePaths } from '../create-manifest.ts'; describe('createManifest', () => { beforeEach(() => { diff --git a/src/commands/create-manifest/create-manifest.ts b/src/commands/create-manifest/create-manifest.ts index e23bf1c32..d273217e6 100644 --- a/src/commands/create-manifest/create-manifest.ts +++ b/src/commands/create-manifest/create-manifest.ts @@ -7,6 +7,7 @@ import { gzipSync } from 'zlib'; import { CliInfo } from '../../cli.info.ts'; import { getActionLocation } from '../../utils/action.storage.ts'; import type { ActionCopy } from '../../utils/actions.ts'; +import type { FileFilter } from '../../utils/chunk.ts'; import { getFiles } from '../../utils/chunk.ts'; import { config, registerCli, verbose } from '../common.ts'; diff --git a/src/commands/generate-path/__test__/generate.path.test.ts b/src/commands/generate-path/__test__/generate.path.test.ts index 6ed923d6c..a76239415 100644 --- a/src/commands/generate-path/__test__/generate.path.test.ts +++ b/src/commands/generate-path/__test__/generate.path.test.ts @@ -1,9 +1,10 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { SampleCollection } from "../../generate-path/__test__/sample.ts"; -import { FakeCogTiff } from "../../tileindex-validate/__test__/tileindex.validate.data.ts"; -import { extractEpsg, extractGsd, generatePath, PathMetadata } from "../path.generate.ts"; +import { SampleCollection } from '../../generate-path/__test__/sample.ts'; +import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.ts'; +import type { PathMetadata } from '../path.generate.ts'; +import { extractEpsg, extractGsd, generatePath } from '../path.generate.ts'; describe('GeneratePathImagery', () => { it('Should match - urban aerial from slug', () => { diff --git a/src/commands/generate-path/__test__/sample.ts b/src/commands/generate-path/__test__/sample.ts index 813e488c3..041b1ba60 100644 --- a/src/commands/generate-path/__test__/sample.ts +++ b/src/commands/generate-path/__test__/sample.ts @@ -1,6 +1,6 @@ import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.ts'; +import type { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index ecc6827e3..45143e3b3 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -1,6 +1,6 @@ import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { Tiff } from '@cogeotiff/core'; +import type { Tiff } from '@cogeotiff/core'; import { command, option, positional, string } from 'cmd-ts'; import type { StacCollection, StacItem } from 'stac-ts'; diff --git a/src/commands/group/__test__/group.test.ts b/src/commands/group/__test__/group.test.ts index 1f8334a83..e15126c9b 100644 --- a/src/commands/group/__test__/group.test.ts +++ b/src/commands/group/__test__/group.test.ts @@ -4,7 +4,7 @@ import { before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { commandGroup, groupItems } from "../group.ts"; +import { commandGroup, groupItems } from '../group.ts'; describe('groupItems', () => { it('should group items', () => { diff --git a/src/commands/group/group.ts b/src/commands/group/group.ts index af53df0e4..b5890edf3 100644 --- a/src/commands/group/group.ts +++ b/src/commands/group/group.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { isArgo } from "../../utils/argo.ts"; -import { config, forceOutput, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { isArgo } from '../../utils/argo.ts'; +import { config, forceOutput, registerCli, verbose } from '../common.ts'; /** Chunk an array into a group size * @example diff --git a/src/commands/index.ts b/src/commands/index.ts index 5f8f2fa06..1ef2735e3 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,22 +1,22 @@ import { subcommands } from 'cmd-ts'; -import { CliInfo } from "../cli.info.ts"; -import { basemapsCreatePullRequest } from "./basemaps-github/create-pr.ts"; -import { basemapsCreateMapSheet } from "./basemaps-mapsheet/create-mapsheet.ts"; -import { commandCopy } from "./copy/copy.ts"; -import { commandCreateManifest } from "./create-manifest/create-manifest.ts"; -import { commandGeneratePath } from "./generate-path/path.generate.ts"; -import { commandGroup } from "./group/group.ts"; -import { commandLdsFetch } from "./lds-fetch-layer/lds.fetch.layer.ts"; -import { commandList } from "./list/list.ts"; -import { commandMapSheetCoverage } from "./mapsheet-coverage/mapsheet.coverage.ts"; -import { commandPrettyPrint } from "./pretty-print/pretty.print.ts"; -import { commandStacCatalog } from "./stac-catalog/stac.catalog.ts"; -import { commandStacGithubImport } from "./stac-github-import/stac.github.import.ts"; -import { commandStacSetup } from "./stac-setup/stac.setup.ts"; -import { commandStacSync } from "./stac-sync/stac.sync.ts"; -import { commandStacValidate } from "./stac-validate/stac.validate.ts"; -import { commandTileIndexValidate } from "./tileindex-validate/tileindex.validate.ts"; +import { CliInfo } from '../cli.info.ts'; +import { basemapsCreatePullRequest } from './basemaps-github/create-pr.ts'; +import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.ts'; +import { commandCopy } from './copy/copy.ts'; +import { commandCreateManifest } from './create-manifest/create-manifest.ts'; +import { commandGeneratePath } from './generate-path/path.generate.ts'; +import { commandGroup } from './group/group.ts'; +import { commandLdsFetch } from './lds-fetch-layer/lds.fetch.layer.ts'; +import { commandList } from './list/list.ts'; +import { commandMapSheetCoverage } from './mapsheet-coverage/mapsheet.coverage.ts'; +import { commandPrettyPrint } from './pretty-print/pretty.print.ts'; +import { commandStacCatalog } from './stac-catalog/stac.catalog.ts'; +import { commandStacGithubImport } from './stac-github-import/stac.github.import.ts'; +import { commandStacSetup } from './stac-setup/stac.setup.ts'; +import { commandStacSync } from './stac-sync/stac.sync.ts'; +import { commandStacValidate } from './stac-validate/stac.validate.ts'; +import { commandTileIndexValidate } from './tileindex-validate/tileindex.validate.ts'; export const AllCommands = { copy: commandCopy, diff --git a/src/commands/list/list.ts b/src/commands/list/list.ts index 9a8f6a6e5..926d00893 100644 --- a/src/commands/list/list.ts +++ b/src/commands/list/list.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { getFiles } from "../../utils/chunk.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const CommandListArgs = { config, diff --git a/src/commands/mapsheet-coverage/mapsheet.coverage.ts b/src/commands/mapsheet-coverage/mapsheet.coverage.ts index 581db3a58..7f10e609c 100644 --- a/src/commands/mapsheet-coverage/mapsheet.coverage.ts +++ b/src/commands/mapsheet-coverage/mapsheet.coverage.ts @@ -1,7 +1,7 @@ import { basename } from 'node:path/posix'; import { gzipSync } from 'node:zlib'; -import { ConfigTileSetRaster } from '@basemaps/config'; +import type { ConfigTileSetRaster } from '@basemaps/config'; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { Area, truncate } from '@linzjs/geojson'; diff --git a/src/commands/pretty-print/pretty.print.ts b/src/commands/pretty-print/pretty.print.ts index 14ba0c934..b3e1a286a 100644 --- a/src/commands/pretty-print/pretty.print.ts +++ b/src/commands/pretty-print/pretty.print.ts @@ -3,11 +3,11 @@ import { command, option, optional, positional, string } from 'cmd-ts'; import { basename } from 'path'; import prettier from 'prettier'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { getFiles } from "../../utils/chunk.ts"; -import { DEFAULT_PRETTIER_FORMAT } from "../../utils/config.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { config, registerCli, verbose } from '../common.ts'; function isJson(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/commands/stac-catalog/__test__/stac.catalog.test.ts b/src/commands/stac-catalog/__test__/stac.catalog.test.ts index 82769c4c9..f916b1b9a 100644 --- a/src/commands/stac-catalog/__test__/stac.catalog.test.ts +++ b/src/commands/stac-catalog/__test__/stac.catalog.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createLinks, makeRelative } from "../stac.catalog.ts"; +import { createLinks, makeRelative } from '../stac.catalog.ts'; describe('stacCatalog', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-github-import/__test__/stac.link.test.ts b/src/commands/stac-github-import/__test__/stac.link.test.ts index 9aa9a6be2..fd39880b8 100644 --- a/src/commands/stac-github-import/__test__/stac.link.test.ts +++ b/src/commands/stac-github-import/__test__/stac.link.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { sortLinks } from "../stac.github.import.ts"; +import { sortLinks } from '../stac.github.import.ts'; function shuffle(array: T[]): T[] { let currentIndex = array.length; diff --git a/src/commands/stac-github-import/stac.github.import.ts b/src/commands/stac-github-import/stac.github.import.ts index 7da852d58..e4220694f 100644 --- a/src/commands/stac-github-import/stac.github.import.ts +++ b/src/commands/stac-github-import/stac.github.import.ts @@ -1,5 +1,6 @@ import { fsa } from '@chunkd/fs'; -import { command, oneOf, option, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { command, oneOf, option, string } from 'cmd-ts'; import type * as st from 'stac-ts'; import { CliInfo } from '../../cli.info.ts'; diff --git a/src/commands/stac-setup/__test__/sample.ts b/src/commands/stac-setup/__test__/sample.ts index fcae3c9c8..1ea87ffbe 100644 --- a/src/commands/stac-setup/__test__/sample.ts +++ b/src/commands/stac-setup/__test__/sample.ts @@ -1,6 +1,6 @@ import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.ts'; +import type { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/stac-setup/__test__/stac.setup.test.ts b/src/commands/stac-setup/__test__/stac.setup.test.ts index 75217a597..987385651 100644 --- a/src/commands/stac-setup/__test__/stac.setup.test.ts +++ b/src/commands/stac-setup/__test__/stac.setup.test.ts @@ -4,10 +4,11 @@ import { afterEach, before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { GeospatialDataCategory } from "../../../utils/metadata.ts"; -import { MeterAsString } from "../../common.ts"; -import { commandStacSetup, formatDate, slugFromMetadata, SlugMetadata } from "../stac.setup.ts"; -import { SampleCollection } from "./sample.ts"; +import type { GeospatialDataCategory } from '../../../utils/metadata.ts'; +import { MeterAsString } from '../../common.ts'; +import type { SlugMetadata } from '../stac.setup.ts'; +import { commandStacSetup, formatDate, slugFromMetadata } from '../stac.setup.ts'; +import { SampleCollection } from './sample.ts'; describe('stac-setup', () => { const mem = new FsMemory(); diff --git a/src/commands/stac-setup/stac.setup.ts b/src/commands/stac-setup/stac.setup.ts index 10f16c15d..163f53bf9 100644 --- a/src/commands/stac-setup/stac.setup.ts +++ b/src/commands/stac-setup/stac.setup.ts @@ -3,11 +3,11 @@ import { command, option, optional, string } from 'cmd-ts'; import type { StacCollection } from 'stac-ts'; import ulid from 'ulid'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { GeospatialDataCategory, StacCollectionLinz } from "../../utils/metadata.ts"; -import { slugify } from "../../utils/slugify.ts"; -import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import type { GeospatialDataCategory, StacCollectionLinz } from '../../utils/metadata.ts'; +import { slugify } from '../../utils/slugify.ts'; +import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from '../common.ts'; export interface SlugMetadata { geospatialCategory: GeospatialDataCategory; diff --git a/src/commands/stac-sync/__test__/stac.sync.test.ts b/src/commands/stac-sync/__test__/stac.sync.test.ts index 3a796602f..f40c05bed 100644 --- a/src/commands/stac-sync/__test__/stac.sync.test.ts +++ b/src/commands/stac-sync/__test__/stac.sync.test.ts @@ -4,8 +4,8 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { hashBuffer, HashKey } from "../../../utils/hash.ts"; -import { synchroniseFiles } from "../stac.sync.ts"; +import { hashBuffer, HashKey } from '../../../utils/hash.ts'; +import { synchroniseFiles } from '../stac.sync.ts'; describe('stacSync', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-sync/stac.sync.ts b/src/commands/stac-sync/stac.sync.ts index 97753f50b..a97a12bb8 100644 --- a/src/commands/stac-sync/stac.sync.ts +++ b/src/commands/stac-sync/stac.sync.ts @@ -1,13 +1,14 @@ -import { FileInfo } from '@chunkd/core'; +import type { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; -import { command, positional, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { command, positional, string } from 'cmd-ts'; -import { CliInfo } from "../../cli.info.ts"; -import { logger } from "../../log.ts"; -import { md } from "../../readme/markdown.ts"; -import { annotateExample } from "../../readme/readme.example.ts"; -import { hashBuffer, HashKey } from "../../utils/hash.ts"; -import { config, registerCli, verbose } from "../common.ts"; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { md } from '../../readme/markdown.ts'; +import { annotateExample } from '../../readme/readme.example.ts'; +import { hashBuffer, HashKey } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; const S3Path: Type = { async from(str) { diff --git a/src/commands/stac-validate/stac.validate.ts b/src/commands/stac-validate/stac.validate.ts index 741cc357b..1b242c88f 100644 --- a/src/commands/stac-validate/stac.validate.ts +++ b/src/commands/stac-validate/stac.validate.ts @@ -1,5 +1,6 @@ import { fsa } from '@chunkd/fs'; -import Ajv, { DefinedError, ValidateFunction } from 'ajv'; +import type { DefinedError, ValidateFunction } from 'ajv'; +import Ajv from 'ajv'; import { fastFormats } from 'ajv-formats/dist/formats.js'; import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; import { createHash } from 'crypto'; diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts index aeed5afe8..5a65f15ba 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts @@ -1,6 +1,7 @@ -import { SampleFormat, Size, Source, Tiff, TiffImage, TiffTag } from '@cogeotiff/core'; +import type { Size, Source, TiffImage } from '@cogeotiff/core'; +import { SampleFormat, Tiff, TiffTag } from '@cogeotiff/core'; -import { MapSheet } from "../../../utils/mapsheet.ts"; +import { MapSheet } from '../../../utils/mapsheet.ts'; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts index d97138a9d..ca6cec2f1 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts @@ -4,14 +4,15 @@ import { before, beforeEach, describe, it } from 'node:test'; import { Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { BBox } from '@linzjs/geojson'; -import { FeatureCollection } from 'geojson'; - -import { logger } from "../../../log.ts"; -import { MapSheetData } from "../../../utils/__test__/mapsheet.data.ts"; -import { FileListEntry } from "../../../utils/filelist.ts"; -import { GridSize, MapSheet } from "../../../utils/mapsheet.ts"; -import { createTiff } from "../../common.ts"; +import type { BBox } from '@linzjs/geojson'; +import type { FeatureCollection } from 'geojson'; + +import { logger } from '../../../log.ts'; +import { MapSheetData } from '../../../utils/__test__/mapsheet.data.ts'; +import type { FileListEntry } from '../../../utils/filelist.ts'; +import type { GridSize } from '../../../utils/mapsheet.ts'; +import { MapSheet } from '../../../utils/mapsheet.ts'; +import { createTiff } from '../../common.ts'; import { commandTileIndexValidate, extractTiffLocations, @@ -22,8 +23,8 @@ import { TiffLoader, validate8BitsTiff, validatePreset, -} from "../tileindex.validate.ts"; -import { FakeCogTiff } from "./tileindex.validate.data.ts"; +} from '../tileindex.validate.ts'; +import { FakeCogTiff } from './tileindex.validate.data.ts'; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/tileindex.validate.ts b/src/commands/tileindex-validate/tileindex.validate.ts index 58ffc9906..bc40907c2 100644 --- a/src/commands/tileindex-validate/tileindex.validate.ts +++ b/src/commands/tileindex-validate/tileindex.validate.ts @@ -1,17 +1,21 @@ import { Bounds, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { Size, Tiff, TiffTag } from '@cogeotiff/core'; +import type { Size, Tiff } from '@cogeotiff/core'; +import { TiffTag } from '@cogeotiff/core'; import type { BBox } from '@linzjs/geojson'; -import { boolean, command, flag, number, option, optional, restPositionals, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { boolean, command, flag, number, option, optional, restPositionals, string } from 'cmd-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; import { isArgo } from '../../utils/argo.ts'; import { extractBandInformation } from '../../utils/band.ts'; -import { FileFilter, getFiles } from '../../utils/chunk.ts'; +import type { FileFilter } from '../../utils/chunk.ts'; +import { getFiles } from '../../utils/chunk.ts'; import { createFileList } from '../../utils/filelist.ts'; import { findBoundingBox } from '../../utils/geotiff.ts'; -import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.ts'; +import type { GridSize } from '../../utils/mapsheet.ts'; +import { GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.ts'; import { config, createTiff, forceOutput, registerCli, verbose } from '../common.ts'; import { CommandListArgs } from '../list/list.ts'; diff --git a/src/fs.register.ts b/src/fs.register.ts index ceab27d84..b4d390a48 100644 --- a/src/fs.register.ts +++ b/src/fs.register.ts @@ -3,7 +3,8 @@ import { setTimeout } from 'node:timers/promises'; import { S3Client } from '@aws-sdk/client-s3'; import { fsa } from '@chunkd/fs'; import { FsAwsS3 } from '@chunkd/source-aws'; -import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3'; +import type { S3LikeV3 } from '@chunkd/source-aws-v3'; +import { FsAwsS3V3 } from '@chunkd/source-aws-v3'; import type { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; import { logger } from './log.ts'; diff --git a/src/index.ts b/src/index.ts index c427613c5..75905afe2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,8 +3,8 @@ process.env['AWS_NODEJS_CONNECTION_REUSE_ENABLED'] = '1'; import { run } from 'cmd-ts'; -import { cmd } from "./commands/index.ts"; -import { logger } from "./log.ts"; +import { cmd } from './commands/index.ts'; +import { logger } from './log.ts'; const startTime = performance.now(); run(cmd, process.argv.slice(2)) diff --git a/src/readme/readme.generate.ts b/src/readme/readme.generate.ts index 16005fa69..539325d87 100644 --- a/src/readme/readme.generate.ts +++ b/src/readme/readme.generate.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; -import type { HelpTopic, ProvidesHelp } from "cmd-ts/dist/cjs/helpdoc.ts"; +import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.ts'; import { writeFileSync } from 'fs'; import * as prettier from 'prettier'; -import { AllCommands } from "../commands/index.ts"; -import { commandHasExample, ExampleSymbol } from "./readme.example.ts"; +import { AllCommands } from '../commands/index.ts'; +import { commandHasExample, ExampleSymbol } from './readme.example.ts'; const AnsiRemove = /\u001b\[.*?m/g; function hasHelp(f: unknown): f is ProvidesHelp { diff --git a/src/utils/__test__/argo.test.ts b/src/utils/__test__/argo.test.ts index 7d3f985db..9d7f3a0e0 100644 --- a/src/utils/__test__/argo.test.ts +++ b/src/utils/__test__/argo.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { beforeEach, describe, it } from 'node:test'; -import { getActionLocation } from "../action.storage.ts"; +import { getActionLocation } from '../action.storage.ts'; describe('argoLocation', () => { beforeEach(() => { diff --git a/src/utils/__test__/band.test.ts b/src/utils/__test__/band.test.ts index 42351d2f0..1c294c7bb 100644 --- a/src/utils/__test__/band.test.ts +++ b/src/utils/__test__/band.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { createTiff } from "../../commands/common.ts"; -import { extractBandInformation } from "../band.ts"; +import { createTiff } from '../../commands/common.ts'; +import { extractBandInformation } from '../band.ts'; describe('extractBandInformation', () => { it('should extract basic band information (8-bit)', async () => { diff --git a/src/utils/__test__/chunk.test.ts b/src/utils/__test__/chunk.test.ts index 3aa3b4aca..cf7bfe48a 100644 --- a/src/utils/__test__/chunk.test.ts +++ b/src/utils/__test__/chunk.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { getFiles, splitPaths } from "../chunk.ts"; +import { getFiles, splitPaths } from '../chunk.ts'; describe('splitPaths', () => { it('should split on ;', () => { diff --git a/src/utils/__test__/config.test.ts b/src/utils/__test__/config.test.ts index 8bbf991d6..380d5d117 100644 --- a/src/utils/__test__/config.test.ts +++ b/src/utils/__test__/config.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import prettier from 'prettier'; -import { DEFAULT_PRETTIER_FORMAT } from "../config.ts"; +import { DEFAULT_PRETTIER_FORMAT } from '../config.ts'; describe('DefaultPrettierFormat', () => { it('should be the same prettier format with @linzjs/style', async () => { diff --git a/src/utils/__test__/date.test.ts b/src/utils/__test__/date.test.ts index 6e9efe5a4..754859445 100644 --- a/src/utils/__test__/date.test.ts +++ b/src/utils/__test__/date.test.ts @@ -2,7 +2,7 @@ import { describe, it } from 'node:test'; import assert from 'assert'; -import { getPacificAucklandYearMonthDay } from "../date.ts"; +import { getPacificAucklandYearMonthDay } from '../date.ts'; describe('getPacificAucklandYearMonthDay', () => { it('should format as yyyy-mm-dd', () => { diff --git a/src/utils/__test__/filelist.test.ts b/src/utils/__test__/filelist.test.ts index 252619ea3..153fc6694 100644 --- a/src/utils/__test__/filelist.test.ts +++ b/src/utils/__test__/filelist.test.ts @@ -1,8 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { TiffLocation } from "../../commands/tileindex-validate/tileindex.validate.ts"; -import { createFileList, FileListEntry } from "../filelist.ts"; +import type { TiffLocation } from '../../commands/tileindex-validate/tileindex.validate.ts'; +import type { FileListEntry } from '../filelist.ts'; +import { createFileList } from '../filelist.ts'; describe('createFileList', () => { const locationTestOne: TiffLocation = { bands: [], bbox: [0, 0, 0, 0], tileNames: [], source: 'input1' }; diff --git a/src/utils/__test__/filter.test.ts b/src/utils/__test__/filter.test.ts index 9970eb69a..bf3cc00d5 100644 --- a/src/utils/__test__/filter.test.ts +++ b/src/utils/__test__/filter.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; -import { asyncFilter } from "../chunk.ts"; +import { asyncFilter } from '../chunk.ts'; describe('AsyncFilter', () => { function makeGenerator(list: string[]): () => AsyncGenerator<{ path: string }> { diff --git a/src/utils/__test__/geotiff.test.ts b/src/utils/__test__/geotiff.test.ts index 9eaeb9de8..9adf7df15 100644 --- a/src/utils/__test__/geotiff.test.ts +++ b/src/utils/__test__/geotiff.test.ts @@ -3,10 +3,10 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { Source, Tiff, TiffImage } from '@cogeotiff/core'; +import type { Source, Tiff, TiffImage } from '@cogeotiff/core'; -import { createTiff } from "../../commands/common.ts"; -import { findBoundingBox, parseTfw, PixelIsPoint } from "../geotiff.ts"; +import { createTiff } from '../../commands/common.ts'; +import { findBoundingBox, parseTfw, PixelIsPoint } from '../geotiff.ts'; describe('geotiff', () => { describe('parseTfw', () => { diff --git a/src/utils/__test__/hash.test.ts b/src/utils/__test__/hash.test.ts index 28d92fc5e..7a71d9e96 100644 --- a/src/utils/__test__/hash.test.ts +++ b/src/utils/__test__/hash.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { Readable } from 'node:stream'; import { describe, it } from 'node:test'; -import { hashBuffer, hashStream } from "../hash.ts"; +import { hashBuffer, hashStream } from '../hash.ts'; describe('hashBuffer', () => { it('should return the expecting digest', () => { diff --git a/src/utils/__test__/mapsheet.test.ts b/src/utils/__test__/mapsheet.test.ts index b4872ef6c..b2bc8437a 100644 --- a/src/utils/__test__/mapsheet.test.ts +++ b/src/utils/__test__/mapsheet.test.ts @@ -1,8 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { MapSheet, MapTileIndex, SheetRanges } from "../mapsheet.ts"; -import { MapSheetData } from "./mapsheet.data.ts"; +import type { MapTileIndex } from '../mapsheet.ts'; +import { MapSheet, SheetRanges } from '../mapsheet.ts'; +import { MapSheetData } from './mapsheet.data.ts'; describe('MapSheets', () => { it('should extract MapTileIndex from 1:500 tile filename', () => { diff --git a/src/utils/__test__/slugify.test.ts b/src/utils/__test__/slugify.test.ts index dda166481..85bef5ed0 100644 --- a/src/utils/__test__/slugify.test.ts +++ b/src/utils/__test__/slugify.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { slugify } from "../slugify.ts"; +import { slugify } from '../slugify.ts'; const slugChars = 'abcdefghijklmnopqrstuvwxyz0123456789_.-'; diff --git a/src/utils/band.ts b/src/utils/band.ts index 5be62b36f..7349dd64f 100644 --- a/src/utils/band.ts +++ b/src/utils/band.ts @@ -1,4 +1,5 @@ -import { SampleFormat, Tiff, TiffImage, TiffTag } from '@cogeotiff/core'; +import type { Tiff, TiffImage } from '@cogeotiff/core'; +import { SampleFormat, TiffTag } from '@cogeotiff/core'; function getDataType(i: SampleFormat): string { switch (i) { diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts index 1201249c2..573a372d4 100644 --- a/src/utils/chunk.ts +++ b/src/utils/chunk.ts @@ -1,7 +1,7 @@ import { fsa } from '@chunkd/fs'; -import { parseSize } from "../commands/common.ts"; -import { logger } from "../log.ts"; +import { parseSize } from '../commands/common.ts'; +import { logger } from '../log.ts'; export interface FileSizeInfo { path: string; diff --git a/src/utils/concurrent.queue.ts b/src/utils/concurrent.queue.ts index 8d2b46b1b..f71ec4f33 100644 --- a/src/utils/concurrent.queue.ts +++ b/src/utils/concurrent.queue.ts @@ -1,4 +1,5 @@ -import pLimit, { LimitFunction } from 'p-limit'; +import type { LimitFunction } from 'p-limit'; +import pLimit from 'p-limit'; export class ConcurrentQueue { Q: LimitFunction; diff --git a/src/utils/config.ts b/src/utils/config.ts index ad43f4253..12162fc9e 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -1,4 +1,4 @@ -import prettier from 'prettier'; +import type prettier from 'prettier'; export const DEFAULT_PRETTIER_FORMAT: prettier.Options = { semi: true, diff --git a/src/utils/filelist.ts b/src/utils/filelist.ts index 0aff66000..9817f82b4 100644 --- a/src/utils/filelist.ts +++ b/src/utils/filelist.ts @@ -1,4 +1,4 @@ -import { TiffLocation } from "../commands/tileindex-validate/tileindex.validate.ts"; +import type { TiffLocation } from '../commands/tileindex-validate/tileindex.validate.ts'; export type FileListEntry = { output: string; diff --git a/src/utils/geotiff.ts b/src/utils/geotiff.ts index 89afe4086..74eeceae4 100644 --- a/src/utils/geotiff.ts +++ b/src/utils/geotiff.ts @@ -1,7 +1,8 @@ import { fsa } from '@chunkd/fs'; -import { RasterTypeKey, Tiff, TiffTagGeo } from '@cogeotiff/core'; +import type { Tiff } from '@cogeotiff/core'; +import { RasterTypeKey, TiffTagGeo } from '@cogeotiff/core'; -import { urlToString } from "../commands/common.ts"; +import { urlToString } from '../commands/common.ts'; /** * Attempt to parse a tiff world file diff --git a/src/utils/hash.stream.ts b/src/utils/hash.stream.ts index 42136011e..d39e9a5a9 100644 --- a/src/utils/hash.stream.ts +++ b/src/utils/hash.stream.ts @@ -1,5 +1,7 @@ -import { createHash, Hash } from 'crypto'; -import { Transform, TransformCallback } from 'stream'; +import type { Hash } from 'crypto'; +import { createHash } from 'crypto'; +import type { TransformCallback } from 'stream'; +import { Transform } from 'stream'; export class HashTransform extends Transform { /** hash function in use */ diff --git a/src/utils/hash.ts b/src/utils/hash.ts index 7527b5848..70e04c8fb 100644 --- a/src/utils/hash.ts +++ b/src/utils/hash.ts @@ -1,5 +1,6 @@ -import { BinaryLike, createHash } from 'crypto'; -import { Readable } from 'stream'; +import type { BinaryLike } from 'crypto'; +import { createHash } from 'crypto'; +import type { Readable } from 'stream'; /** Key concatenated to 'x-amz-meta-' */ export const HashKey = 'multihash'; diff --git a/tsconfig.json b/tsconfig.json index c8c83f010..07817dabf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,8 @@ "compilerOptions": { "lib": ["ES2022"], "target": "ES2022", - "outDir": "build" + "outDir": "build", + "noEmit": true, + "allowImportingTsExtensions": true } } From 7dfc4ce57a4259d99c82d6e16b56144cf163ec96 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 12:49:54 +1300 Subject: [PATCH 5/9] refactor: update node to 23 --- .github/workflows/pull-request-container.yml | 2 +- .github/workflows/push.yml | 2 +- .github/workflows/release-please.yml | 2 +- Dockerfile | 2 +- package-lock.json | 6 +++--- package.json | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull-request-container.yml b/.github/workflows/pull-request-container.yml index 6b6bbf595..23a909f7b 100644 --- a/.github/workflows/pull-request-container.yml +++ b/.github/workflows/pull-request-container.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Setup GIT version id: version diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7cc599a9a..bedcb3982 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Generate Readme run: | diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index a1356b4cf..5b858876f 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -36,7 +36,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Setup GIT version id: version diff --git a/Dockerfile b/Dockerfile index c68c3d5bc..2307c89c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:23-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 +FROM node:23-slim RUN apt-get update && apt-get install openssh-client git -y diff --git a/package-lock.json b/package-lock.json index 24c2caee5..987346738 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@linzjs/argo-tasks", - "version": "4.12.0", + "version": "4.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@linzjs/argo-tasks", - "version": "4.12.0", + "version": "4.11.0", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.440.0", @@ -41,7 +41,7 @@ "stac-ts": "^1.0.4" }, "engines": { - "node": "^20.13.1" + "node": "^23.9.0" } }, "node_modules/@aws-crypto/crc32": { diff --git a/package.json b/package.json index 1b4009a0c..dfef83c59 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@linzjs/argo-tasks", "private": true, - "version": "4.12.0", + "version": "4.11.0", "main": "build/src/index.js", "types": "build/src/index.d.ts", "repository": "git@github.com:linz/argo-tasks.git", @@ -22,17 +22,17 @@ "build": "tsc", "docs": "node build/src/readme/readme.generate.js", "lint": "npx eslint . --ignore-path .gitignore", - "test": "node --test build/" + "test": "node --test" }, "publishConfig": { "access": "restricted" }, "type": "module", "engines": { - "node": "^20.13.1" + "node": "^23.9.0" }, "files": [ - "build/src/**" + "src/" ], "dependencies": { "@aws-sdk/client-s3": "^3.440.0", From 8ce76828c00e58b7fdc39c66e00809f3489e8c7b Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 13:05:18 +1300 Subject: [PATCH 6/9] refactor: fix readme location --- package.json | 2 +- src/readme/readme.generate.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index dfef83c59..7cf54800a 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ }, "scripts": { "build": "tsc", - "docs": "node build/src/readme/readme.generate.js", + "docs": "node src/readme/readme.generate.ts", "lint": "npx eslint . --ignore-path .gitignore", "test": "node --test" }, diff --git a/src/readme/readme.generate.ts b/src/readme/readme.generate.ts index 539325d87..578805892 100644 --- a/src/readme/readme.generate.ts +++ b/src/readme/readme.generate.ts @@ -1,6 +1,7 @@ +import { writeFileSync } from 'node:fs'; + import { fsa } from '@chunkd/fs'; import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.ts'; -import { writeFileSync } from 'fs'; import * as prettier from 'prettier'; import { AllCommands } from '../commands/index.ts'; From 052d4db5a859584ea00ed5f61d3debb5b12fcd16 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 11 Mar 2025 13:20:59 +1300 Subject: [PATCH 7/9] refactor: cleanup unused code --- .eslintrc.cjs | 1 + src/utils/mapsheet.chatham.islands.ts | 144 -------------------------- 2 files changed, 1 insertion(+), 144 deletions(-) delete mode 100644 src/utils/mapsheet.chatham.islands.ts diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 70982d67a..d66d34acf 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -9,6 +9,7 @@ testOverrides.rules['@typescript-eslint/no-floating-promises'] = 'off'; const tsOverrides = cfg.overrides.find((ovr) => ovr.files.find((f) => f.endsWith('*.ts'))); tsOverrides.rules['@typescript-eslint/require-await'] = 'off'; +// node23 needs "import type" when importing types tsOverrides.rules['@typescript-eslint/consistent-type-imports'] = 'error'; module.exports = cfg; diff --git a/src/utils/mapsheet.chatham.islands.ts b/src/utils/mapsheet.chatham.islands.ts deleted file mode 100644 index beb168800..000000000 --- a/src/utils/mapsheet.chatham.islands.ts +++ /dev/null @@ -1,144 +0,0 @@ -// import { Bounds, MapSheetRegex, MapTileIndex } from './mapsheet.js'; - -// export const CiSheetRegex = /(?CI\d{2})(_(?\d+)_(?\d+))?/; - -// export interface MapSheetCi { -// name: string; -// x: number; -// y: number; -// } - -// export const MapSheetChathamIslands = { -// /** Height of Topo 1:50k map sheets (meters) */ -// height: 36_000, -// /** Width of Topo 1:50k map sheets (meters) */ -// width: 24_000, - -// gridSizeMax: 50_000, -// scale: 50_000, - -// /** -// * Map sheets are laid out in a pattern resembling: -// * -// * ``` -// * [CI01] [CI02] [CI03] -// * [CI04] [CI05] -// * [CI06] -// * ``` -// */ -// origins: [ -// // First row -// { name: 'CI01', x: 3458000, y: 5176000 }, -// { name: 'CI02', x: 3482000, y: 5176000 }, -// { name: 'CI03', x: 3506000, y: 5176000 }, - -// // Second row -// { name: 'CI04', x: 3482000, y: 5140000 }, -// { name: 'CI05', x: 3506000, y: 5140000 }, - -// // Third row -// { name: 'CI06', x: 3506000, y: 5104000 }, -// ], - -// getMapTileIndex(str: string): MapTileIndex | null { -// const match = str.match(CiSheetRegex); -// if (match == null) return null; - -// const sheetCode = match?.groups?.['sheetCode']; -// if (sheetCode == null) return null; - -// const gridSize = Number(match?.groups?.['gridSize'] ?? this.gridSizeMax); -// const out: MapTileIndex = { -// mapSheet: sheetCode, -// gridSize: gridSize, -// x: -1, -// y: -1, -// name: match[0], -// origin: { x: 0, y: 0 }, -// width: 0, -// height: 0, -// bbox: [0, 0, 0, 0], -// }; -// const mapSheetOffset = this.offset(sheetCode); - -// if (out.gridSize === this.gridSizeMax) { -// out.y = mapSheetOffset.y; -// out.x = mapSheetOffset.x; -// out.origin = mapSheetOffset; -// out.width = this.width; -// out.height = this.height; -// // As in CITM negative Y goes north, the minY is actually the bottom right point -// out.bbox = [out.origin.x, out.origin.y - out.height, out.origin.x + out.width, out.origin.y]; -// return out; -// } - -// // 1:500 has X/Y is 3 digits not 2 -// if (out.gridSize === 500) { -// out.y = Number(match?.groups?.['tileId']?.slice(0, 3)); -// out.x = Number(match?.groups?.['tileId']?.slice(3)); -// } else { -// out.y = Number(match?.groups?.['tileId']?.slice(0, 2)); -// out.x = Number(match?.groups?.['tileId']?.slice(2)); -// } -// if (isNaN(out.gridSize) || isNaN(out.x) || isNaN(out.y)) return null; - -// const tileOffset = this.tileSize(out.gridSize, out.x, out.y); -// out.origin.x = mapSheetOffset.x + tileOffset.x; -// out.origin.y = mapSheetOffset.y - tileOffset.y; -// out.width = tileOffset.width; -// out.height = tileOffset.height; -// // As in NZTM negative Y goes north, the minY is actually the bottom right point -// out.bbox = [out.origin.x, out.origin.y - tileOffset.height, out.origin.x + tileOffset.width, out.origin.y]; - -// return out; -// }, -// tileSize(gridSize: number, x: number, y: number): Bounds { -// const scale = gridSize / this.gridSizeMax; -// const offsetX = this.width * scale; -// const offsetY = this.height * scale; -// return { x: (x - 1) * offsetX, y: (y - 1) * offsetY, width: offsetX, height: offsetY }; -// }, -// offset(sheet: string): { x: number; y: number } { -// for (const o of this.origins) if (sheet == o.name) return { x: o.x, y: o.y }; -// throw new Error('Sheet not found'); -// }, - -// getMapSheet(x: number, y: number): MapSheetCi { -// for (const o of MapSheetChathamIslands.origins) { -// if (o.x < x && x < o.x + this.width) { -// if (o.y > y && y > o.y - this.height) { -// return o; -// } -// } -// } - -// throw new Error(`Unable to find mapsheet for point: ${x},${y}`); -// }, - -// getTileName(x: number, y: number, gridSize: number): string { -// const ms = this.getMapSheet(x, y); - -// if (gridSize === MapSheetChathamIslands.gridSizeMax) return ms.name; - -// const tilesPerMapSheet = Math.floor(MapSheetChathamIslands.gridSizeMax / gridSize); -// const tileWidth = Math.floor(MapSheetChathamIslands.width / tilesPerMapSheet); -// const tileHeight = Math.floor(MapSheetChathamIslands.height / tilesPerMapSheet); - -// const nbDigits = gridSize === 500 ? 3 : 2; - -// const tileX = Math.round(Math.floor((x - ms.x) / tileWidth + 1)); -// const tileY = Math.round(Math.floor((ms.y - y) / tileHeight + 1)); -// const tileId = `${`${tileY}`.padStart(nbDigits, '0')}${`${tileX}`.padStart(nbDigits, '0')}`; -// return `${ms.name}_${gridSize}_${tileId}`; -// }, -// }; - -// /** -// * 3518000.19287 -// 5085999.80449 -// */ - -// // const pt = [3522800.00752, 5085999.99359, 3525200.00752, 5089599.99359] as const; - -// console.log(MapSheetChathamIslands.getTileName(3518000.19287, 5085999.80449, 5000)); -// console.log(MapSheetChathamIslands.getMapTileIndex('SN8066_CI06_5000_0606.tfw')); From 0ec30636eccbb4810e00757defb538139e384401 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 18 Mar 2025 14:21:18 +1300 Subject: [PATCH 8/9] feat: add --since and --until to filter based off timestamps --- src/commands/list/README.md | 21 +++++++----- src/commands/list/list.ts | 23 +++++++++++++ src/utils/__test__/date.test.ts | 39 ++++++++++++++++++++- src/utils/__test__/filter.test.ts | 57 +++++++++++++++++++++++++++++++ src/utils/chunk.ts | 22 ++++++++++-- src/utils/date.ts | 46 +++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 12 deletions(-) diff --git a/src/commands/list/README.md b/src/commands/list/README.md index 7feee05c5..8f050f833 100644 --- a/src/commands/list/README.md +++ b/src/commands/list/README.md @@ -14,15 +14,18 @@ list [...location] ### Options -| Usage | Description | Options | -| ------------------ | ------------------------------------------------------- | -------- | -| --config | Location of role configuration file | optional | -| --include | Include files eg ".\*.tiff?$" | optional | -| --exclude | Exclude files eg ".\*.prj$" | optional | -| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | -| --group | Group files into this number per group | optional | -| --limit | Limit the file count to this amount, -1 is no limit | optional | -| --output | Output location for the listing | optional | +| Usage | Description | Options | +| --------------------------- | --------------------------------------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --include | Include files eg ".\*.tiff?$" | optional | +| --exclude | Exclude files eg ".\*.prj$" | optional | +| --since | Include files since a timestamp or relative (eg "42m" for 42 minutes) | optional | +| --until | Include files until a timestamp or relative (eg "42m" for 42 minutes) | optional | +| --max-items-listed | Maximum allowed items to be listed, -1 for unlimited | optional | +| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | +| --group | Group files into this number per group | optional | +| --limit | Limit the file count to this amount, -1 is no limit | optional | +| --output | Output location for the listing | optional | ### Flags diff --git a/src/commands/list/list.ts b/src/commands/list/list.ts index 926d00893..be422cd66 100644 --- a/src/commands/list/list.ts +++ b/src/commands/list/list.ts @@ -1,9 +1,11 @@ import { fsa } from '@chunkd/fs'; +import { FsAwsS3V3 } from '@chunkd/source-aws-v3'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; import { CliInfo } from '../../cli.info.ts'; import { logger } from '../../log.ts'; import { getFiles } from '../../utils/chunk.ts'; +import { RelativeDate } from '../../utils/date.ts'; import { config, registerCli, verbose } from '../common.ts'; export const CommandListArgs = { @@ -11,6 +13,26 @@ export const CommandListArgs = { verbose, include: option({ type: optional(string), long: 'include', description: 'Include files eg ".*.tiff?$"' }), exclude: option({ type: optional(string), long: 'exclude', description: 'Exclude files eg ".*.prj$"' }), + since: option({ + type: optional(RelativeDate), + long: 'since', + description: 'Include files since a timestamp or relative (eg "42m" for 42 minutes)', + }), + until: option({ + type: optional(RelativeDate), + long: 'until', + description: 'Include files until a timestamp or relative (eg "42m" for 42 minutes)', + }), + + maxItemList: option({ + type: number, + long: 'max-items-listed', + description: 'Maximum allowed items to be listed, -1 for unlimited', + defaultValue() { + return -1; + }, + }), + groupSize: option({ type: optional(string), long: 'group-size', @@ -32,6 +54,7 @@ export const commandList = command({ description: 'List and group files into collections of tasks', args: CommandListArgs, async handler(args) { + FsAwsS3V3.MaxListCount = args.maxItemList > 0 ? args.maxItemList / 1000 : args.maxItemList; registerCli(this, args); if (args.location.length === 0) { logger.error('List:Error:NoLocationProvided'); diff --git a/src/utils/__test__/date.test.ts b/src/utils/__test__/date.test.ts index 754859445..6def66e00 100644 --- a/src/utils/__test__/date.test.ts +++ b/src/utils/__test__/date.test.ts @@ -2,7 +2,7 @@ import { describe, it } from 'node:test'; import assert from 'assert'; -import { getPacificAucklandYearMonthDay } from '../date.ts'; +import { getPacificAucklandYearMonthDay, RelativeDate } from '../date.ts'; describe('getPacificAucklandYearMonthDay', () => { it('should format as yyyy-mm-dd', () => { @@ -12,3 +12,40 @@ describe('getPacificAucklandYearMonthDay', () => { assert.equal(getPacificAucklandYearMonthDay('2012-06-15T12:00:00Z'), '2012-06-16'); }); }); + +describe('relativeDateParser', () => { + const OneMinuteMs = 60 * 1000; + const OneHourMs = 60 * OneMinuteMs; + const OneDayMs = 24 * OneHourMs; + + it('should parse ISO Dates', async () => { + assert.equal((await RelativeDate.from('2023-02-10T00:00:00.000Z')).toISOString(), '2023-02-10T00:00:00.000Z'); + assert.equal((await RelativeDate.from('2023-02-10')).toISOString(), '2023-02-10T00:00:00.000Z'); + }); + + it('should parse known formats', async () => { + const parseTime = (s: string): Promise => RelativeDate.from(s).then((f) => f.getTime()); + + // Validate times are approximatly the same + const approxEqual = (a: number, b: number, diff: number): void => assert.ok(Math.abs(a - b) < diff); + + approxEqual(await parseTime('23m'), new Date(new Date().getTime() - OneMinuteMs * 23).getTime(), OneMinuteMs); + approxEqual(await parseTime('23M'), new Date(new Date().getTime() - OneMinuteMs * 23).getTime(), OneMinuteMs); + + approxEqual(await parseTime('23h'), new Date(new Date().getTime() - OneHourMs * 23).getTime(), OneMinuteMs); + approxEqual(await parseTime('23H'), new Date(new Date().getTime() - OneHourMs * 23).getTime(), OneMinuteMs); + + approxEqual(await parseTime('23d'), new Date(new Date().getTime() - OneDayMs * 23).getTime(), OneHourMs); + approxEqual(await parseTime('23D'), new Date(new Date().getTime() - OneDayMs * 23).getTime(), OneHourMs); + }); + + it('should error with invalid formats', async () => { + const parseError = (s: string): Promise => + RelativeDate.from(s) + .then(String) + .catch((f) => String(f)); + + assert.equal(await parseError('23Z'), 'Error: Unable to parse date from: 23Z'); + assert.equal(await parseError('abc'), 'Error: Invalid date: abc'); + }); +}); diff --git a/src/utils/__test__/filter.test.ts b/src/utils/__test__/filter.test.ts index bf3cc00d5..b869a5aa3 100644 --- a/src/utils/__test__/filter.test.ts +++ b/src/utils/__test__/filter.test.ts @@ -4,6 +4,7 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { asyncFilter } from '../chunk.ts'; +import { RelativeDate } from '../date.ts'; describe('AsyncFilter', () => { function makeGenerator(list: string[]): () => AsyncGenerator<{ path: string }> { @@ -12,6 +13,28 @@ describe('AsyncFilter', () => { }; } + /** + * Generate a list of files with last modified dates starting at 1 hour ago then increasing by one hour + * for every item in the list + * + * @example + * makeGeneratorWithDates(['a', 'b', 'c']) + * // a 1 hour ago + * // b 2 hours ago + * // c 3 hourss ago + * + * @param list + * @returns + */ + function makeGeneratorWithDates(list: string[]): () => AsyncGenerator<{ path: string; lastModified?: string }> { + const startTime = new Date().getTime(); + let index = 1; + return async function* gen(): AsyncGenerator<{ path: string; lastModified: string }> { + for (const path of list) + yield { path, lastModified: new Date(startTime - 60 * 60 * 1000 * index++).toISOString() }; + }; + } + it('should include all', async () => { const gen = makeGenerator(['a.tiff', 'b.tiff', 'c.tiff']); const result = await fsa.toArray(asyncFilter(gen())); @@ -47,4 +70,38 @@ describe('AsyncFilter', () => { const result = await fsa.toArray(asyncFilter(gen(), { exclude: '^a', include: '.tiff$' })); assert.deepEqual(result, [{ path: 'CA.TIFF' }]); }); + + const filterDates = async ( + opt: { since?: string; until?: string }, + files = ['A.tiff', 'B.tiff', 'C.tiff'], + ): Promise => { + const gen = makeGeneratorWithDates(files); + return fsa + .toArray( + asyncFilter(gen(), { + since: opt.since ? await RelativeDate.from(opt.since) : undefined, + until: opt.until ? await RelativeDate.from(opt.until) : undefined, + }), + ) + .then((m) => m.map((f) => f.path)); + }; + + it('should include since', async () => { + assert.deepEqual(await filterDates({ since: '1m' }), []); + assert.deepEqual(await filterDates({ since: '61m' }), ['A.tiff']); + assert.deepEqual(await filterDates({ since: '121m' }), ['A.tiff', 'B.tiff']); + assert.deepEqual(await filterDates({ since: '181m' }), ['A.tiff', 'B.tiff', 'C.tiff']); + }); + + it('should exclude until', async () => { + assert.deepEqual(await filterDates({ until: '1m' }), ['A.tiff', 'B.tiff', 'C.tiff']); + assert.deepEqual(await filterDates({ until: '61m' }), ['B.tiff', 'C.tiff']); + assert.deepEqual(await filterDates({ until: '121m' }), ['C.tiff']); + assert.deepEqual(await filterDates({ until: '181m' }), []); + }); + it('should include and exclude', async () => { + assert.deepEqual(await filterDates({ until: '1m', since: '61m' }), ['A.tiff']); + assert.deepEqual(await filterDates({ until: '61m', since: '121m' }), ['B.tiff']); + assert.deepEqual(await filterDates({ until: '61m', since: '181m' }), ['B.tiff', 'C.tiff']); + }); }); diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts index 573a372d4..fc0225f5e 100644 --- a/src/utils/chunk.ts +++ b/src/utils/chunk.ts @@ -8,14 +8,23 @@ export interface FileSizeInfo { size?: number; } -export async function* asyncFilter( +export async function* asyncFilter( source: AsyncGenerator, - opts?: { include?: string; exclude?: string }, + opts?: { include?: string; exclude?: string; since?: Date; until?: Date }, ): AsyncGenerator { const include = opts?.include ? new RegExp(opts.include.toLowerCase(), 'i') : true; const exclude = opts?.exclude ? new RegExp(opts.exclude.toLowerCase(), 'i') : undefined; + + const sinceTime = opts?.since?.toISOString(); + const untilTime = opts?.until?.toISOString(); for await (const f of source) { const testPath = f.path.toLowerCase(); + + if (f.lastModified) { + if (sinceTime && sinceTime > f.lastModified) continue; + if (untilTime && untilTime < f.lastModified) continue; + } + if (exclude && exclude.test(testPath)) continue; if (include === true) yield f; else if (include.test(testPath)) yield f; @@ -102,6 +111,15 @@ export type FileFilter = { * @default 1 */ sizeMin?: number; + + /** + * Filter files with a modified time since this time. + */ + since?: Date; + /** + * Filter files with a modified time until this time + */ + until?: Date; }; export async function getFiles(paths: string[], args: FileFilter = {}): Promise { const limit = args.limit ?? -1; // no limit by default diff --git a/src/utils/date.ts b/src/utils/date.ts index 036f24115..9a3050b27 100644 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -1,3 +1,7 @@ +import assert from 'node:assert'; + +import type { Type } from 'cmd-ts'; + /** * Convert time zone-aware date/time string to Pacific/Auckland time zone string * @@ -15,3 +19,45 @@ export function getPacificAucklandYearMonthDay(dateTimeString?: string | null): return pacificAucklandDateTimeString.slice(0, 10); } + +/** + * Parse a date using relative times + * + * use for arguments like + * - `--since=23m` Since 23 minutes ago + * - `--until=30d` Until 30 days ago + * + * @example + * "30s" // 30 seconds ago + * "23m" // 23 minutes ago + * "24h" // 24 hours ago + * "90d" // 90 days ago + */ +export const RelativeDate: Type = { + async from(s: string): Promise { + // Support ISO dates, eg "2023-02-10" or "2023-02-10T12:00:00.000Z" + const d = new Date(s); + if (!Number.isNaN(d.getTime())) return d; + + const scale = s.toLowerCase().slice(-1); + const digits = Number.parseInt(s.slice(0, -1)); + if (Number.isNaN(digits)) throw new Error(`Invalid date: ${s}`); + + const current = new Date(); + switch (scale) { + case 's': + current.setUTCSeconds(current.getUTCSeconds() - digits); + return current; + case 'm': + current.setUTCMinutes(current.getUTCMinutes() - digits); + return current; + case 'h': + current.setUTCHours(current.getUTCHours() - digits); + return current; + case 'd': + current.setUTCDate(current.getUTCDate() - digits); + return current; + } + throw new Error(`Unable to parse date from: ${s}`); + }, +}; From 80e8a4c0f5ec40da1349f55ea4a4841ce51f78d4 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 18 Mar 2025 14:25:19 +1300 Subject: [PATCH 9/9] refactor: cleanup lint --- src/utils/date.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/date.ts b/src/utils/date.ts index 9a3050b27..a5889f0b1 100644 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -1,5 +1,3 @@ -import assert from 'node:assert'; - import type { Type } from 'cmd-ts'; /**