diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 328fcd0ac..d66d34acf 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -7,6 +7,9 @@ testOverrides.rules['@typescript-eslint/no-floating-promises'] = 'off'; // Async functions are required in a few places without awaits eg `async foo() { throw Bar }` const tsOverrides = cfg.overrides.find((ovr) => ovr.files.find((f) => f.endsWith('*.ts'))); -tsOverrides.rules['@typescript-eslint/require-await'] = 'off' +tsOverrides.rules['@typescript-eslint/require-await'] = 'off'; + +// node23 needs "import type" when importing types +tsOverrides.rules['@typescript-eslint/consistent-type-imports'] = 'error'; module.exports = cfg; diff --git a/.github/workflows/pull-request-container.yml b/.github/workflows/pull-request-container.yml index 6b6bbf595..23a909f7b 100644 --- a/.github/workflows/pull-request-container.yml +++ b/.github/workflows/pull-request-container.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Setup GIT version id: version diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7cc599a9a..bedcb3982 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Generate Readme run: | diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index a1356b4cf..5b858876f 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -36,7 +36,7 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 with: - node-version: 20.x + node-version: 23.x - name: Setup GIT version id: version diff --git a/Dockerfile b/Dockerfile index 1ce559342..2307c89c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 +FROM node:23-slim RUN apt-get update && apt-get install openssh-client git -y @@ -16,9 +16,9 @@ ENV NODE_ENV production ADD package.json package-lock.json /app/ RUN npm install --omit=dev -ADD build/src /app/ +ADD src /app/ # Cache of copy of the STAC JSON schemas by triggering a validation run -RUN node /app/index.js stac-validate https://nz-imagery.s3-ap-southeast-2.amazonaws.com/new-zealand/new-zealand_2020-2021_10m/rgb/2193/collection.json +RUN node /app/index.ts stac-validate https://nz-imagery.s3-ap-southeast-2.amazonaws.com/new-zealand/new-zealand_2020-2021_10m/rgb/2193/collection.json -ENTRYPOINT ["node", "/app/index.js"] +ENTRYPOINT ["node", "/app/index.ts"] diff --git a/package-lock.json b/package-lock.json index 3d8668b0b..987346738 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@linzjs/argo-tasks", - "version": "4.12.0", + "version": "4.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@linzjs/argo-tasks", - "version": "4.12.0", + "version": "4.11.0", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.440.0", @@ -38,10 +38,10 @@ "@linzjs/style": "^5.4.0", "@types/node": "^20.12.10", "@types/prettier": "^3.0.0", - "stac-ts": "^1.0.3" + "stac-ts": "^1.0.4" }, "engines": { - "node": "^20.13.1" + "node": "^23.9.0" } }, "node_modules/@aws-crypto/crc32": { @@ -5987,10 +5987,11 @@ } }, "node_modules/stac-ts": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stac-ts/-/stac-ts-1.0.3.tgz", - "integrity": "sha512-QYqE6CFQ5XuWds3/Pi/07HGvWLMMQSesTcrylWF/Hp/ZjvBsSdDf0uokvHsz1iA/6JyZuvekfFlElRz4rsQR+w==", - "dev": true + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/stac-ts/-/stac-ts-1.0.4.tgz", + "integrity": "sha512-5n09H4bmBfrjIkjcDs60joV9zEX55GU29WTCNB++YzRawUZ1cCDG7a/M7bDDyR7VcQz9nguRUnRCz/JWdsFuBQ==", + "dev": true, + "license": "MIT" }, "node_modules/stream-browserify": { "version": "3.0.0", diff --git a/package.json b/package.json index 6acb0b1ef..7cf54800a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@linzjs/argo-tasks", "private": true, - "version": "4.12.0", + "version": "4.11.0", "main": "build/src/index.js", "types": "build/src/index.d.ts", "repository": "git@github.com:linz/argo-tasks.git", @@ -16,23 +16,23 @@ "@linzjs/style": "^5.4.0", "@types/node": "^20.12.10", "@types/prettier": "^3.0.0", - "stac-ts": "^1.0.3" + "stac-ts": "^1.0.4" }, "scripts": { "build": "tsc", - "docs": "node build/src/readme/readme.generate.js", + "docs": "node src/readme/readme.generate.ts", "lint": "npx eslint . --ignore-path .gitignore", - "test": "node --test build/" + "test": "node --test" }, "publishConfig": { "access": "restricted" }, "type": "module", "engines": { - "node": "^20.13.1" + "node": "^23.9.0" }, "files": [ - "build/src/**" + "src/" ], "dependencies": { "@aws-sdk/client-s3": "^3.440.0", diff --git a/src/__test__/eai_againRetry.test.ts b/src/__test__/eai_againRetry.test.ts index 3f1decac4..0842065dd 100644 --- a/src/__test__/eai_againRetry.test.ts +++ b/src/__test__/eai_againRetry.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { beforeEach, describe, it } from 'node:test'; -import { BuildHandler, MetadataBearer } from '@smithy/types'; +import type { BuildHandler, MetadataBearer } from '@smithy/types'; -import { eaiAgainBuilder } from '../fs.register.js'; +import { eaiAgainBuilder } from '../fs.register.ts'; let callCount = 0; diff --git a/src/__test__/fqdn.test.ts b/src/__test__/fqdn.test.ts index 64e7f3ff7..feee5268f 100644 --- a/src/__test__/fqdn.test.ts +++ b/src/__test__/fqdn.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { FinalizeHandler, MetadataBearer } from '@smithy/types'; +import type { FinalizeHandler, MetadataBearer } from '@smithy/types'; -import { fqdn } from '../fs.register.js'; +import { fqdn } from '../fs.register.ts'; describe('fqdnMiddleware', () => { const fakeNext: FinalizeHandler = () => { diff --git a/src/__test__/fs.test.ts b/src/__test__/fs.test.ts index 8d1c3d675..39f7854e2 100644 --- a/src/__test__/fs.test.ts +++ b/src/__test__/fs.test.ts @@ -1,14 +1,15 @@ import { after, before, beforeEach, describe, it } from 'node:test'; import { CompositeError } from '@chunkd/core'; -import { FileSystemAbstraction, fsa } from '@chunkd/fs'; -import { FsAwsS3 } from '@chunkd/source-aws'; -import { S3LikeV3 } from '@chunkd/source-aws-v3'; +import type { FileSystemAbstraction } from '@chunkd/fs'; +import { fsa } from '@chunkd/fs'; +import type { FsAwsS3 } from '@chunkd/source-aws'; +import type { S3LikeV3 } from '@chunkd/source-aws-v3'; import { FsMemory } from '@chunkd/source-memory'; -import { InitializeMiddleware, MetadataBearer } from '@smithy/types'; +import type { InitializeMiddleware, MetadataBearer } from '@smithy/types'; import assert from 'assert'; -import { registerFileSystem } from '../fs.register.js'; +import { registerFileSystem } from '../fs.register.ts'; export class HttpError extends Error { statusCode: number; diff --git a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts index 2b97626ab..307ceaa58 100644 --- a/src/commands/basemaps-github/__test__/create-pr-handler.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr-handler.test.ts @@ -4,12 +4,11 @@ import { afterEach, it } from 'node:test'; import { TileSetType } from '@basemaps/config/build/config/tile.set.js'; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { StacVersion } from 'stac-ts'; +import type { StacVersion } from 'stac-ts'; -import { anySlug } from '../../../utils/__test__/slugify.test.js'; -import { GithubApi } from '../../../utils/github.js'; -import { basemapsCreatePullRequest, ConfigType, LinzBasemapsSourceCollectionRel } from '../create-pr.js'; -import { Category } from '../make.cog.github.js'; +import { anySlug } from '../../../utils/__test__/slugify.test.ts'; +import { GithubApi } from '../../../utils/github.ts'; +import { basemapsCreatePullRequest, LinzBasemapsSourceCollectionRel } from '../create-pr.ts'; const originalEnv = Object.assign({}, process.env); @@ -62,8 +61,8 @@ await it('basemapsCreatePullRequest.handler should handle S3 target', async (t) target: targetUrlsString, repository: 'any-owner/any-repository', verbose: false, - category: Category.Satellite, - configType: ConfigType.Raster, + category: 'Satellite Imagery', + configType: 'raster', individual: false, vector: false, ticket: 'any ticket', diff --git a/src/commands/basemaps-github/__test__/create-pr.test.ts b/src/commands/basemaps-github/__test__/create-pr.test.ts index b8f76525f..2aaf58c4f 100644 --- a/src/commands/basemaps-github/__test__/create-pr.test.ts +++ b/src/commands/basemaps-github/__test__/create-pr.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { parseTargetUrl, targetInfo } from '../create-pr.js'; +import type { targetInfo } from '../create-pr.ts'; +import { parseTargetUrl } from '../create-pr.ts'; describe('parseTargetUrl', () => { it('Should parse the correct target vector Urls', async () => { diff --git a/src/commands/basemaps-github/create-pr.ts b/src/commands/basemaps-github/create-pr.ts index d338b0545..c2e9b7ee6 100644 --- a/src/commands/basemaps-github/create-pr.ts +++ b/src/commands/basemaps-github/create-pr.ts @@ -1,25 +1,23 @@ import { standardizeLayerName } from '@basemaps/config'; -import { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; -import { Epsg, EpsgCode } from '@basemaps/geo'; +import type { ConfigLayer } from '@basemaps/config/build/config/tile.set.js'; +import type { EpsgCode } from '@basemaps/geo'; +import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { boolean, command, flag, oneOf, option, optional, string } from 'cmd-ts'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { registerCli, verbose } from '../common.js'; -import { Category, MakeCogGithub } from './make.cog.github.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { registerCli, verbose } from '../common.ts'; +import { Categories, MakeCogGithub } from './make.cog.github.ts'; export const ValidTargetBuckets: Set = new Set(['linz-basemaps', 'linz-basemaps-staging']); export const ValidSourceBuckets: Set = new Set(['nz-imagery', 'linz-imagery', 'nz-elevation']); export const LinzBasemapsSourceCollectionRel = 'linz_basemaps:source_collection'; -export enum ConfigType { - Raster = 'raster', - Vector = 'vector', - Elevation = 'elevation', -} +export const ConfigTypes = ['raster', 'elevation', 'vector'] as const; +export type ConfigType = (typeof ConfigTypes)[number]; export function assertValidBucket(bucket: string, validBuckets: Set): void { // Validate the target information @@ -202,9 +200,9 @@ export const CommandCreatePRArgs = { description: 'New layers locations as array of strings import into basemaps-config', }), category: option({ - type: optional(oneOf(Object.values(Category))), + type: optional(oneOf(Object.values(Categories))), long: 'category', - description: [...Object.values(Category)].join(', '), + description: [...Object.values(Categories)].join(', '), }), repository: option({ type: string, @@ -214,10 +212,10 @@ export const CommandCreatePRArgs = { defaultValueIsSerializable: true, }), configType: option({ - type: optional(oneOf(Object.values(ConfigType))), + type: optional(oneOf(ConfigTypes)), long: 'config-type', - description: `Basemaps config file type, includes ${[...Object.values(ConfigType)].join(', ')}`, - defaultValue: () => ConfigType.Raster, + description: `Basemaps config file type, includes ${ConfigTypes.join(', ')}`, + defaultValue: () => 'raster' as const, }), individual: flag({ type: boolean, @@ -246,7 +244,7 @@ export const basemapsCreatePullRequest = command({ async handler(args) { registerCli(this, args); const target = args.target; - const category = args.category ?? Category.Other; + const category = args.category ?? Categories.Other; let targets: string[]; try { targets = JSON.parse(target) as string[]; @@ -256,15 +254,15 @@ export const basemapsCreatePullRequest = command({ const layer: ConfigLayer = { name: '', title: '' }; let region; - const configType = args.vector ? ConfigType.Vector : args.configType; - if (configType === ConfigType.Vector) { + const configType = args.vector ? 'vector' : args.configType; + if (configType === 'vector') { for (const target of targets) { const info = await parseVectorTargetInfo(target); layer.name = info.name; layer.title = info.title; layer[info.epsg] = target; } - } else if (configType === ConfigType.Elevation) { + } else if (configType === 'elevation') { for (const target of targets) { const info = await parseElevationTargetInfo(target, args.individual); layer.name = info.name; @@ -287,11 +285,11 @@ export const basemapsCreatePullRequest = command({ if (layer.name === '' || layer.title === '') throw new Error('Failed to find the imagery name or title.'); const git = new MakeCogGithub(layer.name, args.repository, args.ticket); - if (configType === ConfigType.Vector) { + if (configType === 'vector') { await git.updateVectorTileSet(layer.name, layer, args.individual); - } else if (configType === ConfigType.Raster) { + } else if (configType === 'raster') { await git.updateRasterTileSet(layer.name, layer, category, args.individual, region); - } else if (configType === ConfigType.Elevation) { + } else if (configType === 'elevation') { await git.updateElevationTileSet(layer.name, layer, args.individual, region); } else throw new Error(`Invalid Config File target: ${configType}`); }, diff --git a/src/commands/basemaps-github/make.cog.github.ts b/src/commands/basemaps-github/make.cog.github.ts index 48f848bb8..ce7a938c4 100644 --- a/src/commands/basemaps-github/make.cog.github.ts +++ b/src/commands/basemaps-github/make.cog.github.ts @@ -1,27 +1,37 @@ import { DefaultColorRampOutput, DefaultTerrainRgbOutput, standardizeLayerName } from '@basemaps/config'; -import { +import type { ConfigLayer, ConfigTileSet, ConfigTileSetRaster, ConfigTileSetVector, - TileSetType, } from '@basemaps/config/build/config/tile.set.js'; +import { TileSetType } from '@basemaps/config/build/config/tile.set.js'; import { fsa } from '@chunkd/fs'; -import { logger } from '../../log.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { GithubApi } from '../../utils/github.js'; -import { prettyPrint } from '../pretty-print/pretty.print.js'; +import { logger } from '../../log.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { GithubApi } from '../../utils/github.ts'; +import { prettyPrint } from '../pretty-print/pretty.print.ts'; -export enum Category { - Urban = 'Urban Aerial Photos', - Rural = 'Rural Aerial Photos', - Satellite = 'Satellite Imagery', - Elevation = 'Elevation', - Event = 'Event', - Scanned = 'Scanned Aerial Imagery', - Other = 'New Aerial Photos', -} +export const Categories = { + Urban: 'Urban Aerial Photos', + Rural: 'Rural Aerial Photos', + Satellite: 'Satellite Imagery', + Elevation: 'Elevation', + Event: 'Event', + Scanned: 'Scanned Aerial Imagery', + Other: 'New Aerial Photos', +} as const; + +export type Category = (typeof Categories)[keyof typeof Categories]; + +// | 'Urban Aerial Photos' +// | 'Rural Aerial Photos' +// | 'Satellite Imagery' +// | 'Elevation' +// | 'Event' +// | 'Scanned Aerial Imagery' +// | 'New Aerial Photos'; export interface CategorySetting { minZoom?: number; @@ -29,13 +39,13 @@ export interface CategorySetting { } export const DefaultCategorySetting: Record = { - [Category.Urban]: { minZoom: 14 }, - [Category.Rural]: { minZoom: 13 }, - [Category.Satellite]: { minZoom: 5 }, - [Category.Elevation]: { minZoom: 9 }, - [Category.Scanned]: { minZoom: 0, maxZoom: 32 }, - [Category.Other]: {}, - [Category.Event]: {}, + 'Urban Aerial Photos': { minZoom: 14 }, + ['Rural Aerial Photos']: { minZoom: 13 }, + ['Satellite Imagery']: { minZoom: 5 }, + ['Elevation']: { minZoom: 9 }, + ['Scanned Aerial Imagery']: { minZoom: 0, maxZoom: 32 }, + ['New Aerial Photos']: {}, + ['Event']: {}, }; const botEmail = 'basemaps@linz.govt.nz'; @@ -245,16 +255,16 @@ export class MakeCogGithub { this.setDefaultConfig(layer, category); // Set layer zoom level and add to latest order - if (category === Category.Rural) { + if (category === 'Rural Aerial Photos') { for (let i = 0; i < tileSet.layers.length; i++) { // Add new layer above the first Urban - if (tileSet.layers[i]?.category === Category.Urban) { + if (tileSet.layers[i]?.category === 'Urban Aerial Photos') { // Find first valid Urban and insert new record above that. tileSet.layers.splice(i, 0, layer); break; } } - } else if (category === Category.Other) { + } else if (category === 'New Aerial Photos') { // Add new layer at the bottom tileSet.layers.push(layer); } else { @@ -296,19 +306,19 @@ export class MakeCogGithub { async prepareElevationTileSetConfig(layer: ConfigLayer, tileSet?: ConfigTileSetRaster): Promise { if (tileSet == null) { // Prepare individual elevation tileset config - const targetLayer = { ...layer, Category: Category.Elevation, minZoom: 0, maxZoom: 32 }; + const targetLayer = { ...layer, Category: 'Elevation', minZoom: 0, maxZoom: 32 }; return { type: TileSetType.Raster, id: `ts_${layer.name}`, name: layer.name, title: layer.title, - category: Category.Elevation, + category: 'Elevation', layers: [targetLayer], outputs: [DefaultTerrainRgbOutput, DefaultColorRampOutput], }; } - this.setDefaultConfig(layer, Category.Elevation); + this.setDefaultConfig(layer, 'Elevation'); // Remove elevation layer.category if already exists if (layer.category) delete layer.category; diff --git a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts index e50ca54af..666878b87 100644 --- a/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts +++ b/src/commands/basemaps-mapsheet/__test__/create-mapsheet.test.ts @@ -1,10 +1,11 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; -import { FeatureCollection } from 'geojson'; +import type { ConfigImagery, ConfigTileSet } from '@basemaps/config'; +import { ConfigProviderMemory } from '@basemaps/config'; +import type { FeatureCollection } from 'geojson'; -import { createMapSheet } from '../create-mapsheet.js'; +import { createMapSheet } from '../create-mapsheet.ts'; describe('copyFiles', () => { const rest: FeatureCollection = { diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index 66d168707..fde04678e 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -1,15 +1,16 @@ -import { ConfigBundled, ConfigImagery, ConfigProviderMemory, ConfigTileSet } from '@basemaps/config'; +import type { ConfigBundled, ConfigImagery, ConfigTileSet } from '@basemaps/config'; +import { ConfigProviderMemory } from '@basemaps/config'; import { Bounds, EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; import * as fgb from 'flatgeobuf/lib/mjs/geojson.js'; -import { FeatureCollection, MultiPolygon } from 'geojson'; +import type { FeatureCollection, MultiPolygon } from 'geojson'; import { promisify } from 'util'; import { gunzip } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { registerCli, verbose } from '../common.ts'; const gunzipProm = promisify(gunzip); diff --git a/src/commands/common.ts b/src/commands/common.ts index 513d55b87..54d1bce05 100644 --- a/src/commands/common.ts +++ b/src/commands/common.ts @@ -1,13 +1,14 @@ import { fsa } from '@chunkd/fs'; import { Tiff } from '@cogeotiff/core'; -import { boolean, flag, option, optional, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { boolean, flag, option, optional, string } from 'cmd-ts'; import pLimit from 'p-limit'; import { fileURLToPath, pathToFileURL } from 'url'; -import { CliInfo } from '../cli.info.js'; -import { registerFileSystem } from '../fs.register.js'; -import { logger, registerLogger } from '../log.js'; -import { isArgo } from '../utils/argo.js'; +import { CliInfo } from '../cli.info.ts'; +import { registerFileSystem } from '../fs.register.ts'; +import { logger, registerLogger } from '../log.ts'; +import { isArgo } from '../utils/argo.ts'; export const config = option({ long: 'config', diff --git a/src/commands/copy/__test__/copy.test.ts b/src/commands/copy/__test__/copy.test.ts index de3ca47dd..9770e531b 100644 --- a/src/commands/copy/__test__/copy.test.ts +++ b/src/commands/copy/__test__/copy.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { worker } from '../copy-worker.js'; +import { worker } from '../copy-worker.ts'; describe('copyFiles', () => { const memory = new FsMemory(); diff --git a/src/commands/copy/copy-worker.ts b/src/commands/copy/copy-worker.ts index 6160614ed..3600f1f3a 100644 --- a/src/commands/copy/copy-worker.ts +++ b/src/commands/copy/copy-worker.ts @@ -1,17 +1,17 @@ import { performance } from 'node:perf_hooks'; import { parentPort, threadId } from 'node:worker_threads'; -import { FileInfo } from '@chunkd/core'; +import type { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; import { WorkerRpc } from '@wtrpc/core'; -import { logger } from '../../log.js'; -import { ConcurrentQueue } from '../../utils/concurrent.queue.js'; -import { HashKey, hashStream } from '../../utils/hash.js'; -import { HashTransform } from '../../utils/hash.stream.js'; -import { registerCli } from '../common.js'; -import { isTiff } from '../tileindex-validate/tileindex.validate.js'; -import { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.js'; +import { logger } from '../../log.ts'; +import { ConcurrentQueue } from '../../utils/concurrent.queue.ts'; +import { HashTransform } from '../../utils/hash.stream.ts'; +import { HashKey, hashStream } from '../../utils/hash.ts'; +import { registerCli } from '../common.ts'; +import { isTiff } from '../tileindex-validate/tileindex.validate.ts'; +import type { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.ts'; const Q = new ConcurrentQueue(10); diff --git a/src/commands/copy/copy.ts b/src/commands/copy/copy.ts index cae1181c1..73e2e50d6 100644 --- a/src/commands/copy/copy.ts +++ b/src/commands/copy/copy.ts @@ -4,11 +4,11 @@ import { boolean, command, flag, number, option, restPositionals, string } from import { performance } from 'perf_hooks'; import * as z from 'zod'; -import { CliInfo } from '../../cli.info.js'; -import { logger, logId } from '../../log.js'; -import { ActionCopy } from '../../utils/actions.js'; -import { config, registerCli, verbose } from '../common.js'; -import { CopyContract } from './copy-rpc.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger, logId } from '../../log.ts'; +import type { ActionCopy } from '../../utils/actions.ts'; +import { config, registerCli, verbose } from '../common.ts'; +import type { CopyContract } from './copy-rpc.ts'; const CopyValidator = z.object({ source: z.string(), target: z.string() }); const CopyManifest = z.array(CopyValidator); diff --git a/src/commands/create-manifest/__test__/create-manifest.test.ts b/src/commands/create-manifest/__test__/create-manifest.test.ts index 26f9a2b7d..d864b76cd 100644 --- a/src/commands/create-manifest/__test__/create-manifest.test.ts +++ b/src/commands/create-manifest/__test__/create-manifest.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createManifest, validatePaths } from '../create-manifest.js'; +import { createManifest, validatePaths } from '../create-manifest.ts'; describe('createManifest', () => { beforeEach(() => { diff --git a/src/commands/create-manifest/create-manifest.ts b/src/commands/create-manifest/create-manifest.ts index 91a325dad..d273217e6 100644 --- a/src/commands/create-manifest/create-manifest.ts +++ b/src/commands/create-manifest/create-manifest.ts @@ -4,11 +4,12 @@ import { createHash } from 'crypto'; import path from 'path'; import { gzipSync } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { getActionLocation } from '../../utils/action.storage.js'; -import { ActionCopy } from '../../utils/actions.js'; -import { FileFilter, getFiles } from '../../utils/chunk.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { getActionLocation } from '../../utils/action.storage.ts'; +import type { ActionCopy } from '../../utils/actions.ts'; +import type { FileFilter } from '../../utils/chunk.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const commandCreateManifest = command({ name: 'create-manifest', diff --git a/src/commands/generate-path/__test__/generate.path.test.ts b/src/commands/generate-path/__test__/generate.path.test.ts index 9780145fa..a76239415 100644 --- a/src/commands/generate-path/__test__/generate.path.test.ts +++ b/src/commands/generate-path/__test__/generate.path.test.ts @@ -1,9 +1,10 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { SampleCollection } from '../../generate-path/__test__/sample.js'; -import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.js'; -import { extractEpsg, extractGsd, generatePath, PathMetadata } from '../path.generate.js'; +import { SampleCollection } from '../../generate-path/__test__/sample.ts'; +import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.ts'; +import type { PathMetadata } from '../path.generate.ts'; +import { extractEpsg, extractGsd, generatePath } from '../path.generate.ts'; describe('GeneratePathImagery', () => { it('Should match - urban aerial from slug', () => { diff --git a/src/commands/generate-path/__test__/sample.ts b/src/commands/generate-path/__test__/sample.ts index 486a7a246..041b1ba60 100644 --- a/src/commands/generate-path/__test__/sample.ts +++ b/src/commands/generate-path/__test__/sample.ts @@ -1,6 +1,6 @@ -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.js'; +import type { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index 3332400f1..45143e3b3 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -1,13 +1,14 @@ import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { Tiff } from '@cogeotiff/core'; +import type { Tiff } from '@cogeotiff/core'; import { command, option, positional, string } from 'cmd-ts'; -import { StacCollection, StacItem } from 'stac-ts'; +import type { StacCollection, StacItem } from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.js'; -import { config, createTiff, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import type { StacCollectionLinz } from '../../utils/metadata.ts'; +import { GeospatialDataCategories } from '../../utils/metadata.ts'; +import { config, createTiff, registerCli, verbose } from '../common.ts'; export interface PathMetadata { targetBucketName: string; diff --git a/src/commands/group/__test__/group.test.ts b/src/commands/group/__test__/group.test.ts index 112564b90..e15126c9b 100644 --- a/src/commands/group/__test__/group.test.ts +++ b/src/commands/group/__test__/group.test.ts @@ -4,7 +4,7 @@ import { before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { commandGroup, groupItems } from '../group.js'; +import { commandGroup, groupItems } from '../group.ts'; describe('groupItems', () => { it('should group items', () => { diff --git a/src/commands/group/group.ts b/src/commands/group/group.ts index b20c70c0d..b5890edf3 100644 --- a/src/commands/group/group.ts +++ b/src/commands/group/group.ts @@ -1,10 +1,10 @@ import { fsa } from '@chunkd/fs'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { config, forceOutput, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { isArgo } from '../../utils/argo.ts'; +import { config, forceOutput, registerCli, verbose } from '../common.ts'; /** Chunk an array into a group size * @example diff --git a/src/commands/index.ts b/src/commands/index.ts index 42b70b354..1ef2735e3 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,22 +1,22 @@ import { subcommands } from 'cmd-ts'; -import { CliInfo } from '../cli.info.js'; -import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; -import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { commandCopy } from './copy/copy.js'; -import { commandCreateManifest } from './create-manifest/create-manifest.js'; -import { commandGeneratePath } from './generate-path/path.generate.js'; -import { commandGroup } from './group/group.js'; -import { commandLdsFetch } from './lds-fetch-layer/lds.fetch.layer.js'; -import { commandList } from './list/list.js'; -import { commandMapSheetCoverage } from './mapsheet-coverage/mapsheet.coverage.js'; -import { commandPrettyPrint } from './pretty-print/pretty.print.js'; -import { commandStacCatalog } from './stac-catalog/stac.catalog.js'; -import { commandStacGithubImport } from './stac-github-import/stac.github.import.js'; -import { commandStacSetup } from './stac-setup/stac.setup.js'; -import { commandStacSync } from './stac-sync/stac.sync.js'; -import { commandStacValidate } from './stac-validate/stac.validate.js'; -import { commandTileIndexValidate } from './tileindex-validate/tileindex.validate.js'; +import { CliInfo } from '../cli.info.ts'; +import { basemapsCreatePullRequest } from './basemaps-github/create-pr.ts'; +import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.ts'; +import { commandCopy } from './copy/copy.ts'; +import { commandCreateManifest } from './create-manifest/create-manifest.ts'; +import { commandGeneratePath } from './generate-path/path.generate.ts'; +import { commandGroup } from './group/group.ts'; +import { commandLdsFetch } from './lds-fetch-layer/lds.fetch.layer.ts'; +import { commandList } from './list/list.ts'; +import { commandMapSheetCoverage } from './mapsheet-coverage/mapsheet.coverage.ts'; +import { commandPrettyPrint } from './pretty-print/pretty.print.ts'; +import { commandStacCatalog } from './stac-catalog/stac.catalog.ts'; +import { commandStacGithubImport } from './stac-github-import/stac.github.import.ts'; +import { commandStacSetup } from './stac-setup/stac.setup.ts'; +import { commandStacSync } from './stac-sync/stac.sync.ts'; +import { commandStacValidate } from './stac-validate/stac.validate.ts'; +import { commandTileIndexValidate } from './tileindex-validate/tileindex.validate.ts'; export const AllCommands = { copy: commandCopy, diff --git a/src/commands/lds-fetch-layer/lds.fetch.layer.ts b/src/commands/lds-fetch-layer/lds.fetch.layer.ts index 1956dc6f7..cd7b89632 100644 --- a/src/commands/lds-fetch-layer/lds.fetch.layer.ts +++ b/src/commands/lds-fetch-layer/lds.fetch.layer.ts @@ -1,11 +1,11 @@ import { fsa } from '@chunkd/fs'; import { command, option, restPositionals, string } from 'cmd-ts'; -import * as stac from 'stac-ts'; +import type * as stac from 'stac-ts'; import { createGunzip } from 'zlib'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { config, registerCli, verbose } from '../common.ts'; function getTargetPath(source: string, path: string): string { if (path.startsWith('./')) return fsa.join(source, path.slice(2)); diff --git a/src/commands/list/README.md b/src/commands/list/README.md index 7feee05c5..8f050f833 100644 --- a/src/commands/list/README.md +++ b/src/commands/list/README.md @@ -14,15 +14,18 @@ list [...location] ### Options -| Usage | Description | Options | -| ------------------ | ------------------------------------------------------- | -------- | -| --config | Location of role configuration file | optional | -| --include | Include files eg ".\*.tiff?$" | optional | -| --exclude | Exclude files eg ".\*.prj$" | optional | -| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | -| --group | Group files into this number per group | optional | -| --limit | Limit the file count to this amount, -1 is no limit | optional | -| --output | Output location for the listing | optional | +| Usage | Description | Options | +| --------------------------- | --------------------------------------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --include | Include files eg ".\*.tiff?$" | optional | +| --exclude | Exclude files eg ".\*.prj$" | optional | +| --since | Include files since a timestamp or relative (eg "42m" for 42 minutes) | optional | +| --until | Include files until a timestamp or relative (eg "42m" for 42 minutes) | optional | +| --max-items-listed | Maximum allowed items to be listed, -1 for unlimited | optional | +| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | +| --group | Group files into this number per group | optional | +| --limit | Limit the file count to this amount, -1 is no limit | optional | +| --output | Output location for the listing | optional | ### Flags diff --git a/src/commands/list/list.ts b/src/commands/list/list.ts index ef28bc216..be422cd66 100644 --- a/src/commands/list/list.ts +++ b/src/commands/list/list.ts @@ -1,16 +1,38 @@ import { fsa } from '@chunkd/fs'; +import { FsAwsS3V3 } from '@chunkd/source-aws-v3'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getFiles } from '../../utils/chunk.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { RelativeDate } from '../../utils/date.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const CommandListArgs = { config, verbose, include: option({ type: optional(string), long: 'include', description: 'Include files eg ".*.tiff?$"' }), exclude: option({ type: optional(string), long: 'exclude', description: 'Exclude files eg ".*.prj$"' }), + since: option({ + type: optional(RelativeDate), + long: 'since', + description: 'Include files since a timestamp or relative (eg "42m" for 42 minutes)', + }), + until: option({ + type: optional(RelativeDate), + long: 'until', + description: 'Include files until a timestamp or relative (eg "42m" for 42 minutes)', + }), + + maxItemList: option({ + type: number, + long: 'max-items-listed', + description: 'Maximum allowed items to be listed, -1 for unlimited', + defaultValue() { + return -1; + }, + }), + groupSize: option({ type: optional(string), long: 'group-size', @@ -32,6 +54,7 @@ export const commandList = command({ description: 'List and group files into collections of tasks', args: CommandListArgs, async handler(args) { + FsAwsS3V3.MaxListCount = args.maxItemList > 0 ? args.maxItemList / 1000 : args.maxItemList; registerCli(this, args); if (args.location.length === 0) { logger.error('List:Error:NoLocationProvided'); diff --git a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts index 2ae260b5b..fb23e9ed8 100644 --- a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts +++ b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts @@ -5,10 +5,10 @@ import { Nztm2000QuadTms, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; import { featuresToMultiPolygon } from '@linzjs/geojson'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { MapSheet } from '../../../utils/mapsheet.js'; -import { commandMapSheetCoverage, isLargeRegion } from '../mapsheet.coverage.js'; +import { MapSheet } from '../../../utils/mapsheet.ts'; +import { commandMapSheetCoverage, isLargeRegion } from '../mapsheet.coverage.ts'; // convert a collection of map sheets into a multipolygon function mapSheetToGeoJson(...sheetCodes: string[]): GeoJSON.Feature { diff --git a/src/commands/mapsheet-coverage/mapsheet.coverage.ts b/src/commands/mapsheet-coverage/mapsheet.coverage.ts index 2eda7e362..7f10e609c 100644 --- a/src/commands/mapsheet-coverage/mapsheet.coverage.ts +++ b/src/commands/mapsheet-coverage/mapsheet.coverage.ts @@ -1,23 +1,23 @@ +import { basename } from 'node:path/posix'; import { gzipSync } from 'node:zlib'; -import { ConfigTileSetRaster } from '@basemaps/config'; +import type { ConfigTileSetRaster } from '@basemaps/config'; import { EpsgCode } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { Area, truncate } from '@linzjs/geojson'; import buffer from '@turf/buffer'; import { command, number, option, optional, string } from 'cmd-ts'; import pLimit from 'p-limit'; -import { basename } from 'path/posix'; import pc from 'polygon-clipping'; -import { StacCollection, StacItem } from 'stac-ts'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getPacificAucklandYearMonthDay } from '../../utils/date.js'; -import { createFileList } from '../../utils/filelist.js'; -import { hashStream } from '../../utils/hash.js'; -import { MapSheet } from '../../utils/mapsheet.js'; -import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from '../common.js'; +import type { StacCollection, StacItem } from 'stac-ts'; + +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getPacificAucklandYearMonthDay } from '../../utils/date.ts'; +import { createFileList } from '../../utils/filelist.ts'; +import { hashStream } from '../../utils/hash.ts'; +import { MapSheet } from '../../utils/mapsheet.ts'; +import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from '../common.ts'; /** Datasets to skip */ const Skip = new Set([ diff --git a/src/commands/pretty-print/pretty.print.ts b/src/commands/pretty-print/pretty.print.ts index 77ade3a07..b3e1a286a 100644 --- a/src/commands/pretty-print/pretty.print.ts +++ b/src/commands/pretty-print/pretty.print.ts @@ -3,11 +3,11 @@ import { command, option, optional, positional, string } from 'cmd-ts'; import { basename } from 'path'; import prettier from 'prettier'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { getFiles } from '../../utils/chunk.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { config, registerCli, verbose } from '../common.ts'; function isJson(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/commands/stac-catalog/__test__/stac.catalog.test.ts b/src/commands/stac-catalog/__test__/stac.catalog.test.ts index 8a9a02fca..f916b1b9a 100644 --- a/src/commands/stac-catalog/__test__/stac.catalog.test.ts +++ b/src/commands/stac-catalog/__test__/stac.catalog.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { createLinks, makeRelative } from '../stac.catalog.js'; +import { createLinks, makeRelative } from '../stac.catalog.ts'; describe('stacCatalog', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-catalog/stac.catalog.ts b/src/commands/stac-catalog/stac.catalog.ts index d5098bf8b..bc496de4b 100644 --- a/src/commands/stac-catalog/stac.catalog.ts +++ b/src/commands/stac-catalog/stac.catalog.ts @@ -1,12 +1,12 @@ import { fsa } from '@chunkd/fs'; import { command, option, positional, string } from 'cmd-ts'; import { isAbsolute } from 'path'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { hashBuffer } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { hashBuffer } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; /** is a path a URL */ export function isUrl(path: string): boolean { diff --git a/src/commands/stac-github-import/__test__/stac.link.test.ts b/src/commands/stac-github-import/__test__/stac.link.test.ts index 2b0604fc3..fd39880b8 100644 --- a/src/commands/stac-github-import/__test__/stac.link.test.ts +++ b/src/commands/stac-github-import/__test__/stac.link.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { sortLinks } from '../stac.github.import.js'; +import { sortLinks } from '../stac.github.import.ts'; function shuffle(array: T[]): T[] { let currentIndex = array.length; diff --git a/src/commands/stac-github-import/stac.github.import.ts b/src/commands/stac-github-import/stac.github.import.ts index 6426abcff..e4220694f 100644 --- a/src/commands/stac-github-import/stac.github.import.ts +++ b/src/commands/stac-github-import/stac.github.import.ts @@ -1,13 +1,14 @@ import { fsa } from '@chunkd/fs'; -import { command, oneOf, option, string, Type } from 'cmd-ts'; -import * as st from 'stac-ts'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; -import { GithubApi } from '../../utils/github.js'; -import { config, registerCli, verbose } from '../common.js'; -import { prettyPrint } from '../pretty-print/pretty.print.js'; +import type { Type } from 'cmd-ts'; +import { command, oneOf, option, string } from 'cmd-ts'; +import type * as st from 'stac-ts'; + +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.ts'; +import { GithubApi } from '../../utils/github.ts'; +import { config, registerCli, verbose } from '../common.ts'; +import { prettyPrint } from '../pretty-print/pretty.print.ts'; const Url: Type = { async from(str) { diff --git a/src/commands/stac-setup/__test__/sample.ts b/src/commands/stac-setup/__test__/sample.ts index 999adbfa2..1ea87ffbe 100644 --- a/src/commands/stac-setup/__test__/sample.ts +++ b/src/commands/stac-setup/__test__/sample.ts @@ -1,6 +1,6 @@ -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../../../utils/metadata.js'; +import type { StacCollectionLinz } from '../../../utils/metadata.ts'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/stac-setup/__test__/stac.setup.test.ts b/src/commands/stac-setup/__test__/stac.setup.test.ts index 94c36d1a7..987385651 100644 --- a/src/commands/stac-setup/__test__/stac.setup.test.ts +++ b/src/commands/stac-setup/__test__/stac.setup.test.ts @@ -4,10 +4,11 @@ import { afterEach, before, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { GeospatialDataCategory } from '../../../utils/metadata.js'; -import { MeterAsString } from '../../common.js'; -import { commandStacSetup, formatDate, slugFromMetadata, SlugMetadata } from '../stac.setup.js'; -import { SampleCollection } from './sample.js'; +import type { GeospatialDataCategory } from '../../../utils/metadata.ts'; +import { MeterAsString } from '../../common.ts'; +import type { SlugMetadata } from '../stac.setup.ts'; +import { commandStacSetup, formatDate, slugFromMetadata } from '../stac.setup.ts'; +import { SampleCollection } from './sample.ts'; describe('stac-setup', () => { const mem = new FsMemory(); diff --git a/src/commands/stac-setup/stac.setup.ts b/src/commands/stac-setup/stac.setup.ts index e9656665b..163f53bf9 100644 --- a/src/commands/stac-setup/stac.setup.ts +++ b/src/commands/stac-setup/stac.setup.ts @@ -1,13 +1,13 @@ import { fsa } from '@chunkd/fs'; import { command, option, optional, string } from 'cmd-ts'; -import { StacCollection } from 'stac-ts'; +import type { StacCollection } from 'stac-ts'; import ulid from 'ulid'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { GeospatialDataCategory, StacCollectionLinz } from '../../utils/metadata.js'; -import { slugify } from '../../utils/slugify.js'; -import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import type { GeospatialDataCategory, StacCollectionLinz } from '../../utils/metadata.ts'; +import { slugify } from '../../utils/slugify.ts'; +import { config, MeterAsString, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from '../common.ts'; export interface SlugMetadata { geospatialCategory: GeospatialDataCategory; diff --git a/src/commands/stac-sync/__test__/stac.sync.test.ts b/src/commands/stac-sync/__test__/stac.sync.test.ts index 8e985f5ab..f40c05bed 100644 --- a/src/commands/stac-sync/__test__/stac.sync.test.ts +++ b/src/commands/stac-sync/__test__/stac.sync.test.ts @@ -4,8 +4,8 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { hashBuffer, HashKey } from '../../../utils/hash.js'; -import { synchroniseFiles } from '../stac.sync.js'; +import { hashBuffer, HashKey } from '../../../utils/hash.ts'; +import { synchroniseFiles } from '../stac.sync.ts'; describe('stacSync', () => { const fs = new FsMemory(); diff --git a/src/commands/stac-sync/stac.sync.ts b/src/commands/stac-sync/stac.sync.ts index fe1f4bfe5..a97a12bb8 100644 --- a/src/commands/stac-sync/stac.sync.ts +++ b/src/commands/stac-sync/stac.sync.ts @@ -1,13 +1,14 @@ -import { FileInfo } from '@chunkd/core'; +import type { FileInfo } from '@chunkd/core'; import { fsa } from '@chunkd/fs'; -import { command, positional, string, Type } from 'cmd-ts'; +import type { Type } from 'cmd-ts'; +import { command, positional, string } from 'cmd-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { md } from '../../readme/markdown.js'; -import { annotateExample } from '../../readme/readme.example.js'; -import { hashBuffer, HashKey } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { md } from '../../readme/markdown.ts'; +import { annotateExample } from '../../readme/readme.example.ts'; +import { hashBuffer, HashKey } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; const S3Path: Type = { async from(str) { diff --git a/src/commands/stac-validate/__test__/stac.validate.test.ts b/src/commands/stac-validate/__test__/stac.validate.test.ts index a86468348..20908e8bb 100644 --- a/src/commands/stac-validate/__test__/stac.validate.test.ts +++ b/src/commands/stac-validate/__test__/stac.validate.test.ts @@ -3,7 +3,7 @@ import { before, beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; import { getStacSchemaUrl, @@ -13,7 +13,7 @@ import { validateAssets, validateLinks, validateStacChecksum, -} from '../stac.validate.js'; +} from '../stac.validate.ts'; describe('stacValidate', function () { it('listLocation', () => { diff --git a/src/commands/stac-validate/stac.validate.ts b/src/commands/stac-validate/stac.validate.ts index ae189fa06..1b242c88f 100644 --- a/src/commands/stac-validate/stac.validate.ts +++ b/src/commands/stac-validate/stac.validate.ts @@ -1,17 +1,18 @@ import { fsa } from '@chunkd/fs'; -import Ajv, { DefinedError, ValidateFunction } from 'ajv'; +import type { DefinedError, ValidateFunction } from 'ajv'; +import Ajv from 'ajv'; import { fastFormats } from 'ajv-formats/dist/formats.js'; import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; import { createHash } from 'crypto'; import { dirname, join } from 'path'; import { performance } from 'perf_hooks'; -import * as st from 'stac-ts'; +import type * as st from 'stac-ts'; -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { ConcurrentQueue } from '../../utils/concurrent.queue.js'; -import { hashStream, Sha256Prefix } from '../../utils/hash.js'; -import { config, registerCli, verbose } from '../common.js'; +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { ConcurrentQueue } from '../../utils/concurrent.queue.ts'; +import { hashStream, Sha256Prefix } from '../../utils/hash.ts'; +import { config, registerCli, verbose } from '../common.ts'; export const commandStacValidate = command({ name: 'stac-validate', diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts index 115fc751a..5a65f15ba 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.data.ts @@ -1,6 +1,7 @@ -import { SampleFormat, Size, Source, Tiff, TiffImage, TiffTag } from '@cogeotiff/core'; +import type { Size, Source, TiffImage } from '@cogeotiff/core'; +import { SampleFormat, Tiff, TiffTag } from '@cogeotiff/core'; -import { MapSheet } from '../../../utils/mapsheet.js'; +import { MapSheet } from '../../../utils/mapsheet.ts'; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts index c0fcc31ae..ca6cec2f1 100644 --- a/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts +++ b/src/commands/tileindex-validate/__test__/tileindex.validate.test.ts @@ -4,14 +4,15 @@ import { before, beforeEach, describe, it } from 'node:test'; import { Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { BBox } from '@linzjs/geojson'; -import { FeatureCollection } from 'geojson'; - -import { logger } from '../../../log.js'; -import { MapSheetData } from '../../../utils/__test__/mapsheet.data.js'; -import { FileListEntry } from '../../../utils/filelist.js'; -import { GridSize, MapSheet } from '../../../utils/mapsheet.js'; -import { createTiff } from '../../common.js'; +import type { BBox } from '@linzjs/geojson'; +import type { FeatureCollection } from 'geojson'; + +import { logger } from '../../../log.ts'; +import { MapSheetData } from '../../../utils/__test__/mapsheet.data.ts'; +import type { FileListEntry } from '../../../utils/filelist.ts'; +import type { GridSize } from '../../../utils/mapsheet.ts'; +import { MapSheet } from '../../../utils/mapsheet.ts'; +import { createTiff } from '../../common.ts'; import { commandTileIndexValidate, extractTiffLocations, @@ -22,8 +23,8 @@ import { TiffLoader, validate8BitsTiff, validatePreset, -} from '../tileindex.validate.js'; -import { FakeCogTiff } from './tileindex.validate.data.js'; +} from '../tileindex.validate.ts'; +import { FakeCogTiff } from './tileindex.validate.data.ts'; /* eslint-disable @typescript-eslint/no-explicit-any */ diff --git a/src/commands/tileindex-validate/tileindex.validate.ts b/src/commands/tileindex-validate/tileindex.validate.ts index 20330d6e8..bc40907c2 100644 --- a/src/commands/tileindex-validate/tileindex.validate.ts +++ b/src/commands/tileindex-validate/tileindex.validate.ts @@ -1,19 +1,23 @@ import { Bounds, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { Size, Tiff, TiffTag } from '@cogeotiff/core'; -import { BBox } from '@linzjs/geojson'; -import { boolean, command, flag, number, option, optional, restPositionals, string, Type } from 'cmd-ts'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { extractBandInformation } from '../../utils/band.js'; -import { FileFilter, getFiles } from '../../utils/chunk.js'; -import { createFileList } from '../../utils/filelist.js'; -import { findBoundingBox } from '../../utils/geotiff.js'; -import { GridSize, GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.js'; -import { config, createTiff, forceOutput, registerCli, verbose } from '../common.js'; -import { CommandListArgs } from '../list/list.js'; +import type { Size, Tiff } from '@cogeotiff/core'; +import { TiffTag } from '@cogeotiff/core'; +import type { BBox } from '@linzjs/geojson'; +import type { Type } from 'cmd-ts'; +import { boolean, command, flag, number, option, optional, restPositionals, string } from 'cmd-ts'; + +import { CliInfo } from '../../cli.info.ts'; +import { logger } from '../../log.ts'; +import { isArgo } from '../../utils/argo.ts'; +import { extractBandInformation } from '../../utils/band.ts'; +import type { FileFilter } from '../../utils/chunk.ts'; +import { getFiles } from '../../utils/chunk.ts'; +import { createFileList } from '../../utils/filelist.ts'; +import { findBoundingBox } from '../../utils/geotiff.ts'; +import type { GridSize } from '../../utils/mapsheet.ts'; +import { GridSizes, MapSheet, MapSheetTileGridSize } from '../../utils/mapsheet.ts'; +import { config, createTiff, forceOutput, registerCli, verbose } from '../common.ts'; +import { CommandListArgs } from '../list/list.ts'; export function isTiff(x: string): boolean { const search = x.toLowerCase(); diff --git a/src/fs.register.ts b/src/fs.register.ts index 492b6f667..b4d390a48 100644 --- a/src/fs.register.ts +++ b/src/fs.register.ts @@ -3,10 +3,11 @@ import { setTimeout } from 'node:timers/promises'; import { S3Client } from '@aws-sdk/client-s3'; import { fsa } from '@chunkd/fs'; import { FsAwsS3 } from '@chunkd/source-aws'; -import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3'; -import { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; +import type { S3LikeV3 } from '@chunkd/source-aws-v3'; +import { FsAwsS3V3 } from '@chunkd/source-aws-v3'; +import type { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types'; -import { logger } from './log.js'; +import { logger } from './log.ts'; /** Check to see if hostname exists inside of a object */ function hasHostName(x: unknown): x is { hostname: string } { diff --git a/src/index.ts b/src/index.ts index ca42249b4..75905afe2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,8 +3,8 @@ process.env['AWS_NODEJS_CONNECTION_REUSE_ENABLED'] = '1'; import { run } from 'cmd-ts'; -import { cmd } from './commands/index.js'; -import { logger } from './log.js'; +import { cmd } from './commands/index.ts'; +import { logger } from './log.ts'; const startTime = performance.now(); run(cmd, process.argv.slice(2)) diff --git a/src/readme/readme.generate.ts b/src/readme/readme.generate.ts index 7f7fb39a0..578805892 100644 --- a/src/readme/readme.generate.ts +++ b/src/readme/readme.generate.ts @@ -1,10 +1,11 @@ +import { writeFileSync } from 'node:fs'; + import { fsa } from '@chunkd/fs'; -import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.js'; -import { writeFileSync } from 'fs'; +import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.ts'; import * as prettier from 'prettier'; -import { AllCommands } from '../commands/index.js'; -import { commandHasExample, ExampleSymbol } from './readme.example.js'; +import { AllCommands } from '../commands/index.ts'; +import { commandHasExample, ExampleSymbol } from './readme.example.ts'; const AnsiRemove = /\u001b\[.*?m/g; function hasHelp(f: unknown): f is ProvidesHelp { diff --git a/src/utils/__test__/argo.test.ts b/src/utils/__test__/argo.test.ts index 07910a55e..9d7f3a0e0 100644 --- a/src/utils/__test__/argo.test.ts +++ b/src/utils/__test__/argo.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { beforeEach, describe, it } from 'node:test'; -import { getActionLocation } from '../action.storage.js'; +import { getActionLocation } from '../action.storage.ts'; describe('argoLocation', () => { beforeEach(() => { diff --git a/src/utils/__test__/band.test.ts b/src/utils/__test__/band.test.ts index b861b8b17..1c294c7bb 100644 --- a/src/utils/__test__/band.test.ts +++ b/src/utils/__test__/band.test.ts @@ -1,8 +1,8 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { createTiff } from '../../commands/common.js'; -import { extractBandInformation } from '../band.js'; +import { createTiff } from '../../commands/common.ts'; +import { extractBandInformation } from '../band.ts'; describe('extractBandInformation', () => { it('should extract basic band information (8-bit)', async () => { diff --git a/src/utils/__test__/chunk.test.ts b/src/utils/__test__/chunk.test.ts index cf53c2fa4..cf7bfe48a 100644 --- a/src/utils/__test__/chunk.test.ts +++ b/src/utils/__test__/chunk.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { getFiles, splitPaths } from '../chunk.js'; +import { getFiles, splitPaths } from '../chunk.ts'; describe('splitPaths', () => { it('should split on ;', () => { diff --git a/src/utils/__test__/config.test.ts b/src/utils/__test__/config.test.ts index b5060b82f..380d5d117 100644 --- a/src/utils/__test__/config.test.ts +++ b/src/utils/__test__/config.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import prettier from 'prettier'; -import { DEFAULT_PRETTIER_FORMAT } from '../config.js'; +import { DEFAULT_PRETTIER_FORMAT } from '../config.ts'; describe('DefaultPrettierFormat', () => { it('should be the same prettier format with @linzjs/style', async () => { diff --git a/src/utils/__test__/date.test.ts b/src/utils/__test__/date.test.ts index aa2535c14..6def66e00 100644 --- a/src/utils/__test__/date.test.ts +++ b/src/utils/__test__/date.test.ts @@ -2,7 +2,7 @@ import { describe, it } from 'node:test'; import assert from 'assert'; -import { getPacificAucklandYearMonthDay } from '../date.js'; +import { getPacificAucklandYearMonthDay, RelativeDate } from '../date.ts'; describe('getPacificAucklandYearMonthDay', () => { it('should format as yyyy-mm-dd', () => { @@ -12,3 +12,40 @@ describe('getPacificAucklandYearMonthDay', () => { assert.equal(getPacificAucklandYearMonthDay('2012-06-15T12:00:00Z'), '2012-06-16'); }); }); + +describe('relativeDateParser', () => { + const OneMinuteMs = 60 * 1000; + const OneHourMs = 60 * OneMinuteMs; + const OneDayMs = 24 * OneHourMs; + + it('should parse ISO Dates', async () => { + assert.equal((await RelativeDate.from('2023-02-10T00:00:00.000Z')).toISOString(), '2023-02-10T00:00:00.000Z'); + assert.equal((await RelativeDate.from('2023-02-10')).toISOString(), '2023-02-10T00:00:00.000Z'); + }); + + it('should parse known formats', async () => { + const parseTime = (s: string): Promise => RelativeDate.from(s).then((f) => f.getTime()); + + // Validate times are approximatly the same + const approxEqual = (a: number, b: number, diff: number): void => assert.ok(Math.abs(a - b) < diff); + + approxEqual(await parseTime('23m'), new Date(new Date().getTime() - OneMinuteMs * 23).getTime(), OneMinuteMs); + approxEqual(await parseTime('23M'), new Date(new Date().getTime() - OneMinuteMs * 23).getTime(), OneMinuteMs); + + approxEqual(await parseTime('23h'), new Date(new Date().getTime() - OneHourMs * 23).getTime(), OneMinuteMs); + approxEqual(await parseTime('23H'), new Date(new Date().getTime() - OneHourMs * 23).getTime(), OneMinuteMs); + + approxEqual(await parseTime('23d'), new Date(new Date().getTime() - OneDayMs * 23).getTime(), OneHourMs); + approxEqual(await parseTime('23D'), new Date(new Date().getTime() - OneDayMs * 23).getTime(), OneHourMs); + }); + + it('should error with invalid formats', async () => { + const parseError = (s: string): Promise => + RelativeDate.from(s) + .then(String) + .catch((f) => String(f)); + + assert.equal(await parseError('23Z'), 'Error: Unable to parse date from: 23Z'); + assert.equal(await parseError('abc'), 'Error: Invalid date: abc'); + }); +}); diff --git a/src/utils/__test__/filelist.test.ts b/src/utils/__test__/filelist.test.ts index 346b74f51..153fc6694 100644 --- a/src/utils/__test__/filelist.test.ts +++ b/src/utils/__test__/filelist.test.ts @@ -1,8 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { TiffLocation } from '../../commands/tileindex-validate/tileindex.validate.js'; -import { createFileList, FileListEntry } from '../filelist.js'; +import type { TiffLocation } from '../../commands/tileindex-validate/tileindex.validate.ts'; +import type { FileListEntry } from '../filelist.ts'; +import { createFileList } from '../filelist.ts'; describe('createFileList', () => { const locationTestOne: TiffLocation = { bands: [], bbox: [0, 0, 0, 0], tileNames: [], source: 'input1' }; diff --git a/src/utils/__test__/filter.test.ts b/src/utils/__test__/filter.test.ts index fb379b9aa..b869a5aa3 100644 --- a/src/utils/__test__/filter.test.ts +++ b/src/utils/__test__/filter.test.ts @@ -3,7 +3,8 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; -import { asyncFilter } from '../chunk.js'; +import { asyncFilter } from '../chunk.ts'; +import { RelativeDate } from '../date.ts'; describe('AsyncFilter', () => { function makeGenerator(list: string[]): () => AsyncGenerator<{ path: string }> { @@ -12,6 +13,28 @@ describe('AsyncFilter', () => { }; } + /** + * Generate a list of files with last modified dates starting at 1 hour ago then increasing by one hour + * for every item in the list + * + * @example + * makeGeneratorWithDates(['a', 'b', 'c']) + * // a 1 hour ago + * // b 2 hours ago + * // c 3 hourss ago + * + * @param list + * @returns + */ + function makeGeneratorWithDates(list: string[]): () => AsyncGenerator<{ path: string; lastModified?: string }> { + const startTime = new Date().getTime(); + let index = 1; + return async function* gen(): AsyncGenerator<{ path: string; lastModified: string }> { + for (const path of list) + yield { path, lastModified: new Date(startTime - 60 * 60 * 1000 * index++).toISOString() }; + }; + } + it('should include all', async () => { const gen = makeGenerator(['a.tiff', 'b.tiff', 'c.tiff']); const result = await fsa.toArray(asyncFilter(gen())); @@ -47,4 +70,38 @@ describe('AsyncFilter', () => { const result = await fsa.toArray(asyncFilter(gen(), { exclude: '^a', include: '.tiff$' })); assert.deepEqual(result, [{ path: 'CA.TIFF' }]); }); + + const filterDates = async ( + opt: { since?: string; until?: string }, + files = ['A.tiff', 'B.tiff', 'C.tiff'], + ): Promise => { + const gen = makeGeneratorWithDates(files); + return fsa + .toArray( + asyncFilter(gen(), { + since: opt.since ? await RelativeDate.from(opt.since) : undefined, + until: opt.until ? await RelativeDate.from(opt.until) : undefined, + }), + ) + .then((m) => m.map((f) => f.path)); + }; + + it('should include since', async () => { + assert.deepEqual(await filterDates({ since: '1m' }), []); + assert.deepEqual(await filterDates({ since: '61m' }), ['A.tiff']); + assert.deepEqual(await filterDates({ since: '121m' }), ['A.tiff', 'B.tiff']); + assert.deepEqual(await filterDates({ since: '181m' }), ['A.tiff', 'B.tiff', 'C.tiff']); + }); + + it('should exclude until', async () => { + assert.deepEqual(await filterDates({ until: '1m' }), ['A.tiff', 'B.tiff', 'C.tiff']); + assert.deepEqual(await filterDates({ until: '61m' }), ['B.tiff', 'C.tiff']); + assert.deepEqual(await filterDates({ until: '121m' }), ['C.tiff']); + assert.deepEqual(await filterDates({ until: '181m' }), []); + }); + it('should include and exclude', async () => { + assert.deepEqual(await filterDates({ until: '1m', since: '61m' }), ['A.tiff']); + assert.deepEqual(await filterDates({ until: '61m', since: '121m' }), ['B.tiff']); + assert.deepEqual(await filterDates({ until: '61m', since: '181m' }), ['B.tiff', 'C.tiff']); + }); }); diff --git a/src/utils/__test__/geotiff.test.ts b/src/utils/__test__/geotiff.test.ts index 1d6a5dedd..9adf7df15 100644 --- a/src/utils/__test__/geotiff.test.ts +++ b/src/utils/__test__/geotiff.test.ts @@ -3,10 +3,10 @@ import { describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; -import { Source, Tiff, TiffImage } from '@cogeotiff/core'; +import type { Source, Tiff, TiffImage } from '@cogeotiff/core'; -import { createTiff } from '../../commands/common.js'; -import { findBoundingBox, parseTfw, PixelIsPoint } from '../geotiff.js'; +import { createTiff } from '../../commands/common.ts'; +import { findBoundingBox, parseTfw, PixelIsPoint } from '../geotiff.ts'; describe('geotiff', () => { describe('parseTfw', () => { diff --git a/src/utils/__test__/hash.test.ts b/src/utils/__test__/hash.test.ts index 755f0e8fd..7a71d9e96 100644 --- a/src/utils/__test__/hash.test.ts +++ b/src/utils/__test__/hash.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { Readable } from 'node:stream'; import { describe, it } from 'node:test'; -import { hashBuffer, hashStream } from '../hash.js'; +import { hashBuffer, hashStream } from '../hash.ts'; describe('hashBuffer', () => { it('should return the expecting digest', () => { diff --git a/src/utils/__test__/mapsheet.test.ts b/src/utils/__test__/mapsheet.test.ts index c07264887..b2bc8437a 100644 --- a/src/utils/__test__/mapsheet.test.ts +++ b/src/utils/__test__/mapsheet.test.ts @@ -1,8 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { MapSheet, MapTileIndex, SheetRanges } from '../mapsheet.js'; -import { MapSheetData } from './mapsheet.data.js'; +import type { MapTileIndex } from '../mapsheet.ts'; +import { MapSheet, SheetRanges } from '../mapsheet.ts'; +import { MapSheetData } from './mapsheet.data.ts'; describe('MapSheets', () => { it('should extract MapTileIndex from 1:500 tile filename', () => { diff --git a/src/utils/__test__/slugify.test.ts b/src/utils/__test__/slugify.test.ts index bb52fa4e1..85bef5ed0 100644 --- a/src/utils/__test__/slugify.test.ts +++ b/src/utils/__test__/slugify.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { slugify } from '../slugify.js'; +import { slugify } from '../slugify.ts'; const slugChars = 'abcdefghijklmnopqrstuvwxyz0123456789_.-'; diff --git a/src/utils/band.ts b/src/utils/band.ts index 5be62b36f..7349dd64f 100644 --- a/src/utils/band.ts +++ b/src/utils/band.ts @@ -1,4 +1,5 @@ -import { SampleFormat, Tiff, TiffImage, TiffTag } from '@cogeotiff/core'; +import type { Tiff, TiffImage } from '@cogeotiff/core'; +import { SampleFormat, TiffTag } from '@cogeotiff/core'; function getDataType(i: SampleFormat): string { switch (i) { diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts index 56eb8806d..fc0225f5e 100644 --- a/src/utils/chunk.ts +++ b/src/utils/chunk.ts @@ -1,21 +1,30 @@ import { fsa } from '@chunkd/fs'; -import { parseSize } from '../commands/common.js'; -import { logger } from '../log.js'; +import { parseSize } from '../commands/common.ts'; +import { logger } from '../log.ts'; export interface FileSizeInfo { path: string; size?: number; } -export async function* asyncFilter( +export async function* asyncFilter( source: AsyncGenerator, - opts?: { include?: string; exclude?: string }, + opts?: { include?: string; exclude?: string; since?: Date; until?: Date }, ): AsyncGenerator { const include = opts?.include ? new RegExp(opts.include.toLowerCase(), 'i') : true; const exclude = opts?.exclude ? new RegExp(opts.exclude.toLowerCase(), 'i') : undefined; + + const sinceTime = opts?.since?.toISOString(); + const untilTime = opts?.until?.toISOString(); for await (const f of source) { const testPath = f.path.toLowerCase(); + + if (f.lastModified) { + if (sinceTime && sinceTime > f.lastModified) continue; + if (untilTime && untilTime < f.lastModified) continue; + } + if (exclude && exclude.test(testPath)) continue; if (include === true) yield f; else if (include.test(testPath)) yield f; @@ -102,6 +111,15 @@ export type FileFilter = { * @default 1 */ sizeMin?: number; + + /** + * Filter files with a modified time since this time. + */ + since?: Date; + /** + * Filter files with a modified time until this time + */ + until?: Date; }; export async function getFiles(paths: string[], args: FileFilter = {}): Promise { const limit = args.limit ?? -1; // no limit by default diff --git a/src/utils/concurrent.queue.ts b/src/utils/concurrent.queue.ts index 8d2b46b1b..f71ec4f33 100644 --- a/src/utils/concurrent.queue.ts +++ b/src/utils/concurrent.queue.ts @@ -1,4 +1,5 @@ -import pLimit, { LimitFunction } from 'p-limit'; +import type { LimitFunction } from 'p-limit'; +import pLimit from 'p-limit'; export class ConcurrentQueue { Q: LimitFunction; diff --git a/src/utils/config.ts b/src/utils/config.ts index ad43f4253..12162fc9e 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -1,4 +1,4 @@ -import prettier from 'prettier'; +import type prettier from 'prettier'; export const DEFAULT_PRETTIER_FORMAT: prettier.Options = { semi: true, diff --git a/src/utils/date.ts b/src/utils/date.ts index 036f24115..a5889f0b1 100644 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -1,3 +1,5 @@ +import type { Type } from 'cmd-ts'; + /** * Convert time zone-aware date/time string to Pacific/Auckland time zone string * @@ -15,3 +17,45 @@ export function getPacificAucklandYearMonthDay(dateTimeString?: string | null): return pacificAucklandDateTimeString.slice(0, 10); } + +/** + * Parse a date using relative times + * + * use for arguments like + * - `--since=23m` Since 23 minutes ago + * - `--until=30d` Until 30 days ago + * + * @example + * "30s" // 30 seconds ago + * "23m" // 23 minutes ago + * "24h" // 24 hours ago + * "90d" // 90 days ago + */ +export const RelativeDate: Type = { + async from(s: string): Promise { + // Support ISO dates, eg "2023-02-10" or "2023-02-10T12:00:00.000Z" + const d = new Date(s); + if (!Number.isNaN(d.getTime())) return d; + + const scale = s.toLowerCase().slice(-1); + const digits = Number.parseInt(s.slice(0, -1)); + if (Number.isNaN(digits)) throw new Error(`Invalid date: ${s}`); + + const current = new Date(); + switch (scale) { + case 's': + current.setUTCSeconds(current.getUTCSeconds() - digits); + return current; + case 'm': + current.setUTCMinutes(current.getUTCMinutes() - digits); + return current; + case 'h': + current.setUTCHours(current.getUTCHours() - digits); + return current; + case 'd': + current.setUTCDate(current.getUTCDate() - digits); + return current; + } + throw new Error(`Unable to parse date from: ${s}`); + }, +}; diff --git a/src/utils/filelist.ts b/src/utils/filelist.ts index 1ae97fbf1..9817f82b4 100644 --- a/src/utils/filelist.ts +++ b/src/utils/filelist.ts @@ -1,4 +1,4 @@ -import { TiffLocation } from '../commands/tileindex-validate/tileindex.validate.js'; +import type { TiffLocation } from '../commands/tileindex-validate/tileindex.validate.ts'; export type FileListEntry = { output: string; diff --git a/src/utils/geotiff.ts b/src/utils/geotiff.ts index d0fc044d8..74eeceae4 100644 --- a/src/utils/geotiff.ts +++ b/src/utils/geotiff.ts @@ -1,7 +1,8 @@ import { fsa } from '@chunkd/fs'; -import { RasterTypeKey, Tiff, TiffTagGeo } from '@cogeotiff/core'; +import type { Tiff } from '@cogeotiff/core'; +import { RasterTypeKey, TiffTagGeo } from '@cogeotiff/core'; -import { urlToString } from '../commands/common.js'; +import { urlToString } from '../commands/common.ts'; /** * Attempt to parse a tiff world file diff --git a/src/utils/github.ts b/src/utils/github.ts index 1e4d38318..e697fff4e 100644 --- a/src/utils/github.ts +++ b/src/utils/github.ts @@ -1,8 +1,8 @@ import { Octokit } from '@octokit/core'; import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'; -import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; +import type { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; -import { logger } from '../log.js'; +import { logger } from '../log.ts'; export interface Blob { path: string; diff --git a/src/utils/hash.stream.ts b/src/utils/hash.stream.ts index 42136011e..d39e9a5a9 100644 --- a/src/utils/hash.stream.ts +++ b/src/utils/hash.stream.ts @@ -1,5 +1,7 @@ -import { createHash, Hash } from 'crypto'; -import { Transform, TransformCallback } from 'stream'; +import type { Hash } from 'crypto'; +import { createHash } from 'crypto'; +import type { TransformCallback } from 'stream'; +import { Transform } from 'stream'; export class HashTransform extends Transform { /** hash function in use */ diff --git a/src/utils/hash.ts b/src/utils/hash.ts index 7527b5848..70e04c8fb 100644 --- a/src/utils/hash.ts +++ b/src/utils/hash.ts @@ -1,5 +1,6 @@ -import { BinaryLike, createHash } from 'crypto'; -import { Readable } from 'stream'; +import type { BinaryLike } from 'crypto'; +import { createHash } from 'crypto'; +import type { Readable } from 'stream'; /** Key concatenated to 'x-amz-meta-' */ export const HashKey = 'multihash'; diff --git a/tsconfig.json b/tsconfig.json index c8c83f010..07817dabf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,8 @@ "compilerOptions": { "lib": ["ES2022"], "target": "ES2022", - "outDir": "build" + "outDir": "build", + "noEmit": true, + "allowImportingTsExtensions": true } }