feat(cli-config): generate QA preview URLs by pipeline BM-1407

packages/cli-config/src/__tests__/action.import.test.ts

@@ -0,0 +1,111 @@
+import { before, beforeEach, describe, it, TestContext } from 'node:test';
+
+import { ConfigProviderMemory, getAllImagery, sha256base58 } from '@basemaps/config';
+import { ConfigJson } from '@basemaps/config-loader';
+import { ConfigImageryTiff } from '@basemaps/config-loader/build/json/tiff.config.js';
+import { Epsg, EpsgCode, GoogleTms, TileMatrixSet, TileMatrixSets } from '@basemaps/geo';
+import { fsa, FsMemory, LogConfig } from '@basemaps/shared';
+import pLimit from 'p-limit';
+
+import assert from 'assert';
+import { prepareUrls } from '../cli/action.import.js';
+import { TsElevation } from './config.diff.data.js';
+
+function splitUrlIntoParts(e: URL): { tileMatrix: TileMatrixSet; name: string; gsd: number } {
+  const parts = e.pathname.slice(1).split('/');
+
+  if (e.hostname === 'linz-basemaps') {
+    // /3857/:name/:id/
+    let partOffset = 0;
+    // /elevation/3857/:name/:id/
+    if (parts[0] === 'elevation') partOffset++;
+
+    return {
+      tileMatrix: TileMatrixSets.get(Epsg.parseCode(parts[partOffset]) as EpsgCode),
+      name: parts[partOffset + 1],
+      gsd: parseFloat(parts[partOffset + 1].split('_').at(-1) ?? '0'),
+    };
+  }
+  if (e.hostname === 'nz-imagery' || e.hostname === 'nz-elevation') {
+    return {
+      tileMatrix: TileMatrixSets.get(Epsg.parseCode(parts[3]) as EpsgCode),
+      name: parts[1],
+      gsd: 1,
+    };
+  }
+
+  throw new Error('Unable to parse path: ' + e.href);
+}
+
+describe('action.import', () => {
+  const fsMem = new FsMemory();
+  const elevationJson = JSON.stringify(TsElevation);
+
+  before(() => {
+    // Imagery is linked from s3, overwrite s3 links to just use memory
+    fsa.register('s3://', fsMem);
+    fsa.register('source://', fsMem);
+
+    LogConfig.get().level = 'silent';
+  });
+
+  async function stubLoadConfig(t: TestContext, source: string): Promise<ConfigProviderMemory> {
+    t.mock.method(ConfigJson.prototype, 'initImageryFromTiffUrl', (url: URL) => {
+      const parts = splitUrlIntoParts(url);
+
+      const imagery: ConfigImageryTiff = {
+        v: 2,
+        id: `im_${sha256base58(url.href)}`,
+        name: parts.name,
+        title: parts.name,
+        updatedAt: Date.now(),
+        projection: parts.tileMatrix.projection.code,
+        tileMatrix: parts.tileMatrix.identifier ?? 'none',
+        gsd: parts.gsd,
+        uri: url.href,
+        url: url,
+        bounds: { x: 0, y: 0, width: 0, height: 0 },
+        bands: [
+          { type: 'uint8', color: 'red' },
+          { type: 'uint8', color: 'green' },
+          { type: 'uint8', color: 'blue' },
+          { type: 'uint8', color: 'alpha' },
+        ],
+        files: [],
+      };
+      return Promise.resolve(imagery);
+    });
+
+    const mem = await ConfigJson.fromUrl(new URL(source), pLimit(1), LogConfig.get());
+    mem.createVirtualTileSets();
+    return mem;
+  }
+
+  describe('elevation', () => {
+    beforeEach(async () => {
+      fsMem.files.clear();
+      await fsa.write(fsa.toUrl('source://config/tileset/elevation.json'), elevationJson);
+    });
+
+    it.skip('should prepare a url for each pipeline', async (t) => {
+      const cfg = await stubLoadConfig(t, 'source://config/');
+
+      const tsElevation = await cfg.TileSet.get('elevation');
+      assert.ok(tsElevation, 'Should have loaded the elevation tile set');
+      assert.equal(tsElevation.layers.length, 3);
+
+      const all3857 = await getAllImagery(cfg, tsElevation.layers, [Epsg.Google]);
+      assert.equal(Array.from(all3857.keys()).length, 3);
+
+      for (const id of all3857.keys()) {
+        const urls = await prepareUrls(id, cfg, GoogleTms, 'FAKE_CONFIG_PATH');
+
+        // ISSUE: doesn't produce the expected QA links for elevation imagery.
+        // we should see `color-ramp` and `terrain-rgb` keys, but we don't.
+        // the `import` CLI command itself doesn't see this issue.
+        // I suspect I need to re-work the `stubLoadConfig` function.
+        console.log({ id, urls: JSON.stringify(urls) });
+      }
+    });
+  });
+});

packages/cli-config/src/cli/action.import.ts

@@ -10,7 +10,7 @@ import {
   ConfigTileSetVector,
   TileSetType,
 } from '@basemaps/config';
-import { GoogleTms, Nztm2000QuadTms, TileMatrixSet } from '@basemaps/geo';
+import { EpsgCode, GoogleTms, Nztm2000QuadTms, TileMatrixSet } from '@basemaps/geo';
 import {
   Env,
   fsa,
@@ -234,16 +234,29 @@ export async function outputChange(
     } else await outputNewLayers(mem, layer, inserts, configPath, true);
   }
 
-  if (inserts.length > 0) outputMarkdown.push('# 🟩🟩 Aerial Imagery Inserts 🟩🟩', ...inserts);
-  if (updates.length > 0) outputMarkdown.push('# 🟡🟡 Aerial Imagery Updates 🟡🟡', ...updates);
+  if (inserts.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>🟩🟩 Aerial Imagery Inserts 🟩🟩</b></summary>\n');
+    outputMarkdown.push(...inserts);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
+  if (updates.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>🟡🟡 Aerial Imagery Updates 🟡🟡</b></summary>\n');
+    outputMarkdown.push(...updates);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
 
   // Some layers were not removed from the old config so they no longer exist in the new config
   if (oldData.layers.length > 0) {
-    outputMarkdown.push(
-      '# 🚨🚨 Aerial Imagery Deletes 🚨🚨',
-      ' Basemaps layers will be removed for the following layers: ',
-      ...oldData.layers.map((m) => `- ${m.title}`),
-    );
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>🚨🚨 Aerial Imagery Deletes 🚨🚨</b></summary>\n');
+    outputMarkdown.push('### Basemaps layers will be removed for the following layers:\n');
+    outputMarkdown.push(...oldData.layers.map((m) => `- ${m.title}`));
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
   }
 
   // Output for individual tileset config changes or inserts
@@ -262,8 +275,20 @@ export async function outputChange(
     else await outputNewLayers(mem, layer, individualInserts, configPath);
   }
 
-  if (individualInserts.length > 0) outputMarkdown.push('# Individual Inserts', ...individualInserts);
-  if (individualUpdates.length > 0) outputMarkdown.push('# Individual Updates', ...individualUpdates);
+  if (individualInserts.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>Individual Inserts</b></summary>\n');
+    outputMarkdown.push(...individualInserts);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
+  if (individualUpdates.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>Individual Updates</b></summary>\n');
+    outputMarkdown.push(...individualUpdates);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
 
   // Output for vector config changes
   const vectorUpdate = [];
@@ -307,8 +332,20 @@ export async function outputChange(
     }
   }
 
-  if (styleUpdate.length > 0) outputMarkdown.push('# Vector Style Update', ...styleUpdate);
-  if (vectorUpdate.length > 0) outputMarkdown.push('# Vector Data Update', ...vectorUpdate);
+  if (styleUpdate.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>Vector Style Update</b></summary>\n');
+    outputMarkdown.push(...styleUpdate);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
+  if (vectorUpdate.length > 0) {
+    outputMarkdown.push('<details>\n');
+    outputMarkdown.push('<summary><b>Vector Data Update</b></summary>\n');
+    outputMarkdown.push(...vectorUpdate);
+    outputMarkdown.push('</details>\n');
+    outputMarkdown.push('---\n');
+  }
 
   return outputMarkdown.join('\n');
 }
@@ -326,18 +363,31 @@ async function outputNewLayers(
   layer: ConfigLayer,
   inserts: string[],
   configPath: string,
-  aerial?: boolean,
+  isAerial?: boolean,
 ): Promise<void> {
-  inserts.push(`\n### ${layer.name}\n`);
-  if (layer[2193]) {
-    const urls = await prepareUrl(layer[2193], mem, Nztm2000QuadTms, configPath);
-    inserts.push(` - [NZTM2000Quad](${urls.layer})`);
-    if (aerial) inserts.push(` - [Aerial](${urls.tag})`);
-  }
-  if (layer[3857]) {
-    const urls = await prepareUrl(layer[3857], mem, GoogleTms, configPath);
-    inserts.push(` - [WebMercatorQuad](${urls.layer})`);
-    if (aerial) inserts.push(` - [Aerial](${urls.tag})`);
+  inserts.push(`### ${layer.name}`);
+  inserts.push(layer.title);
+
+  for (const { code, tms } of [
+    { code: EpsgCode.Nztm2000, tms: Nztm2000QuadTms },
+    { code: EpsgCode.Google, tms: GoogleTms },
+  ]) {
+    if (layer[code] == null) continue;
+    inserts.push(`- ${code} (${tms.identifier})`);
+
+    const urls = await prepareUrls(layer[code], mem, tms, configPath);
+
+    // append URLs grouped by pipeline
+    for (const [pipeline, formats] of Object.entries(urls).sort()) {
+      inserts.push(`  - Pipeline: ${pipeline}`);
+
+      for (const [format, { aerial, individual }] of Object.entries(formats).sort()) {
+        inserts.push(`    - Format: ${format}`);
+
+        if (isAerial) inserts.push(`      - [Aerial](${aerial})`);
+        inserts.push(`      - [Individual](${individual})`);
+      }
+    }
   }
 }
 
@@ -356,62 +406,109 @@ async function outputUpdatedLayers(
   existing: ConfigLayer,
   updates: string[],
   configPath: string,
-  aerial?: boolean,
+  isAerial?: boolean,
 ): Promise<void> {
-  let zoom = undefined;
-  if (layer.minZoom !== existing.minZoom || layer.maxZoom !== existing.maxZoom) {
-    zoom = ' - Zoom level updated.';
-    if (layer.minZoom !== existing.minZoom) zoom += ` min zoom ${existing.minZoom} -> ${layer.minZoom}`;
-    if (layer.maxZoom !== existing.maxZoom) zoom += ` max zoom ${existing.maxZoom} -> ${layer.maxZoom}`;
-  }
+  let zoomChanged = false;
+  const zoomChange: string[] = [];
 
-  const change: string[] = [`\n### ${layer.name}\n`];
-  if (layer[2193]) {
-    if (layer[2193] !== existing[2193]) {
-      const urls = await prepareUrl(layer[2193], mem, Nztm2000QuadTms, configPath);
-      change.push(`- Layer update [NZTM2000Quad](${urls.layer})`);
-      if (aerial) updates.push(` - [Aerial](${urls.tag})`);
-    }
+  if (layer.minZoom !== existing.minZoom) {
+    zoomChange.push(`min zoom ${existing.minZoom} -> ${layer.minZoom}`);
+    zoomChanged = true;
+  }
 
-    if (zoom) {
-      const urls = await prepareUrl(layer[2193], mem, Nztm2000QuadTms, configPath);
-      zoom += ` [NZTM2000Quad](${urls.tag})`;
-    }
+  if (layer.maxZoom !== existing.maxZoom) {
+    zoomChange.push(`max zoom ${existing.maxZoom} -> ${layer.maxZoom}`);
+    zoomChanged = true;
   }
-  if (layer[3857]) {
-    if (layer[3857] !== existing[3857]) {
-      const urls = await prepareUrl(layer[3857], mem, GoogleTms, configPath);
-      change.push(`- Layer update [WebMercatorQuad](${urls.layer})`);
-      if (aerial) updates.push(` - [Aerial](${urls.tag})`);
-    }
 
-    if (zoom) {
-      const urls = await prepareUrl(layer[3857], mem, GoogleTms, configPath);
-      zoom += ` [WebMercatorQuad](${urls.tag})`;
+  const changes: string[] = [`### ${layer.name}`];
+  changes.push(layer.title);
+
+  for (const { code, tms } of [
+    { code: EpsgCode.Nztm2000, tms: Nztm2000QuadTms },
+    { code: EpsgCode.Google, tms: GoogleTms },
+  ]) {
+    if (layer[code] == null) continue;
+    changes.push(`- ${code} (${tms.identifier})`);
+
+    const urls = await prepareUrls(layer[code], mem, tms, configPath);
+
+    // append URLs grouped by pipeline
+    for (const [pipeline, formats] of Object.entries(urls).sort()) {
+      changes.push(`  - Pipeline: ${pipeline}`);
+
+      for (const [format, { aerial, individual }] of Object.entries(formats).sort()) {
+        changes.push(`    - Format: ${format}`);
+
+        if (isAerial) changes.push(`      - [Aerial](${aerial})`);
+        changes.push(`      - [Individual](${individual})`);
+
+        if (zoomChanged) zoomChange.push(`[${code}](${aerial})`);
+      }
     }
   }
 
-  if (zoom) change.push(`${zoom}\n`);
-  if (change.length > 1) updates.push(change.join(''));
+  if (zoomChange.length > 1) changes.push(`${zoomChange.join('\n')}\n`);
+  if (changes.length > 1) updates.push(changes.join('\n'));
 }
 
 /**
  * Prepare QA urls with center location
  */
-async function prepareUrl(
+export async function prepareUrls(
   id: string,
   mem: BasemapsConfigProvider,
   tileMatrix: TileMatrixSet,
   configPath: string,
-): Promise<{ layer: string; tag: string }> {
+): Promise<{ [pipeline: string]: { [format: string]: { aerial: string; individual: string } } }> {
   const configImagery = await mem.Imagery.get(id);
   if (configImagery == null) throw new Error(`Failed to find imagery config from config bundle file. Id: ${id}`);
 
   const center = getPreviewUrl({ imagery: configImagery });
-  const urls = {
-    layer: `${PublicUrlBase}?config=${configPath}&i=${center.name}&p=${tileMatrix.identifier}&debug#@${center.location.lat},${center.location.lon},z${center.location.zoom}`,
-    tag: `${PublicUrlBase}?config=${configPath}&p=${tileMatrix.identifier}&debug#@${center.location.lat},${center.location.lon},z${center.location.zoom}`,
-  };
+  const urls: { [pipeline: string]: { [format: string]: { aerial: string; individual: string } } } = {};
+
+  const tileset = ConfigProviderMemory.imageryToTileSet(configImagery);
+  const outputs = tileset.outputs;
+
+  const aerial = `${PublicUrlBase}@${center.location.lat},${center.location.lon},z${center.location.zoom}?c=${configPath}&p=${tileMatrix.identifier}&debug`;
+  const individual = `${PublicUrlBase}@${center.location.lat},${center.location.lon},z${center.location.zoom}?c=${configPath}&p=${tileMatrix.identifier}&i=${center.name}&debug`;
+
+  if (outputs == null) {
+    urls['default'] = {
+      default: {
+        aerial,
+        individual,
+      },
+    };
+
+    return urls;
+  }
+
+  for (const output of outputs) {
+    const pipeline = output.name;
+    const formats = output.format;
+
+    if (formats == null) {
+      urls[pipeline] = {
+        default: {
+          aerial: `${aerial}&pipeline=${pipeline}`,
+          individual: `${individual}&pipeline=${pipeline}`,
+        },
+      };
+
+      return urls;
+    }
+
+    urls[pipeline] = {};
+
+    for (const format of formats) {
+      urls[pipeline][format] = {
+        aerial: `${aerial}&pipeline=${pipeline}&format=${format}`,
+        individual: `${individual}&pipeline=${pipeline}&format=${format}`,
+      };
+    }
+  }
+
   return urls;
 }