Merge pull request #3677 from kbase/URO-352-dts-import

URO-352 DTS importer setup for narrative

Author: briehl

kbase-extension/static/kbase/config/staging_upload.json

@@ -52,6 +52,10 @@
             "id": "import_specification",
             "name": "Import Specification"
         },
+        {
+            "id": "dts_manifest",
+            "name": "Data Transfer Service Manifest"
+        },
         {
             "id": "decompress",
             "name": "Decompress/Unpack"
@@ -62,13 +66,15 @@
         "fastq_reads_interleaved",
         "fastq_reads_noninterleaved",
         "gff_metagenome",
+        "gff_genome",
         "assembly",
         "genbank_genome"
     ],
     "app_info": {
         "web_upload": {
             "app_id": "kb_uploadmethods/upload_web_file"
         },
+        "dts_manifest": { },
         "import_specification": { },
         "test_fastq_reads": {
             "app_id": "NarrativeTest/example_reads_upload",

kbase-extension/static/kbase/js/api/StagingServiceClient.js

@@ -21,7 +21,10 @@ define(['RestAPIClient'], (RestAPIClient) => {
                 rename: { method: 'post', path: 'rename/${path}' },
                 decompress: { method: 'patch', path: 'decompress/${path}' },
                 importer_mappings: { method: 'get', path: 'importer_mappings/?${file_list}' },
-                bulkSpecification: { method: 'get', path: 'bulk_specification/?files=${files}' },
+                bulkSpecification: {
+                    method: 'get',
+                    path: 'bulk_specification/?files=${files}&${flag}',
+                },
                 write_bulk_specification: { method: 'post', path: 'write_bulk_specification/' },
             },
         });

kbase-extension/static/kbase/js/widgets/narrative_core/upload/importSetup.js

@@ -14,7 +14,7 @@ define([
 
     /**
      * This makes a call to the Staging Service to fetch information from bulk specification files.
-     * This then gets processed through `processSpreadsheetFileData` before being returned.
+     * This then gets processed through `processBulkImportSpecData` before being returned.
      * @param {Array[string]} files - array of file path names to treat as import specifications
      * @returns Promise that resolves into information that can be used to open a bulk import cell.
      * This has the format:
@@ -32,45 +32,18 @@ define([
      *   }
      * }
      * @throws Errors.ImportSetupError if an error occurs in either data fetching from the Staging
-     *   Service, or in the initial parsing done by `processSpreadsheetFileData`
+     *   Service, or in the initial parsing done by `processBulkImportSpecData`
      */
-    function getSpreadsheetFileInfo(files) {
-        if (!files || files.length === 0) {
-            return Promise.resolve({});
-        }
-        const stagingUrl = Config.url('staging_api_url');
-        const stagingServiceClient = new StagingServiceClient({
-            root: stagingUrl,
-            token: Runtime.make().authToken(),
-        });
-
-        // This is overkill, but a little future proofing. We have to make a GET call with an
-        // undetermined number of files, so if there are more than we can allow in the URL, gotta break that
-        // into multiple calls.
+    function getBulkImportFileInfo(xsvFiles, dtsFiles) {
+        // dtsFiles gets wiped out by the spec requester.
+        const dtsFileList = [...dtsFiles];
 
-        // a little cheating here to figure out the length allowance. Maybe it should be in the client?
-        const maxQueryLength = 2048 - stagingUrl.length - '/bulk_specification/?files='.length;
-        const bulkSpecProms = [];
-
-        while (files.length) {
-            const fileBatch = [];
-            let remainingLength = maxQueryLength;
-            while (
-                files.length &&
-                remainingLength - files[0].length - 1 >= 0 // -1 is for the comma
-            ) {
-                const nextFile = files.shift();
-                fileBatch.push(nextFile);
-                remainingLength -= nextFile.length + 1;
-            }
-            bulkSpecProms.push(
-                stagingServiceClient.bulkSpecification({
-                    files: encodeURIComponent(fileBatch.join(',')),
-                })
-            );
-        }
+        // noting here that these are effectively jQuery promises, so we can't just
+        // make this an async/await function, but need to wrap with Promise.all.
+        const xsvFileProms = requestBulkImportSpec(xsvFiles);
+        const dtsFileProms = requestBulkImportSpec(dtsFiles, 'dts');
 
-        return Promise.all(bulkSpecProms)
+        return Promise.all([...xsvFileProms, ...dtsFileProms])
             .then((result) => {
                 // join results of all calls together
                 const errors = [];
@@ -92,6 +65,13 @@ define([
                             } else {
                                 allCalls.types[dataType] = callResult.types[dataType];
                                 allCalls.files[dataType] = callResult.files[dataType];
+                                // These files have the username in there. We don't want that.
+                                // So need to compare after stripping them out.
+                                const fileName = allCalls.files[dataType].file;
+                                const pathIdx = fileName.indexOf('/');
+                                const strippedFile =
+                                    pathIdx !== -1 ? fileName.substring(pathIdx + 1) : fileName;
+                                allCalls.files[dataType].isDts = dtsFileList.includes(strippedFile);
                             }
                         });
                         return allCalls;
@@ -123,10 +103,54 @@ define([
                 );
             })
             .then((result) => {
-                return processSpreadsheetFileData(result);
+                return processBulkImportSpecData(result);
             });
     }
 
+    function requestBulkImportSpec(files, flag = '') {
+        /**
+         * This returns an array of jQuery Promises, which is what
+         * the staging service client uses, so it can't be (easily)
+         * cast into async/await.
+         */
+        if (!files || files.length === 0) {
+            return [];
+        }
+        const stagingUrl = Config.url('staging_api_url');
+        const stagingServiceClient = new StagingServiceClient({
+            root: stagingUrl,
+            token: Runtime.make().authToken(),
+        });
+        // This is overkill, but a little future proofing. We have to make a GET call with an
+        // undetermined number of files, so if there are more than we can allow in the URL, gotta break that
+        // into multiple calls.
+
+        // a little cheating here to figure out the length allowance. Maybe it should be in the client?
+        const path = '/bulk_specification/?' + flag + 'files=&';
+        const maxQueryLength = 2048 - stagingUrl.length - path.length;
+        const bulkSpecProms = [];
+
+        while (files.length) {
+            const fileBatch = [];
+            let remainingLength = maxQueryLength;
+            while (
+                files.length &&
+                remainingLength - files[0].length - 1 >= 0 // -1 is for the comma
+            ) {
+                const nextFile = files.shift();
+                fileBatch.push(nextFile);
+                remainingLength -= nextFile.length + 1;
+            }
+            bulkSpecProms.push(
+                stagingServiceClient.bulkSpecification({
+                    files: encodeURIComponent(fileBatch.join(',')),
+                    flag,
+                })
+            );
+        }
+        return bulkSpecProms;
+    }
+
     /**
      * This function does some preprocessing on the spreadsheet file data. Specifically,
      * those parameters that are static dropdowns or checkboxes need to translate their input
@@ -153,10 +177,14 @@ define([
      * TODO: also return the fetched app specs to avoid fetching them twice?
      * @param {Object} data
      */
-    async function processSpreadsheetFileData(data) {
+    async function processBulkImportSpecData(data) {
         // map from given datatype to app id.
         // if any data types are missing, record that
         // if any data types are not bulk import ready, record that, too.
+
+        if (Object.keys(data.types).length === 0 && Object.keys(data.files).length === 0) {
+            return data;
+        }
         const appIdToType = {};
         const dataTypeErrors = [];
         Object.keys(data.types).forEach((dataType) => {
@@ -253,19 +281,58 @@ define([
             {}
         );
 
+        /*
+         * Map from datatype to all file parameters. These are found (as in the bulk import cell)
+         * by looking for those params that are dynamic dropdowns that look at ftp_staging.
+         */
+        const typeToFileParams = Object.entries(appIdToType).reduce(
+            (_typeToFileParams, [appId, dataType]) => {
+                const spec = appIdToSpec[appId].appSpec;
+                const specParams = spec.parameters.filter((param) => {
+                    return (
+                        param.dynamic_dropdown_options &&
+                        param.dynamic_dropdown_options.data_source === 'ftp_staging'
+                    );
+                });
+                _typeToFileParams[dataType] = specParams.map((param) => param.id);
+                return _typeToFileParams;
+            },
+            {}
+        );
+
         /*
          * Now, update all parameters in place.
          * For each set of parameters in each type, look at the translated spec parameters.
          * If any of those are in the given parameter set, do the translation.
+         *
+         * If the datatype comes from a DTS manifest file, adjust the file paths to always be
+         * relative to the subdirectory of that file.
          */
         Object.values(appIdToType).forEach((dataType) => {
             const specParams = typeToAlteredParams[dataType];
+            const fileParams = typeToFileParams[dataType];
+            let filePrefix = '';
+            if (data.files[dataType].isDts) {
+                const file = data.files[dataType].file;
+                const parts = file.split('/');
+                if (parts.length > 2) {
+                    filePrefix = parts.slice(1, -1).join('/') + '/';
+                }
+            }
+
             data.types[dataType] = data.types[dataType].map((parameterSet) => {
                 Object.keys(parameterSet).forEach((paramId) => {
                     const value = parameterSet[paramId];
                     if (specParams[paramId] && value in specParams[paramId]) {
                         parameterSet[paramId] = specParams[paramId][value];
                     }
+                    if (
+                        data.files[dataType].isDts &&
+                        fileParams.includes(paramId) &&
+                        parameterSet[paramId] !== null
+                    ) {
+                        parameterSet[paramId] = filePrefix + parameterSet[paramId];
+                    }
                 });
                 return parameterSet;
             });
@@ -390,6 +457,7 @@ define([
         const bulkFiles = {};
         const singleFiles = [];
         const xsvFiles = [];
+        const dtsFiles = [];
         fileInfo.forEach((file) => {
             const importType = file.type;
             if (bulkIds.has(importType)) {
@@ -404,11 +472,13 @@ define([
                 bulkFiles[importType].files.push(file.name);
             } else if (importType === 'import_specification') {
                 xsvFiles.push(file.name);
+            } else if (importType === 'dts_manifest') {
+                dtsFiles.push(file.name);
             } else {
                 singleFiles.push(file);
             }
         });
-        return getSpreadsheetFileInfo(xsvFiles)
+        return getBulkImportFileInfo(xsvFiles, dtsFiles)
             .then((result) => {
                 if (result.types) {
                     Object.keys(result.types).forEach((dataType) => {