Introduce controller tests, implement validation service, and perform refactoring of utility files

Author: priyanshu-kun

src/server/api/controllers/CrawlingController.js

@@ -1,5 +1,5 @@
-const { cleanDB } = require('../utils/CleanDBUtils');
-const { saveIndexFiles } = require('../utils/SaveIntoDBUtils');
+const { cleanDB } = require('../utils/DBUtils');
+const { saveIndexFiles } = require('../utils/DBUtils');
 const { selectDataSources } = require('../utils/SelectDataSourcesUtils');
 
 module.exports = {

src/server/api/controllers/DownloadAndProcessIndexFilesController.js

@@ -1,12 +1,11 @@
-const { cleanDB } = require('../utils/CleanDBUtils');
+const { cleanDB } = require('../utils/DBUtils');
+const { emptyQueue, pushDataToQueue } = require('../utils/QueueUtils');
 const { connectRabbitMQ } = require('../utils/ConnectRabbitMQUtils');
-const { emptyQueue } = require('../utils/EmptyQueueUtils');
 const {
   fetchIndexFilesFromDBUtils,
-} = require('../utils/FetchIndexFilesFromDBUtils');
-const { pushDataToQueue } = require('../utils/PushDataToQueue');
+} = require('../utils/DBUtils');
 const { startWorker } = require('../workers/DownloadAndProcessWorker');
-const { removeDist } = require('../utils/RemoveDist');
+const { removeDist } = require('../utils/FileHandlingUtils');
 
 module.exports = {
   /**
@@ -31,23 +30,26 @@ module.exports = {
       const indexFileUrls = await fetchIndexFilesFromDBUtils({ skip, limit, sort });
 
       if (indexFileUrls.length === 0) {
-        return res.notFound('Before proceeding, execute the initial endpoint to obtain a response from it');
+        return res.badRequest('Before proceeding, execute the initial endpoint to obtain a response from it');
       }
 
+
       // Empty the queue and clean the database for storing latest data
       await emptyQueue();
       await cleanDB(APIsDefinitionsModel);
       removeDist();
 
       const { channel, QUEUE_NAME, connection } = await connectRabbitMQ();
+
       pushDataToQueue(indexFileUrls, QUEUE_NAME, channel);
 
-      await channel.close();
-      await connection.close();
 
       // Start the worker to process the downloaded files
       await startWorker();
 
+      await channel.close();
+      await connection.close();
+
       return res
         .status(202)
         .json({ message: 'Downloading has been started in background.' });

src/server/api/services/DownloadService.js

@@ -1,4 +1,5 @@
 const axios = require('axios');
+const readline = require('readline');
 const { PROGRESS_BAR_WIDTH } = require('../constants/Constants');
 const { formatSizeUnits } = require('../utils/FormatSizeUnitsUtils');
 
@@ -28,16 +29,15 @@ module.exports = {
             .repeat(Math.round(percent / (100 / PROGRESS_BAR_WIDTH)))
             .padEnd(PROGRESS_BAR_WIDTH, ' ');
 
-          process.stdout.clearLine();
-          process.stdout.cursorTo(0);
+          readline.clearLine(process.stdout);
+          readline.cursorTo(process.stdout, 0);
 
           process.stdout.write(
             `Progress: [${progress}] ${percent}% (${downloadedSize}/${totalSize})`
           );
         },
       });
     } catch (error) {
-      console.error(error);
       throw error;
     }
   },

src/server/api/services/ParserService.js

@@ -1,4 +1,4 @@
-const { URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX } = require('../constants/Constants');
+const { URL_REGEX, API_DEFINITION_REGEX } = require('../constants/Constants');
 
 module.exports = {
   /**
@@ -8,7 +8,7 @@ module.exports = {
    * @param {string} text - The input text to parse and extract URLs from.
    * @throws {Error} If there is an error during the parsing process, it will be thrown.
  */
-  parsing: function (text) {
+  parsingService: function (text) {
     try {
       const matchUrls = text.match(URL_REGEX);
       matchUrls.forEach((url) => {

src/server/api/services/ValidateService.js

@@ -1,16 +1,11 @@
-const { downloadFile } = require('./DownloadService');
-
-const OpenAPISchemaValidator = require('openapi-schema-validator').default;
+const SwaggerParser = require('swagger-parser');
 
 module.exports = {
-  validate: async function (url) {
+  validatingService: async function (url) {
     try {
-      downloadFile(url).then(response => {
-        
-      }).catch(error => {
-        throw error;
-      });
-    } catch (error) {
+      return await SwaggerParser.validate(url);
+    }
+    catch(error) {
       throw error;
     }
   },

src/server/api/utils/CheckAndCreateDistFiles.js

@@ -0,0 +1,92 @@
+const { validatingService } = require('../services/ValidateService');
+
+module.exports = {
+  /**
+   * Asynchronously cleans the database by destroying all records of the specified model.
+   *
+   * @function cleanDB
+   * @param {Object} model - The database model to clean (e.g., Waterline model).
+   * @throws {Error} If there is an error during the database cleaning process, it will be thrown.
+ */
+  cleanDB: async function (model) {
+    try {
+      await model.destroy({});
+      console.log('\nDatabase is cleaned.\n');
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  /**
+   * Asynchronously fetches index files from the database using the specified filter options.
+   *
+   * @function fetchIndexFilesFromDBUtils
+   * @param {Object} filter - An object containing filter options for fetching index files.
+   * @param {number} [filter.skip] - The number of index files to skip before returning results (default: 0).
+   * @param {number} [filter.limit] - The maximum number of index files to return (default: Infinity).
+   * @param {'aes' | 'des'} [filter.sort] - The sorting order for the index files (default: 'aes').
+   *                                        Use 'aes' for ascending and 'des' for descending.
+   * @returns {Promise<Array>} A Promise that resolves with an array of index files fetched from the database.
+   * @throws {Error} If there is an error during the database query, it will be thrown.
+ */
+  fetchIndexFilesFromDBUtils: async function (filter) {
+    try {
+      const orderMap = {
+        'aes': 1,
+        'des': -1
+      };
+
+      const skp = filter.skip === undefined ? 0 : parseInt(filter.skip);
+      const lmt = filter.limit === undefined ? Infinity : parseInt(filter.limit);
+      const srt = filter.sort === undefined ? 1 : orderMap[filter.sort];
+
+      return await IndexFilesModel.find({})
+        .sort({ URL: srt })
+        .skip(skp)
+        .limit(lmt);
+
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  /**
+   * Asynchronously saves a list of index file URLs to the database.
+   *
+   * @function saveIndexFiles
+   * @param {Array<string>} indexFiles - An array of index file URLs to be saved to the database.
+   * @returns {Promise<void>} A Promise that resolves when all index file URLs have been saved to the database.
+   * @throws {Error} If there is an error during the database insertion process, it will be thrown.
+ */
+  saveIndexFiles: async function (indexFiles) {
+    try {
+      indexFiles.forEach(async url => {
+        await IndexFilesModel.create({URL: url});
+      });
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  /**
+   * Asynchronously saves API definitions to a file and database.
+   *
+   * @function saveAPIsDefinitions
+   * @param {string[]} definitions - An array of API definitions (URLs) to be saved.
+   * @throws {Error} If there's an error while saving the definitions to the file or the database.
+   * @returns {Promise<void>} A Promise that resolves once all definitions have been saved.
+ */
+  saveAPIsDefinitions: async function(definitions) {
+    try {
+      definitions.forEach(async url => {
+        validatingService(url).then(async () => {
+          await APIsDefinitionsModel.create({url});
+        }).catch(error => {
+          console.log('Invalid definition!',error.message);
+        });
+      });
+    } catch(error) {
+      throw error;
+    }
+  }
+}