Merge pull request #39 from DIG-Network/release/v0.0.1-alpha.40

Release/v0.0.1 alpha.40

Author: MichaelTaylor3D

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.40](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.39...v0.0.1-alpha.40) (2024-10-10)
+
 ### [0.0.1-alpha.39](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.38...v0.0.1-alpha.39) (2024-10-08)
 
 ### [0.0.1-alpha.38](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.37...v0.0.1-alpha.38) (2024-10-03)

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-chia-cli",
-  "version": "0.0.1-alpha.39",
+  "version": "0.0.1-alpha.40",
   "description": "",
   "type": "commonjs",
   "main": "./dist/cli.js",
@@ -24,7 +24,7 @@
     "LICENSE"
   ],
   "dependencies": {
-    "@dignetwork/dig-sdk": "^0.0.1-alpha.167",
+    "@dignetwork/dig-sdk": "^0.0.1-alpha.169",
     "bip39": "^3.1.0",
     "datalayer-driver": "^0.1.21",
     "inquirer": "^10.1.8",

package.json

@@ -1,7 +1,8 @@
 import path from "path";
 import fs from "fs";
 import {
-  addDirectory,
+  listFilesRecursively,
+  asyncPool,
   calculateFolderSize,
   waitForPromise,
   DataStore,
@@ -29,34 +30,51 @@ export const commit = async (): Promise<void> => {
 
     const digConfig = await loadDigConfig(process.cwd());
 
-    await addDirectory(
-      dataStore.Tree,
-      path.join(process.cwd(), digConfig.deploy_dir)
+    // Group into 1000 files per batch to make management and upload easier
+    const fileGroups = listFilesRecursively(
+      path.join(process.cwd(), digConfig.deploy_dir),
+      1000
     );
 
-    const newRootHash = dataStore.Tree.commit();
+    const concurrencyLimit = 10;
 
-    if (!newRootHash) {
-      return;
-    }
+    for (const files of fileGroups) {
+      await asyncPool(concurrencyLimit, files as string[], async (file) => {
+        const filePath = path.join(process.cwd(), digConfig.deploy_dir, file);
+        const relativePath = file.replace(/\\/g, "/");
+        const stream = fs.createReadStream(filePath);
+        await dataStore.Tree.upsertKey(
+          stream,
+          Buffer.from(relativePath).toString("hex")
+        );
+      });
 
-    const totalBytes = calculateFolderSize(
-      path.resolve(STORE_PATH, dataStore.StoreId)
-    );
+      const newRootHash = dataStore.Tree.commit();
 
-    console.log(
-      `Updating store metadata with new root hash: ${newRootHash}, bytes: ${totalBytes}`
-    );
+      if (!newRootHash) {
+        return;
+      }
 
-    const updatedStoreInfo = await dataStore.updateMetadata({
-      ...latestStore.metadata,
-      rootHash: Buffer.from(newRootHash, "hex"),
-      bytes: totalBytes,
-    });
+      const totalBytes = calculateFolderSize(
+        path.resolve(STORE_PATH, dataStore.StoreId)
+      );
 
-    await FullNodePeer.waitForConfirmation(
-      updatedStoreInfo.coin.parentCoinInfo
-    );
+      console.log(
+        `Updating store metadata with new root hash: ${newRootHash}, bytes: ${totalBytes}`
+      );
+
+      const updatedStoreInfo = await dataStore.updateMetadata({
+        ...latestStore.metadata,
+        rootHash: Buffer.from(newRootHash, "hex"),
+        bytes: totalBytes,
+      });
+
+      await FullNodePeer.waitForConfirmation(
+        updatedStoreInfo.coin.parentCoinInfo
+      );
+
+      await dataStore.fetchCoinInfo();
+    }
 
     await waitForPromise(
       () => dataStore.fetchCoinInfo(),