Merge pull request #97 from DIG-Network/release/v0.0.1-alpha.107

Release/v0.0.1 alpha.107

Author: MichaelTaylor3D

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.107](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.106...v0.0.1-alpha.107) (2024-10-01)
+
+
+### Features
+
+* add store info precache ([27b11e4](https://github.com/DIG-Network/dig-chia-sdk/commit/27b11e4e9334cb7fe318685e06bb7c0ec980dca2))
+
 ### [0.0.1-alpha.106](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.105...v0.0.1-alpha.106) (2024-10-01)
 
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.106",
+  "version": "0.0.1-alpha.107",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",

package.json

@@ -7,8 +7,9 @@ import https from "https";
 import os from "os";
 import path from "path";
 import ProgressStream from "progress-stream";
+import * as zlib from "zlib";
 
-import { asyncPool } from "../utils/asyncPool";
+import { asyncPool } from "../utils/promiseUtils";
 import { createSpinner } from "nanospinner";
 import { getFilePathFromSha256 } from "../utils/hashUtils";
 import { getOrCreateSSLCerts } from "../utils/ssl";
@@ -18,7 +19,7 @@ import { PassThrough } from "stream";
 import { promptCredentials } from "../utils/credentialsUtils";
 import { STORE_PATH } from "../utils/config";
 import { Wallet, DataStore } from "../blockchain";
-import { formatHost } from '../utils/network';
+import { formatHost } from "../utils/network";
 
 // Helper function to trim long filenames with ellipsis and ensure consistent padding
 function formatFilename(filename: string | undefined, maxLength = 30): string {
@@ -121,7 +122,9 @@ export class PropagationServer {
         httpsAgent: this.createHttpsAgent(),
       };
 
-      let url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/${this.storeId}`;
+      let url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/${this.storeId}`;
       if (rootHash) {
         url += `?hasRootHash=${rootHash}`;
       }
@@ -194,7 +197,9 @@ export class PropagationServer {
         };
       }
 
-      const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/upload/${this.storeId}?roothash=${rootHash}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/upload/${this.storeId}?roothash=${rootHash}`;
       const response = await axios.post(url, formData, config);
 
       this.sessionId = response.data.sessionId;
@@ -223,7 +228,9 @@ export class PropagationServer {
         httpsAgent: this.createHttpsAgent(),
       };
 
-      const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${filename}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/upload/${this.storeId}/${this.sessionId}/${filename}`;
       const response = await axios.head(url, config);
 
       // Check for 'x-file-exists' header
@@ -246,7 +253,7 @@ export class PropagationServer {
    * Upload a file to the server by sending a PUT request.
    * Logs progress using a local cli-progress bar.
    */
-  async uploadFile(label: string, dataPath: string) {
+  async uploadFile(label: string, dataPath: string, uncompress: boolean = false) {
     const filePath = path.join(STORE_PATH, this.storeId, dataPath);
 
     const { nonce, fileExists } = await this.getFileNonce(dataPath);
@@ -306,8 +313,20 @@ export class PropagationServer {
         PropagationServer.inactivityTimeout
       );
 
-      // Pipe the read stream through the progress stream into the PassThrough stream
-      fileReadStream.pipe(progressStream).pipe(passThroughStream);
+      // Decide whether to uncompress the file during upload
+      if (uncompress) {
+        // Create a gunzip (uncompression) stream
+        const gunzip = zlib.createGunzip();
+
+        // Pipe the streams: fileReadStream -> gunzip -> progressStream -> passThroughStream
+        fileReadStream
+          .pipe(gunzip)
+          .pipe(progressStream)
+          .pipe(passThroughStream);
+      } else {
+        // Pipe the streams: fileReadStream -> progressStream -> passThroughStream
+        fileReadStream.pipe(progressStream).pipe(passThroughStream);
+      }
 
       // Use form-data to construct the request body
       const formData = new FormData();
@@ -327,7 +346,9 @@ export class PropagationServer {
         maxBodyLength: Infinity,
       };
 
-      const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
 
       // Create a promise that resolves when the progress stream ends
       const progressPromise = new Promise<void>((resolve, reject) => {
@@ -369,7 +390,9 @@ export class PropagationServer {
             : undefined,
       };
 
-      const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/commit/${this.storeId}/${this.sessionId}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/commit/${this.storeId}/${this.sessionId}`;
       const response = await axios.post(url, {}, config);
 
       spinner.success({
@@ -424,8 +447,7 @@ export class PropagationServer {
     await propagationServer.startUploadSession(rootHash);
 
     const dataStore = DataStore.from(storeId);
-    const files = await dataStore.getFileSetForRootHash(rootHash);
-
+    const files = await dataStore.getFileSetForRootHash(rootHash, true);
     // Prepare upload tasks
     const uploadTasks = files.map((file) => ({
       label: file.name,
@@ -436,7 +458,7 @@ export class PropagationServer {
     const concurrencyLimit = 10; // Adjust this number as needed
 
     await asyncPool(concurrencyLimit, uploadTasks, async (task) => {
-      await propagationServer.uploadFile(task.label, task.dataPath);
+      await propagationServer.uploadFile(task.label, task.dataPath, true);
     });
 
     // Commit the session after all files have been uploaded
@@ -454,7 +476,9 @@ export class PropagationServer {
    * Logs progress using a local cli-progress bar.
    */
   async fetchFile(dataPath: string): Promise<Buffer> {
-    const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
+    const url = `https://${formatHost(this.ipAddress)}:${
+      PropagationServer.port
+    }/fetch/${this.storeId}/${dataPath}`;
     const config: AxiosRequestConfig = {
       responseType: "stream",
       httpsAgent: this.createHttpsAgent(),
@@ -545,7 +569,9 @@ export class PropagationServer {
         httpsAgent: this.createHttpsAgent(),
       };
 
-      const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        PropagationServer.port
+      }/store/${this.storeId}/${rootHash}/${dataPath}`;
       const response = await axios.head(url, config);
 
       // Check the headers for file existence and size
@@ -575,7 +601,9 @@ export class PropagationServer {
     rootHash: string,
     baseDir: string
   ) {
-    const url = `https://${formatHost(this.ipAddress)}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
+    const url = `https://${formatHost(this.ipAddress)}:${
+      PropagationServer.port
+    }/fetch/${this.storeId}/${dataPath}`;
     let downloadPath = path.join(baseDir, dataPath);
 
     // Ensure that the directory for the file exists

src/DigNetwork/PropagationServer.ts

@@ -37,6 +37,7 @@ import { FileCache } from "../utils/FileCache";
 import { DataStoreSerializer } from "./DataStoreSerializer";
 import NodeCache from "node-cache";
 import { MAIN_NET_GENISES_CHALLENGE } from "../utils/config";
+import { StoreInfoCacheUpdater } from "./StoreInfoCacheUpdater";
 
 // Initialize the cache with a TTL of 180 seconds (3 minutes)
 const rootHistoryCache = new NodeCache({ stdTTL: 180 });
@@ -311,77 +312,39 @@ export class DataStore {
     latestHash: Buffer;
   }> {
     try {
+      // Kick off the updater instance in the backgounf if not already running
+      StoreInfoCacheUpdater.initInstance();
+
       // Initialize the cache for the current storeId's coin info
       const storeCoinCache = new FileCache<{
         latestStore: ReturnType<DataStoreSerializer["serialize"]>;
         latestHeight: number;
         latestHash: string;
       }>(`stores`);
-
+  
       // Try to get cached store info
       const cachedInfo = storeCoinCache.get(this.storeId);
-
+  
       if (cachedInfo) {
-        try {
-          const {
-            latestStore: serializedStore,
-            latestHeight: previousHeight,
-            latestHash: previousHash,
-          } = cachedInfo;
-
-          // Deserialize the stored data using DataStoreSerializer
-          const { latestStore: previousInfo } = DataStoreSerializer.deserialize(
-            {
-              latestStore: serializedStore,
-              latestHeight: previousHeight.toString(),
-              latestHash: previousHash,
-            }
-          );
-
-          // Sync with peer if necessary
-          const peer = await FullNodePeer.connect();
-          const { latestStore, latestHeight } = await peer.syncStore(
-            previousInfo,
-            previousHeight,
-            Buffer.from(previousHash, "hex"),
-            false
-          );
-          const latestHash = await peer.getHeaderHash(latestHeight);
-
-          // Serialize the store data for caching
-          const serializedLatestStore = new DataStoreSerializer(
-            latestStore,
-            latestHeight,
-            latestHash
-          ).serialize();
-
-          // Cache updated store info
-          storeCoinCache.set(this.storeId, {
-            latestStore: serializedLatestStore,
-            latestHeight,
-            latestHash: latestHash.toString("hex"),
+        // If we have cached info, return it directly
+        const { latestStore, latestHeight } =
+          DataStoreSerializer.deserialize({
+            latestStore: cachedInfo.latestStore,
+            latestHeight: cachedInfo.latestHeight.toString(),
+            latestHash: cachedInfo.latestHash,
           });
-
-          return { latestStore, latestHeight, latestHash };
-        } catch {
-          // Return cached info if sync fails
-          const { latestStore, latestHeight, latestHash } =
-            DataStoreSerializer.deserialize({
-              latestStore: cachedInfo.latestStore,
-              latestHeight: cachedInfo.latestHeight.toString(),
-              latestHash: cachedInfo.latestHash,
-            });
-          return {
-            latestStore,
-            latestHeight,
-            latestHash: latestHash,
-          };
-        }
+  
+        return {
+          latestStore,
+          latestHeight: cachedInfo.latestHeight,
+          latestHash: Buffer.from(cachedInfo.latestHash, "hex"),
+        };
       }
-
+  
+      // If no cached info, proceed to fetch and cache it
       // Use getCreationHeight to retrieve height and hash information
       const { createdAtHeight, createdAtHash } = await this.getCreationHeight();
-
+  
       // Sync store from peer
       const peer = await FullNodePeer.connect();
       const { latestStore, latestHeight } = await peer.syncStoreFromLauncherId(
@@ -390,29 +353,29 @@ export class DataStore {
         createdAtHash,
         false
       );
-
+  
       const latestHash = await peer.getHeaderHash(latestHeight);
-
+  
       // Serialize the latest store info for caching
       const serializedLatestStore = new DataStoreSerializer(
         latestStore,
         latestHeight,
         latestHash
       ).serialize();
-
+  
       // Cache the latest store info
       storeCoinCache.set(this.storeId, {
         latestStore: serializedLatestStore,
         latestHeight,
         latestHash: latestHash.toString("hex"),
       });
-
+  
       return { latestStore, latestHeight, latestHash };
     } catch (error) {
       console.trace("Failed to fetch coin info", error);
       throw error;
     }
-  }
+  }  
 
   public async cacheStoreCreationHeight(): Promise<{
     createdAtHeight: number;
@@ -621,7 +584,8 @@ export class DataStore {
    * @returns {Promise<{ fileName: string, file: string }[]>} - An array of unique file objects.
    */
   public async getFileSetForRootHash(
-    rootHash: string
+    rootHash: string,
+    skipIntegirtyCheck: boolean = false
   ): Promise<{ name: string; path: string }[]> {
     const datFilePath = path.join(STORE_PATH, this.storeId, `${rootHash}.dat`);
     const datFileContent = JSON.parse(fs.readFileSync(datFilePath, "utf-8"));
@@ -637,16 +601,20 @@ export class DataStore {
       );
 
       // Perform the integrity check
-      const integrityCheck =
-        await DataIntegrityTree.validateKeyIntegrityWithForeignTree(
-          fileKey,
-          datFileContent.files[fileKey].sha256,
-          datFileContent,
-          rootHash,
-          path.join(STORE_PATH, this.storeId, "data")
-        );
+      let integrityCheck;
+
+      if (!skipIntegirtyCheck) {
+        integrityCheck =
+          await DataIntegrityTree.validateKeyIntegrityWithForeignTree(
+            fileKey,
+            datFileContent.files[fileKey].sha256,
+            datFileContent,
+            rootHash,
+            path.join(STORE_PATH, this.storeId, "data")
+          );
+      }
 
-      if (integrityCheck) {
+      if (integrityCheck || skipIntegirtyCheck) {
         // Add the file to the Set
         filesInvolved.add({
           name: Buffer.from(fileKey, "hex").toString("utf-8"),