Merge pull request #141 from DIG-Network/release/v0.0.1-alpha.156

Release/v0.0.1 alpha.156

Author: MichaelTaylor3D

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.156](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.155...v0.0.1-alpha.156) (2024-10-07)
+
+
+### Features
+
+* additional caching measure to optimize requests ([e2519b1](https://github.com/DIG-Network/dig-chia-sdk/commit/e2519b1cb9fed7889b0026079cfeb3e86c72c363))
+
 ### [0.0.1-alpha.155](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.154...v0.0.1-alpha.155) (2024-10-06)
 
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.155",
+  "version": "0.0.1-alpha.156",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",

package.json

@@ -4,6 +4,10 @@ import { URL } from "url";
 import { Readable } from "stream";
 import { getOrCreateSSLCerts } from "../utils/ssl";
 import { formatHost } from "../utils/network";
+import NodeCache from "node-cache";
+
+const hasRootHashCache = new NodeCache({ stdTTL: 86400 });
+const wellKnownCache = new NodeCache({ stdTTL: 86400 });
 
 export class ContentServer {
   private ipAddress: string;
@@ -30,7 +34,9 @@ export class ContentServer {
     challengeHex?: string
   ): Promise<string> {
     // Construct the base URL
-    let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`;
+    let url = `https://${formatHost(this.ipAddress)}:${
+      ContentServer.port
+    }/chia.${this.storeId}.${rootHash}/${key}`;
 
     // If a challenge is provided, append it as a query parameter
     if (challengeHex) {
@@ -40,13 +46,12 @@ export class ContentServer {
     return this.fetchWithRetries(url);
   }
 
-   // New method to get only the first chunk of the content
-   public async getKeyChunk(
-    key: string,
-    rootHash: string,
-  ): Promise<Buffer> {
+  // New method to get only the first chunk of the content
+  public async getKeyChunk(key: string, rootHash: string): Promise<Buffer> {
     // Construct the base URL
-    let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`;
+    let url = `https://${formatHost(this.ipAddress)}:${
+      ContentServer.port
+    }/chia.${this.storeId}.${rootHash}/${key}`;
     return this.fetchFirstChunk(url);
   }
 
@@ -65,15 +70,44 @@ export class ContentServer {
     }
   }
 
-  // Method to get the .well-known information
+  /**
+   * Fetches and caches the .well-known information for the store's IP address.
+   *
+   * @returns A promise that resolves to the .well-known JSON data.
+   */
   public async getWellKnown(): Promise<any> {
-    const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/.well-known`;
-    return this.fetchJson(url);
+    // Construct the cache key based on ipAddress
+    const cacheKey = `${this.ipAddress}-wellknown`;
+
+    // Check if the result is already cached
+    const cachedResult = wellKnownCache.get<any>(cacheKey);
+    if (cachedResult !== undefined) {
+      return cachedResult;
+    }
+
+    // If not cached, proceed to fetch the .well-known information
+    const url = `https://${formatHost(this.ipAddress)}:${
+      ContentServer.port
+    }/.well-known`;
+
+    try {
+      const data = await this.fetchJson(url);
+      wellKnownCache.set(cacheKey, data);
+      return data;
+    } catch (error: any) {
+      console.error(
+        `Error fetching .well-known information for ${this.ipAddress}:`,
+        error.message
+      );
+      throw error; // Propagate the error after logging
+    }
   }
 
   // Method to get the list of known stores
   public async getKnownStores(): Promise<any> {
-    const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/.well-known/stores`;
+    const url = `https://${formatHost(this.ipAddress)}:${
+      ContentServer.port
+    }/.well-known/stores`;
     return this.fetchJson(url);
   }
 
@@ -85,53 +119,103 @@ export class ContentServer {
 
   // Method to get the index of keys in a store
   public async getKeysIndex(rootHash?: string): Promise<any> {
-    let udi = `chia.${this.storeId}`;
+    try {
+      let udi = `chia.${this.storeId}`;
 
-    if (rootHash) {
-      udi += `.${rootHash}`;
-    }
+      if (rootHash) {
+        udi += `.${rootHash}`;
+      }
 
-    const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}`;
-    return this.fetchJson(url);
+      const url = `https://${formatHost(this.ipAddress)}:${
+        ContentServer.port
+      }/${udi}`;
+      return this.fetchJson(url);
+    } catch (error: any) {
+      if (rootHash) {
+        hasRootHashCache.del(`${this.storeId}-${rootHash}`);
+      }
+      throw error;
+    }
   }
 
   // Method to check if a specific key exists (HEAD request)
   public async headKey(
     key: string,
     rootHash?: string
   ): Promise<{ success: boolean; headers?: http.IncomingHttpHeaders }> {
-    let udi = `chia.${this.storeId}`;
+    try {
+      let udi = `chia.${this.storeId}`;
 
-    if (rootHash) {
-      udi += `.${rootHash}`;
-    }
+      if (rootHash) {
+        udi += `.${rootHash}`;
+      }
 
-    const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`;
-    return this.head(url);
+      const url = `https://${formatHost(this.ipAddress)}:${
+        ContentServer.port
+      }/${udi}/${key}`;
+      return this.head(url);
+    } catch (error: any) {
+      if (rootHash) {
+        hasRootHashCache.del(`${this.storeId}-${rootHash}`);
+      }
+      throw error;
+    }
   }
 
   // Method to check if a specific store exists (HEAD request)
   public async headStore(options?: { hasRootHash: string }): Promise<{
     success: boolean;
     headers?: http.IncomingHttpHeaders;
   }> {
-    let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}`;
+    try {
+      let url = `https://${formatHost(this.ipAddress)}:${
+        ContentServer.port
+      }/chia.${this.storeId}`;
 
-    if (options?.hasRootHash) {
-      url += `?hasRootHash=${options.hasRootHash}`;
-    }
+      if (options?.hasRootHash) {
+        url += `?hasRootHash=${options.hasRootHash}`;
+      }
 
-    return this.head(url);
+      return this.head(url);
+    } catch (error: any) {
+      if (options?.hasRootHash) {
+        hasRootHashCache.del(`${this.storeId}-${options.hasRootHash}`);
+      }
+      throw error;
+    }
   }
 
+  /**
+   * Checks if the store has the specified rootHash.
+   * Utilizes caching to improve performance.
+   *
+   * @param rootHash - The root hash to check.
+   * @returns A promise that resolves to true if the root hash exists, otherwise false.
+   */
   public async hasRootHash(rootHash: string): Promise<boolean> {
+    // Construct the cache key using storeId and rootHash
+    const cacheKey = `${this.storeId}-${rootHash}`;
+
+    // Check if the result is already cached
+    const cachedResult = hasRootHashCache.get<boolean>(cacheKey);
+    if (cachedResult !== undefined) {
+      return cachedResult;
+    }
+
+    // If not cached, perform the headStore request
     const { success, headers } = await this.headStore({
       hasRootHash: rootHash,
     });
-    if (success) {
-      return headers?.["x-has-root-hash"] === "true";
+
+    // Determine if the store has the root hash
+    const hasHash = success && headers?.["x-has-root-hash"] === "true";
+
+    // Only cache the result if the store has the root hash
+    if (hasHash) {
+      hasRootHashCache.set(cacheKey, true);
     }
-    return false;
+
+    return hasHash;
   }
 
   public streamKey(key: string, rootHash?: string): Promise<Readable> {
@@ -142,7 +226,9 @@ export class ContentServer {
     }
 
     return new Promise((resolve, reject) => {
-      const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`;
+      const url = `https://${formatHost(this.ipAddress)}:${
+        ContentServer.port
+      }/${udi}/${key}`;
       const urlObj = new URL(url);
 
       const requestOptions = {
@@ -164,6 +250,7 @@ export class ContentServer {
             reject(new Error("Redirected without a location header"));
           }
         } else {
+          hasRootHashCache.del(`${this.storeId}-${rootHash}`);
           reject(
             new Error(
               `Failed to retrieve data from ${url}. Status code: ${response.statusCode}`
@@ -483,13 +570,14 @@ export class ContentServer {
           response.headers.location
         ) {
           // Handle redirects
-          const redirectUrl = new URL(response.headers.location, url).toString(); // Resolve relative URLs
+          const redirectUrl = new URL(
+            response.headers.location,
+            url
+          ).toString(); // Resolve relative URLs
           if (timeout) {
             clearTimeout(timeout);
           }
-          this.fetchFirstChunk(redirectUrl)
-            .then(resolve)
-            .catch(reject);
+          this.fetchFirstChunk(redirectUrl).then(resolve).catch(reject);
         } else {
           if (timeout) {
             clearTimeout(timeout);

src/DigNetwork/ContentServer.ts

@@ -20,6 +20,10 @@ import { promptCredentials } from "../utils/credentialsUtils";
 import { STORE_PATH } from "../utils/config";
 import { Wallet, DataStore } from "../blockchain";
 import { formatHost } from "../utils/network";
+import NodeCache from "node-cache";
+
+// Initialize cache with a TTL of 1 week (604800 seconds)
+const storeExistsCache = new NodeCache({ stdTTL: 86400 });
 
 // Helper function to trim long filenames with ellipsis and ensure consistent padding
 function formatFilename(filename: string | undefined, maxLength = 30): string {
@@ -140,7 +144,10 @@ export class PropagationServer {
 
       try {
         const response = await axios.post(url, data, config);
-        console.log(green(`✔ Successfully pinged peer: ${this.ipAddress}`));
+        console.log(
+          green(`✔ Successfully pinged peer: ${this.ipAddress}`),
+          response
+        );
         return response.data;
       } catch (error: any) {
         console.error(red(`✖ Failed to ping peer: ${this.ipAddress}`));
@@ -181,14 +188,34 @@ export class PropagationServer {
   }
 
   /**
-   * Check if the store and optional root hash exist by making a HEAD request.
+   * Checks if the store exists and optionally if a specific rootHash exists.
+   * Utilizes caching to improve performance.
+   *
+   * @param rootHash - (Optional) The root hash to check.
+   * @returns A promise that resolves to an object containing store existence and root hash existence.
    */
-  async checkStoreExists(
+  public async checkStoreExists(
     rootHash?: string
   ): Promise<{ storeExists: boolean; rootHashExists: boolean }> {
+    // Construct the cache key
+    const cacheKey = rootHash
+      ? `${this.storeId}-${rootHash}`
+      : `${this.storeId}-nohash`;
+
+    // Check if the result is already cached
+    const cachedResult = storeExistsCache.get<{
+      storeExists: boolean;
+      rootHashExists: boolean;
+    }>(cacheKey);
+    if (cachedResult !== undefined) {
+      return cachedResult;
+    }
+
+    // If not cached, proceed with the HTTP request
     const spinner = createSpinner(
       `Checking if store ${this.storeId} exists...`
     ).start();
+
     try {
       const config: AxiosRequestConfig = {
         httpsAgent: this.createHttpsAgent(),
@@ -221,7 +248,11 @@ export class PropagationServer {
         });
       }
 
-      return { storeExists, rootHashExists };
+      const result = { storeExists, rootHashExists };
+
+      storeExistsCache.set(cacheKey, result);
+
+      return result;
     } catch (error: any) {
       spinner.error({ text: red("Error checking if store exists:") });
       console.error(red(error.message));
@@ -625,6 +656,7 @@ export class PropagationServer {
 
       return Buffer.concat(dataBuffers);
     } catch (error) {
+      storeExistsCache.del(`${this.storeId}-nohash`);
       throw error;
     } finally {
       if (progressBar) {
@@ -773,6 +805,8 @@ export class PropagationServer {
         console.log("integrity check");
       }
     } catch (error) {
+      storeExistsCache.del(`${this.storeId}-nohash`);
+      storeExistsCache.del(`${this.storeId}-${rootHash}`);
       throw error;
     } finally {
       if (progressBar) {

src/DigNetwork/PropagationServer.ts

@@ -109,7 +109,6 @@ export class PeerRanker {
   private async measureBandwidth(ip: string): Promise<number> {
     const cachedMetrics = peerCache.get<PeerMetrics>(ip);
     if (cachedMetrics && cachedMetrics.bandwidth) {
-      console.log(`Bandwidth for IP ${ip} retrieved from cache.`);
       return cachedMetrics.bandwidth;
     }