support clob and blob object handling

Author: mazen-r

src/client.ts

@@ -92,6 +92,42 @@ export class ScrapflyClient {
     return new errors.ScrapflyError(message, args);
   }
 
+  /**
+   * Handle clob and blob large objects
+   */
+  async handleLargeObjects(result: any): Promise<ScrapeResult> {
+    const format = result.format
+    if (format === 'clob' || format === 'blob') {
+      let response: Response;
+      try {
+        const url = new URL(result.content);
+        const params = { key: this.key };
+        url.search = new URLSearchParams(params).toString();
+        response = await this.fetch({
+          url: url.toString(),
+          method: 'GET',
+          headers: {
+            'user-agent': this.ua,
+            'accept-encoding': 'gzip, deflate, br',
+            accept: 'application/json',
+          },
+        });
+      } catch (e) {
+        log.error('error', e);
+        throw e;
+      }
+      const content: string = await response.text();
+      result.content = content
+      if (format === 'clob') {
+        result.format = 'text'
+      }
+      if (format === 'blob') {
+        result.format = 'binary'
+      }
+    }
+    return result
+  }
+
   /**
    * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
    */
@@ -173,6 +209,10 @@ export class ScrapflyClient {
       }
       throw new errors.ApiHttpClientError(JSON.stringify(data));
     }
+
+    const content = await this.handleLargeObjects(data.result)
+    data.result = content
+
     const result = this.handleResponse(
       response,
       new ScrapeResult({