diff --git a/src/client.ts b/src/client.ts index 3ae2880..37dc83c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -92,6 +92,41 @@ export class ScrapflyClient { return new errors.ScrapflyError(message, args); } + /** + * Handle clob and blob large objects + */ + async handleLargeObjects(result: any, format: "clob" | "blob"): Promise { + let response: Response; + + try { + const url = new URL(result.content); + const params = { key: this.key }; + url.search = new URLSearchParams(params).toString(); + response = await this.fetch({ + url: url.toString(), + method: 'GET', + headers: { + 'user-agent': this.ua, + 'accept-encoding': 'gzip, deflate, br', + accept: 'application/json', + }, + }); + } catch (e) { + log.error('error', e); + throw e; + } + + const content: string = await response.text(); + result.content = content + if (format === 'clob') { + result.format = 'text' + } + if (format === 'blob') { + result.format = 'binary' + } + return result + } + /** * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError */ @@ -173,6 +208,12 @@ export class ScrapflyClient { } throw new errors.ApiHttpClientError(JSON.stringify(data)); } + + const content_format = data.result.format + if (content_format === 'clob' || content_format === 'blob') { + data.result = await this.handleLargeObjects(data.result, content_format) + } + const result = this.handleResponse( response, new ScrapeResult({