- Added the ability to return the header information in the request sent to scrape.do. It can be used in case of need for values such as server-timing, scrape.do-rid in the header that are not provided with the library.

- Added 2 new tests to check if the sdo values in the header are returned correctly and to check if the html returns an error.
- playWithBrowser Test has been made more detailed.

Author: mdervisaygan

.env

@@ -0,0 +1,2 @@
+# Only the TOKEN element is required for the test.
+TOKEN=<your-api-token>

package-lock.json

@@ -39,6 +39,7 @@
   },
   "dependencies": {
     "@types/qs": "^6.9.17",
+    "dotenv": "^16.4.5",
     "qs": "^6.13.0"
   }
 }

package.json

@@ -59,6 +59,9 @@ export class ScrapeDo {
     try { data = JSON.parse(data) } catch { }
 
     response.data = data
+    
+    // There may be information that you want to get from the returned header. However, since this is not an important part, it is not important to get an error.
+    try { response.sdoResponseHeaders = Object.fromEntries(response.headers); } catch { }
 
     if (ValidStatusCodes.includes(response.status) || ValidStatusCodeRanges.some((range) => response.status >= range.min && response.status <= range.max)) {
       return response
@@ -143,14 +146,16 @@ export class ScrapeDo {
       params,
     })
       .then((response: MakeRequestResponse) => {
+        // sdoResponseHeaders will return empty if Object.fromEntries fails. For this reason, .get is used when getting header values.
         const sdoHeaders: Partial<DoResponse> = {
-          cookies: response.headers["Scrape.do-Cookies"],
-          remainingCredits: response.headers["Scrape.do-Remaining-Credits"],
-          requestCost: response.headers["Scrape.do-Request-Cost"],
-          resolvedURL: response.headers["Scrape.do-Resolved-Url"],
-          targetURL: response.headers["Scrape.do-Target-Url"],
-          initialStatusCode: response.headers["Scrape.do-Initial-Status-Code"],
-          targetRedirectedLocation: response.headers["Scrape.do-Target-Redirected-Location"],
+          cookies: response.headers.get("scrape.do-cookies")?.toString(),
+          remainingCredits: response.headers.get("scrape.do-remaining-credits")?.toString(),
+          requestCost: response.headers.get("scrape.do-request-cost")?.toString(),
+          resolvedURL: response.headers.get("scrape.do-resolved-url")?.toString(),
+          targetURL: response.headers.get("scrape.do-target-url")?.toString(),
+          initialStatusCode: response.headers.get("scrape.do-initial-status-code")?.toString(),
+          targetRedirectedLocation: response.headers.get("scrape.do-target-redirected-location")?.toString(),
+          sdoResponseHeaders: response.sdoResponseHeaders
         };
 
         if (options.returnJSON) {

src/lib.ts

@@ -115,6 +115,7 @@ export type DoResponse = {
  * @property {string} [targetURL] - The target URL that was scraped
  * @property {string} [initialStatusCode] - The status code of the initial request before any redirects
  * @property {string} [targetRedirectedLocation] - Final location after following redirects
+ * @property {Record<string, string>} [sdoResponseHeaders] - Additional headers returned in the response
  */
 export interface DoHeaders {
   cookies?: string;
@@ -124,6 +125,7 @@ export interface DoHeaders {
   targetURL?: string;
   initialStatusCode?: string;
   targetRedirectedLocation?: string;
+  sdoResponseHeaders?: Record<string, string>;
 }
 
 /**
@@ -195,8 +197,10 @@ export interface FetchConfig {
 /**
  * Response structure for a fetch request.
  * @property {any} data - Data returned from the request.
+ * @property {Record<string, string>} sdoResponseHeaders - Contains the headers returned from the request sent to Scrape.do. It is independent of the headers in the request sent to the site.
  * @property {Response} response - Response object from the fetch request.
  */
 export interface MakeRequestResponse extends Response {
   data?: any;
+  sdoResponseHeaders?: Record<string, string>;
 }

src/types.ts

@@ -1,9 +1,33 @@
 import { describe, expect, it, test } from "@jest/globals";
 import { ScrapeDo } from "../src/lib";
+import 'dotenv/config'
 
 const TOKEN = process.env.TOKEN || "";
 
 describe("Usability tests", () => {
+
+  test("Should contain scrape.do-prefixed headers in the response header", async () => {
+    const client = new ScrapeDo(TOKEN);
+    const response = await client.sendRequest("GET", {
+      url: "https://httpbin.co/anything"
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response).toHaveProperty('requestCost')
+    expect(response.requestCost).toBeDefined();
+  })
+
+  test("Should handle HTML responses without errors", async () => {
+    const client = new ScrapeDo(TOKEN);
+    const response = await client.sendRequest("GET", {
+      url: "https://httpbin.co/html"
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response).toHaveProperty('content')
+    expect(response.content).toContain("<html");
+  })
+
   test("Should be able to get successful response with extra headers", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
@@ -16,6 +40,7 @@ describe("Usability tests", () => {
     expect(response.statusCode).toBe(200);
     expect(response.content.headers["A123"]).toStrictEqual(["Extra Header"]);
   });
+
   test("Should be able to get successful response with custom headers", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
@@ -28,6 +53,7 @@ describe("Usability tests", () => {
     expect(response.statusCode).toBe(200);
     expect(response.content.headers["A123"]).toStrictEqual(["Custom Header"]);
   });
+
   test("Should be able to get successful response with forward headers", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
@@ -40,6 +66,7 @@ describe("Usability tests", () => {
     expect(response.statusCode).toBe(200);
     expect(response.content.headers["A123"]).toStrictEqual(["Forward Header"]);
   });
+
   test("Should be able to get successful response with cookies", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
@@ -52,6 +79,7 @@ describe("Usability tests", () => {
     expect(response.statusCode).toBe(200);
     expect(response.content.headers["Cookie"]).toStrictEqual(["A123=Cookie"]);
   });
+
   test("Should throw error if setCookies is used with customHeaders", async () => {
     const client = new ScrapeDo(TOKEN);
     await expect(
@@ -66,25 +94,41 @@ describe("Usability tests", () => {
       })
     ).rejects.toThrow("setCookies cannot be used with customHeaders, extraHeaders or forwardHeaders");
   });
+
   test("Should get successful response with render and playWithBrowser", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
-      url: "https://httpbin.co/anything",
+      url: "https://httpbin.co",
       render: true,
       returnJSON: true,
       playWithBrowser: [
         {
           Action: "WaitSelector",
-          WaitSelector: "body",
+          WaitSelector: "a code",
+        },
+        {
+          Action: "ScrollY",
+          Value: 100
         },
+        {
+          Action: "Wait",
+          Timeout: 1000
+        },
+        {
+          Action: "Execute",
+          Execute: "window.scrollY"
+        }
       ],
     });
 
     expect(response.statusCode).toBe(200);
-    expect(response.actionResults).toHaveLength(1);
-    expect(response.actionResults![0].action).toBe("WaitSelector");
-    expect(response.actionResults![0].success).toBe(true);
+    expect(response.actionResults).toHaveLength(4);
+    expect(response.actionResults?.at(-1)?.action).toBe("Execute");
+    expect(response.actionResults?.at(-1)?.success).toBe(true);
+    // Since the scrollTo operation is asynchronous, its value may not be exactly 100. For this reason, it is only checked if it is greater than 0.
+    expect(response.actionResults?.at(-1)?.response).toBeGreaterThan(0);
   });
+
   test("Should get successful response with render and super proxy", async () => {
     const client = new ScrapeDo(TOKEN);
     const response = await client.sendRequest("GET", {
@@ -95,11 +139,13 @@ describe("Usability tests", () => {
 
     expect(response.statusCode).toBe(200);
   });
+
   test("Should get successful response from statistics request", async () => {
     const client = new ScrapeDo(TOKEN);
     const stats = await client.statistics();
 
     expect(stats.IsActive).toBe(true);
     expect(stats.RemainingMonthlyRequest).toBeGreaterThan(0);
   });
+
 });