Calibration with QR codes

Author: divdavem

components/assistive-playwright-client/src/index.ts

@@ -41,7 +41,6 @@ export {
   CalibrationError,
   CalibrationResult,
   CalibrationOptions,
-  Color,
   playwrightCalibrate
 } from "./vm/calibrate";
 export { MouseButton, Key } from "vm-providers";

components/assistive-playwright-client/src/vm/calibrate.ts

@@ -22,7 +22,11 @@ import {
   VM,
   ScreenPosition,
   SimplePosition,
-  findRectangle
+  calibrationQRCodesGenerate,
+  calibrationQRCodesScan,
+  pngToDataURI,
+  CalibrationQRCodesConfig,
+  MouseButton
 } from "vm-providers";
 import { ElementHandle, Frame, Page } from "playwright-core";
 import { createWriteStream } from "fs";
@@ -44,45 +48,14 @@ export class CalibrationError {
    * to give details allowing to understand why the calibration failed.
    *
    * @param screenshot - Cf {@link CalibrationError.screenshot | screenshot}
-   * @param color - Cf {@link CalibrationError.color | color}
-   * @param expectedWidth - Cf {@link CalibrationError.expectedWidth | expectedWidth}
-   * @param expectedHeight - Cf {@link CalibrationError.expectedHeight | expectedHeight}
-   * @param colorTolerance - Cf {@link CalibrationError.colorTolerance | colorTolerance}
    */
   constructor(
     /**
      * Screenshot of the virtual machine, in which the browser viewport could not be found.
      */
-    public screenshot: PNG,
-    /**
-     * Color of the rectangle displayed in the viewport.
-     */
-    public color: Color,
-    /**
-     * Expected width (in pixels) of the rectangle that was looked for in the screenshot.
-     */
-    public expectedWidth: number,
-    /**
-     * Expected height (in pixels) of the rectangle that was looked for in the screenshot.
-     */
-    public expectedHeight: number,
-    /**
-     * Tolerance on the color that was used (as configured in {@link CalibrationOptions.colorTolerance}).
-     */
-    public colorTolerance: number
+    public screenshot: PNG
   ) {}
 
-  /**
-   * Error message.
-   */
-  get message(): string {
-    return `Could not find the ${this.expectedWidth}x${
-      this.expectedHeight
-    } rectangle filled with color ${JSON.stringify(this.color)} (tolerance ${
-      this.colorTolerance
-    })`;
-  }
-
   /**
    * Saves the {@link CalibrationError.screenshot | screenshot} as a file.
    * @param fileName - Full path (or path relative to the current directory) where to store the screenshot.
@@ -181,56 +154,15 @@ export class CalibrationResult implements ScreenPosition {
   }
 }
 
-/**
- * Color, expressed as an array of three numbers between 0 and 255,
- * representing the red, green and blue parts
- * @public
- */
-export type Color = [number, number, number];
-const rgb = (color: Color) => `rgb(${color[0]},${color[1]},${color[2]})`;
-
 /**
  * Options for the calibration, passed to {@link playwrightCalibrate}.
  * @public
  */
-export interface CalibrationOptions {
-  /**
-   * Color of the rectangle to display in the viewport.
-   * Defaults to `[255, 0, 0]` (red)
-   * @defaultValue [255, 0, 0]
-   */
-  calibrationColor?: Color;
+export interface CalibrationOptions extends CalibrationQRCodesConfig {
   /**
-   * Color of the border of the rectangle to be displayed in the viewport.
-   * Defaults to `[100, 100, 100]`
-   * @defaultValue [100, 100, 100]
+   * Whether to skip the click done during calibration.
    */
-  borderColor?: Color;
-  /**
-   * Width (in pixels) of the border of the rectangle to be displayed in the viewport.
-   * Defaults to `30`.
-   * @defaultValue 30
-   */
-  borderWidth?: number;
-  /**
-   * Allowed difference between the color in the screenshot and {@link CalibrationOptions.calibrationColor | calibrationColor}, 0 meaning no difference.
-   * The difference is computed as the sum of the absolute value of the difference for each red, green and blue parts.
-   * Defaults to `[255, 0, 0]`.
-   * @defaultValue [255, 0, 0]
-   */
-  colorTolerance?: number;
-  /**
-   * Extra horizontal space to add at the right of the colored rectangle.
-   * Defaults to `0`.
-   * @defaultValue 0
-   */
-  estimatedXMargin?: number;
-  /**
-   * Extra vertical space to add at the bottom of the colored rectangle.
-   * Defaults to `0`.
-   * @defaultValue 0
-   */
-  estimatedYMargin?: number;
+  skipClick?: boolean;
 }
 
 /**
@@ -248,16 +180,7 @@ export async function playwrightCalibrate(
   frame: Page | Frame,
   options: CalibrationOptions = {}
 ): Promise<CalibrationResult> {
-  const {
-    calibrationColor = [255, 0, 0],
-    borderColor = [100, 100, 100],
-    borderWidth = 30,
-    colorTolerance = 50,
-    estimatedXMargin = 0,
-    estimatedYMargin = 0
-  } = options;
-  const rgbCalibrationColor = rgb(calibrationColor);
-  const rgbBorderColor = rgb(borderColor);
+  const { skipClick, ...qrCodeOptions } = options;
   const elementId = JSON.stringify(createUUIDv4());
   const displayRectangleResult: {
     width: number;
@@ -269,9 +192,7 @@ export async function playwrightCalibrate(
   } = await frame.evaluate(`(() => {
 const div = document.createElement("div");
 div.setAttribute("id", ${elementId});
-div.style.cssText = "display:block;position:fixed;background-color:${rgbCalibrationColor};border:${borderWidth}px solid ${rgbBorderColor};left:0px;top:0px;right:0px;bottom:0px;cursor:none;z-index:999999;";
-div.style.maxWidth = (screen.availWidth - window.screenX - ${estimatedXMargin}) + "px";
-div.style.maxHeight = (screen.availHeight - window.screenY - ${estimatedYMargin}) + "px";
+div.style.cssText = "display:block;position:fixed;left:0px;top:0px;right:0px;bottom:0px;cursor:none;z-index:999999;";
 document.body.appendChild(div);
 return {
   width: div.clientWidth,
@@ -282,7 +203,17 @@ return {
   screenHeight: window.screen.height
 };})()`);
   try {
-    // moves the mouse out of the colored zone
+    const viewportImage = calibrationQRCodesGenerate(
+      displayRectangleResult.width,
+      displayRectangleResult.height,
+      qrCodeOptions
+    );
+    await frame.evaluate(
+      `(() => {const calibrationDIV = document.getElementById(${elementId}); calibrationDIV.style.backgroundImage = ${JSON.stringify(
+        `url("${await pngToDataURI(viewportImage)}")`
+      )};})()`
+    );
+    // moves the mouse out of the way
     await vm.sendMouseMoveEvent({
       x: displayRectangleResult.screenX,
       y: displayRectangleResult.screenY,
@@ -291,30 +222,41 @@ return {
     });
     await wait(1000);
     const image = await vm.takePNGScreenshot();
-    const calibrationResult = findRectangle(
-      image,
-      [...calibrationColor, 255],
-      displayRectangleResult.width,
-      displayRectangleResult.height,
-      colorTolerance
-    );
-    if (!calibrationResult) {
-      throw new CalibrationError(
-        image,
-        calibrationColor,
-        displayRectangleResult.width,
-        displayRectangleResult.height,
-        colorTolerance
+    try {
+      const result = calibrationQRCodesScan(image, qrCodeOptions);
+      if (!skipClick) {
+        // click on the detected QR code:
+        await vm.sendMouseMoveEvent({
+          x: Math.floor(result.qrCode.x + result.qrCode.width / 2),
+          y: Math.floor(result.qrCode.y + result.qrCode.height / 2),
+          screenWidth: image.width,
+          screenHeight: image.height
+        });
+        await wait(100);
+        await vm.sendMouseDownEvent(MouseButton.LEFT);
+        await wait(50);
+        await vm.sendMouseUpEvent(MouseButton.LEFT);
+        await wait(100);
+        // moves again the mouse out of the way
+        await vm.sendMouseMoveEvent({
+          x: displayRectangleResult.screenX,
+          y: displayRectangleResult.screenY,
+          screenWidth: image.width,
+          screenHeight: image.height
+        });
+        await wait(100);
+      }
+      return new CalibrationResult(
+        result.viewport.x - displayRectangleResult.screenX,
+        result.viewport.y - displayRectangleResult.screenY,
+        image.width,
+        image.height,
+        vm,
+        frame
       );
+    } catch (error) {
+      throw new CalibrationError(image);
     }
-    return new CalibrationResult(
-      calibrationResult.x - borderWidth - displayRectangleResult.screenX,
-      calibrationResult.y - borderWidth - displayRectangleResult.screenY,
-      image.width,
-      image.height,
-      vm,
-      frame
-    );
   } finally {
     await frame.evaluate(
       `(() => {const calibrationDIV = document.getElementById(${elementId}); calibrationDIV.parentNode.removeChild(calibrationDIV);})()`

components/assistive-webdriver/jest.config.js

@@ -26,5 +26,6 @@ module.exports = {
     "ts-jest": {
       tsconfig: "tsconfig.test.json"
     }
-  }
+  },
+  testTimeout: 10000
 };

components/assistive-webdriver/src/server/calibration.ts

@@ -23,9 +23,12 @@ import {
   VM,
   ScreenPosition,
   wait,
-  findRectangle,
+  calibrationQRCodesGenerate,
+  calibrationQRCodesScan,
+  pngToDataURI,
   createSubLogFunction,
-  LogFunction
+  LogFunction,
+  MouseButton
 } from "vm-providers";
 import { PublicError } from "./publicError";
 
@@ -57,20 +60,16 @@ export async function webdriverCalibrate(
         log
       );
     }
-    const color: [number, number, number, number] = [255, 0, 0, 255];
-    const rgbColor = `rgb(${color[0]},${color[1]},${color[2]});`;
-    const border = 30;
-    const colorTolerance = 50;
-    const displayRectangleResult = await request(
+    const sizeInfo = await request(
       `${sessionUrl}/execute/sync`,
       {
         body: {
           script: `var div = document.createElement("div"); div.setAttribute("id", "calibrationDIV");
-div.style.cssText = "display:block;position:absolute;background-color:${rgbColor};border:${border}px solid rgb(100, 100, 100);left:0px;top:0px;right:0px;bottom:0px;cursor:none;z-index:999999;";
+div.style.cssText = "display:block;position:absolute;left:0px;top:0px;right:0px;bottom:0px;cursor:none;z-index:999999;";
 document.body.appendChild(div);
 return {
-  width: div.offsetWidth - ${2 * border},
-  height: div.offsetHeight - ${2 * border},
+  width: div.offsetWidth,
+  height: div.offsetHeight,
   screenX: window.screenX,
   screenY: window.screenY
 };`,
@@ -79,27 +78,32 @@ return {
       },
       log
     );
+    const viewportImage = calibrationQRCodesGenerate(
+      sizeInfo.value.width,
+      sizeInfo.value.height
+    );
+    await request(
+      `${sessionUrl}/execute/sync`,
+      {
+        body: {
+          script: `var calibrationDIV = document.getElementById("calibrationDIV"); calibrationDIV.style.backgroundImage = ${JSON.stringify(
+            `url("${await pngToDataURI(viewportImage)}")`
+          )};`,
+          args: []
+        }
+      },
+      log
+    );
     log({
       message: "displayed",
-      rectangle: displayRectangleResult.value
+      result: sizeInfo.value
     });
     await wait(1000);
     const image = await vm.takePNGScreenshot();
-    const calibrationResult = findRectangle(
-      image,
-      color,
-      displayRectangleResult.value.width,
-      displayRectangleResult.value.height,
-      colorTolerance
-    );
-    if (!calibrationResult) {
-      const errorMessage = `could not find the ${
-        displayRectangleResult.value.width
-      }x${
-        displayRectangleResult.value.height
-      } rectangle filled with color ${JSON.stringify(
-        color
-      )} (tolerance ${colorTolerance})`;
+    let calibrationResult;
+    try {
+      calibrationResult = calibrationQRCodesScan(image);
+    } catch (error) {
       if (failedCalibrationFileName) {
         await new Promise<void>((resolve, reject) =>
           pipeline(
@@ -109,16 +113,40 @@ return {
           )
         );
         throw new Error(
-          `${errorMessage}, screenshot recorded as ${failedCalibrationFileName}`
+          `${error}\nScreenshot recorded as ${failedCalibrationFileName}`
         );
       } else {
-        throw new Error(`${errorMessage}, screenshot was not saved.`);
+        throw new Error(`${error}\nScreenshot was not saved.`);
       }
     }
     log({
       message: "success",
-      rectangle: calibrationResult
+      result: calibrationResult
+    });
+    // click on the detected QR code:
+    await vm.sendMouseMoveEvent({
+      x: Math.floor(
+        calibrationResult.qrCode.x + calibrationResult.qrCode.width / 2
+      ),
+      y: Math.floor(
+        calibrationResult.qrCode.y + calibrationResult.qrCode.height / 2
+      ),
+      screenWidth: image.width,
+      screenHeight: image.height
+    });
+    await wait(100);
+    await vm.sendMouseDownEvent(MouseButton.LEFT);
+    await wait(50);
+    await vm.sendMouseUpEvent(MouseButton.LEFT);
+    await wait(100);
+    // moves again the mouse out of the way
+    await vm.sendMouseMoveEvent({
+      x: sizeInfo.value.screenX,
+      y: sizeInfo.value.screenY,
+      screenWidth: image.width,
+      screenHeight: image.height
     });
+    await wait(100);
     await request(
       `${sessionUrl}/execute/sync`,
       {
@@ -130,8 +158,8 @@ return {
       log
     );
     return {
-      x: calibrationResult.x - border - displayRectangleResult.value.screenX,
-      y: calibrationResult.y - border - displayRectangleResult.value.screenY,
+      x: calibrationResult.viewport.x - sizeInfo.value.screenX,
+      y: calibrationResult.viewport.y - sizeInfo.value.screenY,
       screenWidth: image.width,
       screenHeight: image.height
     };