Use unity-changeset package to scrape versions (#55)

* Use unity-changeset package to scrape versions

Author: AndrewKahr

.github/workflows/main.yml

@@ -53,7 +53,7 @@ jobs:
       - name: install dependencies
         run: yarn && yarn --cwd ./functions
       - name: Deploy test to Firebase
-        uses: w9jds/firebase-action@v13.10.2
+        uses: w9jds/firebase-action@v13.25.0
         with:
           args: deploy --only functions:testFunction
         env:
@@ -63,7 +63,7 @@ jobs:
         run: curl -f -s -S -X POST https://testfunction-wbe4ukn6tq-ey.a.run.app
 
       - name: Cleanup Firebase Test
-        uses: w9jds/firebase-action@v13.10.2
+        uses: w9jds/firebase-action@v13.25.0
         if: always()
         with:
           args: functions:delete testFunction --force
@@ -84,13 +84,13 @@ jobs:
       - name: install dependencies
         run: yarn && yarn --cwd ./functions
       - name: Deploy to Firebase
-        uses: w9jds/firebase-action@v13.10.2
+        uses: w9jds/firebase-action@v13.25.0
         with:
           args: deploy
         env:
           GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'
       - name: Cleanup Firebase Test
-        uses: w9jds/firebase-action@v13.10.2
+        uses: w9jds/firebase-action@v13.25.0
         if: always()
         with:
           args: functions:delete testFunction --force

functions/package.json

@@ -16,32 +16,31 @@
   },
   "main": "lib/index.js",
   "dependencies": {
-    "@octokit/auth-app": "^6.1.1",
+    "@octokit/auth-app": "^6.1.3",
     "@octokit/rest": "^20.1.1",
     "eris": "^0.17.2",
-    "firebase-admin": "^12.1.1",
-    "firebase-functions": "^5.0.1",
-    "httpie": "^1.1.2",
-    "jsdom": "^24.1.0",
+    "firebase-admin": "^12.7.0",
+    "firebase-functions": "^6.1.0",
+    "graphql": "^16.9.0",
     "lodash": "^4.17.21",
     "node-fetch": "^2.7.0",
-    "semver": "^7.6.2"
+    "semver": "^7.6.3",
+    "unity-changeset": "^2.3.0"
   },
   "devDependencies": {
     "@octokit/types": "^13.5.0",
-    "@types/jsdom": "^21.1.7",
     "@types/lodash": "^4.17.4",
-    "@types/node": "^20.14.0",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "^2.6.11",
     "@types/semver": "^7.5.8",
-    "@types/ws": "^8.5.10",
+    "@types/ws": "^8.5.13",
     "@typescript-eslint/eslint-plugin": "^7.12.0",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-plugin-import": "^2.29.1",
     "firebase-functions-test": "^3.3.0",
     "jest": "^29.7.0",
-    "typescript": "^5.4.5"
+    "typescript": "^5.6.3"
   },
   "private": true,
   "volta": {

functions/src/logic/ingestUnityVersions/scrapeVersions.ts

@@ -1,53 +1,33 @@
-import { getDocumentFromUrl } from '../utils/get-document-from-url';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
+import { searchChangesets, SearchMode } from 'unity-changeset';
 
-const UNITY_ARCHIVE_URL = 'https://unity.com/releases/editor/archive';
-const unity_version_regex = /unityhub:\/\/(\d+)\.(\d+)\.(\d+[a-zA-Z]\d+)\/(\w+)/g;
+const unity_version_regex = /^(\d+)\.(\d+)\.(\d+)([a-zA-Z]+)(-?\d+)$/;
 
 export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
-  const document = await getDocumentFromUrl(UNITY_ARCHIVE_URL);
+  const unityVersions = await searchChangesets(SearchMode.Default);
 
-  const scripts = document.querySelectorAll('script');
+  if (unityVersions?.length > 0) {
+    return unityVersions
+      .map((unityVersion) => {
+        const match = RegExp(unity_version_regex).exec(unityVersion.version);
+        if (match) {
+          const [_, major, minor, patch, lifecycle, build] = match;
 
-  const allVersions = new Map<string, EditorVersionInfo>();
-
-  for (const script of scripts) {
-    if (script.textContent) {
-      const matches = [...script.textContent.matchAll(unity_version_regex)];
-      if (matches.length > 0) {
-        const versions = matches
-          .filter((match) => {
-            // Filter out prerelease and unsupported versions
-            const [_, major, minor, patch, changeSet] = match;
-            return patch.includes('f') && Number(major) >= 2017;
-          })
-          .map((match) => {
-            const [_, major, minor, patch, changeSet] = match;
-            const version = `${major}.${minor}.${patch}`;
-            if (!allVersions.has(version)) {
-              return {
-                version,
-                changeSet,
-                major: Number(major),
-                minor: Number(minor),
-                patch,
-              };
-            }
-
-            // Return null if version is not unique
+          if (lifecycle !== 'f' || Number(major) < 2017) {
             return null;
-          })
-          .filter((version) => version !== null) as EditorVersionInfo[];
-
-        versions.forEach((it) => {
-          allVersions.set(it.version, it);
-        });
-      }
-    }
-  }
-
-  if (allVersions.size > 0) {
-    return Array.from(allVersions.values());
+          }
+
+          return {
+            version: unityVersion.version,
+            changeSet: unityVersion.changeset,
+            major: Number(major),
+            minor: Number(minor),
+            patch,
+          } as EditorVersionInfo;
+        }
+        return null;
+      })
+      .filter((versionInfo): versionInfo is EditorVersionInfo => versionInfo !== null);
   }
 
   throw new Error('No Unity versions found!');