Exclude repos by glob (#70)

Author: Steven-Nagie

configs/filter.json

@@ -15,7 +15,9 @@
                 "forks": true,
                 "repos": [
                     "my-org/repo1",
-                    "my-org/repo2"
+                    "my-org/repo2",
+                    "my-org/sub-org-1/**",
+                    "my-org/sub-org-*/**"
                 ]
             }
         },
@@ -34,7 +36,9 @@
                 "forks": true,
                 "projects": [
                     "my-group/project1",
-                    "my-group/project2"
+                    "my-group/project2",
+                    "my-org/sub-org-1/**",
+                    "my-org/sub-org-*/**"
                 ]
             }
         },
@@ -53,7 +57,9 @@
                 "forks": true,
                 "repos": [
                     "my-org/repo1",
-                    "my-org/repo2"
+                    "my-org/repo2",
+                    "my-org/sub-org-1/**",
+                    "my-org/sub-org-*/**"
                 ]
             }
         },

packages/backend/src/utils.test.ts

@@ -1,5 +1,62 @@
 import { expect, test } from 'vitest';
-import { arraysEqualShallow, isRemotePath } from './utils';
+import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils';
+import { Repository } from './types';
+
+const testNames: string[] = [
+  "abcdefg/zfmno/ioiwerj/fawdf",
+  "abcdefg/zfmno/ioiwerj/werw",
+  "abcdefg/zfmno/ioiwerj/terne",
+  "abcdefg/zfmno/ioiwerj/asdf45e4r",
+  "abcdefg/zfmno/ioiwerj/ddee",
+  "abcdefg/zfmno/ioiwerj/ccdfeee",
+  "abcdefg/zfmno/sadfaw",
+  "abcdefg/zfmno/ioiwerj/wwe",
+  "abcdefg/ieieiowowieu8383/ieckup-e",
+  "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"
+];
+
+const createRepository = (name: string) => (<Repository>{
+  vcs: 'git',
+  id: name,
+  name: name,
+  path: name,
+  isStale: false,
+  cloneUrl: name,
+  branches: [name],
+  tags: [name]
+});
+
+test('should filter repos by micromatch pattern', () => {
+  // bad glob patterns
+  const unfilteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['/zfmno/']);
+  expect(unfilteredRepos.length).toBe(10);
+  expect(unfilteredRepos.map(r => r.name)).toEqual(testNames);
+  const unfilteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**zfmno**']);
+  expect(unfilteredRepos1.length).toBe(10);
+  expect(unfilteredRepos1.map(r => r.name)).toEqual(testNames);
+
+  // good glob patterns
+  const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/zfmno/**']);
+  expect(filteredRepos.length).toBe(2);
+  expect(filteredRepos.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]);
+  const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/*fmn*/**']);
+  expect(filteredRepos1.length).toBe(2);
+  expect(filteredRepos1.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]);
+});
+
+test('should filter repos by name exact match', () => {
+  const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(1, 9));
+  expect(filteredRepos.length).toBe(2);
+  expect(filteredRepos.map(r => r.name)).toEqual([testNames[0], testNames[9]]);
+
+  const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(3, 5));
+  expect(filteredRepos1.length).toBe(8);
+  expect(filteredRepos1.map(r => r.name)).toEqual([testNames[0], testNames[1], testNames[2], testNames[5], testNames[6], testNames[7], testNames[8], testNames[9]]);
+
+  const filteredRepos2 = excludeReposByName(testNames.map(n => (createRepository(n))), [testNames[0], testNames[7], testNames[9]]);
+  expect(filteredRepos2.length).toBe(7);
+  expect(filteredRepos2.map(r => r.name)).toEqual([...testNames.slice(1, 7), testNames[8]]);
+});
 
 test('should return true for identical arrays', () => {
     expect(arraysEqualShallow([1, 2, 3], [1, 2, 3])).toBe(true);

packages/backend/src/utils.ts

@@ -1,6 +1,7 @@
 import { Logger } from "winston";
 import { AppContext, Repository } from "./types.js";
 import path from 'path';
+import micromatch from "micromatch";
 
 export const measure = async <T>(cb : () => Promise<T>) => {
     const start = Date.now();
@@ -36,10 +37,10 @@ export const excludeArchivedRepos = <T extends Repository>(repos: T[], logger?:
     });
 }
 
+
 export const excludeReposByName = <T extends Repository>(repos: T[], excludedRepoNames: string[], logger?: Logger) => {
-    const excludedRepos = new Set(excludedRepoNames);
     return repos.filter((repo) => {
-        if (excludedRepos.has(repo.name)) {
+        if (micromatch.isMatch(repo.name, excludedRepoNames)) {
             logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.repos contains ${repo.name}`);
             return false;
         }
@@ -90,4 +91,4 @@ export const arraysEqualShallow = <T>(a?: readonly T[], b?: readonly T[]) => {
     }
 
     return true;
-}
+}

schemas/v2/index.json

@@ -146,11 +146,10 @@
                         "repos": {
                             "type": "array",
                             "items": {
-                                "type": "string",
-                                "pattern": "^[\\w.-]+\\/[\\w.-]+$"
+                                "type": "string"
                             },
                             "default": [],
-                            "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'."
+                            "description": "List of individual repositories to exclude from syncing. Glob patterns are supported."
                         }
                     },
                     "additionalProperties": false
@@ -238,16 +237,15 @@
                         "projects": {
                             "type": "array",
                             "items": {
-                                "type": "string",
-                                "pattern": "^[\\w.-]+\\/[\\w.-]+$"
+                                "type": "string"
                             },
                             "default": [],
                             "examples": [
                                 [
                                     "my-group/my-project"
                                 ]
                             ],
-                            "description": "List of individual projects to exclude from syncing. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/"
+                            "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/"
                         }
                     },
                     "additionalProperties": false
@@ -336,11 +334,10 @@
                         "repos": {
                             "type": "array",
                             "items": {
-                                "type": "string",
-                                "pattern": "^[\\w.-]+\\/[\\w.-]+$"
+                                "type": "string"
                             },
                             "default": [],
-                            "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'."
+                            "description": "List of individual repositories to exclude from syncing. Glob patterns are supported."
                         }
                     },
                     "additionalProperties": false
@@ -432,4 +429,4 @@
         }
     },
     "additionalProperties": false
-}
+}