Add new excludeRepos option to the Gitlab provider

Signed-off-by: Dane Elwell <dane.elwell@ans.co.uk>

.changeset/red-radios-promise.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Adds new optional excludeRepos configuration option to the Gitlab catalog provider.

docs/integrations/gitlab/discovery.md

@@ -153,6 +153,7 @@ catalog:
         group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole instance will be scanned
         entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
         projectPattern: '[\s\S]*' # Optional. Filters found projects based on provided patter. Defaults to `[\s\S]*`, which means to not filter anything
+        excludeRepos: [] # Optional. A list of project paths that should be excluded from discovery, e.g. group/subgroup/repo. Should not start or end with a slash.
         schedule: # Same options as in TaskScheduleDefinition. Optional for the Legacy Backend System
           # supports cron, ISO duration, "human duration" as used in code
           frequency: { minutes: 30 }

plugins/catalog-backend-module-gitlab/api-report.md

@@ -110,6 +110,7 @@ export type GitlabProviderConfig = {
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
   skipForkedRepos?: boolean;
+  excludeRepos?: string[];
 };
 
 // @public

plugins/catalog-backend-module-gitlab/config.d.ts

@@ -63,6 +63,11 @@ export interface Config {
            * (Optional) Skip forked repository
            */
           skipForkedRepos?: boolean;
+          /**
+           * (Optional) A list of strings containing the paths of the repositories to skip
+           * Should be in the format group/subgroup/repo, with no leading or trailing slashes.
+           */
+          excludeRepos?: string[];
         };
       };
     };

plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts

@@ -299,6 +299,33 @@ export const config_single_integration_skip_forks: MockObject = {
   },
 };
 
+export const config_single_integration_exclude_repos: MockObject = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'example.com',
+        apiBaseUrl: 'https://example.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'example.com',
+          group: 'group1',
+          excludeRepos: ['group1/test-repo1'],
+          schedule: {
+            frequency: 'PT30M',
+            timeout: 'PT3M',
+          },
+        },
+      },
+    },
+  },
+};
+
 export const config_no_schedule: MockObject = {
   integrations: {
     gitlab: [

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -201,6 +201,11 @@ export type GitlabProviderConfig = {
    * If the project is a fork, skip repository
    */
   skipForkedRepos?: boolean;
+  /**
+   * List of repositories to exclude from discovery, should be the full path to the repository, e.g. `group/project`
+   * Paths should not start or end with a slash.
+   */
+  excludeRepos?: string[];
 };
 
 /**

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -130,6 +130,7 @@ describe('GitlabDiscoveryEntityProvider - configuration', () => {
     );
   });
 });
+
 describe('GitlabDiscoveryEntityProvider - refresh', () => {
   it('should apply full update on scheduled execution', async () => {
     const config = new ConfigReader(mock.config_no_org_integration);
@@ -231,6 +232,38 @@ describe('GitlabDiscoveryEntityProvider - refresh', () => {
     });
   });
 
+  it('should filter repositories that are excluded', async () => {
+    const config = new ConfigReader(
+      mock.config_single_integration_exclude_repos,
+    );
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities_default_branch.filter(
+        entity =>
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('test-repo1') &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
+    });
+  });
+
   // branch and fallback branch are undefined in the config
   it('should ingest catalog from project default branch only', async () => {
     const config = new ConfigReader(mock.config_single_integration);

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -470,6 +470,13 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       return false;
     }
 
+    if (this.config.excludeRepos?.includes(project.path_with_namespace ?? '')) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it is excluded.`,
+      );
+      return false;
+    }
+
     const project_branch =
       this.config.branch ??
       project.default_branch ??

plugins/catalog-backend-module-gitlab/src/providers/config.test.ts

@@ -60,6 +60,7 @@ describe('config', () => {
         allowInherited: false,
         schedule: undefined,
         skipForkedRepos: false,
+        excludeRepos: [],
         restrictUsersToGroup: false,
       }),
     );
@@ -99,6 +100,7 @@ describe('config', () => {
         allowInherited: false,
         schedule: undefined,
         skipForkedRepos: false,
+        excludeRepos: [],
         restrictUsersToGroup: false,
       }),
     );
@@ -139,11 +141,54 @@ describe('config', () => {
         allowInherited: false,
         schedule: undefined,
         restrictUsersToGroup: false,
+        excludeRepos: [],
         skipForkedRepos: true,
       }),
     );
   });
 
+  it('valid config with excludeRepos', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          gitlab: {
+            test: {
+              group: 'group',
+              host: 'host',
+              branch: 'not-master',
+              fallbackBranch: 'main',
+              entityFilename: 'custom-file.yaml',
+              skipForkedRepos: false,
+              excludeRepos: ['foo/bar', 'quz/qux'],
+            },
+          },
+        },
+      },
+    });
+
+    const result = readGitlabConfigs(config);
+    expect(result).toHaveLength(1);
+    result.forEach(r =>
+      expect(r).toStrictEqual({
+        id: 'test',
+        group: 'group',
+        branch: 'not-master',
+        fallbackBranch: 'main',
+        host: 'host',
+        catalogFile: 'custom-file.yaml',
+        projectPattern: /[\s\S]*/,
+        groupPattern: /[\s\S]*/,
+        userPattern: /[\s\S]*/,
+        orgEnabled: false,
+        allowInherited: false,
+        schedule: undefined,
+        restrictUsersToGroup: false,
+        skipForkedRepos: false,
+        excludeRepos: ['foo/bar', 'quz/qux'],
+      }),
+    );
+  });
+
   it('valid config with schedule', () => {
     const config = new ConfigReader({
       catalog: {
@@ -181,6 +226,7 @@ describe('config', () => {
         allowInherited: false,
         skipForkedRepos: false,
         restrictUsersToGroup: false,
+        excludeRepos: [],
         schedule: {
           frequency: { minutes: 30 },
           timeout: {

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -47,6 +47,8 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     config.getOptionalBoolean('allowInherited') ?? false;
   const skipForkedRepos: boolean =
     config.getOptionalBoolean('skipForkedRepos') ?? false;
+  const excludeRepos: string[] =
+    config.getOptionalStringArray('excludeRepos') ?? [];
 
   const schedule = config.has('schedule')
     ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
@@ -68,6 +70,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     orgEnabled,
     allowInherited,
     skipForkedRepos,
+    excludeRepos,
     restrictUsersToGroup,
   };
 }