Add 'includeUsersWithoutSeat' flag to enable ingestion of non-paid users from Gitlab

Signed-off-by: Hghtwr <johannes.sonner@outlook.com>

.changeset/spotty-jokes-boil.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Added includeUsersWithoutSeat allows to import users without paid seat, e.g. for Gitlab Free on SaaS. Defaults to false

docs/integrations/gitlab/org.md

@@ -231,6 +231,8 @@ of the top-level group for the configured group path will be ingested.
 
 In both cases (SaaS & self hosted), you can limit the ingested users to users directly assigned to the group defined in your `app-config.yaml` by setting the configuration key `restrictUsersToGroup: true`. This is especially useful when you have a large user base that you don't want to import by default.
 
+On SaaS, you can choose to include users to be ingested that do not have a paid seat. This can be useful when using a free version of Gitlab, or when you use Guest Users on Gitlab Ultimate. Unfortunately, this will also lead to some technical users that might be imported into your user base. While project & group access tokens are filtered, service accounts will remain. [Learn more about Billable Users](https://docs.gitlab.com/ee/subscriptions/self_managed/index.html#billable-users).
+
 ```yaml
 catalog:
   providers:
@@ -239,7 +241,8 @@ catalog:
         host: gitlab.com ## Could also be self hosted.
         orgEnabled: true
         group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped)
-        restrictUsersToGroup: true # Backstage will ingest only users directly assigned to org/teams.
+        restrictUsersToGroup: true # Optional: Backstage will ingest only users directly assigned to org/teams.
+        includeUsersWithoutSeat: true # Optional: Include users without paid seat, only valid for SaaS
 ```
 
 ### Limiting `User` and `Group` entity ingestion in the provider

plugins/catalog-backend-module-gitlab/api-report.md

@@ -112,6 +112,7 @@ export type GitlabProviderConfig = {
   schedule?: SchedulerServiceTaskScheduleDefinition;
   skipForkedRepos?: boolean;
   excludeRepos?: string[];
+  includeUsersWithoutSeat?: boolean;
 };
 
 // @public

plugins/catalog-backend-module-gitlab/src/__testUtils__/mocks.ts

@@ -696,6 +696,31 @@ export const config_org_group_restrictUsers_true_saas = {
   },
 };
 
+export const config_org_group_includeUsersWithoutSeat_true_saas = {
+  integrations: {
+    gitlab: [
+      {
+        host: 'gitlab.com',
+        apiBaseUrl: 'https://gitlab.com/api/v4',
+        token: '1234',
+      },
+    ],
+  },
+  catalog: {
+    providers: {
+      gitlab: {
+        'test-id': {
+          host: 'gitlab.com',
+          group: 'group1',
+          orgEnabled: true,
+          skipForkedRepos: true,
+          includeUsersWithoutSeat: true,
+        },
+      },
+    },
+  },
+};
+
 export const config_org_group_selfHosted = {
   integrations: {
     gitlab: [
@@ -937,6 +962,40 @@ export const all_saas_users_response: MockObject[] = [
     is_using_seat: false,
     membership_state: 'active',
   },
+  {
+    access_level: 50,
+    created_at: '2023-07-15T08:58:34.984Z',
+    expires_at: '2023-10-26',
+    id: 54,
+    username: 'project_100_bot_23dc8057bef66e05181f39be4652577c',
+    name: 'Token Bot',
+    state: 'active',
+    avatar_url: 'https://secure.gravatar.com/',
+    web_url:
+      'https://gitlab.com/project_100_bot_23dc8057bef66e05181f39be4652577c',
+    group_saml_identity: null,
+    is_using_seat: false,
+    membership_state: 'active',
+  },
+  {
+    access_level: 30,
+    created_at: '2023-07-19T08:58:34.984Z',
+    expires_at: null,
+    id: 34,
+    username: 'testuser3',
+    name: 'Test User 3',
+    state: 'active',
+    avatar_url: 'https://secure.gravatar.com/',
+    web_url: 'https://gitlab.com/testuser3',
+    email: 'testuser3@example.com',
+    group_saml_identity: {
+      provider: 'group_saml',
+      extern_uid: '53',
+      saml_provider_id: 1,
+    },
+    is_using_seat: false,
+    membership_state: 'active',
+  },
 ];
 
 export const all_groups_response: GitLabGroup[] = [
@@ -2050,6 +2109,88 @@ export const expected_full_org_scan_entities_saas: MockObject[] = [
   },
 ];
 
+export const expected_full_org_scan_entities_includeUsersWithoutSeat_saas: MockObject[] =
+  [
+    {
+      entity: {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://gitlab.com/testuser1',
+            'backstage.io/managed-by-origin-location':
+              'url:https://gitlab.com/testuser1',
+            'gitlab.com/user-login': 'https://gitlab.com/testuser1',
+            'gitlab.com/saml-external-uid': '51',
+          },
+          name: 'testuser1',
+        },
+        spec: {
+          memberOf: [],
+          profile: {
+            displayName: 'Test User 1',
+            email: 'testuser1@example.com',
+            picture: 'https://secure.gravatar.com/',
+          },
+        },
+      },
+      locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
+    },
+    {
+      entity: {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://gitlab.com/testuser2',
+            'backstage.io/managed-by-origin-location':
+              'url:https://gitlab.com/testuser2',
+            'gitlab.com/user-login': 'https://gitlab.com/testuser2',
+            'gitlab.com/saml-external-uid': '52',
+          },
+          name: 'testuser2',
+        },
+        spec: {
+          memberOf: [],
+          profile: {
+            displayName: 'Test User 2',
+            email: 'testuser2@example.com',
+            picture: 'https://secure.gravatar.com/',
+          },
+        },
+      },
+      locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
+    },
+    {
+      entity: {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://gitlab.com/testuser3',
+            'backstage.io/managed-by-origin-location':
+              'url:https://gitlab.com/testuser3',
+            'gitlab.com/user-login': 'https://gitlab.com/testuser3',
+            'gitlab.com/saml-external-uid': '53',
+          },
+          name: 'testuser3',
+        },
+        spec: {
+          memberOf: [],
+          profile: {
+            displayName: 'Test User 3',
+            email: 'testuser3@example.com',
+            picture: 'https://secure.gravatar.com/',
+          },
+        },
+      },
+      locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id',
+    },
+  ];
+
 export const subgroup_saas_users_response: MockObject[] = [
   {
     access_level: 30,

plugins/catalog-backend-module-gitlab/src/lib/client.ts

@@ -137,12 +137,24 @@ export class GitLabClient {
   async listSaaSUsers(
     groupPath: string,
     options?: CommonListOptions,
+    includeUsersWithoutSeat?: boolean | false,
   ): Promise<PagedResponse<GitLabUser>> {
     return this.listGroupMembers(groupPath, {
       ...options,
+      active: true, // Users with seat are always active but for users without seat we need to filter
       show_seat_info: true,
     }).then(resp => {
-      resp.items = resp.items.filter(user => user.is_using_seat);
+      // Filter is optional to allow to import Gitlab Free users without seats
+      // https://github.com/backstage/backstage/issues/26438
+      // Filter out API tokens https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#bot-users-for-projects
+      if (includeUsersWithoutSeat) {
+        const regex = /^(?:project|group)_(\w+)_bot_(\w+)$/;
+        resp.items = resp.items.filter(user => {
+          return !regex.test(user.username);
+        });
+      } else {
+        resp.items = resp.items.filter(user => user.is_using_seat);
+      }
       return resp;
     });
   }

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -237,6 +237,13 @@ export type GitlabProviderConfig = {
    * Paths should not start or end with a slash.
    */
   excludeRepos?: string[];
+
+  /**
+   * If true, users without a seat will be included in the catalog.
+   * Group/Application Access Tokens are still filtered out but you might find service accounts or other users without a seat.
+   * Defaults to `false`
+   */
+  includeUsersWithoutSeat?: boolean;
 };
 
 /**

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts

@@ -400,6 +400,40 @@ describe('GitlabOrgDiscoveryEntityProvider - refresh', () => {
     });
   });
 
+  // This needs to return all users, including those without a seat but filter out the bot users
+  it('SaaS: should get users without a seat if includeUsersWithoutSeat true', async () => {
+    const config = new ConfigReader(
+      mock.config_org_group_includeUsersWithoutSeat_true_saas,
+    );
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+    expect(provider.getProviderName()).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id',
+    );
+
+    await provider.connect(entityProviderConnection);
+
+    const taskDef = schedule.getTasks()[0];
+    expect(taskDef.id).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id:refresh',
+    );
+    await (taskDef.fn as () => Promise<void>)();
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities:
+        mock.expected_full_org_scan_entities_includeUsersWithoutSeat_saas, //
+    });
+  });
+
   // This should return all members of the self-hosted instance regardless of the group set -> expected_full_members_group_org_scan_entities
   // All instance members, but only the group entities below the config.group
   it('Self-hosted: should get all instance users when restrictUsersToGroup is not set', async () => {

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -400,7 +400,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
         ? rootGroupSplit[rootGroupSplit.length - 1]
         : rootGroupSplit[0];
       users = paginated<GitLabUser>(
-        options => this.gitLabClient.listSaaSUsers(rootGroup, options),
+        options =>
+          this.gitLabClient.listSaaSUsers(
+            rootGroup,
+            options,
+            this.config.includeUsersWithoutSeat,
+          ),
         {
           page: 1,
           per_page: 100,

plugins/catalog-backend-module-gitlab/src/providers/config.test.ts

@@ -63,6 +63,7 @@ describe('config', () => {
         skipForkedRepos: false,
         excludeRepos: [],
         restrictUsersToGroup: false,
+        includeUsersWithoutSeat: false,
       }),
     );
   });
@@ -78,6 +79,7 @@ describe('config', () => {
               branch: 'not-master',
               fallbackBranch: 'main',
               entityFilename: 'custom-file.yaml',
+              includeUsersWithoutSeat: true,
             },
           },
         },
@@ -104,6 +106,7 @@ describe('config', () => {
         skipForkedRepos: false,
         excludeRepos: [],
         restrictUsersToGroup: false,
+        includeUsersWithoutSeat: true,
       }),
     );
   });
@@ -146,6 +149,7 @@ describe('config', () => {
         restrictUsersToGroup: false,
         excludeRepos: [],
         skipForkedRepos: true,
+        includeUsersWithoutSeat: false,
       }),
     );
   });
@@ -189,6 +193,7 @@ describe('config', () => {
         restrictUsersToGroup: false,
         skipForkedRepos: false,
         excludeRepos: ['foo/bar', 'quz/qux'],
+        includeUsersWithoutSeat: false,
       }),
     );
   });
@@ -232,6 +237,7 @@ describe('config', () => {
         skipForkedRepos: false,
         restrictUsersToGroup: false,
         excludeRepos: [],
+        includeUsersWithoutSeat: false,
         schedule: {
           frequency: { minutes: 30 },
           timeout: {

plugins/catalog-backend-module-gitlab/src/providers/config.ts

@@ -60,6 +60,9 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
   const restrictUsersToGroup =
     config.getOptionalBoolean('restrictUsersToGroup') ?? false;
 
+  const includeUsersWithoutSeat =
+    config.getOptionalBoolean('includeUsersWithoutSeat') ?? false;
+
   return {
     id,
     group,
@@ -77,6 +80,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
     skipForkedRepos,
     excludeRepos,
     restrictUsersToGroup,
+    includeUsersWithoutSeat,
   };
 }