get group members via graphQL

The new `getGroupMembers` method replaces the existing `getUserMemberships` on
`GitLabClient`.

Signed-off-by: Jamie Klassen <jklassen@vmware.com>

.changeset/hungry-lies-cry.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+The gitlab org data integration now makes use of the GraphQL API to determine
+the relationships between imported User and Group entities, effectively making
+this integration usable without an administrator account's Personal Access
+Token.

docs/integrations/gitlab/org.md

@@ -2,15 +2,14 @@
 id: org
 title: GitLab Organizational Data
 sidebar_label: Org Data
-description: Importing users and groups from a GitLab organization into Backstage
+description: Importing users and groups from GitLab into Backstage
 ---
 
-The Backstage catalog can be set up to ingest organizational data - users and
-teams - directly from an organization in GitLab. The result
-is a hierarchy of
+The Backstage catalog can be set up to ingest organizational data -- users and
+groups -- directly from GitLab. The result is a hierarchy of
 [`User`](../../features/software-catalog/descriptor-format.md#kind-user) and
-[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) kind
-entities that mirror your org setup.
+[`Group`](../../features/software-catalog/descriptor-format.md#kind-group)
+entities that mirrors your org setup.
 
 ```yaml
 integrations:
@@ -19,10 +18,11 @@ integrations:
       token: ${GITLAB_TOKEN}
 ```
 
-This will query all users and groups from your gitlab installation. Depending on the size
-of the Gitlab Instance, this can take some time and resources.
+This will query all users and groups from your GitLab instance. Depending on the
+amount of data, this can take significant time and resources.
 
-The token that is used for the Organization Integration, has to be an Admin Personal Access Token (PAT).
+The token used must have the `read_api` scope, and the Users and Groups fetched
+will be those visible to the account which provisioned the token.
 
 ```yaml
 catalog:
@@ -35,6 +35,7 @@ catalog:
         groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything
 ```
 
-When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group
-when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will
-be `avengers-gotg`.
+When the `group` parameter is provided, the corresponding path prefix will be
+stripped out from each matching group when computing the unique entity name.
+e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be
+`avengers-gotg`.

plugins/catalog-backend-module-gitlab/src/lib/client.test.ts

@@ -18,10 +18,10 @@ import { ConfigReader } from '@backstage/config';
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { readGitLabIntegrationConfig } from '@backstage/integration';
 import { getVoidLogger } from '@backstage/backend-common';
-import { rest } from 'msw';
+import { graphql, rest } from 'msw';
 import { setupServer, SetupServerApi } from 'msw/node';
 import { GitLabClient, paginated } from './client';
-import { GitLabUser } from './types';
+import { GitLabGroup, GitLabUser } from './types';
 
 const server = setupServer();
 setupRequestMockHandlers(server);
@@ -31,6 +31,7 @@ const MOCK_CONFIG = readGitLabIntegrationConfig(
     host: 'example.com',
     token: 'test-token',
     apiBaseUrl: 'https://example.com/api/v4',
+    baseUrl: 'https://example.com',
   }),
 );
 const FAKE_PAGED_ENDPOINT = `/some-endpoint`;
@@ -338,6 +339,7 @@ describe('GitLabClient', () => {
       },
     ]);
   });
+
   it('listGroups gets all groups in the instance', async () => {
     server.use(
       rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) =>
@@ -395,6 +397,32 @@ describe('GitLabClient', () => {
       },
     ]);
   });
+
+  it('getGroupMembers gets member IDs', async () => {
+    server.use(
+      graphql
+        .link(`${MOCK_CONFIG.baseUrl}/api/graphql`)
+        .operation((_, res, ctx) =>
+          res(
+            ctx.data({
+              group: {
+                groupMembers: {
+                  nodes: [{ user: { id: 'gid://gitlab/User/1' } }],
+                },
+              },
+            }),
+          ),
+        ),
+    );
+    const client = new GitLabClient({
+      config: MOCK_CONFIG,
+      logger: getVoidLogger(),
+    });
+
+    const members = await client.getGroupMembers('group1');
+
+    expect(members).toEqual([1]);
+  });
 });
 
 describe('paginated', () => {

plugins/catalog-backend-module-gitlab/src/lib/client.ts

@@ -20,7 +20,7 @@ import {
   GitLabIntegrationConfig,
 } from '@backstage/integration';
 import { Logger } from 'winston';
-import { GitLabGroup, GitLabMembership, GitLabUser } from './types';
+import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types';
 
 export type CommonListOptions = {
   [key: string]: string | number | boolean | undefined;
@@ -91,30 +91,37 @@ export class GitLabClient {
     return this.pagedRequest(`/groups`, options);
   }
 
-  async getUserMemberships(userId: number): Promise<GitLabMembership[]> {
-    const endpoint: string = `/users/${encodeURIComponent(userId)}/memberships`;
-    const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
-    request.searchParams.append('per_page', '100');
+  async getGroupMembers(groupPath: string): Promise<number[]> {
+    const response: GitLabGroupMembersResponse = await fetch(
+      `${this.config.baseUrl}/api/graphql`,
+      {
+        method: 'POST',
+        headers: {
+          ...getGitLabRequestOptions(this.config).headers,
+          ['Content-Type']: 'application/json',
+        },
+        body: JSON.stringify({
+          variables: { group: groupPath },
+          query: `query($group: ID!) {
+          group(fullPath: $group) {
+            groupMembers(first: 10, relations: [DIRECT]) {
+              nodes {
+                user {
+                  id
+                }
+              }
+            }
+          }
+        }`,
+        }),
+      },
+    ).then(r => r.json());
+    this.logger.debug(`got GraphQL response: ${JSON.stringify(response)}`);
 
-    const response = await fetch(request.toString(), {
-      headers: getGitLabRequestOptions(this.config).headers,
-      method: 'GET',
-    });
-
-    if (!response.ok) {
-      if (response.status >= 500) {
-        this.logger.debug(
-          `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
-            response.status
-          } - ${response.statusText}`,
-        );
-      }
-      return [];
-    }
-
-    return response.json().then(items => {
-      return items as GitLabMembership[];
-    });
+    return response.data.group.groupMembers.nodes.map(
+      (node: { user: { id: string } }) =>
+        Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')),
+    );
   }
 
   /**

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -50,11 +50,19 @@ export type GitLabGroup = {
   parent_id?: number;
 };
 
-export type GitLabMembership = {
-  source_id: number;
-  source_name: string;
-  source_type: string;
-  access_level: number;
+export type GitLabGroupMembersResponse = {
+  errors: { message: string }[];
+  data: {
+    group: {
+      groupMembers: {
+        nodes: { user: { id: string } }[];
+        pageInfo: {
+          endCursor: string;
+          hasNextPage: boolean;
+        };
+      };
+    };
+  };
 };
 
 export type GitlabProviderConfig = {

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts

@@ -23,7 +23,7 @@ import {
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
-import { rest } from 'msw';
+import { graphql, rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider';
 
@@ -318,20 +318,22 @@ describe('GitlabOrgDiscoveryEntityProvider', () => {
         ];
         return res(ctx.json(response));
       }),
-      rest.get(
-        `https://api.gitlab.example/api/v4/users/1/memberships`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              source_id: 2,
-              source_name: 'Group 2',
-              source_type: 'Namespace',
-              access_level: 50,
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
+      graphql
+        .link('https://test-gitlab/api/graphql')
+        .operation(async (req, res, ctx) =>
+          res(
+            ctx.data({
+              group: {
+                groupMembers: {
+                  nodes:
+                    req.variables.group === 'group1/group2'
+                      ? [{ user: { id: 'gid://gitlab/User/1' } }]
+                      : [],
+                },
+              },
+            }),
+          ),
+        ),
     );
 
     await provider.connect(entityProviderConnection);

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -185,7 +185,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       },
     );
 
-    const idMappedGroup: { [groupId: number]: GitLabGroup } = {};
+    const idMappedUser: { [userId: number]: GitLabUser } = {};
 
     const res: Result = {
       scanned: 0,
@@ -197,6 +197,21 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       matches: [],
     };
 
+    for await (const user of users) {
+      if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
+        continue;
+      }
+
+      res.scanned++;
+
+      if (user.state !== 'active') {
+        continue;
+      }
+
+      idMappedUser[user.id] = user;
+      res.matches.push(user);
+    }
+
     for await (const group of groups) {
       if (!this.config.groupPattern.test(group.full_path ?? '')) {
         continue;
@@ -212,35 +227,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       groupRes.scanned++;
       groupRes.matches.push(group);
 
-      idMappedGroup[group.id] = group;
-    }
-
-    for await (const user of users) {
-      if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
-        continue;
-      }
-
-      res.scanned++;
-
-      if (user.state !== 'active') {
-        continue;
-      }
-
-      const memberships = await client.getUserMemberships(user.id);
-      const userGroups: GitLabGroup[] = [];
-
-      for (const i of memberships) {
-        if (
-          i.source_type === 'Namespace' &&
-          idMappedGroup.hasOwnProperty(i.source_id)
-        ) {
-          userGroups.push(idMappedGroup[i.source_id]);
+      for (const id of await client.getGroupMembers(group.full_path)) {
+        const user = idMappedUser[id];
+        if (user) {
+          user.groups = (user.groups ?? []).concat(group);
         }
       }
-
-      user.groups = userGroups;
-
-      res.matches.push(user);
     }
 
     const groupsWithUsers = groupRes.matches.filter(group => {