diff --git a/.changeset/hungry-lies-cry.md b/.changeset/hungry-lies-cry.md new file mode 100644 index 0000000000..712aef6172 --- /dev/null +++ b/.changeset/hungry-lies-cry.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +The gitlab org data integration now makes use of the GraphQL API to determine +the relationships between imported User and Group entities, effectively making +this integration usable without an administrator account's Personal Access +Token. diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 7ddc5202b5..d78b0ed388 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -2,15 +2,14 @@ id: org title: GitLab Organizational Data sidebar_label: Org Data -description: Importing users and groups from a GitLab organization into Backstage +description: Importing users and groups from GitLab into Backstage --- -The Backstage catalog can be set up to ingest organizational data - users and -teams - directly from an organization in GitLab. The result -is a hierarchy of +The Backstage catalog can be set up to ingest organizational data -- users and +groups -- directly from GitLab. The result is a hierarchy of [`User`](../../features/software-catalog/descriptor-format.md#kind-user) and -[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) kind -entities that mirror your org setup. +[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) +entities that mirrors your org setup. ```yaml integrations: @@ -19,10 +18,11 @@ integrations: token: ${GITLAB_TOKEN} ``` -This will query all users and groups from your gitlab installation. Depending on the size -of the Gitlab Instance, this can take some time and resources. +This will query all users and groups from your GitLab instance. Depending on the +amount of data, this can take significant time and resources. -The token that is used for the Organization Integration, has to be an Admin Personal Access Token (PAT). +The token used must have the `read_api` scope, and the Users and Groups fetched +will be those visible to the account which provisioned the token. ```yaml catalog: @@ -35,6 +35,7 @@ catalog: groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` -When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group -when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will -be `avengers-gotg`. +When the `group` parameter is provided, the corresponding path prefix will be +stripped out from each matching group when computing the unique entity name. +e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be +`avengers-gotg`. diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 4ac7d23390..66c04f6cc4 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -18,10 +18,10 @@ import { ConfigReader } from '@backstage/config'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; -import { rest } from 'msw'; +import { graphql, rest } from 'msw'; import { setupServer, SetupServerApi } from 'msw/node'; import { GitLabClient, paginated } from './client'; -import { GitLabUser } from './types'; +import { GitLabGroup, GitLabUser } from './types'; const server = setupServer(); setupRequestMockHandlers(server); @@ -31,6 +31,7 @@ const MOCK_CONFIG = readGitLabIntegrationConfig( host: 'example.com', token: 'test-token', apiBaseUrl: 'https://example.com/api/v4', + baseUrl: 'https://example.com', }), ); const FAKE_PAGED_ENDPOINT = `/some-endpoint`; @@ -338,6 +339,7 @@ describe('GitLabClient', () => { }, ]); }); + it('listGroups gets all groups in the instance', async () => { server.use( rest.get(`${MOCK_CONFIG.apiBaseUrl}/groups`, (_, res, ctx) => @@ -395,6 +397,32 @@ describe('GitLabClient', () => { }, ]); }); + + it('getGroupMembers gets member IDs', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [{ user: { id: 'gid://gitlab/User/1' } }], + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const members = await client.getGroupMembers('group1'); + + expect(members).toEqual([1]); + }); }); describe('paginated', () => { diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index e36c0d72c6..74a3a4be71 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -20,7 +20,7 @@ import { GitLabIntegrationConfig, } from '@backstage/integration'; import { Logger } from 'winston'; -import { GitLabGroup, GitLabMembership, GitLabUser } from './types'; +import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types'; export type CommonListOptions = { [key: string]: string | number | boolean | undefined; @@ -91,30 +91,37 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } - async getUserMemberships(userId: number): Promise { - const endpoint: string = `/users/${encodeURIComponent(userId)}/memberships`; - const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); - request.searchParams.append('per_page', '100'); + async getGroupMembers(groupPath: string): Promise { + const response: GitLabGroupMembersResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath }, + query: `query($group: ID!) { + group(fullPath: $group) { + groupMembers(first: 10, relations: [DIRECT]) { + nodes { + user { + id + } + } + } + } + }`, + }), + }, + ).then(r => r.json()); + this.logger.debug(`got GraphQL response: ${JSON.stringify(response)}`); - const response = await fetch(request.toString(), { - headers: getGitLabRequestOptions(this.config).headers, - method: 'GET', - }); - - if (!response.ok) { - if (response.status >= 500) { - this.logger.debug( - `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ - response.status - } - ${response.statusText}`, - ); - } - return []; - } - - return response.json().then(items => { - return items as GitLabMembership[]; - }); + return response.data.group.groupMembers.nodes.map( + (node: { user: { id: string } }) => + Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + ); } /** diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index c3fc8a4ede..b4480efdc2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -50,11 +50,19 @@ export type GitLabGroup = { parent_id?: number; }; -export type GitLabMembership = { - source_id: number; - source_name: string; - source_type: string; - access_level: number; +export type GitLabGroupMembersResponse = { + errors: { message: string }[]; + data: { + group: { + groupMembers: { + nodes: { user: { id: string } }[]; + pageInfo: { + endCursor: string; + hasNextPage: boolean; + }; + }; + }; + }; }; export type GitlabProviderConfig = { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 5a323a8dec..88f8d58bf2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -23,7 +23,7 @@ import { import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; -import { rest } from 'msw'; +import { graphql, rest } from 'msw'; import { setupServer } from 'msw/node'; import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider'; @@ -318,20 +318,22 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ]; return res(ctx.json(response)); }), - rest.get( - `https://api.gitlab.example/api/v4/users/1/memberships`, - (_req, res, ctx) => { - const response = [ - { - source_id: 2, - source_name: 'Group 2', - source_type: 'Namespace', - access_level: 50, - }, - ]; - return res(ctx.json(response)); - }, - ), + graphql + .link('https://test-gitlab/api/graphql') + .operation(async (req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: + req.variables.group === 'group1/group2' + ? [{ user: { id: 'gid://gitlab/User/1' } }] + : [], + }, + }, + }), + ), + ), ); await provider.connect(entityProviderConnection); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index acfa784c2e..12cdf40e0c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -185,7 +185,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }, ); - const idMappedGroup: { [groupId: number]: GitLabGroup } = {}; + const idMappedUser: { [userId: number]: GitLabUser } = {}; const res: Result = { scanned: 0, @@ -197,6 +197,21 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { matches: [], }; + for await (const user of users) { + if (!this.config.userPattern.test(user.email ?? user.username ?? '')) { + continue; + } + + res.scanned++; + + if (user.state !== 'active') { + continue; + } + + idMappedUser[user.id] = user; + res.matches.push(user); + } + for await (const group of groups) { if (!this.config.groupPattern.test(group.full_path ?? '')) { continue; @@ -212,35 +227,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - idMappedGroup[group.id] = group; - } - - for await (const user of users) { - if (!this.config.userPattern.test(user.email ?? user.username ?? '')) { - continue; - } - - res.scanned++; - - if (user.state !== 'active') { - continue; - } - - const memberships = await client.getUserMemberships(user.id); - const userGroups: GitLabGroup[] = []; - - for (const i of memberships) { - if ( - i.source_type === 'Namespace' && - idMappedGroup.hasOwnProperty(i.source_id) - ) { - userGroups.push(idMappedGroup[i.source_id]); + for (const id of await client.getGroupMembers(group.full_path)) { + const user = idMappedUser[id]; + if (user) { + user.groups = (user.groups ?? []).concat(group); } } - - user.groups = userGroups; - - res.matches.push(user); } const groupsWithUsers = groupRes.matches.filter(group => {