From ac15c0cb1f316a86eb2143fff27e2cffdd4ef025 Mon Sep 17 00:00:00 2001 From: Dominik Pfaffenbauer Date: Sun, 22 Jan 2023 19:39:59 +0100 Subject: [PATCH] implement gitlab org catalog provider Signed-off-by: Dominik Pfaffenbauer --- docs/integrations/gitlab/org.md | 29 + .../src/index.ts | 5 +- .../src/lib/client.ts | 36 ++ .../src/lib/types.ts | 29 + .../GitlabOrgDiscoveryEntityProvider.test.ts | 514 ++++++++++++++++++ .../GitlabOrgDiscoveryEntityProvider.ts | 363 +++++++++++++ .../src/providers/config.test.ts | 9 + .../src/providers/config.ts | 10 + .../src/providers/index.ts | 1 + 9 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 docs/integrations/gitlab/org.md create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md new file mode 100644 index 0000000000..caf0851477 --- /dev/null +++ b/docs/integrations/gitlab/org.md @@ -0,0 +1,29 @@ +--- +id: org +title: GitLab Org +sidebar_label: Org Data +description: Importing users and groups from a GitLab organization into Backstage +--- + +The Backstage catalog can be set up to ingest organizational data - users and +teams - directly from an organization in GitLab. The result +is a hierarchy of +[`User`](../../features/software-catalog/descriptor-format.md#kind-user) and +[`Group`](../../features/software-catalog/descriptor-format.md#kind-group) kind +entities that mirror your org setup. + +```yaml +integrations: + gitlab: + - host: gitlab.com + token: ${GITLAB_TOKEN} +``` + +```yaml +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab.com + orgEnabled: true +``` diff --git a/plugins/catalog-backend-module-gitlab/src/index.ts b/plugins/catalog-backend-module-gitlab/src/index.ts index 27aac9e78f..30efac4924 100644 --- a/plugins/catalog-backend-module-gitlab/src/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/index.ts @@ -21,5 +21,8 @@ */ export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; -export { GitlabDiscoveryEntityProvider } from './providers'; +export { + GitlabDiscoveryEntityProvider, + GitlabOrgDiscoveryEntityProvider, +} from './providers'; export { gitlabDiscoveryEntityProviderCatalogModule } from './service/GitlabDiscoveryEntityProviderCatalogModule'; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index adf0a78e8f..b86de7bf1d 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -20,12 +20,14 @@ import { GitLabIntegrationConfig, } from '@backstage/integration'; import { Logger } from 'winston'; +import { GitLabGroup, GitLabMembership, GitLabUser } from './types'; export type ListOptions = { [key: string]: string | number | boolean | undefined; group?: string; per_page?: number | undefined; page?: number | undefined; + active?: boolean; }; export type PagedResponse = { @@ -63,6 +65,40 @@ export class GitLabClient { return this.pagedRequest(`/projects`, options); } + async listUsers(options?: ListOptions): Promise> { + return this.pagedRequest(`/users`, options); + } + + async listGroups(options?: ListOptions): Promise> { + return this.pagedRequest(`/groups`, options); + } + + async getUserMemberships(userId: number): Promise { + const endpoint: string = `/users/${encodeURIComponent(userId)}/memberships`; + const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); + request.searchParams.append('per_page', '100'); + + const response = await fetch(request.toString(), { + headers: getGitLabRequestOptions(this.config).headers, + method: 'GET', + }); + + if (!response.ok) { + if (response.status >= 500) { + this.logger.debug( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return []; + } + + return response.json().then(items => { + return items as GitLabMembership[]; + }); + } + /** * General existence check. * diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 86fe1aa385..97ea87ea98 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -31,6 +31,32 @@ export type GitLabProject = { path_with_namespace?: string; }; +export type GitLabUser = { + id: number; + username: string; + name: string; + email: string; + active: boolean; + web_url: string; + avatar_url: string; + groups?: GitLabGroup[]; +}; + +export type GitLabGroup = { + id: number; + name: string; + full_path: string; + description?: string; + parent_id?: number; +}; + +export type GitLabMembership = { + source_id: number; + source_name: string; + source_type: string; + access_level: number; +}; + export type GitlabProviderConfig = { host: string; group: string; @@ -38,5 +64,8 @@ export type GitlabProviderConfig = { branch: string; catalogFile: string; projectPattern: RegExp; + userPattern: RegExp; + groupPattern: RegExp; + orgEnabled?: boolean; schedule?: TaskScheduleDefinition; }; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts new file mode 100644 index 0000000000..74649f41da --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -0,0 +1,514 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +const server = setupServer(); + +describe('GitlabOrgDiscoveryEntityProvider', () => { + setupRequestMockHandlers(server); + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple discovery config with org disabled', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + }, + }, + }, + }, + }); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple discovery config with org enabled', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + }, + }, + }, + }, + }); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('multiple discovery configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + }, + 'second-test': { + host: 'test-gitlab', + group: 'second-group', + orgEnabled: true, + }, + }, + }, + }, + }); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + expect(providers[1].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:second-test', + ); + }); + + it('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + + server.use( + rest.get( + `https://api.gitlab.example/api/v4/groups/test-group/projects`, + (_req, res, ctx) => { + const response = [ + { + id: 123, + default_branch: 'master', + archived: false, + last_activity_at: new Date().toString(), + web_url: 'https://api.gitlab.example/test-group/test-repo', + path_with_namespace: 'test-group/test-repo', + }, + ]; + return res(ctx.json(response)); + }, + ), + rest.get(`https://api.gitlab.example/api/v4/users`, (_req, res, ctx) => { + const response = [ + { + id: 1, + username: 'test1', + name: 'Test Testit', + state: 'active', + avatar_url: 'https://secure.gravatar.com/', + web_url: 'https://gitlab.example/test1', + created_at: '2023-01-19T07:27:03.333Z', + bio: '', + location: null, + public_email: null, + skype: '', + linkedin: '', + twitter: '', + website_url: '', + organization: null, + job_title: '', + pronouns: null, + bot: false, + work_information: null, + followers: 0, + following: 0, + is_followed: false, + local_time: null, + last_sign_in_at: '2023-01-19T07:27:49.601Z', + confirmed_at: '2023-01-19T07:27:02.905Z', + last_activity_on: '2023-01-19', + email: 'test@example.com', + theme_id: 1, + color_scheme_id: 1, + projects_limit: 100000, + current_sign_in_at: '2023-01-19T09:09:10.676Z', + identities: [], + can_create_group: true, + can_create_project: true, + two_factor_enabled: false, + external: false, + private_profile: false, + commit_email: 'test@example.com', + is_admin: false, + note: '', + }, + ]; + return res(ctx.json(response)); + }), + rest.get(`https://api.gitlab.example/api/v4/groups`, (_req, res, ctx) => { + const response = [ + { + id: 1, + web_url: 'https://gitlab.example/groups/group1', + name: 'group1', + path: 'group1', + description: '', + visibility: 'internal', + share_with_group_lock: false, + require_two_factor_authentication: false, + two_factor_grace_period: 48, + project_creation_level: 'developer', + auto_devops_enabled: null, + subgroup_creation_level: 'owner', + emails_disabled: null, + mentions_disabled: null, + lfs_enabled: true, + default_branch_protection: 2, + avatar_url: null, + request_access_enabled: false, + full_name: '8020', + full_path: '8020', + created_at: '2017-06-19T06:42:34.160Z', + parent_id: null, + }, + { + id: 2, + web_url: 'https://gitlab.example/groups/group1/group2', + name: 'group2', + path: 'group1/group2', + description: 'Group2', + visibility: 'internal', + share_with_group_lock: false, + require_two_factor_authentication: false, + two_factor_grace_period: 48, + project_creation_level: 'developer', + auto_devops_enabled: null, + subgroup_creation_level: 'owner', + emails_disabled: null, + mentions_disabled: null, + lfs_enabled: true, + request_access_enabled: false, + full_name: 'group2', + full_path: 'group1/group2', + created_at: '2017-12-07T13:20:40.675Z', + parent_id: null, + }, + ]; + return res(ctx.json(response)); + }), + rest.get( + `https://api.gitlab.example/api/v4/users/1/memberships`, + (_req, res, ctx) => { + const response = [ + { + source_id: 2, + source_name: 'Group 2', + source_type: 'Namespace', + access_level: 50, + }, + ]; + return res(ctx.json(response)); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:test-gitlab/test1', + 'backstage.io/managed-by-origin-location': + 'url:test-gitlab/test1', + 'test-gitlab/user-login': 'https://gitlab.example/test1', + }, + name: 'test1', + }, + spec: { + memberOf: ['group1-group2'], + profile: { + displayName: 'Test Testit', + email: 'test@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:test-gitlab/teams/group1-group2', + 'backstage.io/managed-by-origin-location': + 'url:test-gitlab/teams/group1-group2', + 'test-gitlab/team-path': 'group1/group2', + }, + description: 'Group2', + name: 'group1-group2', + }, + spec: { + children: [], + profile: { + displayName: 'group2', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + }, + }, + }, + }, + }); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + }, + }, + }, + }, + }); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); + + it('single simple provider config with schedule in config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'test-gitlab', + group: 'test-group', + orgEnabled: true, + schedule: { + frequency: 'PT30M', + timeout: 'PT3M', + }, + }, + }, + }, + }, + }); + const providers = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); + }); +}); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts new file mode 100644 index 0000000000..ca4f9ec719 --- /dev/null +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -0,0 +1,363 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { GitLabIntegration, ScmIntegrations } from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; +import { + GitLabClient, + GitlabProviderConfig, + paginated, + readGitlabConfigs, +} from '../lib'; +import { GitLabGroup, GitLabUser } from '../lib/types'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + Entity, + UserEntity, + GroupEntity, +} from '@backstage/catalog-model'; +import { merge } from 'lodash'; + +type Result = { + scanned: number; + matches: GitLabUser[]; +}; + +type GroupResult = { + scanned: number; + matches: GitLabGroup[]; +}; + +/** + * Discovers entity definition files in the groups of a Gitlab instance. + * @public + */ +export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { + private readonly config: GitlabProviderConfig; + private readonly integration: GitLabIntegration; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + }, + ): GitlabOrgDiscoveryEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + const providerConfigs = readGitlabConfigs(config); + const integrations = ScmIntegrations.fromConfig(config).gitlab; + const providers: GitlabOrgDiscoveryEntityProvider[] = []; + + providerConfigs.forEach(providerConfig => { + const integration = integrations.byHost(providerConfig.host); + if (!integration) { + throw new Error( + `No gitlab integration found that matches host ${providerConfig.host}`, + ); + } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + + if (!providerConfig.orgEnabled) { + return; + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + providers.push( + new GitlabOrgDiscoveryEntityProvider({ + ...options, + config: providerConfig, + integration, + taskRunner, + }), + ); + }); + return providers; + } + + private constructor(options: { + config: GitlabProviderConfig; + integration: GitLabIntegration; + logger: Logger; + taskRunner: TaskRunner; + }) { + this.config = options.config; + this.integration = options.integration; + this.logger = options.logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(options.taskRunner); + } + + getProviderName(): string { + return `GitlabOrgDiscoveryEntityProvider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + private createScheduleFn(taskRunner: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: GitlabOrgDiscoveryEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(`${this.getProviderName()} refresh failed`, error); + } + }, + }); + }; + } + + async refresh(logger: Logger): Promise { + if (!this.connection) { + throw new Error( + `Gitlab discovery connection not initialized for ${this.getProviderName()}`, + ); + } + + const client = new GitLabClient({ + config: this.integration.config, + logger: logger, + }); + + const users = paginated(options => client.listUsers(options), { + page: 1, + per_page: 50, + active: true, + }); + + const groups = paginated( + options => client.listGroups(options), + { + page: 1, + per_page: 50, + }, + ); + + const idMappedGroup: { [groupId: number]: GitLabGroup } = {}; + + const res: Result = { + scanned: 0, + matches: [], + }; + + const groupRes: GroupResult = { + scanned: 0, + matches: [], + }; + + for await (const group of groups) { + if (!this.config.groupPattern.test(group.full_path ?? '')) { + continue; + } + + groupRes.scanned++; + groupRes.matches.push(group); + + idMappedGroup[group.id] = group; + } + + for await (const user of users) { + if (!this.config.userPattern.test(user.email ?? '')) { + continue; + } + + res.scanned++; + + if (user.active) { + continue; + } + + const memberships = await client.getUserMemberships(user.id); + const userGroups: GitLabGroup[] = []; + + for (const i of memberships) { + if ( + i.source_type === 'Namespace' && + idMappedGroup.hasOwnProperty(i.source_id) + ) { + userGroups.push(idMappedGroup[i.source_id]); + } + } + + user.groups = userGroups; + + res.matches.push(user); + } + + const groupsWithUsers = groupRes.matches.filter(group => { + return ( + res.matches.filter(x => { + return !!x.groups?.find(y => y.id === group.id); + }).length > 0 + ); + }); + + const userEntities = res.matches.map(p => + this.createUserEntity(p, this.integration.config.host), + ); + const groupEntities = this.createGroupEntities( + groupsWithUsers, + this.integration.config.host, + ); + + await this.connection.applyMutation({ + type: 'full', + entities: [...userEntities, ...groupEntities].map(entity => ({ + locationKey: this.getProviderName(), + entity: this.withLocations(this.integration.config.host, entity), + })), + }); + } + + private createGroupEntities( + groupResult: GitLabGroup[], + host: string, + ): GroupEntity[] { + const idMapped: { [groupId: number]: GitLabGroup } = {}; + const entities: GroupEntity[] = []; + + for (const group of groupResult) { + idMapped[group.id] = group; + } + + for (const group of groupResult) { + const entity = this.createGroupEntity(group, host); + + if (group.parent_id && idMapped.hasOwnProperty(group.parent_id)) { + entity.spec.parent = idMapped[group.parent_id].full_path; + } + + entities.push(entity); + } + + return entities; + } + + private withLocations(host: string, entity: Entity): Entity { + const location = + entity.kind === 'Group' + ? `url:${host}/teams/${entity.metadata.name}` + : `url:${host}/${entity.metadata.name}`; + return merge( + { + metadata: { + annotations: { + [ANNOTATION_LOCATION]: location, + [ANNOTATION_ORIGIN_LOCATION]: location, + }, + }, + }, + entity, + ) as Entity; + } + + private createUserEntity(user: GitLabUser, host: string): UserEntity { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${host}/user-login`] = user.web_url; + + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: user.username, + annotations: annotations, + }, + spec: { + profile: { + email: user.email, + displayName: user.name, + picture: user.avatar_url, + }, + memberOf: [], + }, + }; + + if (user.groups) { + for (const group of user.groups) { + if (!entity.spec.memberOf) { + entity.spec.memberOf = []; + } + entity.spec.memberOf.push(group.full_path.replace('/', '-')); + } + } + + return entity; + } + + private createGroupEntity(group: GitLabGroup, host: string): GroupEntity { + const annotations: { [annotationName: string]: string } = {}; + + annotations[`${host}/team-path`] = group.full_path; + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: group.full_path.replace('/', '-'), + annotations: annotations, + }, + spec: { + type: 'team', + children: [], + profile: { + displayName: group.name, + }, + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + + return entity; + } +} diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts index 6b337bf7cd..e72c38e9d1 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.test.ts @@ -54,6 +54,9 @@ describe('config', () => { host: 'host', catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, schedule: undefined, }), ); @@ -85,6 +88,9 @@ describe('config', () => { host: 'host', catalogFile: 'custom-file.yaml', projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, schedule: undefined, }), ); @@ -120,6 +126,9 @@ describe('config', () => { host: 'host', catalogFile: 'catalog-info.yaml', projectPattern: /[\s\S]*/, + groupPattern: /[\s\S]*/, + userPattern: /[\s\S]*/, + orgEnabled: false, schedule: { frequency: Duration.fromISO('PT30M'), timeout: { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/config.ts b/plugins/catalog-backend-module-gitlab/src/providers/config.ts index 3dfc00745e..9980d59023 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/config.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/config.ts @@ -35,6 +35,13 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { const projectPattern = new RegExp( config.getOptionalString('projectPattern') ?? /[\s\S]*/, ); + const userPattern = new RegExp( + config.getOptionalString('userPattern') ?? /[\s\S]*/, + ); + const groupPattern = new RegExp( + config.getOptionalString('grupPattern') ?? /[\s\S]*/, + ); + const orgEnabled: boolean = config.getOptionalBoolean('orgEnabled') ?? false; const schedule = config.has('schedule') ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) @@ -47,7 +54,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig { host, catalogFile, projectPattern, + userPattern, + groupPattern, schedule, + orgEnabled, }; } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/index.ts b/plugins/catalog-backend-module-gitlab/src/providers/index.ts index e7cb00a73f..2091b3f47a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/index.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/index.ts @@ -15,3 +15,4 @@ */ export { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider'; +export { GitlabOrgDiscoveryEntityProvider } from './GitlabOrgDiscoveryEntityProvider';