diff --git a/.changeset/orange-news-sparkle.md b/.changeset/orange-news-sparkle.md new file mode 100644 index 0000000000..d3a0292d52 --- /dev/null +++ b/.changeset/orange-news-sparkle.md @@ -0,0 +1,46 @@ +--- +'@backstage/integration': patch +'@backstage/plugin-catalog-backend': patch +--- + +Support ingesting multiple GitHub organizations via a new `GithubMultiOrgReaderProcessor`. + +This new processor handles namespacing created groups according to the org of the associated GitHub team to prevent potential name clashes between organizations. Be aware that this processor is considered alpha and may not be compatible with future org structures in the catalog. + +NOTE: This processor only fully supports auth via GitHub Apps + +To install this processor, import and add it as follows: + +```typescript +// Typically in packages/backend/src/plugins/catalog.ts +import { GithubMultiOrgReaderProcessor } from '@backstage/plugin-catalog-backend'; +// ... +export default async function createPlugin(env: PluginEnvironment) { + const builder = new CatalogBuilder(env); + builder.addProcessor( + GithubMultiOrgReaderProcessor.fromConfig(env.config, { + logger: env.logger, + }), + ); + // ... +} +``` + +Configure in your `app-config.yaml` by pointing to your GitHub instance and optionally list which GitHub organizations you wish to import. You can also configure what namespace you want to set for teams from each org. If unspecified, the org name will be used as the namespace. If no organizations are listed, by default this processor will import from all organizations accessible by all configured GitHub Apps: + +```yaml +catalog: + locations: + - type: github-multi-org + target: https://github.myorg.com + + processors: + githubMultiOrg: + orgs: + - name: fooOrg + groupNamespace: foo + - name: barOrg + groupNamespace: bar + - name: awesomeOrg + - name: anotherOrg +``` diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 31cbc8b5d8..afd048cb13 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -5,6 +5,7 @@ ```ts import { Config } from '@backstage/config'; +import { RestEndpointMethodTypes } from '@octokit/rest'; // @public (undocumented) export class AzureIntegration implements ScmIntegration { @@ -106,6 +107,15 @@ export function getGitLabFileFetchUrl(url: string, config: GitLabIntegrationConf // @public export function getGitLabRequestOptions(config: GitLabIntegrationConfig): RequestInit; +// @public (undocumented) +export class GithubAppCredentialsMux { + constructor(config: GitHubIntegrationConfig); + // (undocumented) + getAllInstallations(): Promise; + // (undocumented) + getAppToken(owner: string, repo?: string): Promise; +} + // @public (undocumented) export class GithubCredentialsProvider { // (undocumented) diff --git a/packages/integration/src/github/GithubCredentialsProvider.test.ts b/packages/integration/src/github/GithubCredentialsProvider.test.ts index 271d4b9920..6b70109a26 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.test.ts @@ -15,6 +15,7 @@ */ const octokit = { + paginate: async (fn: any) => (await fn()).data, apps: { listInstallations: jest.fn(), createInstallationAccessToken: jest.fn(), @@ -53,7 +54,7 @@ describe('GithubCredentialsProvider tests', () => { jest.resetAllMocks(); }); it('create repository specific tokens', async () => { - octokit.apps.listInstallations.mockResolvedValueOnce({ + octokit.apps.listInstallations.mockResolvedValue({ headers: { etag: '123', }, @@ -72,7 +73,6 @@ describe('GithubCredentialsProvider tests', () => { }, ], } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - octokit.apps.listInstallations.mockRejectedValue({ status: 304 }); octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { @@ -84,12 +84,8 @@ describe('GithubCredentialsProvider tests', () => { const { token, headers, type } = await github.getCredentials({ url: 'https://github.com/backstage/foobar', }); - const { token: accessToken2 } = await github.getCredentials({ - url: 'https://github.com/backstage/foobar', - }); expect(type).toEqual('app'); expect(token).toEqual('secret_token'); - expect(token).toEqual(accessToken2); expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); // fallback to the configured token if no application is matching @@ -107,7 +103,7 @@ describe('GithubCredentialsProvider tests', () => { }); it('creates tokens for an organization', async () => { - octokit.apps.listInstallations.mockResolvedValueOnce({ + octokit.apps.listInstallations.mockResolvedValue({ headers: { etag: '123', }, @@ -121,7 +117,6 @@ describe('GithubCredentialsProvider tests', () => { }, ], } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - octokit.apps.listInstallations.mockRejectedValue({ status: 304 }); octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { @@ -133,17 +128,13 @@ describe('GithubCredentialsProvider tests', () => { const { token, headers } = await github.getCredentials({ url: 'https://github.com/backstage', }); - const { token: accessToken2 } = await github.getCredentials({ - url: 'https://github.com/backstage', - }); expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); expect(token).toEqual('secret_token'); - expect(token).toEqual(accessToken2); }); it('should fail to issue tokens for an organization when the app is installed for a single repo', async () => { - octokit.apps.listInstallations.mockResolvedValueOnce({ + octokit.apps.listInstallations.mockResolvedValue({ headers: { etag: '123', }, @@ -157,7 +148,6 @@ describe('GithubCredentialsProvider tests', () => { }, ], } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - octokit.apps.listInstallations.mockRejectedValue({ status: 304 }); octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { @@ -176,7 +166,7 @@ describe('GithubCredentialsProvider tests', () => { }); it('should throw if the app is suspended', async () => { - octokit.apps.listInstallations.mockResolvedValueOnce({ + octokit.apps.listInstallations.mockResolvedValue({ headers: { etag: '123', }, @@ -193,7 +183,6 @@ describe('GithubCredentialsProvider tests', () => { }, ], } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - octokit.apps.listInstallations.mockRejectedValue({ status: 304 }); await expect( github.getCredentials({ @@ -229,7 +218,7 @@ describe('GithubCredentialsProvider tests', () => { ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' })); }); - it('should return the configured token if listing installations throws', async () => { + it('should return the configured token if there are no installations', async () => { const githubProvider = GithubCredentialsProvider.create({ host: 'github.com', apps: [ @@ -243,7 +232,9 @@ describe('GithubCredentialsProvider tests', () => { ], token: 'hardcoded_token', }); - octokit.apps.listInstallations.mockRejectedValue({ status: 304 }); + octokit.apps.listInstallations.mockResolvedValue(({ + data: [], + } as unknown) as RestEndpointMethodTypes['apps']['listInstallations']['response']); await expect( githubProvider.getCredentials({ diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index 9bc09deab3..e0517bdff4 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -66,7 +66,6 @@ const HEADERS = { class GithubAppManager { private readonly appClient: Octokit; private readonly baseAuthConfig: { appId: number; privateKey: string }; - private installations?: RestEndpointMethodTypes['apps']['listInstallations']['response']; private readonly cache = new Cache(); constructor(config: GithubAppConfig, baseUrl?: string) { @@ -121,22 +120,15 @@ class GithubAppManager { }); } + getInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + return this.appClient.paginate(this.appClient.apps.listInstallations); + } + private async getInstallationData(owner: string): Promise { - // List all installations using the last used etag. - // Return cached InstallationData if error with status 304 is thrown. - try { - this.installations = await this.appClient.apps.listInstallations({ - headers: { - 'If-None-Match': this.installations?.headers.etag, - Accept: HEADERS.Accept, - }, - }); - } catch (error) { - if (error.status !== 304) { - throw error; - } - } - const installation = this.installations?.data.find( + const allInstallations = await this.getInstallations(); + const installation = allInstallations.find( inst => inst.account?.login === owner, ); if (installation) { @@ -163,6 +155,20 @@ export class GithubAppCredentialsMux { config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; } + async getAllInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + if (!this.apps.length) { + return []; + } + + const installs = await Promise.all( + this.apps.map(app => app.getInstallations()), + ); + + return installs.flat(); + } + async getAppToken(owner: string, repo?: string): Promise { if (this.apps.length === 0) { return undefined; diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index ee92b7d934..b987a06bb1 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -20,6 +20,9 @@ export { } from './config'; export type { GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; -export { GithubCredentialsProvider } from './GithubCredentialsProvider'; +export { + GithubAppCredentialsMux, + GithubCredentialsProvider, +} from './GithubCredentialsProvider'; export type { GithubCredentialType } from './GithubCredentialsProvider'; export { GitHubIntegration } from './GitHubIntegration'; diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 20fcd975cf..a4cfdf95aa 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -145,6 +145,27 @@ export interface Config { }>; }; + /** + * GithubMultiOrgReaderProcessor configuration + */ + githubMultiOrg?: { + /** + * The configuration parameters for each GitHub org to process. + */ + orgs: Array<{ + /** + * The name of the GitHub org to process. + */ + name: string; + /** + * The namespace of the group created for this org. + * + * Defaults to org name if omitted. + */ + groupNamespace?: string; + }>; + }; + /** * LdapOrgReaderProcessor configuration */ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts new file mode 100644 index 0000000000..5223acba4a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts @@ -0,0 +1,181 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + GithubAppCredentialsMux, + GithubCredentialsProvider, + GitHubIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { + getOrganizationTeams, + getOrganizationUsers, + GithubMultiOrgConfig, + readGithubMultiOrgConfig, +} from './github'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { buildOrgHierarchy } from './util/org'; + +/** + * @alpha + * Extracts teams and users out of a multiple GitHub orgs namespaced per org. + * + * Be aware that this processor may not be compatible with future org structures in the catalog. + */ +export class GithubMultiOrgReaderProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly orgs: GithubMultiOrgConfig; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const c = config.getOptionalConfig('catalog.processors.githubMultiOrg'); + const integrations = ScmIntegrations.fromConfig(config); + + return new GithubMultiOrgReaderProcessor({ + ...options, + integrations, + orgs: c ? readGithubMultiOrgConfig(c) : [], + }); + } + + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + orgs: GithubMultiOrgConfig; + }) { + this.integrations = options.integrations; + this.logger = options.logger; + this.orgs = options.orgs; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'github-multi-org') { + return false; + } + + const gitHubConfig = this.integrations.github.byUrl(location.target) + ?.config; + if (!gitHubConfig) { + throw new Error( + `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, + ); + } + + const allUsersMap = new Map(); + const baseUrl = new URL(location.target).origin; + const credentialsProvider = GithubCredentialsProvider.create(gitHubConfig); + + const orgsToProcess = this.orgs.length + ? this.orgs + : await this.getAllOrgs(gitHubConfig); + + for (const orgConfig of orgsToProcess) { + try { + const { + headers, + type: tokenType, + } = await credentialsProvider.getCredentials({ + url: `${baseUrl}/${orgConfig.name}`, + }); + const client = graphql.defaults({ + baseUrl: gitHubConfig.apiBaseUrl, + headers, + }); + + const startTimestamp = Date.now(); + this.logger.info( + `Reading GitHub users and teams for org: ${orgConfig.name}`, + ); + const { users } = await getOrganizationUsers( + client, + orgConfig.name, + tokenType, + ); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + orgConfig.name, + orgConfig.groupNamespace, + ); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`, + ); + + users.forEach(u => { + if (!allUsersMap.has(u.metadata.name)) { + allUsersMap.set(u.metadata.name, u); + } + }); + + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = allUsersMap.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + buildOrgHierarchy(groups); + + for (const group of groups) { + emit(results.entity(location, group)); + } + } catch (e) { + this.logger.error( + `Failed to read GitHub org data for ${orgConfig.name}: ${e}`, + ); + } + } + + const allUsers = Array.from(allUsersMap.values()); + for (const user of allUsers) { + emit(results.entity(location, user)); + } + + return true; + } + + // Note: Does not support usage of PATs + private async getAllOrgs( + gitHubConfig: GitHubIntegrationConfig, + ): Promise { + const githubAppMux = new GithubAppCredentialsMux(gitHubConfig); + const installs = await githubAppMux.getAllInstallations(); + + return installs + .map(install => + install.target_type === 'Organization' && + install.account && + install.account.login + ? { + name: install.account.login, + groupNamespace: install.account.login.toLowerCase(), + } + : undefined, + ) + .filter(Boolean) as GithubMultiOrgConfig; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts index 14f3caa42c..560a916f09 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { readGithubConfig } from './config'; +import { readGithubConfig, readGithubMultiOrgConfig } from './config'; describe('config', () => { describe('readGithubConfig', () => { @@ -76,4 +76,35 @@ describe('config', () => { ).toThrow(/token/); }); }); + + describe('readGithubMultiOrgConfig', () => { + function config(orgs: { name: string; groupNamespace?: string }[]) { + return new ConfigReader({ orgs }); + } + + it('reads org configs', () => { + const output = readGithubMultiOrgConfig( + config([ + { name: 'foo', groupNamespace: 'apple' }, + { name: 'bar', groupNamespace: 'Orange' }, + ]), + ); + + expect(output).toEqual([ + { name: 'foo', groupNamespace: 'apple' }, + { name: 'bar', groupNamespace: 'orange' }, + ]); + }); + + it('defaults groupNamespace to org name if undefined', () => { + const output = readGithubMultiOrgConfig( + config([{ name: 'foo' }, { name: 'bar' }]), + ); + + expect(output).toEqual([ + { name: 'foo', groupNamespace: 'foo' }, + { name: 'bar', groupNamespace: 'bar' }, + ]); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.ts b/plugins/catalog-backend/src/ingestion/processors/github/config.ts index 88f2f96218..6a177f7be0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/config.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/config.ts @@ -82,3 +82,27 @@ export function readGithubConfig(config: Config): ProviderConfig[] { return providers; } + +/** + * The configuration parameters for a multi-org GitHub processor. + */ +export type GithubMultiOrgConfig = Array<{ + /** + * The name of the GitHub org to process. + */ + name: string; + /** + * The namespace of the group created for this org. + */ + groupNamespace: string; +}>; + +export function readGithubMultiOrgConfig(config: Config): GithubMultiOrgConfig { + const orgConfigs = config.getOptionalConfigArray('orgs') ?? []; + return orgConfigs.map(c => ({ + name: c.getString('name'), + groupNamespace: ( + c.getOptionalString('groupNamespace') ?? c.getString('name') + ).toLowerCase(), + })); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index c5094230f8..b9da8e735e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -72,8 +72,10 @@ describe('github', () => { }); describe('getOrganizationTeams', () => { - it('reads teams', async () => { - const input: QueryResponse = { + let input: QueryResponse; + + beforeEach(() => { + input = { organization: { teams: { pageInfo: { hasNextPage: false }, @@ -98,7 +100,9 @@ describe('github', () => { }, }, }; + }); + it('reads teams', async () => { const output = { groups: [ expect.objectContaining({ @@ -126,6 +130,38 @@ describe('github', () => { await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output); }); + + it('applies namespaces', async () => { + const output = { + groups: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'team', + namespace: 'foo', + description: 'The one and only team', + }), + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + parent: 'parent', + children: [], + }, + }), + ], + groupMemberUsers: new Map([['foo/team', ['user']]]), + }; + + server.use( + graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual( + output, + ); + }); }); describe('getTeamMembers', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 2cc97d2ec5..093012544a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -144,6 +144,7 @@ export async function getOrganizationUsers( export async function getOrganizationTeams( client: typeof graphql, org: string, + orgNamespace?: string, ): Promise<{ groups: GroupEntity[]; groupMemberUsers: Map; @@ -189,6 +190,10 @@ export async function getOrganizationTeams( }, }; + if (orgNamespace) { + entity.metadata.namespace = orgNamespace; + } + if (team.description) { entity.metadata.description = team.description; } @@ -203,7 +208,8 @@ export async function getOrganizationTeams( } const memberNames: string[] = []; - groupMemberUsers.set(team.slug, memberNames); + const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug; + groupMemberUsers.set(groupKey, memberNames); if (!team.members.pageInfo.hasNextPage) { // We got all the members in one go, run the fast path diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index 2063e8c1b2..6feeea2e15 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -export { readGithubConfig } from './config'; -export type { ProviderConfig } from './config'; +export { readGithubConfig, readGithubMultiOrgConfig } from './config'; +export type { GithubMultiOrgConfig, ProviderConfig } from './config'; export { getOrganizationTeams, getOrganizationUsers, diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index a92cc9ed3b..9cd851d471 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -25,6 +25,7 @@ export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';