diff --git a/.changeset/polite-plants-exercise.md b/.changeset/polite-plants-exercise.md new file mode 100644 index 0000000000..42df6875e6 --- /dev/null +++ b/.changeset/polite-plants-exercise.md @@ -0,0 +1,28 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Port `GithubOrgReaderProcessor` to support configuration via +[`integrations`](https://backstage.io/docs/integrations/github/locations) in +addition to [`catalog.processors.githubOrg.providers`](https://backstage.io/docs/integrations/github/org#configuration). +The `integrations` package supports authentication with both personal access +tokens and GitHub apps. + +This deprecates the `catalog.processors.githubOrg.providers` configuration. If +you still have a configuration for providers the processor keeps working, but +consider moving the [`integrations` configuration](https://backstage.io/docs/integrations/github/locations) +as the providers will be removed in the future. You might need to allow +additional scopes for the credentials. + +If you want to stay with providers for now, this introduces a small breaking +change, previously if you had no provider configured, one for GitHub was automatically added. To keep the behavior, add a +default provider for GitHub: + +```yaml +catalog: + processors: + githubOrg: + providers: + - target: https://github.com + apiBaseUrl: https://api.github.com +``` diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 564dd7511e..a913473ff8 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -114,6 +114,8 @@ export interface Config { processors?: { /** * GithubOrgReaderProcessor configuration + * + * @deprecated Configure an GitHub integration instead. */ githubOrg?: { /** diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 19a08d219f..9b724def28 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -16,6 +16,11 @@ import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; +import { + GitHubIntegration, + ScmIntegrations, + ScmIntegrationsGroup, +} from '@backstage/integration'; import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor'; describe('GithubOrgReaderProcessor', () => { @@ -29,6 +34,20 @@ describe('GithubOrgReaderProcessor', () => { }); describe('implementation', () => { + let integrations: ScmIntegrations; + let github: jest.Mocked>; + + beforeEach(() => { + github = { + byHost: jest.fn(), + byUrl: jest.fn(), + list: jest.fn(), + }; + integrations = ({ + github, + } as Partial) as ScmIntegrations; + }); + it('rejects unknown types', async () => { const processor = new GithubOrgReaderProcessor({ providers: [ @@ -37,6 +56,7 @@ describe('GithubOrgReaderProcessor', () => { apiBaseUrl: 'https://api.github.com', }, ], + integrations, logger: getVoidLogger(), }); const location: LocationSpec = { @@ -48,7 +68,7 @@ describe('GithubOrgReaderProcessor', () => { ).resolves.toBeFalsy(); }); - it('rejects unknown targets', async () => { + it('rejects unknown targets from providers', async () => { const processor = new GithubOrgReaderProcessor({ providers: [ { @@ -56,6 +76,24 @@ describe('GithubOrgReaderProcessor', () => { apiBaseUrl: 'https://api.github.com', }, ], + integrations, + logger: getVoidLogger(), + }); + const location: LocationSpec = { + type: 'github-org', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, + ); + }); + + it('rejects unknown targets from integrations', async () => { + const processor = new GithubOrgReaderProcessor({ + providers: [], + integrations, logger: getVoidLogger(), }); const location: LocationSpec = { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 6a83d705f6..db61886b89 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -16,6 +16,10 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; import { @@ -28,22 +32,33 @@ import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { buildOrgHierarchy } from './util/org'; +type GraphQL = typeof graphql; + /** * Extracts teams and users out of a GitHub org. */ export class GithubOrgReaderProcessor implements CatalogProcessor { private readonly providers: ProviderConfig[]; + private readonly integrations: ScmIntegrations; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + return new GithubOrgReaderProcessor({ ...options, providers: readGithubConfig(config), + integrations, }); } - constructor(options: { providers: ProviderConfig[]; logger: Logger }) { + constructor(options: { + providers: ProviderConfig[]; + integrations: ScmIntegrations; + logger: Logger; + }) { this.providers = options.providers; + this.integrations = options.integrations; this.logger = options.logger; } @@ -56,24 +71,8 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return false; } - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, - ); - } - + const client = await this.createClient(location.target); const { org } = parseUrl(location.target); - const client = !provider.token - ? graphql - : graphql.defaults({ - baseUrl: provider.apiBaseUrl, - headers: { - authorization: `token ${provider.token}`, - }, - }); // Read out all of the raw data const startTimestamp = Date.now(); @@ -112,6 +111,65 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return true; } + + private async createClient(orgUrl: string): Promise { + let client = this.createClientFromProvider(orgUrl); + + if (!client) { + client = await this.createClientFromIntegrations(orgUrl); + } + + if (!client) { + throw new Error( + `There is no GitHub Org provider that matches ${orgUrl}. Please add a configuration for an integration or add an entry for it under catalog.processors.githubOrg.providers.`, + ); + } + + return client; + } + + private createClientFromProvider(orgUrl: string): GraphQL | undefined { + const provider = this.providers.find(p => + orgUrl.startsWith(`${p.target}/`), + ); + + if (!provider) { + return undefined; + } + + this.logger.warn( + 'GithubOrgReaderProcessor uses provider defined in catalog.processors.githubOrg.providers, migrate to integrations instead.', + ); + + return !provider.token + ? graphql + : graphql.defaults({ + baseUrl: provider.apiBaseUrl, + headers: { + authorization: `token ${provider.token}`, + }, + }); + } + + private async createClientFromIntegrations( + orgUrl: string, + ): Promise { + const gitHubConfig = this.integrations.github.byUrl(orgUrl)?.config; + if (!gitHubConfig) { + return undefined; + } + + const credentialsProvider = GithubCredentialsProvider.create(gitHubConfig); + + const { headers } = await credentialsProvider.getCredentials({ + url: orgUrl, + }); + + return graphql.defaults({ + baseUrl: gitHubConfig.apiBaseUrl, + headers, + }); + } } /* diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts index 14f3caa42c..b39a7a7ff9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts @@ -27,16 +27,6 @@ describe('config', () => { }); } - it('adds a default GitHub entry when missing', () => { - const output = readGithubConfig(config([])); - expect(output).toEqual([ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ]); - }); - it('injects the correct GitHub API base URL when missing', () => { const output = readGithubConfig( config([{ target: 'https://github.com' }]), diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.ts b/plugins/catalog-backend/src/ingestion/processors/github/config.ts index 88f2f96218..7b16448b9e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/config.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/config.ts @@ -71,14 +71,5 @@ export function readGithubConfig(config: Config): ProviderConfig[] { providers.push({ target, apiBaseUrl, token }); } - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { - providers.push({ - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }); - } - return providers; }