Merge pull request #12822 from brentg-telus/github-entity-provider-3

feat: github entity provider
This commit is contained in:
Ben Lambert
2022-07-29 11:19:44 +02:00
committed by GitHub
15 changed files with 795 additions and 11 deletions
@@ -40,6 +40,24 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
): Promise<boolean>;
}
// @public
export class GitHubEntityProvider implements EntityProvider {
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): GitHubEntityProvider[];
// (undocumented)
getProviderName(): string;
// (undocumented)
refresh(logger: Logger): Promise<void>;
}
// @public
export type GithubMultiOrgConfig = Array<{
name: string;
@@ -20,9 +20,10 @@
* @packageDocumentation
*/
export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';
export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider';
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
export { GitHubEntityProvider } from './providers/GitHubEntityProvider';
export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider';
export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider';
export type { GithubMultiOrgConfig } from './lib';
@@ -22,9 +22,9 @@ import {
} from '@backstage/integration';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
import { getOrganizationRepositories } from './lib';
import { getOrganizationRepositories } from '../lib';
jest.mock('./lib');
jest.mock('../lib');
const mockGetOrganizationRepositories =
getOrganizationRepositories as jest.MockedFunction<
typeof getOrganizationRepositories
@@ -29,7 +29,7 @@ import {
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import { getOrganizationRepositories } from './lib';
import { getOrganizationRepositories } from '../lib';
/**
* Extracts repositories out of a GitHub org.
@@ -37,7 +37,7 @@ import {
getOrganizationUsers,
GithubMultiOrgConfig,
readGithubMultiOrgConfig,
} from './lib';
} from '../lib';
/**
* Extracts teams and users out of a multiple GitHub orgs namespaced per org.
@@ -36,7 +36,7 @@ import {
getOrganizationTeams,
getOrganizationUsers,
parseGitHubOrgUrl,
} from './lib';
} from '../lib';
type GraphQL = typeof graphql;
@@ -0,0 +1,185 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GitHubEntityProvider } from './GitHubEntityProvider';
import * as helpers from '../lib/github';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
getTasks() {
return this.tasks;
}
run(task: TaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
const logger = getVoidLogger();
describe('GitHubEntityProvider', () => {
afterEach(() => jest.resetAllMocks());
it('no provider config', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({});
const providers = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(0);
});
it('single simple provider config', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const providers = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual('github-provider:default');
});
it('multiple provider configs', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org1',
},
anotherProvider: {
organization: 'test-org2',
},
},
},
},
});
const providers = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(2);
expect(providers[0].getProviderName()).toEqual(
'github-provider:myProvider',
);
expect(providers[1].getProviderName()).toEqual(
'github-provider:anotherProvider',
);
});
it('apply full update on scheduled execution', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-21936a3d1e926b8bb3b00ac4398dc9a8dbb90b45',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toBeCalledTimes(1);
expect(entityProviderConnection.applyMutation).toBeCalledWith({
type: 'full',
entities: expectedEntities,
});
});
});
@@ -0,0 +1,253 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import {
GithubCredentialsProvider,
ScmIntegrations,
GitHubIntegrationConfig,
GitHubIntegration,
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import {
EntityProvider,
EntityProviderConnection,
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import {
readProviderConfigs,
GitHubEntityProviderConfig,
} from './GitHubEntityProviderConfig';
import { getOrganizationRepositories, Repository } from '../lib/github';
/**
* Discovers catalog files located in [GitHub](https://github.com).
* The provider will search your GitHub account and register catalog files matching the configured path
* as Location entity and via following processing steps add all contained catalog entities.
* This can be useful as an alternative to static locations or manually adding things to the catalog.
*
* @public
*/
export class GitHubEntityProvider implements EntityProvider {
private readonly config: GitHubEntityProviderConfig;
private readonly logger: Logger;
private readonly integration: GitHubIntegrationConfig;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): GitHubEntityProvider[] {
const integrations = ScmIntegrations.fromConfig(config);
const integration = integrations.github.byHost('github.com');
if (!integration) {
throw new Error(
`There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`,
);
}
return readProviderConfigs(config).map(
providerConfig =>
new GitHubEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
),
);
}
private constructor(
config: GitHubEntityProviderConfig,
integration: GitHubIntegration,
logger: Logger,
schedule: TaskRunner,
) {
this.config = config;
this.integration = integration.config;
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
this.githubCredentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
this.scheduleFn = this.createScheduleFn(schedule);
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
getProviderName(): string {
return `github-provider:${this.config.id}`;
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
return await this.scheduleFn();
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
class: GitHubEntityProvider.prototype.constructor.name,
taskId,
taskInstanceId: uuid.v4(),
});
try {
await this.refresh(logger);
} catch (error) {
logger.error(error);
}
},
});
};
}
async refresh(logger: Logger) {
if (!this.connection) {
throw new Error('Not initialized');
}
const targets = await this.findCatalogFiles();
const matchingTargets = this.matchesFilters(targets);
const entities = matchingTargets
.map(repository => this.createLocationUrl(repository))
.map(GitHubEntityProvider.toLocationSpec)
.map(location => {
return {
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
};
});
await this.connection.applyMutation({
type: 'full',
entities,
});
logger.info(
`Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`,
);
}
// go to the server and get all of the repositories
private async findCatalogFiles(): Promise<Repository[]> {
const organization = this.config.organization;
const host = this.integration.host;
const orgUrl = `https://${host}/${organization}`;
const { headers } = await this.githubCredentialsProvider.getCredentials({
url: orgUrl,
});
const client = graphql.defaults({
baseUrl: this.integration.apiBaseUrl,
headers,
});
const { repositories } = await getOrganizationRepositories(
client,
organization,
);
return repositories;
}
private matchesFilters(repositories: Repository[]) {
const repositoryFilter = this.config.filters?.repository;
const matchingRepositories = repositories.filter(r => {
return !r.isArchived && repositoryFilter
? repositoryFilter?.test(r.name)
: true && r.defaultBranchRef?.name;
});
return matchingRepositories;
}
private createLocationUrl(repository: Repository): string {
const branch =
this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
const catalogFile = this.config.catalogPath.substring(
this.config.catalogPath.lastIndexOf('/') + 1,
);
return `${repository.url}/blob/${branch}/${catalogFile}`;
}
private static toLocationSpec(target: string): LocationSpec {
return {
type: 'url',
target: target,
presence: 'optional',
};
}
}
/*
* Helpers
*/
export function parseUrl(urlString: string): {
org: string;
repoSearchPath: RegExp;
catalogPath: string;
branch: string;
host: string;
} {
const url = new URL(urlString);
const path = url.pathname.substr(1).split('/');
// /backstage/techdocs-*/blob/master/catalog-info.yaml
// can also be
// /backstage
if (path.length > 2 && path[0].length && path[1].length) {
return {
org: decodeURIComponent(path[0]),
repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
branch: decodeURIComponent(path[3]),
host: url.host,
};
} else if (path.length === 1 && path[0].length) {
return {
org: decodeURIComponent(path[0]),
repoSearchPath: escapeRegExp('*'),
catalogPath: '/catalog-info.yaml',
branch: '-',
host: url.host,
};
}
throw new Error(`Failed to parse ${urlString}`);
}
export function escapeRegExp(str: string): RegExp {
return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
}
@@ -0,0 +1,115 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readProviderConfigs } from './GitHubEntityProviderConfig';
describe('readProviderConfigs', () => {
afterEach(() => jest.resetAllMocks());
it('no provider config', () => {
const config = new ConfigReader({});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(0);
});
it('single simple provider config', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(1);
expect(providerConfigs[0].id).toEqual('default');
expect(providerConfigs[0].organization).toEqual('test-org');
});
it('multiple provider configs', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
providerOrganizationOnly: {
organization: 'test-org1',
},
providerCustomCatalogPath: {
organization: 'test-org2',
catalogPath: 'custom/path/catalog-info.yaml',
},
providerWithRepositoryFilter: {
organization: 'test-org3',
filters: {
repository: 'repository.*filter',
},
},
providerWithBranchFilter: {
organization: 'test-org4',
filters: {
branch: 'branch-name',
},
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(4);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
filters: {
repository: undefined,
branch: undefined,
},
});
expect(providerConfigs[1]).toEqual({
id: 'providerCustomCatalogPath',
organization: 'test-org2',
catalogPath: 'custom/path/catalog-info.yaml',
filters: {
repository: undefined,
branch: undefined,
},
});
expect(providerConfigs[2]).toEqual({
id: 'providerWithRepositoryFilter',
organization: 'test-org3', // organization
catalogPath: '/catalog-info.yaml', // file
filters: {
repository: /^repository.*filter$/, // repo
branch: undefined, // branch
},
});
expect(providerConfigs[3]).toEqual({
id: 'providerWithBranchFilter',
organization: 'test-org4',
catalogPath: '/catalog-info.yaml',
filters: {
repository: undefined,
branch: 'branch-name',
},
});
});
});
@@ -0,0 +1,90 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
const DEFAULT_PROVIDER_ID = 'default';
export type GitHubEntityProviderConfig = {
id: string;
catalogPath: string;
organization: string;
filters?: {
repository?: RegExp;
branch?: string;
};
};
export function readProviderConfigs(
config: Config,
): GitHubEntityProviderConfig[] {
const providersConfig = config.getOptionalConfig('catalog.providers.github');
if (!providersConfig) {
return [];
}
if (providersConfig.has('organization')) {
// simple/single config variant
return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
}
return providersConfig.keys().map(id => {
const providerConfig = providersConfig.getConfig(id);
return readProviderConfig(id, providerConfig);
});
}
function readProviderConfig(
id: string,
config: Config,
): GitHubEntityProviderConfig {
const organization = config.getString('organization');
const catalogPath =
config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
const repositoryPattern = config.getOptionalString('filters.repository');
const branchPattern = config.getOptionalString('filters.branch');
return {
id,
catalogPath,
organization,
filters: {
repository: repositoryPattern
? compileRegExp(repositoryPattern)
: undefined,
branch: branchPattern || undefined,
},
};
}
/**
* Compiles a RegExp while enforcing the pattern to contain
* the start-of-line and end-of-line anchors.
*
* @param pattern
*/
function compileRegExp(pattern: string): RegExp {
let fullLinePattern = pattern;
if (!fullLinePattern.startsWith('^')) {
fullLinePattern = `^${fullLinePattern}`;
}
if (!fullLinePattern.endsWith('$')) {
fullLinePattern = `${fullLinePattern}$`;
}
return new RegExp(fullLinePattern);
}
@@ -42,7 +42,7 @@ import {
getOrganizationTeams,
getOrganizationUsers,
parseGitHubOrgUrl,
} from './lib';
} from '../lib';
/**
* Options for {@link GitHubOrgEntityProvider}.