paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Externalize repository processing for BitbucketDiscoveryProcessor

Browse Source 
Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>
This commit is contained in:
Mathias Åhsberg
2021-04-12 15:04:22 +00:00
parent d269bf83c8
commit c2306f898d
9 changed files with 441 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/stale-carpets-poke.md
+30
View File
@@ -0,0 +1,30 @@
---
'@backstage/plugin-catalog-backend': patch
---
Externalize repository processing for BitbucketDiscoveryProcessor.
Add an extension point where you can customize how a matched Bitbucket repository should
be processed. This can for example be used if you want to generate the catalog-info.yaml
automatically based on other files in a repository, while taking advantage of the
build-in repository crawling functionality.
`BitbucketDiscoveryProcessor.fromConfig` now takes an optional parameter `options.parser` where
you can customize the logic for each repository found. The default parser has the same
behaviour as before, where it emits an optional location for the matched repository
and lets the other processors take care of further processing.
```typescript
const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({
client,
repository,
}) {
// Custom logic for interpret the matching repository.
// See defaultRepositoryParser for an example
};
const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, {
parser: customRepositoryParser,
logger: env.logger,
});
```
docs/integrations/bitbucket/discovery.md
+26
View File
@@ -39,3 +39,29 @@ The target is composed of four parts:
- The path within each repository to find the catalog YAML file. This will
usually be `/catalog-info.yaml` or a similar variation for catalog files
stored in the root directory of each repository.
## Custom repository processing
The Bitbucket Discovery Processor will by default emit a location for each
matching repository for further processing by other processors. However, it is
possible to override this functionality and take full control of how each
matching repository is processed.
`BitbucketDiscoveryProcessor.fromConfig` takes an optional parameter
`options.parser` where you can set your own parser to be used for each matched
repository.
```typescript
const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({
client,
repository,
}) {
// Custom logic for interpret the matching repository.
// See defaultRepositoryParser for an example
};
const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, {
parser: customRepositoryParser,
logger: env.logger,
});
```
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
+223 -127
View File
@@ -14,13 +14,15 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
BitbucketDiscoveryProcessor,
readBitbucketOrg,
} from './BitbucketDiscoveryProcessor';
import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
import { ConfigReader } from '@backstage/config';
import { LocationSpec } from '@backstage/catalog-model';
import { BitbucketClient, PagedResponse } from './bitbucket';
import {
BitbucketClient,
BitbucketRepositoryParser,
PagedResponse,
} from './bitbucket';
import { results } from './index';
function pagedResponse(values: any): PagedResponse<any> {
return {
@@ -30,11 +32,6 @@ function pagedResponse(values: any): PagedResponse<any> {
}
describe('BitbucketDiscoveryProcessor', () => {
const client: jest.Mocked<BitbucketClient> = {
listProjects: jest.fn(),
listRepositories: jest.fn(),
} as any;
afterEach(() => jest.resetAllMocks());
describe('reject unrelated entries', () => {
@@ -81,137 +78,236 @@ describe('BitbucketDiscoveryProcessor', () => {
});
describe('handles repositories', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
},
}),
{ logger: getVoidLogger() },
);
it('output all repositories', async () => {
const target =
'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{
slug: 'backstage',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
},
],
},
},
]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{
slug: 'demo',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(2);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(
pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
);
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{
slug: 'backstage',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
},
],
},
},
]),
);
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{
slug: 'demo',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
},
optional: true,
});
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
},
optional: true,
});
});
it('output repositories with wildcards', async () => {
const target =
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{ slug: 'backstage' },
{
slug: 'techdocs-cli',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
},
],
},
},
{
slug: 'techdocs-container',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{ slug: 'backstage' },
{
slug: 'techdocs-cli',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
},
],
},
},
{
slug: 'techdocs-container',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
},
optional: true,
});
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
},
optional: true,
});
});
it('filter unrelated repositories', async () => {
const target =
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }]),
);
client.listRepositories.mockResolvedValue(
pagedResponse([
{ slug: 'abstest' },
{ slug: 'testxyz' },
{
slug: 'test',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/test',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValue(
pagedResponse([
{ slug: 'abstest' },
{ slug: 'testxyz' },
{
slug: 'test',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/test',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
},
optional: true,
});
});
});
describe('Custom repository parser', () => {
const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) {
yield results.location(
{
type: 'custom-location-type',
target: 'custom-target',
},
true,
);
};
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
},
}),
{ parser: customRepositoryParser, logger: getVoidLogger() },
);
it('use custom repository parser', async () => {
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValue(pagedResponse([{ slug: 'test' }]));
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'custom-location-type',
target: 'custom-target',
},
optional: true,
});
});
});
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
+26 -19
View File
@@ -21,15 +21,24 @@ import {
ScmIntegrations,
} from '@backstage/integration';
import { LocationSpec } from '@backstage/catalog-model';
import { BitbucketClient, paginated } from './bitbucket';
import {
Repository,
BitbucketRepositoryParser,
BitbucketClient,
defaultRepositoryParser,
paginated,
} from './bitbucket';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { results } from './index';
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrationRegistry;
private readonly parser: BitbucketRepositoryParser;
private readonly logger: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: { parser?: BitbucketRepositoryParser; logger: Logger },
) {
const integrations = ScmIntegrations.fromConfig(config);
return new BitbucketDiscoveryProcessor({
@@ -40,9 +49,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
parser?: BitbucketRepositoryParser;
logger: Logger;
}) {
this.integrations = options.integrations;
this.parser = options.parser || defaultRepositoryParser;
this.logger = options.logger;
}
@@ -73,18 +84,18 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
const startTimestamp = Date.now();
this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
const { catalogPath } = parseUrl(location.target);
const result = await readBitbucketOrg(client, location.target);
for (const repository of result.matches) {
emit(
results.location(
repository,
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
),
);
for await (const entity of this.parser({
client: client,
repository: repository,
path: catalogPath,
})) {
emit(entity);
}
}
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -100,7 +111,7 @@ export async function readBitbucketOrg(
client: BitbucketClient,
target: string,
): Promise<Result> {
const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
const { projectSearchPath, repoSearchPath } = parseUrl(target);
const projects = paginated(options => client.listProjects(options));
const result: Result = {
scanned: 0,
@@ -116,12 +127,8 @@ export async function readBitbucketOrg(
);
for await (const repository of repositories) {
result.scanned++;
if (repoSearchPath.test(repository.slug)) {
result.matches.push({
type: 'url',
target: `${repository.links.self[0].href}${catalogPath}`,
});
result.matches.push(repository);
}
}
}
@@ -152,5 +159,5 @@ function escapeRegExp(str: string): RegExp {
type Result = {
scanned: number;
matches: LocationSpec[];
matches: Repository[];
};
plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { defaultRepositoryParser } from './BitbucketRepositoryParser';
import { Project, Repository } from './types';
import { BitbucketClient } from './client';
import { results } from '../index';
describe('BitbucketRepositoryParser', () => {
describe('defaultRepositoryParser', () => {
it('emits location', async () => {
const browseUrl =
'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse';
const path = '/catalog-info.yaml';
const expected = [
results.location(
{
type: 'url',
target: `${browseUrl}${path}`,
},
true,
),
];
const actual = await defaultRepositoryParser({
client: {} as BitbucketClient,
repository: {
project: {} as Project,
slug: 'repo-slug',
links: {
self: [{ href: browseUrl }],
},
} as Repository,
path: path,
});
let i = 0;
for await (const entity of actual) {
expect(entity).toStrictEqual(expected[i]);
i++;
}
});
});
});
plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Repository } from './types';
import { CatalogProcessorResult } from '../types';
import { results } from '../index';
import { BitbucketClient } from './client';
export type BitbucketRepositoryParser = (options: {
client: BitbucketClient;
repository: Repository;
path: string;
}) => AsyncIterable<CatalogProcessorResult>;
export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({
repository,
path,
}) {
yield results.location(
{
type: 'url',
target: `${repository.links.self[0].href}${path}`,
},
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
);
};
plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
+9
View File
@@ -41,6 +41,15 @@ export class BitbucketClient {
);
}
async getRaw(
projectKey: string,
repo: string,
path: string,
): Promise<Response> {
const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`;
return fetch(request, getBitbucketRequestOptions(this.config));
}
private async pagedRequest(
endpoint: string,
options?: ListOptions,
plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts
+3
View File
@@ -15,3 +15,6 @@
*/
export { BitbucketClient, paginated } from './client';
export type { PagedResponse } from './client';
export * from './types';
export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';
export { defaultRepositoryParser } from './BitbucketRepositoryParser';
plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts
+28
View File
@@ -0,0 +1,28 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type Project = {
key: string;
};
export type Repository = {
project: Project;
slug: string;
links: Record<string, Link[]>;
};
export type Link = {
href: string;
};