feat: add BitbucketServerEntityProvider (new plugin)

Adds a new entity provider `BitbucketServerEntityProvider` as a new plugin.

Signed-off-by: Otto Nordander <otto.nordander@resurs.se>
This commit is contained in:
Otto Nordander
2022-07-21 14:23:29 +02:00
parent 73cee58fc2
commit f7607f9d85
19 changed files with 1572 additions and 0 deletions
+51
View File
@@ -0,0 +1,51 @@
---
'@backstage/plugin-catalog-backend-module-bitbucket-server': minor
---
Add new plugin catalog-backend-module-bitbucket-server which adds the `BitbucketServerEntityProvider`.
The entity provider is meant as a replacement for the `BitbucketDiscoveryProcessor` to be used with Bitbucket Server (Bitbucket Cloud already has a replacement).
**Before:**
```typescript
// packages/backend/src/plugins/catalog.ts
builder.addProcessor(
BitbucketDiscoveryProcessor.fromConfig(env.config, { logger: env.logger }),
);
```
```yaml
# app-config.yaml
catalog:
locations:
- type: bitbucket-discovery
target: 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog-info.yaml
```
**After:**
```typescript
// packages/backend/src/plugins/catalog.ts
builder.addEntityProvider(
BitbucketServerEntityProvider.fromConfig(env.config, {
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 3 },
}),
}),
);
```
```yaml
# app-config.yaml
catalog:
providers:
bitbucketServer:
yourProviderId: # identifies your ingested dataset
catalogPath: /catalog-info.yaml # default value
filters: # optional
projectKey: '.*' # optional; RegExp
repoSlug: '.*' # optional; RegExp
```
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
@@ -0,0 +1 @@
# @backstage/plugin-catalog-backend-module-bitbucket-server
@@ -0,0 +1,9 @@
# Catalog Backend Module for Bitbucket Server
This is an extension module to the catalog-backend plugin,
providing extensions targeted at Bitbucket Server offerings.
## Getting started
See [Backstage documentation](https://backstage.io/docs/integrations/bitbucketServer/discovery)
for details on how to install and configure the plugin.
@@ -0,0 +1,104 @@
## API Report File for "@backstage/plugin-catalog-backend-module-bitbucket-server"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { BitbucketServerIntegrationConfig } from '@backstage/integration';
import { Config } from '@backstage/config';
import { Entity } from '@backstage/catalog-model';
import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { TaskRunner } from '@backstage/backend-tasks';
// @public
export class BitbucketServerClient {
constructor(options: { config: BitbucketServerIntegrationConfig });
// (undocumented)
static fromConfig(options: {
config: BitbucketServerIntegrationConfig;
}): BitbucketServerClient;
// (undocumented)
getRepository(options: {
projectKey: string;
repo: string;
}): Promise<BitbucketServerRepository>;
// (undocumented)
listProjects(options: {
listOptions?: BitbucketServerListOptions;
}): Promise<BitbucketServerPagedResponse<BitbucketServerProject>>;
// (undocumented)
listRepositories(options: {
projectKey: string;
listOptions?: BitbucketServerListOptions;
}): Promise<BitbucketServerPagedResponse<BitbucketServerRepository>>;
// (undocumented)
resolvePath(options: { projectKey: string; repo: string; path: string }): {
path: string;
};
}
// @public
export class BitbucketServerEntityProvider implements EntityProvider {
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
parser?: BitbucketServerLocationParser;
},
): BitbucketServerEntityProvider[];
// (undocumented)
getProviderName(): string;
// (undocumented)
refresh(logger: Logger): Promise<void>;
}
// @public (undocumented)
export type BitbucketServerListOptions = {
[key: string]: number | undefined;
limit?: number | undefined;
start?: number | undefined;
};
// @public
export type BitbucketServerLocationParser = (options: {
client: BitbucketServerClient;
location: LocationSpec;
logger: Logger;
}) => AsyncIterable<Entity>;
// @public (undocumented)
export type BitbucketServerPagedResponse<T> = {
size: number;
limit: number;
start: number;
isLastPage: boolean;
values: T[];
nextPageStart: number;
};
// @public (undocumented)
export type BitbucketServerProject = {
key: string;
};
// @public (undocumented)
export type BitbucketServerRepository = {
project: {
key: string;
};
slug: string;
description: string;
links: Record<
string,
{
href: string;
}[]
>;
};
```
@@ -0,0 +1,80 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface Config {
catalog?: {
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* BitbucketServerEntityProvider configuration
*
* Uses "default" as default id for the single config variant.
*/
bitbucketServer?:
| {
/**
* (Optional) Path to the catalog file. Default to "/catalog-info.yaml".
* @visibility frontend
*/
catalogPath?: string;
/**
* (Optional) Filters applied to discovered catalog files in repositories.
* @visibility frontend
*/
filters?: {
/**
* (Optional) Filter for the repository slug.
* @visibility frontend
*/
repoSlug?: RegExp;
/**
* (Optional) Filter for the project key.
* @visibility frontend
*/
projectKey?: RegExp;
};
}
| Record<
string,
{
/**
* (Optional) Path to the catalog file. Default to "/catalog-info.yaml".
* @visibility frontend
*/
catalogPath?: string;
/**
* (Optional) Filters applied to discovered catalog files in repositories.
* @visibility frontend
*/
filters?: {
/**
* (Optional) Filter for the repository slug.
* @visibility frontend
*/
repoSlug?: RegExp;
/**
* (Optional) Filter for the project key.
* @visibility frontend
*/
projectKey?: RegExp;
};
}
>;
};
};
}
@@ -0,0 +1,56 @@
{
"name": "@backstage/plugin-catalog-backend-module-bitbucket-server",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin-module"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/catalog-backend-module-bitbucket-server"
},
"keywords": [
"backstage"
],
"scripts": {
"start": "backstage-cli package start",
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"clean": "backstage-cli package clean",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/backend-common": "^0.15.0-next.0",
"@backstage/backend-tasks": "^0.3.4-next.0",
"@backstage/catalog-model": "^1.0.1",
"@backstage/config": "^1.0.0",
"@backstage/errors": "^1.0.0",
"@backstage/integration": "^1.3.0-next.0",
"@backstage/plugin-catalog-backend": "^1.3.1-next.0",
"cross-fetch": "^3.1.5",
"uuid": "^8.0.0",
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.27-next.0",
"@backstage/cli": "^0.18.1-next.0",
"msw": "^0.35.0"
},
"files": [
"dist",
"config.d.ts"
],
"configSchema": "config.d.ts"
}
@@ -0,0 +1,31 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A Backstage catalog backend module that helps integrate towards Bitbucket Server
*
* @packageDocumentation
*/
export { BitbucketServerClient } from './lib';
export type {
BitbucketServerProject,
BitbucketServerRepository,
BitbucketServerPagedResponse,
BitbucketServerListOptions,
} from './lib';
export { BitbucketServerEntityProvider } from './providers';
export type { BitbucketServerLocationParser } from './providers';
@@ -0,0 +1,182 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { BitbucketServerIntegrationConfig } from '@backstage/integration';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import {
BitbucketServerClient,
BitbucketServerPagedResponse,
paginated,
} from './BitbucketServerClient';
import { BitbucketServerProject, BitbucketServerRepository } from './types';
const server = setupServer();
describe('BitbucketServerClient', () => {
const config: BitbucketServerIntegrationConfig = {
host: 'bitbucket.mycompany.com',
apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0',
username: 'test-user',
password: 'test-pw',
};
const client = BitbucketServerClient.fromConfig({
config: config,
});
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
afterAll(() => server.close());
afterEach(() => server.resetHandlers());
it('listProjects', async () => {
server.use(
rest.get(`${config.apiBaseUrl}/projects`, (req, res, ctx) => {
if (
req.headers.get('authorization') !== 'Basic dGVzdC11c2VyOnRlc3QtcHc='
) {
return res(ctx.status(400));
}
const response: BitbucketServerPagedResponse<BitbucketServerProject> = {
size: 1,
limit: 25,
isLastPage: true,
start: 0,
nextPageStart: 0,
values: [
{
key: 'test-project',
},
],
};
return res(ctx.json(response));
}),
);
const projects = paginated(options =>
client.listProjects({ listOptions: options }),
);
const results = [];
for await (const project of projects) {
results.push(project);
}
expect(results).toHaveLength(1);
expect(results[0].key).toEqual('test-project');
});
it('listRepositories', async () => {
server.use(
rest.get(
`${config.apiBaseUrl}/projects/test-project/repos`,
(req, res, ctx) => {
if (
req.headers.get('authorization') !==
'Basic dGVzdC11c2VyOnRlc3QtcHc='
) {
return res(ctx.status(400));
}
const response: BitbucketServerPagedResponse<BitbucketServerRepository> =
{
size: 1,
limit: 25,
isLastPage: true,
start: 0,
nextPageStart: 0,
values: [
{
project: {
key: 'test-project',
},
slug: 'test-repository',
description: 'A test repository.',
links: {
self: [
{
href: 'https://bitbucket.mycompany.com/projects/test-project',
},
],
},
},
],
};
return res(ctx.json(response));
},
),
);
const repos = paginated(options =>
client.listRepositories({
projectKey: 'test-project',
listOptions: options,
}),
);
const results = [];
for await (const repo of repos) {
results.push(repo);
}
expect(results).toHaveLength(1);
expect(results[0].project.key).toEqual('test-project');
expect(results[0].slug).toEqual('test-repository');
expect(results[0].links.self[0].href).toEqual(
'https://bitbucket.mycompany.com/projects/test-project',
);
});
it('getRepository', async () => {
server.use(
rest.get(
`${config.apiBaseUrl}/projects/test-project/repos/test-repo`,
(req, res, ctx) => {
if (
req.headers.get('authorization') !==
'Basic dGVzdC11c2VyOnRlc3QtcHc='
) {
return res(ctx.status(400));
}
const response: BitbucketServerRepository = {
project: {
key: 'test-project',
},
slug: 'test-repo',
description: 'A test repository.',
links: {
self: [
{
href: 'https://bitbucket.mycompany.com/projects/test-project',
},
],
},
};
return res(ctx.json(response));
},
),
);
const repo = await client.getRepository({
projectKey: 'test-project',
repo: 'test-repo',
});
expect(repo.project.key).toEqual('test-project');
expect(repo.slug).toEqual('test-repo');
expect(repo.links.self[0].href).toEqual(
'https://bitbucket.mycompany.com/projects/test-project',
);
});
});
@@ -0,0 +1,158 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { fetch, Request } from 'cross-fetch';
import {
BitbucketServerIntegrationConfig,
getBitbucketServerRequestOptions,
} from '@backstage/integration';
import { BitbucketServerProject, BitbucketServerRepository } from './types';
/**
* A client for interacting with a Bitbucket Server instance
*
* @public
*/
export class BitbucketServerClient {
private readonly config: BitbucketServerIntegrationConfig;
static fromConfig(options: {
config: BitbucketServerIntegrationConfig;
}): BitbucketServerClient {
return new BitbucketServerClient(options);
}
constructor(options: { config: BitbucketServerIntegrationConfig }) {
this.config = options.config;
}
async listProjects(options: {
listOptions?: BitbucketServerListOptions;
}): Promise<BitbucketServerPagedResponse<BitbucketServerProject>> {
return this.pagedRequest(
`${this.config.apiBaseUrl}/projects`,
options.listOptions,
);
}
async listRepositories(options: {
projectKey: string;
listOptions?: BitbucketServerListOptions;
}): Promise<BitbucketServerPagedResponse<BitbucketServerRepository>> {
return this.pagedRequest(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
options.projectKey,
)}/repos`,
options.listOptions,
);
}
async getRepository(options: {
projectKey: string;
repo: string;
}): Promise<BitbucketServerRepository> {
const request = `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}`;
const response = await fetch(
request,
getBitbucketServerRequestOptions(this.config),
);
return response.json();
}
resolvePath(options: { projectKey: string; repo: string; path: string }): {
path: string;
} {
const base = new URL(this.config.apiBaseUrl || '');
return {
path: `${base.protocol}//${base.host}/projects/${options.projectKey}/repos/${options.repo}${options.path}`,
};
}
private async pagedRequest(
endpoint: string,
options?: BitbucketServerListOptions,
): Promise<BitbucketServerPagedResponse<any>> {
const request = new URL(endpoint);
for (const key in options) {
if (options[key]) {
request.searchParams.append(key, options[key]!.toString());
}
}
return this.getTypeMapped(request);
}
private async getTypeMapped<T = any>(url: URL): Promise<T> {
return this.get(url).then((response: Response) => {
return response.json() as Promise<T>;
});
}
private async get(url: URL): Promise<Response> {
return this.request(new Request(url.toString(), { method: 'GET' }));
}
private async request(req: Request): Promise<Response> {
return fetch(req, getBitbucketServerRequestOptions(this.config)).then(
(response: Response) => {
if (!response.ok) {
throw new Error(
`Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`,
);
}
return response;
},
);
}
}
/**
* @public
*/
export type BitbucketServerListOptions = {
[key: string]: number | undefined;
limit?: number | undefined;
start?: number | undefined;
};
/**
* @public
*/
export type BitbucketServerPagedResponse<T> = {
size: number;
limit: number;
start: number;
isLastPage: boolean;
values: T[];
nextPageStart: number;
};
export async function* paginated(
request: (
options: BitbucketServerListOptions,
) => Promise<BitbucketServerPagedResponse<any>>,
options?: BitbucketServerListOptions,
) {
const opts = options || { start: 0 };
let res;
do {
res = await request(opts);
opts.start = res.nextPageStart;
for (const item of res.values) {
yield item;
}
} while (!res.isLastPage);
}
@@ -0,0 +1,25 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { BitbucketServerClient, paginated } from './BitbucketServerClient';
export type {
BitbucketServerPagedResponse,
BitbucketServerListOptions,
} from './BitbucketServerClient';
export type {
BitbucketServerRepository,
BitbucketServerProject,
} from './types';
@@ -0,0 +1,35 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** @public */
export type BitbucketServerRepository = {
project: {
key: string;
};
slug: string;
description: string;
links: Record<
string,
{
href: string;
}[]
>;
};
/** @public */
export type BitbucketServerProject = {
key: string;
};
@@ -0,0 +1,368 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { BitbucketServerPagedResponse } from '../lib/BitbucketServerClient';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
getTasks() {
return this.tasks;
}
run(task: TaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
type Project = {
key: string;
repos: [string];
};
function pagedResponse(values: any): BitbucketServerPagedResponse<any> {
return {
values: values,
isLastPage: true,
} as BitbucketServerPagedResponse<any>;
}
const logger = getVoidLogger();
const server = setupServer();
function setupStubs(projects: Project[], baseUrl: string) {
// Stub projects
server.use(
rest.get(`${baseUrl}/rest/api/1.0/projects`, (_, res, ctx) => {
return res(
ctx.json(
pagedResponse(
projects.map(p => {
return { key: p.key };
}),
),
),
);
}),
);
for (const project of projects) {
// Stub list repositories
server.use(
rest.get(
`${baseUrl}/rest/api/1.0/projects/${project.key}/repos`,
(_, res, ctx) => {
const response = [];
for (const repo of project.repos) {
response.push({
slug: repo,
links: {
self: [
{
href: `${baseUrl}/projects/${project.key}/repos/${repo}/browse`,
},
],
},
});
}
return res(ctx.json(pagedResponse(response)));
},
),
);
}
}
describe('BitbucketServerEntityProvider', () => {
setupRequestMockHandlers(server);
afterEach(() => jest.resetAllMocks());
it('no provider config', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({});
const providers = BitbucketServerEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(0);
});
it('rejects no matching integration', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: 'bitbucket.mycompany.com',
},
},
},
},
});
expect(() =>
BitbucketServerEntityProvider.fromConfig(config, { logger, schedule }),
).toThrow(/bitbucket\.mycompany\.com/);
});
it('single simple provider config', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
host: 'bitbucket.mycompany.com',
},
},
},
integrations: {
bitbucketServer: [
{
host: 'bitbucket.mycompany.com',
},
],
},
});
const providers = BitbucketServerEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual(
'bitbucketServer-provider:default',
);
});
it('multiple provider configs', () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
integrations: {
bitbucketServer: [
{
host: 'bitbucket.mycompany.com',
},
],
},
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: 'bitbucket.mycompany.com',
},
secondary: {
host: 'bitbucket.mycompany.com',
},
},
},
},
});
const providers = BitbucketServerEntityProvider.fromConfig(config, {
logger,
schedule,
});
expect(providers).toHaveLength(2);
expect(providers[0].getProviderName()).toEqual(
'bitbucketServer-provider:mainProvider',
);
expect(providers[1].getProviderName()).toEqual(
'bitbucketServer-provider:secondary',
);
});
it('apply full update on scheduled execution with filters', async () => {
const host = 'bitbucket.mycompany.com';
const config = new ConfigReader({
integrations: {
bitbucketServer: [
{
host: host,
},
],
},
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: host,
filters: {
projectKey: 'project-.*',
repoSlug: 'repo-.*',
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = BitbucketServerEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
expect(provider.getProviderName()).toEqual(
'bitbucketServer-provider:mainProvider',
);
setupStubs(
[
{ key: 'project-test', repos: ['repo-test'] },
{ key: 'other-project', repos: ['other-repo'] },
],
`https://${host}`,
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'bitbucketServer-provider:mainProvider',
},
];
expect(entityProviderConnection.applyMutation).toBeCalledTimes(1);
expect(entityProviderConnection.applyMutation).toBeCalledWith({
type: 'full',
entities: expectedEntities,
});
});
it('apply full update on scheduled execution without filters', async () => {
const host = 'bitbucket.mycompany.com';
const config = new ConfigReader({
integrations: {
bitbucketServer: [
{
host: host,
},
],
},
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: host,
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = BitbucketServerEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
expect(provider.getProviderName()).toEqual(
'bitbucketServer-provider:mainProvider',
);
setupStubs(
[
{ key: 'project-test', repos: ['repo-test'] },
{ key: 'other-project', repos: ['other-repo'] },
],
`https://${host}`,
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
'backstage.io/managed-by-origin-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
},
name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6',
},
spec: {
presence: 'optional',
target: `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
type: 'url',
},
},
locationKey: 'bitbucketServer-provider:mainProvider',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
},
name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f',
},
spec: {
presence: 'optional',
target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
type: 'url',
},
},
locationKey: 'bitbucketServer-provider:mainProvider',
},
];
expect(entityProviderConnection.applyMutation).toBeCalledTimes(1);
expect(entityProviderConnection.applyMutation).toBeCalledWith({
type: 'full',
entities: expectedEntities,
});
});
});
@@ -0,0 +1,200 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { TaskRunner } from '@backstage/backend-tasks';
import * as uuid from 'uuid';
import {
BitbucketServerLocationParser,
defaultBitbucketServerLocationParser,
} from './BitbucketServerLocationParser';
import {
BitbucketServerIntegration,
ScmIntegrations,
} from '@backstage/integration';
import { BitbucketServerClient, paginated } from '../lib';
import { InputError } from '@backstage/errors';
import {
BitbucketServerEntityProviderConfig,
readProviderConfigs,
} from './BitbucketServerEntityProviderConfig';
import { Entity } from '@backstage/catalog-model';
/**
* Discovers catalog files located in Bitbucket Server.
* The provider will search your Bitbucket Server instance and register catalog files matching the configured path
* as Location entity and via following processing steps add all contained catalog entities.
* This can be useful as an alternative to static locations or manually adding things to the catalog.
*
* @public
*/
export class BitbucketServerEntityProvider implements EntityProvider {
private readonly integration: BitbucketServerIntegration;
private readonly config: BitbucketServerEntityProviderConfig;
private readonly parser: BitbucketServerLocationParser;
private readonly logger: Logger;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
static fromConfig(
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
parser?: BitbucketServerLocationParser;
},
): BitbucketServerEntityProvider[] {
const integrations = ScmIntegrations.fromConfig(config);
return readProviderConfigs(config).map(providerConfig => {
const integration = integrations.bitbucketServer.byHost(
providerConfig.host,
);
if (!integration) {
throw new InputError(
`No BitbucketServer integration found that matches host ${providerConfig.host}`,
);
}
return new BitbucketServerEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
options.parser,
);
});
}
private constructor(
config: BitbucketServerEntityProviderConfig,
integration: BitbucketServerIntegration,
logger: Logger,
schedule: TaskRunner,
parser?: BitbucketServerLocationParser,
) {
this.integration = integration;
this.config = config;
this.parser = parser || defaultBitbucketServerLocationParser;
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
class: BitbucketServerEntityProvider.prototype.constructor.name,
taskId,
taskInstanceId: uuid.v4(),
});
try {
await this.refresh(logger);
} catch (error) {
logger.error(error);
}
},
});
};
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
getProviderName(): string {
return `bitbucketServer-provider:${this.config.id}`;
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
await this.scheduleFn();
}
async refresh(logger: Logger) {
if (!this.connection) {
throw new Error('Not initialized');
}
logger.info('Discovering catalog files in Bitbucket Server repositories');
const entities = await this.findEntities();
await this.connection.applyMutation({
type: 'full',
entities: entities.map(entity => ({
locationKey: this.getProviderName(),
entity: entity,
})),
});
logger.info(
`Committed ${entities.length} entities for Bitbucket Server repositories`,
);
}
private async findEntities(): Promise<Entity[]> {
const client = BitbucketServerClient.fromConfig({
config: this.integration.config,
});
const projects = paginated(options =>
client.listProjects({ listOptions: options }),
);
const result: Entity[] = [];
for await (const project of projects) {
if (
this.config?.filters?.projectKey &&
!this.config.filters.projectKey.test(project.key)
) {
continue;
}
const repositories = paginated(options =>
client.listRepositories({
projectKey: project.key,
listOptions: options,
}),
);
for await (const repository of repositories) {
if (
this.config?.filters?.repoSlug &&
!this.config.filters.repoSlug.test(repository.slug)
) {
continue;
}
for await (const entity of this.parser({
client,
logger: this.logger,
location: {
type: 'url',
target: `${repository.links.self[0].href}${this.config.catalogPath}`,
presence: 'optional',
},
})) {
result.push(entity);
}
}
}
return result;
}
}
@@ -0,0 +1,122 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readProviderConfigs } from './BitbucketServerEntityProviderConfig';
describe('readProviderConfigs', () => {
afterEach(() => jest.resetAllMocks());
it('no provider config', () => {
const config = new ConfigReader({});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(0);
});
it('single simple provider config', () => {
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
host: 'bitbucket.mycompany.com',
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(1);
expect(providerConfigs[0]).toEqual({
id: 'default',
catalogPath: '/catalog-info.yaml',
host: 'bitbucket.mycompany.com',
filters: {
projectKey: undefined,
repoSlug: undefined,
},
});
});
it('multiple provider configs', () => {
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: 'bitbucket.mycompany.com',
},
secondaryProvider: {
host: 'bitbucket2.mycompany.com',
catalogPath: 'custom/path/catalog-info.yaml',
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(2);
expect(providerConfigs[0]).toEqual({
id: 'mainProvider',
catalogPath: '/catalog-info.yaml',
host: 'bitbucket.mycompany.com',
filters: {
projectKey: undefined,
repoSlug: undefined,
},
});
expect(providerConfigs[1]).toEqual({
id: 'secondaryProvider',
catalogPath: 'custom/path/catalog-info.yaml',
host: 'bitbucket2.mycompany.com',
filters: {
projectKey: undefined,
repoSlug: undefined,
},
});
});
it('single provider config with filters', () => {
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: 'bitbucket.mycompany.com',
filters: {
projectKey: 'project1',
repoSlug: '.*',
},
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(1);
expect(providerConfigs[0]).toEqual({
id: 'mainProvider',
catalogPath: '/catalog-info.yaml',
host: 'bitbucket.mycompany.com',
filters: {
projectKey: /project1/,
repoSlug: /.*/,
},
});
});
});
@@ -0,0 +1,72 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
const DEFAULT_PROVIDER_ID = 'default';
export type BitbucketServerEntityProviderConfig = {
id: string;
host: string;
catalogPath: string;
filters?: {
projectKey?: RegExp;
repoSlug?: RegExp;
};
};
export function readProviderConfigs(
config: Config,
): BitbucketServerEntityProviderConfig[] {
const providersConfig = config.getOptionalConfig(
'catalog.providers.bitbucketServer',
);
if (!providersConfig) {
return [];
}
if (providersConfig.has('host')) {
// simple/single config variant
return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
}
return providersConfig.keys().map(id => {
const providerConfig = providersConfig.getConfig(id);
return readProviderConfig(id, providerConfig);
});
}
function readProviderConfig(
id: string,
config: Config,
): BitbucketServerEntityProviderConfig {
const host = config.getString('host');
const catalogPath =
config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
const projectKeyPattern = config.getOptionalString('filters.projectKey');
const repoSlugPattern = config.getOptionalString('filters.repoSlug');
return {
id,
host,
catalogPath,
filters: {
projectKey: projectKeyPattern ? new RegExp(projectKeyPattern) : undefined,
repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined,
},
};
}
@@ -0,0 +1,42 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import { Entity } from '@backstage/catalog-model';
import { Logger } from 'winston';
import { BitbucketServerClient } from '../lib';
/**
* A custom callback that reacts to finding a location by yielding entities.
* Can be used for custom location/repository parsing logic.
*
* @public
*/
export type BitbucketServerLocationParser = (options: {
client: BitbucketServerClient;
location: LocationSpec;
logger: Logger;
}) => AsyncIterable<Entity>;
export const defaultBitbucketServerLocationParser =
async function* defaultBitbucketServerLocationParser(options: {
location: LocationSpec;
}) {
yield locationSpecToLocationEntity({ location: options.location });
};
@@ -0,0 +1,18 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider';
export type { BitbucketServerLocationParser } from './BitbucketServerLocationParser';
@@ -0,0 +1,17 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};