diff --git a/.changeset/fluffy-cherries-own.md b/.changeset/fluffy-cherries-own.md new file mode 100644 index 0000000000..93d7272b14 --- /dev/null +++ b/.changeset/fluffy-cherries-own.md @@ -0,0 +1,58 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': minor +--- + +Add new plugin `catalog-backend-module-bitbucket-cloud` with `BitbucketCloudEntityProvider`. + +This entity provider is an alternative/replacement to the `BitbucketDiscoveryProcessor` **_(for Bitbucket Cloud only!)_**. +It replaces use cases using `search=true` and should be powerful enough as a complete replacement. + +If any feature for Bitbucket Cloud is missing and preventing you from switching, please raise an issue. + +**Before:** + +```typescript +// packages/backend/src/plugins/catalog.ts + +builder.addProcessor( + BitbucketDiscoveryProcessor.fromConfig(env.config, { logger: env.logger }), +); +``` + +```yaml +# app-config.yaml + +catalog: + locations: + - type: bitbucket-discovery + target: 'https://bitbucket.org/workspaces/workspace-name/projects/apis-*/repos/service-*?search=true&catalogPath=/catalog-info.yaml' +``` + +**After:** + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addEntityProvider( + BitbucketCloudEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + }), +); +``` + +```yaml +# app-config.yaml + +catalog: + providers: + bitbucketCloud: + yourProviderId: # identifies your ingested dataset + catalogPath: /catalog-info.yaml # default value + filters: # optional + projectKey: '^apis-.*$' # optional; RegExp + repoSlug: '^service-.*$' # optional; RegExp + workspace: workspace-name +``` diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucket/locations.md index c0c6afc7a3..878030a29c 100644 --- a/docs/integrations/bitbucket/locations.md +++ b/docs/integrations/bitbucket/locations.md @@ -15,25 +15,7 @@ plugin. ## Bitbucket Cloud -```yaml -integrations: - bitbucketCloud: - - username: ${BITBUCKET_CLOUD_USERNAME} - appPassword: ${BITBUCKET_CLOUD_PASSWORD} -``` - -> Note: A public Bitbucket Cloud provider is added automatically at startup for -> convenience, so you only need to list it if you want to supply credentials. - -Directly under the `bitbucketCloud` key is a list of provider configurations, where -you can list the Bitbucket Cloud providers you want to fetch data from. -In the case of Bitbucket Cloud, you will have up to one entry. - -This one entry will have the following elements: - -- `username`: The Bitbucket Cloud username to use in API requests. If - neither a username nor token are supplied, anonymous access will be used. -- `appPassword`: The app password for the Bitbucket Cloud user. +Please see [the Bitbucket Cloud documentation](../bitbucketCloud/locations.md). ## Bitbucket Server diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md new file mode 100644 index 0000000000..6609707868 --- /dev/null +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -0,0 +1,95 @@ +--- +id: discovery +title: Bitbucket Cloud Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in Bitbucket Cloud +--- + +The Bitbucket Cloud integration has a special entity provider for discovering +catalog files located in [Bitbucket Cloud](https://bitbucket.org). +The provider will search your Bitbucket Cloud account and register catalog files matching the configured path +as Location entity and via following processing steps add all contained catalog entities. +This can be useful as an alternative to static locations or manually adding things to the catalog. + +## Installation + +You will have to add the entity provider in the catalog initialization code of your +backend. The provider is not installed by default, therefore you have to add a +dependency to `@backstage/plugin-catalog-backend-module-bitbucket-cloud` to your backend +package. + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-bitbucket-cloud +``` + +And then add the entity provider to your catalog builder: + +```diff + // In packages/backend/src/plugins/catalog.ts ++ import { BitbucketCloudEntityProvider } from '@backstage/plugin-catalog-backend-module-bitbucket-cloud'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ builder.addEntityProvider( ++ BitbucketCloudEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ }), ++ ); + + // [...] + } +``` + +## Configuration + +To use the entity provider, you'll need a [Bitbucket Cloud integration set up](locations.md). +Very likely a `username` and `appPassword` will be required +(you are restricted to public repositories and a very low rate limit otherwise). + +Additionally, you need to configure your entity provider instance(s): + +```yaml +# app-config.yaml + +catalog: + providers: + bitbucketCloud: + yourProviderId: # identifies your ingested dataset + catalogPath: /catalog-info.yaml # default value + filters: # optional + projectKey: '^apis-.*$' # optional; RegExp + repoSlug: '^service-.*$' # optional; RegExp + workspace: workspace-name +``` + +> **Note:** It is possible but certainly not recommended to skip the provider ID level. +> If you do so, `default` will be used as provider ID. + +- **catalogPath** _(optional)_: + Default: `/catalog-info.yaml`. + Path where to look for `catalog-info.yaml` files. + When started with `/`, it is an absolute path from the repo root. + It supports values as allowed by the `path` filter/modifier + [at Bitbucket Cloud's code search](https://confluence.atlassian.com/bitbucket/code-search-in-bitbucket-873876782.html#Search-Pathmodifier). +- **filters** _(optional)_: + - **projectKey** _(optional)_: + Regular expression used to filter results based on the project key. + - **repoSlug** _(optional)_: + Regular expression used to filter results based on the repo slug. +- **workspace**: + Name of your organization account/workspace. + If you want to add multiple workspaces, you need to add one provider config each. + +## Alternative + +_Deprecated!_ Please raise issues for use cases not covered by the entity provider. + +[You can use the `BitbucketDiscoveryProcessor`.](../bitbucket/discovery.md#bitbucket-cloud) diff --git a/docs/integrations/bitbucketCloud/locations.md b/docs/integrations/bitbucketCloud/locations.md new file mode 100644 index 0000000000..b886a49346 --- /dev/null +++ b/docs/integrations/bitbucketCloud/locations.md @@ -0,0 +1,36 @@ +--- +id: locations +title: Bitbucket Cloud Locations +sidebar_label: Locations +# prettier-ignore +description: Integrating source code stored in Bitbucket Cloud into the Backstage catalog +--- + +The Bitbucket Cloud integration supports loading catalog entities from [bitbucket.org](https://bitbucket.org). +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +```yaml +integrations: + bitbucketCloud: + - username: ${BITBUCKET_CLOUD_USERNAME} + appPassword: ${BITBUCKET_CLOUD_PASSWORD} +``` + +> Note: A public Bitbucket Cloud provider is added automatically at startup for +> convenience, so you only need to list it if you want to supply credentials. + +Directly under the `bitbucketCloud` key is a list of provider configurations, where +you can list the Bitbucket Cloud providers you want to fetch data from. +In the case of Bitbucket Cloud, you will have up to one entry. + +This one entry will have the following elements: + +- `username`: The Bitbucket Cloud username to use in API requests. If + neither a username nor token are supplied, anonymous access will be used. +- `appPassword`: The app password for the Bitbucket Cloud user. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 7bbe67f58f..d285800c58 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -149,6 +149,14 @@ "integrations/bitbucket/discovery" ] }, + { + "type": "subcategory", + "label": "Bitbucket Cloud", + "ids": [ + "integrations/bitbucketCloud/locations", + "integrations/bitbucketCloud/discovery" + ] + }, { "type": "subcategory", "label": "Datadog", diff --git a/mkdocs.yml b/mkdocs.yml index caaee37f29..7200471326 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -97,6 +97,9 @@ nav: - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' - Discovery: 'integrations/bitbucket/discovery.md' + - Bitbucket Cloud: + - Locations: 'integrations/bitbucketCloud/locations.md' + - Discovery: 'integrations/bitbucketCloud/discovery.md' - Datadog: - Installation: 'integrations/datadog-rum/installation.md' - Gerrit: diff --git a/plugins/catalog-backend-module-bitbucket-cloud/.eslintrc.js b/plugins/catalog-backend-module-bitbucket-cloud/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/CHANGELOG.md b/plugins/catalog-backend-module-bitbucket-cloud/CHANGELOG.md new file mode 100644 index 0000000000..d9c2148dcf --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/CHANGELOG.md @@ -0,0 +1 @@ +# @backstage/plugin-catalog-backend-module-bitbucket-cloud diff --git a/plugins/catalog-backend-module-bitbucket-cloud/README.md b/plugins/catalog-backend-module-bitbucket-cloud/README.md new file mode 100644 index 0000000000..11f40776ca --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/README.md @@ -0,0 +1,9 @@ +# Catalog Backend Module for Bitbucket Cloud + +This is an extension module to the catalog-backend plugin, +providing extensions targeted at Bitbucket Cloud offerings. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/bitbucketCloud/discovery) +for details on how to install and configure the plugin. diff --git a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md new file mode 100644 index 0000000000..b886cc224f --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md @@ -0,0 +1,31 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-bitbucket-cloud" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { TaskRunner } from '@backstage/backend-tasks'; + +// @public +export class BitbucketCloudEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule: TaskRunner; + }, + ): BitbucketCloudEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + getTaskId(): string; + // (undocumented) + refresh(logger: Logger): Promise; +} +``` diff --git a/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts new file mode 100644 index 0000000000..29289bb70e --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/config.d.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + catalog?: { + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * BitbucketCloudEntityProvider configuration + * + * Uses "default" as default id for the single config variant. + */ + bitbucketCloud?: + | { + /** + * (Optional) Path to the catalog file. Default to "/catalog-info.yaml". + * @visibility frontend + */ + catalogPath?: string; + /** + * (Required) Your workspace. + * @visibility frontend + */ + workspace: string; + /** + * (Optional) Filters applied to discovered catalog files in repositories. + * @visibility frontend + */ + filters?: { + /** + * (Optional) Filter for the repository slug. + * @visibility frontend + */ + repoSlug?: RegExp; + /** + * (Optional) Filter for the project key. + * @visibility frontend + */ + projectKey?: RegExp; + }; + } + | Record< + string, + { + /** + * (Optional) Path to the catalog file. Default to "/catalog-info.yaml". + * @visibility frontend + */ + catalogPath?: string; + /** + * (Required) Your workspace. + * @visibility frontend + */ + workspace: string; + /** + * (Optional) Filters applied to discovered catalog files in repositories. + * @visibility frontend + */ + filters?: { + /** + * (Optional) Filter for the repository slug. + * @visibility frontend + */ + repoSlug?: RegExp; + /** + * (Optional) Filter for the project key. + * @visibility frontend + */ + projectKey?: RegExp; + }; + } + >; + }; + }; +} diff --git a/plugins/catalog-backend-module-bitbucket-cloud/package.json b/plugins/catalog-backend-module-bitbucket-cloud/package.json new file mode 100644 index 0000000000..8286e0cf00 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/package.json @@ -0,0 +1,55 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-bitbucket-cloud", + "description": "A Backstage catalog backend module that helps integrate towards Bitbucket Cloud", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-bitbucket-cloud" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean", + "start": "backstage-cli package start" + }, + "dependencies": { + "@backstage/backend-tasks": "^0.3.2-next.0", + "@backstage/config": "^1.0.1", + "@backstage/integration": "^1.2.1-next.0", + "@backstage/plugin-bitbucket-cloud-common": "^0.0.0", + "@backstage/plugin-catalog-backend": "^1.2.0-next.0", + "uuid": "^8.0.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-common": "^0.13.6-next.0", + "@backstage/backend-test-utils": "^0.1.25-next.0", + "@backstage/cli": "^0.17.2-next.0", + "msw": "^0.35.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts new file mode 100644 index 0000000000..efb1328643 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts @@ -0,0 +1,281 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { BitbucketCloudEntityProvider } from './BitbucketCloudEntityProvider'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +const server = setupServer(); + +describe('BitbucketCloudEntityProvider', () => { + setupRequestMockHandlers(server); + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = BitbucketCloudEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + }, + }, + }, + }); + const providers = BitbucketCloudEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'bitbucketCloud-provider:default', + ); + }); + + it('multiple provider configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + myProvider: { + workspace: 'test-ws1', + }, + anotherProvider: { + workspace: 'test-ws2', + }, + }, + }, + }, + }); + const providers = BitbucketCloudEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'bitbucketCloud-provider:myProvider', + ); + expect(providers[1].getProviderName()).toEqual( + 'bitbucketCloud-provider:anotherProvider', + ); + }); + + it('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + myProvider: { + workspace: 'test-ws', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + projectKey: 'test-.*', + repoSlug: 'test-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = BitbucketCloudEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketCloud-provider:myProvider', + ); + + server.use( + rest.get( + `https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`, + (_req, res, ctx) => { + const response = { + values: [ + { + // skipped as empty + path_matches: [], + file: { + type: 'commit_file', + path: 'path/to/ignored/file', + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + // skipped as no match with filter + slug: 'repo', + project: { + key: 'test-project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/repo', + }, + }, + }, + }, + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + slug: 'test-repo1', + project: { + // skipped as no match with filter + key: 'project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo1', + }, + }, + }, + }, + }, + }, + { + path_matches: [ + { + match: true, + text: 'catalog-custom.yaml', + }, + ], + file: { + type: 'commit_file', + path: 'custom/path/catalog-custom.yaml', + commit: { + repository: { + slug: 'test-repo2', + project: { + key: 'test-project', + }, + mainbranch: { + name: 'main', + }, + links: { + html: { + href: 'https://bitbucket.org/test-ws/test-repo2', + }, + }, + }, + }, + }, + }, + ], + }; + return res(ctx.json(response)); + }, + ), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketCloud-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://bitbucket.org/test-ws/test-repo2/src/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'bitbucketCloud-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts new file mode 100644 index 0000000000..74d642ed9b --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts @@ -0,0 +1,240 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { + BitbucketCloudIntegration, + ScmIntegrations, +} from '@backstage/integration'; +import { + BitbucketCloudClient, + Models, +} from '@backstage/plugin-bitbucket-cloud-common'; +import { + EntityProvider, + EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-backend'; +import { + BitbucketCloudEntityProviderConfig, + readProviderConfigs, +} from './BitbucketCloudEntityProviderConfig'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; + +const DEFAULT_BRANCH = 'master'; + +/** + * Discovers catalog files located in [Bitbucket Cloud](https://bitbucket.org). + * The provider will search your Bitbucket Cloud account and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * + * @public + */ +export class BitbucketCloudEntityProvider implements EntityProvider { + private readonly client: BitbucketCloudClient; + private readonly config: BitbucketCloudEntityProviderConfig; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule: TaskRunner; + }, + ): BitbucketCloudEntityProvider[] { + const integrations = ScmIntegrations.fromConfig(config); + const integration = integrations.bitbucketCloud.byHost('bitbucket.org'); + if (!integration) { + // this should never happen as we add a default integration, + // but as a general safeguard, e.g. if this approach gets changed + throw new Error('No integration for bitbucket.org available'); + } + + return readProviderConfigs(config).map( + providerConfig => + new BitbucketCloudEntityProvider( + providerConfig, + integration, + options.logger, + options.schedule, + ), + ); + } + + private constructor( + config: BitbucketCloudEntityProviderConfig, + integration: BitbucketCloudIntegration, + logger: Logger, + schedule: TaskRunner, + ) { + this.client = BitbucketCloudClient.fromConfig(integration.config); + this.config = config; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = this.getTaskId(); + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: BitbucketCloudEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `bitbucketCloud-provider:${this.config.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getTaskId} */ + getTaskId(): string { + return `${this.getProviderName()}:refresh`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + async refresh(logger: Logger) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering catalog files in Bitbucket Cloud repositories'); + + const targets = await this.findCatalogFiles(); + const entities = targets + .map(BitbucketCloudEntityProvider.toLocationSpec) + .map(location => locationSpecToLocationEntity({ location })) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + + await this.connection.applyMutation({ + type: 'full', + entities: entities, + }); + + logger.info( + `Committed ${entities.length} Locations for catalog files in Bitbucket Cloud repositories`, + ); + } + + private async findCatalogFiles(): Promise { + const workspace = this.config.workspace; + const catalogPath = this.config.catalogPath; + + const catalogFilename = catalogPath.substring( + catalogPath.lastIndexOf('/') + 1, + ); + + // load all fields relevant for creating refs later, but not more + const fields = [ + // exclude code/content match details + '-values.content_matches', + // include/add relevant repository details + '+values.file.commit.repository.mainbranch.name', + '+values.file.commit.repository.project.key', + '+values.file.commit.repository.slug', + // remove irrelevant links + '-values.*.links', + '-values.*.*.links', + '-values.*.*.*.links', + // ...except the one we need + '+values.file.commit.repository.links.html.href', + ].join(','); + const query = `"${catalogFilename}" path:${catalogPath}`; + const searchResults = this.client + .searchCode(workspace, query, { fields }) + .iterateResults(); + + const result: string[] = []; + + for await (const searchResult of searchResults) { + // not a file match, but a code match + if (searchResult.path_matches!.length === 0) { + continue; + } + + const repository = searchResult.file!.commit!.repository!; + if (this.matchesFilters(repository)) { + result.push( + BitbucketCloudEntityProvider.toUrl( + repository, + searchResult.file!.path!, + ), + ); + } + } + + return result; + } + + private matchesFilters(repository: Models.Repository): boolean { + const filters = this.config.filters; + return ( + !filters || + ((!filters.projectKey || + filters.projectKey.test(repository.project!.key!)) && + (!filters.repoSlug || filters.repoSlug.test(repository.slug!))) + ); + } + + private static toUrl( + repository: Models.Repository, + filePath: string, + ): string { + const repoUrl = repository.links!.html!.href; + const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH; + + return `${repoUrl}/src/${branch}/${filePath}`; + } + + private static toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'required', + }; + } +} diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts new file mode 100644 index 0000000000..a42bd2cde7 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './BitbucketCloudEntityProviderConfig'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + workspace: 'test-ws', + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('default'); + expect(providerConfigs[0].workspace).toEqual('test-ws'); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketCloud: { + providerWorkspaceOnly: { + workspace: 'test-ws1', + }, + providerCustomCatalogPath: { + workspace: 'test-ws2', + catalogPath: 'custom/path/catalog-info.yaml', + }, + providerWithProjectKeyFilter: { + workspace: 'test-ws3', + filters: { + projectKey: 'projectKey.*filter', + }, + }, + providerWithRepoSlugFilter: { + workspace: 'test-ws4', + filters: { + repoSlug: 'repoSlug.*filter', + }, + }, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(4); + expect(providerConfigs[0]).toEqual({ + id: 'providerWorkspaceOnly', + workspace: 'test-ws1', + catalogPath: '/catalog-info.yaml', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + }); + expect(providerConfigs[1]).toEqual({ + id: 'providerCustomCatalogPath', + workspace: 'test-ws2', + catalogPath: 'custom/path/catalog-info.yaml', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + }); + expect(providerConfigs[2]).toEqual({ + id: 'providerWithProjectKeyFilter', + workspace: 'test-ws3', + catalogPath: '/catalog-info.yaml', + filters: { + projectKey: /^projectKey.*filter$/, + repoSlug: undefined, + }, + }); + expect(providerConfigs[3]).toEqual({ + id: 'providerWithRepoSlugFilter', + workspace: 'test-ws4', + catalogPath: '/catalog-info.yaml', + filters: { + projectKey: undefined, + repoSlug: /^repoSlug.*filter$/, + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts new file mode 100644 index 0000000000..95e995f8c3 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProviderConfig.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; +const DEFAULT_PROVIDER_ID = 'default'; + +export type BitbucketCloudEntityProviderConfig = { + id: string; + catalogPath: string; + workspace: string; + filters?: { + projectKey?: RegExp; + repoSlug?: RegExp; + }; +}; + +export function readProviderConfigs( + config: Config, +): BitbucketCloudEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig( + 'catalog.providers.bitbucketCloud', + ); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('workspace')) { + // simple/single config variant + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + const providerConfig = providersConfig.getConfig(id); + + return readProviderConfig(id, providerConfig); + }); +} + +function readProviderConfig( + id: string, + config: Config, +): BitbucketCloudEntityProviderConfig { + const workspace = config.getString('workspace'); + const catalogPath = + config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; + const projectKeyPattern = config.getOptionalString('filters.projectKey'); + const repoSlugPattern = config.getOptionalString('filters.repoSlug'); + + return { + id, + catalogPath, + workspace, + filters: { + projectKey: projectKeyPattern + ? compileRegExp(projectKeyPattern) + : undefined, + repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined, + }, + }; +} + +/** + * Compiles a RegExp while enforcing the pattern to contain + * the start-of-line and end-of-line anchors. + * + * @param pattern + */ +function compileRegExp(pattern: string): RegExp { + let fullLinePattern = pattern; + if (!fullLinePattern.startsWith('^')) { + fullLinePattern = `^${fullLinePattern}`; + } + if (!fullLinePattern.endsWith('$')) { + fullLinePattern = `${fullLinePattern}$`; + } + + return new RegExp(fullLinePattern); +} diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts new file mode 100644 index 0000000000..1c15ad4e8f --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards Bitbucket Cloud + * + * @packageDocumentation + */ + +export { BitbucketCloudEntityProvider } from './BitbucketCloudEntityProvider'; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/setupTests.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/setupTests.ts new file mode 100644 index 0000000000..813cdeaae3 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {};