cli: add template for catalog provider

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

.changeset/better-steaks-act.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+The templates executed with the `yarn new` command now supports templating filenames.

.changeset/loud-carpets-throw.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Added a template for the `yarn new` command to create an catalog entity provider. To add this template to an explicit list in the root `package.json`, use `@backstage/cli/templates/catalog-provider-module`.

.changeset/short-sides-feel.md

@@ -0,0 +1,5 @@
+---
+'@backstage/create-app': patch
+---
+
+Added the new `@backstage/cli/templates/catalog-provider-module` template to the explicit template configuration for the `next-app` template.

docs/frontend-system/building-apps/08-migrating.md

@@ -979,6 +979,7 @@ When creating a new Backstage app with `create-app` and using the `--next` flag
           "@backstage/cli/templates/plugin-common-library",
           "@backstage/cli/templates/web-library",
           "@backstage/cli/templates/node-library",
+          "@backstage/cli/templates/catalog-provider-module",
           "@backstage/cli/templates/scaffolder-backend-module"
         ]
       }

docs/tooling/cli/04-templates.md

@@ -90,6 +90,7 @@ When defining the `templates` array it will override the default set of template
           "@backstage/cli/templates/plugin-common-library",
           "@backstage/cli/templates/web-library",
           "@backstage/cli/templates/node-library",
+          "@backstage/cli/templates/catalog-provider-module",
           "@backstage/cli/templates/scaffolder-backend-module"
         ]
       }

packages/cli/src/modules/new/lib/defaultTemplates.ts

@@ -23,5 +23,6 @@ export const defaultTemplates = [
   '@backstage/cli/templates/plugin-common-library',
   '@backstage/cli/templates/web-library',
   '@backstage/cli/templates/node-library',
+  '@backstage/cli/templates/catalog-provider-module',
   '@backstage/cli/templates/scaffolder-backend-module',
 ];

packages/cli/src/modules/new/lib/execution/writeTemplateContents.ts

@@ -22,6 +22,7 @@ import { PortableTemplate, PortableTemplateInput } from '../types';
 import { ForwardedError, InputError } from '@backstage/errors';
 import { isMonoRepo as getIsMonoRepo } from '@backstage/cli-node';
 import { PortableTemplater } from './PortableTemplater';
+import { isChildPath } from '@backstage/cli-common';
 
 export async function writeTemplateContents(
   template: PortableTemplate,
@@ -63,7 +64,12 @@ export async function writeTemplateContents(
     }
 
     for (const file of template.files) {
-      const destPath = resolvePath(targetDir, file.path);
+      const destPath = resolvePath(targetDir, templater.template(file.path));
+      if (!isChildPath(targetDir, destPath)) {
+        throw new Error(
+          `Path ${destPath} is outside of target directory ${targetDir}`,
+        );
+      }
       await fs.ensureDir(dirname(destPath));
 
       let content =

packages/cli/templates/catalog-provider-module/.eslintrc.js.hbs

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

packages/cli/templates/catalog-provider-module/README.md.hbs

@@ -0,0 +1,5 @@
+# {{packageName}}
+
+The {{fullModuleId}} module for [@backstage/plugin-catalog-backend](https://www.npmjs.com/package/@backstage/plugin-catalog-backend).
+
+_This plugin was created through the Backstage CLI_

packages/cli/templates/catalog-provider-module/config.d.ts.hbs

@@ -0,0 +1,34 @@
+import { SchedulerServiceTaskScheduleDefinitionConfig } from '@backstage/backend-plugin-api';
+
+export interface Config {
+  catalog?: {
+    providers?: {
+      /**
+       * {{providerClass}} configuration.
+       */
+      {{providerVar}}?:
+        | {
+            /**
+             * The target that this provider should consume.
+             */
+            target: string;
+            /**
+             * Overrides the schedule at which this provider runs.
+             */
+            schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
+          }
+        | {
+            [name: string]: {
+              /**
+               * The target that this provider should consume.
+               */
+              target: string;
+              /**
+               * Overrides the schedule at which this provider runs.
+               */
+              schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
+            };
+          };
+    };
+  };
+}

packages/cli/templates/catalog-provider-module/package.json.hbs

@@ -0,0 +1,36 @@
+{
+  "name": "{{packageName}}",
+  "description": "The {{fullModuleId}} module for @backstage/plugin-catalog-backend",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module",
+    "pluginId": "catalog",
+    "pluginPackage": "@backstage/plugin-catalog-backend"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-plugin-api": "{{versionQuery '@backstage/backend-plugin-api'}}",
+    "@backstage/plugin-catalog-node": "{{versionQuery '@backstage/plugin-catalog-node'}}"
+  },
+  "devDependencies": {
+    "@backstage/cli": "{{versionQuery '@backstage/cli'}}",
+    "@backstage/backend-test-utils": "{{versionQuery '@backstage/backend-test-utils'}}"
+  },
+  "files": [
+    "dist"
+  ]
+}

packages/cli/templates/catalog-provider-module/portable-template.yaml

@@ -0,0 +1,9 @@
+name: catalog-provider-module
+role: backend-plugin-module
+description: An Entity Provider module for the Software Catalog
+values:
+  pluginId: catalog
+  fullModuleId: '{{ moduleId }}-provider'
+  moduleVar: '{{ camelCase pluginId }}Module{{ upperFirst ( camelCase moduleId ) }}'
+  providerVar: '{{ camelCase moduleId }}Provider'
+  providerClass: '{{ upperFirst ( camelCase moduleId ) }}Provider'

packages/cli/templates/catalog-provider-module/src/index.ts.hbs

@@ -0,0 +1,8 @@
+/***/
+/**
+ * The {{fullModuleId}} module for @backstage/plugin-catalog-backend
+ *
+ * @packageDocumentation
+ */
+
+export { {{moduleVar}} as default } from './module';

packages/cli/templates/catalog-provider-module/src/module.ts.hbs

@@ -0,0 +1,29 @@
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { {{providerClass}} } from './provider/{{providerClass}}';
+
+export const {{moduleVar}} = createBackendModule({
+  moduleId: '{{fullModuleId}}',
+  pluginId: '{{pluginId}}',
+  register({ registerInit }) {
+    registerInit({
+      deps: {
+        logger: coreServices.logger,
+        config: coreServices.rootConfig,
+        scheduler: coreServices.scheduler,
+        processing: catalogProcessingExtensionPoint,
+      },
+      async init({ logger, scheduler, config, processing }) {
+        processing.addEntityProvider(
+          {{providerClass}}.fromConfig(config, {
+            logger,
+            scheduler,
+          }),
+        );
+      }
+    });
+  },
+})

packages/cli/templates/catalog-provider-module/src/provider/readProviderConfigs.ts.hbs

@@ -0,0 +1,78 @@
+import {
+  readSchedulerServiceTaskScheduleDefinitionFromConfig,
+  SchedulerServiceTaskScheduleDefinition,
+} from '@backstage/backend-plugin-api';
+import { Config } from '@backstage/config';
+
+const DEFAULT_PROVIDER_ID = 'default';
+const DEFAULT_SCHEDULE: SchedulerServiceTaskScheduleDefinition = {
+  frequency: {
+    minutes: 30,
+  },
+  timeout: {
+    minutes: 3,
+  },
+}
+
+export type {{providerClass}}ProviderConfig = {
+  id: string;
+  target: string;
+  schedule: SchedulerServiceTaskScheduleDefinition;
+}
+
+/**
+ * Parses all configured providers.
+ *
+ * @param config - The root of the provider config hierarchy
+ *
+ * @public
+ */
+export function readProviderConfigs(
+  config: Config,
+): {{providerClass}}ProviderConfig[] {
+  const providersConfig = config.getOptionalConfig(
+    'catalog.providers.{{providerVar}}',
+  );
+  if (!providersConfig) {
+    return [];
+  }
+
+  if ((providersConfig).has('target')) {
+    // simple/single config variant
+    return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
+  }
+
+  return providersConfig.keys().map(id => {
+    const providerConfig = providersConfig.getConfig(id);
+
+    return readProviderConfig(id, providerConfig);
+  });
+}
+
+/**
+ * Parses a single configured provider by id.
+ *
+ * @param id - the id of the provider to parse
+ * @param config - The root of the provider config hierarchy
+ *
+ * @public
+ */
+export function readProviderConfig(
+  id: string,
+  config: Config,
+): {{providerClass}}ProviderConfig {
+
+  const target = config.getString('target');
+
+  const schedule = config.has('schedule')
+    ? readSchedulerServiceTaskScheduleDefinitionFromConfig(
+        config.getConfig('schedule'),
+      )
+    : DEFAULT_SCHEDULE;
+
+  return {
+    id,
+    target,
+    schedule,
+  };
+}

packages/cli/templates/catalog-provider-module/src/provider/{{providerClass}}.test.ts.hbs

@@ -0,0 +1,18 @@
+import { {{providerClass}} } from './{{providerClass}}';
+import { mockServices } from '@backstage/backend-test-utils';
+
+describe('{{providerClass}}', () => {
+  it('should read entities from the target', async () => {
+    const logger = mockServices.logger.mock();
+    const provider = new {{providerClass}}({
+      id: 'test',
+      target: 'https://example.com',
+      logger: mockServices.logger.mock(),
+      taskRunner: { run: jest.fn() },
+    });
+
+    const entities = await provider.read({ logger });
+
+    expect(entities).toEqual([]);
+  });
+})

packages/cli/templates/catalog-provider-module/src/provider/{{providerClass}}.ts.hbs

@@ -0,0 +1,109 @@
+import { Config } from '@backstage/config';
+import {
+  DeferredEntity,
+  EntityProvider,
+  EntityProviderConnection,
+} from '@backstage/plugin-catalog-node';
+import * as uuid from 'uuid';
+import { readProviderConfigs } from './readProviderConfigs';
+import {
+  LoggerService,
+  SchedulerService,
+  SchedulerServiceTaskRunner,
+} from '@backstage/backend-plugin-api';
+
+export type {{providerClass}}Options = {
+  /**
+   * The logger to use.
+   */
+  logger: LoggerService;
+
+  /**
+   * Scheduler used to schedule refreshes based on
+   * the schedule config.
+   */
+  scheduler: SchedulerService;
+};
+
+export class {{providerClass}} implements EntityProvider {
+  static fromConfig(
+    configRoot: Config,
+    options: {{providerClass}}Options,
+  ): {{providerClass}}[] {
+    return readProviderConfigs(configRoot).map(providerConfig => {
+      return new {{providerClass}}({
+        id: providerConfig.id,
+        target: providerConfig.target,
+        logger: options.logger,
+        taskRunner: options.scheduler.createScheduledTaskRunner(
+          providerConfig.schedule,
+        ),
+      });
+    });
+  }
+
+  readonly #id: string;
+  readonly #target: string;
+  readonly #logger: LoggerService;
+  readonly #taskRunner: SchedulerServiceTaskRunner;
+
+  constructor(options: {
+    id: string;
+    target: string;
+    logger: LoggerService;
+    taskRunner: SchedulerServiceTaskRunner;
+  }) {
+    this.#id = options.id;
+    this.#target = options.target;
+    this.#logger = options.logger;
+    this.#taskRunner = options.taskRunner;
+  }
+
+  /** {@inheritdoc @backstage/plugin-catalog-node#EntityProvider.getProviderName} */
+  getProviderName() {
+    return `{{providerClass}}:${this.#id}`;
+  }
+
+  /** {@inheritdoc @backstage/plugin-catalog-node#EntityProvider.connect} */
+  async connect(connection: EntityProviderConnection) {
+    const id = `${this.getProviderName()}:refresh`;
+
+    // Schedule a refresh task to be run periodically
+    await this.#taskRunner.run({
+      id,
+      fn: async () => {
+        const logger = this.#logger.child({
+          taskId: id,
+          taskInstanceId: uuid.v4(),
+        });
+
+        try {
+          const entities = await this.read({ logger });
+
+          logger.info(`Read ${entities.length} entities`);
+
+          await connection.applyMutation({
+            type: 'full',
+            entities,
+          });
+        } catch (error) {
+          logger.error(`Refresh failed`, error);
+        }
+      },
+    });
+  }
+
+  /**
+   * Reads entities to be added to the catalog.
+   */
+  async read(options: { logger: LoggerService }): Promise<DeferredEntity[]> {
+    const { logger } = options;
+
+    logger.info(`Reading entities from ${this.#target}`);
+
+    // TODO: Implement entity reading logic from the target
+    const entities: DeferredEntity[] = [];
+
+    return entities;
+  }
+}

packages/create-app/templates/next-app/package.json.hbs

@@ -38,6 +38,7 @@
           "@backstage/cli/templates/plugin-common-library",
           "@backstage/cli/templates/web-library",
           "@backstage/cli/templates/node-library",
+          "@backstage/cli/templates/catalog-provider-module",
           "@backstage/cli/templates/scaffolder-backend-module"
         ]
       }