feat(repo-tools): add a new command that provides dynamic test-driven OpenAPI schema feedback.

Signed-off-by: Aramis Sennyey <aramiss@spotify.com>
Signed-off-by: Aramis <sennyeyaramis@gmail.com>

.changeset/nasty-colts-tickle.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-backend': patch
+'@backstage/plugin-search-backend': patch
+---
+
+Update the OpenAPI spec with more complete error responses and request bodies using Optic. Also, updates the test cases to use the new `supertest` pass through from `@backstage/backend-openapi-utils`.

.changeset/old-chairs-switch.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': patch
+---
+
+Adds a new command `schema openapi test` that performs runtime validation of your OpenAPI specs using your test data. Under the hood, we're using Optic to perform this check, really cool work by them!

.changeset/tricky-vans-behave.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-openapi-utils': patch
+---
+
+Adds a new function `wrapInOpenApiTestServer` that allows for proxied requests at runtime. This will support the new `yarn backstage-repo-tools schema openapi test` command.

.github/vale/Vocab/Backstage/accept.txt

@@ -291,6 +291,7 @@ Preprarer
 productional
 Protobuf
 proxying
+proxied
 Proxying
 pseudonymized
 pubsub

.github/workflows/ci.yml

@@ -222,6 +222,10 @@ jobs:
           BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}
           BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
 
+      # We run the test cases before verifying the specs to prevent any failing tests from causing errors.
+      - name: verify openapi specs against test cases
+        run: yarn backstage-repo-tools schema openapi test
+
       - name: ensure clean working directory
         run: |
           if files=$(git ls-files --exclude-standard --others --modified) && [[ -z "$files" ]]; then

docs/openapi/test-case-validation.md

@@ -0,0 +1,41 @@
+## OpenAPI Validation using Test Cases
+
+This is primarily performed by `backstage-repo-tools schema openapi test`. Any errors found in the generated specs can be either
+
+1. Fixed manually, this is usually relevant for request body or response body changes.
+2. Fixed automatically with `backstage-repo-tools schema openapi test --update`.
+3. Fixing the test case. This can happen where a response is mocked as
+
+```ts
+it('should return the right value', () => {
+  // We will assume that this is the actual response and update the spec accordingly.
+  // Ideally, this should be a fully populated return value.
+  const entity: Entity = {} as any;
+  app.get('/test', () => {
+    return entity;
+  });
+  const response = await request(app).get('/test');
+  expect(response.body).toEqual(entity);
+});
+```
+
+will cause an invalid spec validation. The return value doesn't have all properties as defined in the type. Try to avoid this if possible. Something better would be,
+
+```ts
+it('should return the right value', () => {
+  // We will assume that this is the actual response and update the spec accordingly.
+  // Ideally, this should be a fully populated return value.
+  const entity: Entity = {
+    apiVersion: 'a1',
+    kind: 'k1',
+    metadata: { name: 'n1' },
+  };
+  app.get('/test', () => {
+    return entity;
+  });
+  const response = await request(app).get('/test');
+  expect(response.body).toEqual(entity);
+});
+```
+
+Additionally, for more advanced use cases, you can run `yarn optic capture {PATH_TO_OPENAPI_FILE} --update interactive` and go through the prompts on the screen. Under the hood, the test validation + updating is done by [Optic](https://github.com/opticdev/optic), a great project around supporting OpenAPI specs and development. You can find additional options [here](https://www.useoptic.com/docs/verify-openapi).

package.json

@@ -73,6 +73,7 @@
     "@techdocs/cli": "workspace:*",
     "@types/node": "^18.17.8",
     "@types/webpack": "^5.28.0",
+    "@useoptic/optic": "^0.50.7",
     "command-exists": "^1.2.9",
     "concurrently": "^8.0.0",
     "cross-env": "^7.0.0",

packages/backend-openapi-utils/README.md

@@ -65,7 +65,7 @@ export function createRouter() {
 
 ### Why am I getting `unknown` as the type for a response?
 
-This can happen when you have a `charset` defined in your `response.content` section. Something like `response.content[    'application/json; charset=utf-8:']` will cause this issue.
+This can happen when you have a `charset` defined in your `response.content` section. Something like `response.content['application/json; charset=utf-8:']` will cause this issue.
 
 ## INTERNAL
 

packages/backend-openapi-utils/api-report.md

@@ -5,6 +5,7 @@
 ```ts
 import type { ContentObject } from 'openapi3-ts';
 import type core from 'express-serve-static-core';
+import { Express as Express_2 } from 'express';
 import { FromSchema } from 'json-schema-to-ts';
 import { JSONSchema7 } from 'json-schema-to-ts';
 import { middleware } from 'express-openapi-validator';
@@ -16,6 +17,7 @@ import { RequestHandler } from 'express';
 import type { ResponseObject } from 'openapi3-ts';
 import { Router } from 'express';
 import type { SchemaObject } from 'openapi3-ts';
+import { Server } from 'http';
 
 // @public
 export interface ApiRouter<Doc extends RequiredDoc> extends Router {
@@ -708,4 +710,7 @@ type UnknownIfNever<P> = [P] extends [never] ? unknown : P;
 
 // @public
 type ValueOf<T> = T[keyof T];
+
+// @public
+export const wrapInOpenApiTestServer: (app: Express_2) => Server | Express_2;
 ```

packages/backend-openapi-utils/src/index.ts

@@ -32,3 +32,4 @@ export type {
 } from './utility';
 export type { ApiRouter } from './router';
 export { createValidatedOpenApiRouter } from './stub';
+export { wrapInOpenApiTestServer } from './testUtils';

packages/backend-openapi-utils/src/stub.test.ts

@@ -27,7 +27,6 @@ describe('createRouter', () => {
     status: 'available',
     photoUrls: [],
   };
-
   it('does NOT override originalUrl and basePath after execution', async () => {
     expect.assertions(2);
     const router = createValidatedOpenApiRouter(singlePathSpec);

packages/backend-openapi-utils/src/testUtils.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Express } from 'express';
+import { Server } from 'http';
+
+/**
+ * !!! THIS CURRENTLY ONLY SUPPORTS SUPERTEST !!!
+ * Running against supertest, we need some way to hit the optic proxy. This ensures that
+ *  that happens at runtime when in the context of a `yarn optic capture` command.
+ * @param app - Express router that would be passed to supertest's `request`.
+ * @returns A wrapper around the express router (or the router untouched) that still works with supertest.
+ * @public
+ */
+export const wrapInOpenApiTestServer = (app: Express): Server | Express => {
+  if (process.env.OPTIC_PROXY) {
+    const server = app.listen(+process.env.PORT!);
+    return {
+      ...server,
+      address: () => new URL(process.env.OPTIC_PROXY!),
+    } as any;
+  }
+  return app;
+};

packages/repo-tools/cli-report.md

@@ -73,6 +73,8 @@ Commands:
   verify [paths...]
   generate [paths...]
   lint [options] [paths...]
+  test [options] [paths...]
+  init <paths...>
   help [command]
 ```
 
@@ -85,6 +87,15 @@ Options:
   -h, --help
 ```
 
+### `backstage-repo-tools schema openapi init`
+
+```
+Usage: backstage-repo-tools schema openapi init [options] <paths...>
+
+Options:
+  -h, --help
+```
+
 ### `backstage-repo-tools schema openapi lint`
 
 ```
@@ -95,6 +106,16 @@ Options:
   -h, --help
 ```
 
+### `backstage-repo-tools schema openapi test`
+
+```
+Usage: backstage-repo-tools schema openapi test [options] [paths...]
+
+Options:
+  --update
+  -h, --help
+```
+
 ### `backstage-repo-tools schema openapi verify`
 
 ```

packages/repo-tools/package.json

@@ -43,7 +43,7 @@
     "@stoplight/spectral-formatters": "^1.1.0",
     "@stoplight/spectral-functions": "^1.7.2",
     "@stoplight/spectral-parsers": "^1.0.2",
-    "@stoplight/spectral-rulesets": "^1.16.0",
+    "@stoplight/spectral-rulesets": "^1.18.0",
     "@stoplight/spectral-runtime": "^1.1.2",
     "@stoplight/types": "^13.14.0",
     "chalk": "^4.0.0",
@@ -56,6 +56,7 @@
     "lodash": "^4.17.21",
     "minimatch": "^5.1.1",
     "p-limit": "^3.0.2",
+    "portfinder": "^1.0.32",
     "ts-node": "^10.0.0",
     "yaml-diff-patch": "^2.0.0"
   },

packages/repo-tools/src/commands/index.ts

@@ -49,6 +49,17 @@ function registerSchemaCommand(program: Command) {
       'Fail on any linting severity messages, not just errors.',
     )
     .action(lazy(() => import('./openapi/lint').then(m => m.bulkCommand)));
+
+  openApiCommand
+    .command('test [paths...]')
+    .description('Test OpenAPI schemas against written tests')
+    .option('--update', 'Update the spec on failure.')
+    .action(lazy(() => import('./openapi/test').then(m => m.bulkCommand)));
+
+  openApiCommand
+    .command('init <paths...>')
+    .description('Creates any config needed for the test command.')
+    .action(lazy(() => import('./openapi/test/init').then(m => m.default)));
 }
 
 export function registerCommands(program: Command) {

packages/repo-tools/src/commands/openapi/runner.ts

@@ -18,21 +18,40 @@ import { resolvePackagePaths } from '../../lib/paths';
 import pLimit from 'p-limit';
 import { relative as relativePath } from 'path';
 import { paths as cliPaths } from '../../lib/paths';
+import portFinder from 'portfinder';
 
 export async function runner(
   paths: string[],
-  command: (dir: string) => Promise<void>,
+  command: (dir: string, options?: { port: number }) => Promise<void>,
+  options?: {
+    concurrencyLimit: number;
+    startingPort?: number;
+  },
 ) {
   const packages = await resolvePackagePaths({ paths });
-  const limit = pLimit(5);
-
+  const limit = pLimit(options?.concurrencyLimit ?? 5);
+  let port =
+    options?.startingPort &&
+    (await portFinder.getPortPromise({
+      // Prevent collisions with optic which runs 8000->8999
+      port: options.startingPort,
+      stopPort: options.startingPort + 1_000,
+    }));
   const resultsList = await Promise.all(
     packages.map(pkg =>
       limit(async () => {
         let resultText = '';
         try {
+          if (port)
+            port =
+              options?.startingPort &&
+              (await portFinder.getPortPromise({
+                // Prevent collisions with optic which runs 8000->8999
+                port: port + 1,
+                stopPort: options.startingPort + 1_000,
+              }));
           console.log(`## Processing ${pkg}`);
-          await command(pkg);
+          await command(pkg, port ? { port } : undefined);
         } catch (err) {
           resultText = err.message;
         }

packages/repo-tools/src/commands/openapi/test/index.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { join } from 'path';
+import chalk from 'chalk';
+import { runner } from '../runner';
+import { YAML_SCHEMA_PATH } from '../constants';
+import { paths as cliPaths } from '../../../lib/paths';
+import { exec } from '../../../lib/exec';
+
+async function test(
+  directoryPath: string,
+  { port }: { port: number },
+  options?: { update?: boolean },
+) {
+  const openapiPath = join(directoryPath, YAML_SCHEMA_PATH);
+  if (!(await fs.pathExists(openapiPath))) {
+    return;
+  }
+  const opticConfigFilePath = join(directoryPath, 'optic.yml');
+  if (!(await fs.pathExists(opticConfigFilePath))) {
+    return;
+  }
+
+  try {
+    await exec(
+      `yarn run -T optic capture`,
+      [
+        YAML_SCHEMA_PATH,
+        '--server-override',
+        `http://localhost:${port}`,
+        options?.update ? '--update' : '',
+      ],
+      {
+        cwd: directoryPath,
+        env: {
+          ...process.env,
+          PORT: `${port}`,
+        },
+      },
+    );
+  } catch (err) {
+    // Optic outputs the actual results to stdout, but that will not be added to the message by default.
+    err.message = err.stderr + err.stdout;
+    err.message = (err.message as string)
+      .split('\n')
+      .map(e => e.replace(/.{1} Sending requests to server/, ''))
+      // Remove any lines that are emitted during processing and only show output.
+      .filter(e => !e.includes('PASS'))
+      .filter(e => e.trim())
+      .join('\n');
+    throw err;
+  }
+  if (
+    (await cliPaths.resolveTargetRoot('node_modules/.bin/prettier')) &&
+    options?.update
+  ) {
+    await exec(`yarn prettier`, ['--write', openapiPath]);
+  }
+}
+
+export async function bulkCommand(
+  paths: string[] = [],
+  options: { update?: boolean },
+): Promise<void> {
+  const resultsList = await runner(
+    paths,
+    (dir, runnerOptions) => test(dir, runnerOptions!, options),
+    {
+      concurrencyLimit: 1,
+      startingPort: 9_000,
+    },
+  );
+
+  let failed = false;
+  for (const { relativeDir, resultText } of resultsList) {
+    if (resultText) {
+      console.log();
+      console.log(
+        chalk.red(
+          `OpenAPI runtime validation against tests failed in ${relativeDir}:`,
+        ),
+      );
+      console.log(resultText.trimStart());
+
+      failed = true;
+    }
+  }
+
+  if (failed) {
+    process.exit(1);
+  } else {
+    console.log(chalk.green('Verified all specifications against test data.'));
+  }
+}

packages/repo-tools/src/commands/openapi/test/init.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import { join } from 'path';
+import { YAML_SCHEMA_PATH } from './../constants';
+
+import { paths as cliPaths } from '../../../lib/paths';
+import { runner } from '../runner';
+import chalk from 'chalk';
+import { exec } from '../../../lib/exec';
+
+const ROUTER_TEST_PATHS = [
+  'src/service/router.test.ts',
+  'src/service/createRouter.test.ts',
+];
+
+async function init(directoryPath: string) {
+  const openapiPath = join(directoryPath, YAML_SCHEMA_PATH);
+  if (!(await fs.pathExists(openapiPath))) {
+    throw new Error(
+      `You do not have an OpenAPI YAML file at ${openapiPath}. Please create one and retry this command. If you already have existing test cases for your router, see 'backstage-repo-tools schema openapi test --update'`,
+    );
+  }
+  const opticConfigFilePath = join(directoryPath, 'optic.yml');
+  if (!(await fs.pathExists(opticConfigFilePath))) {
+    throw new Error(`This directory already has an optic.yml file. Exiting.`);
+  }
+  await fs.writeFile(
+    opticConfigFilePath,
+    `ruleset:
+    - breaking-changes
+capture:
+    ${YAML_SCHEMA_PATH}:
+        # 🔧 Runnable example with simple get requests.
+        # Run with "PORT=3000 optic capture ${YAML_SCHEMA_PATH} --update interactive" in '${directoryPath}'
+        # You can change the server and the 'requests' section to experiment
+        server:
+            # This will not be used by 'backstage-repo-tools schema openapi test', but may be useful for interactive updates.
+            url: http://localhost:3000
+        requests:
+            # ℹ️ Requests should be sent to the Optic proxy, the address of which is injected into 'run.command's env as OPTIC_PROXY (or the value of 'run.proxy_variable').
+            run:
+                # 🔧 Specify a command that will generate traffic
+                command: yarn backstage-cli package test --no-watch ${ROUTER_TEST_PATHS.map(
+                  e => `"${e}"`,
+                ).join(',')} 
+  `,
+  );
+  if (await cliPaths.resolveTargetRoot('node_modules/.bin/prettier')) {
+    await exec(`yarn prettier`, ['--write', opticConfigFilePath]);
+  }
+}
+
+export default async function initCommand(paths: string[] = []) {
+  const resultsList = await runner(paths, dir => init(dir), {
+    concurrencyLimit: 5,
+  });
+
+  let failed = false;
+  for (const { relativeDir, resultText } of resultsList) {
+    if (resultText) {
+      console.log();
+      console.log(
+        chalk.red(`Failed to initialize ${relativeDir} for OpenAPI commands.`),
+      );
+      console.log(resultText.trimStart());
+
+      failed = true;
+    }
+  }
+
+  if (failed) {
+    process.exit(1);
+  } else {
+    console.log(chalk.green(`All directories have already been configured.`));
+  }
+}

packages/repo-tools/src/lib/exec.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { promisify } from 'util';
+import { ExecOptions, exec as execCb } from 'child_process';
+
+const execPromise = promisify(execCb);
+
+export const exec = (
+  command: string,
+  options: string[] = [],
+  execOptions?: ExecOptions,
+) => {
+  return execPromise(
+    [
+      command,
+      ...options.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)),
+    ].join(' '),
+    execOptions,
+  );
+};

plugins/catalog-backend/optic.yml

@@ -0,0 +1,15 @@
+ruleset:
+  - breaking-changes
+capture:
+  src/schema/openapi.yaml:
+    # 🔧 Runnable example with simple get requests.
+    # Run with "PORT=3000 optic capture src/schema/openapi.yaml --update interactive" in 'plugins/catalog-backend'
+    # You can change the server and the 'requests' section to experiment
+    server:
+      # This will not be used by 'backstage-repo-tools schema openapi test', but may be useful for interactive updates.
+      url: http://localhost:3000
+    requests:
+      # ℹ️ Requests should be sent to the Optic proxy, the address of which is injected into 'run.command's env as OPTIC_PROXY (or the value of 'run.proxy_variable').
+      run:
+        # 🔧 Specify a command that will generate traffic
+        command: yarn backstage-cli package test --no-watch src/service/router.test.ts src/service/createRouter.test.ts

plugins/catalog-backend/src/schema/openapi.generated.ts

@@ -191,12 +191,69 @@ export const spec = {
       },
     },
     requestBodies: {},
-    responses: {},
+    responses: {
+      ErrorResponse: {
+        description: 'An error response from the backend.',
+        content: {
+          'application/json': {
+            schema: {
+              $ref: '#/components/schemas/Error',
+            },
+          },
+        },
+      },
+    },
     schemas: {
+      Error: {
+        type: 'object',
+        properties: {
+          error: {
+            type: 'object',
+            properties: {
+              name: {
+                type: 'string',
+              },
+              message: {
+                type: 'string',
+              },
+              stack: {
+                type: 'string',
+              },
+              code: {
+                type: 'string',
+              },
+            },
+            required: ['name', 'message'],
+          },
+          request: {
+            type: 'object',
+            properties: {
+              method: {
+                type: 'string',
+              },
+              url: {
+                type: 'string',
+              },
+            },
+            required: ['method', 'url'],
+          },
+          response: {
+            type: 'object',
+            properties: {
+              statusCode: {
+                type: 'number',
+              },
+            },
+            required: ['statusCode'],
+          },
+        },
+        required: ['error', 'response'],
+      },
       JsonObject: {
         type: 'object',
         properties: {},
         description: 'A type representing all allowed JSON object values.',
+        additionalProperties: true,
       },
       MapStringString: {
         type: 'object',
@@ -698,38 +755,6 @@ export const spec = {
         required: ['type', 'target'],
         additionalProperties: false,
       },
-      SerializedError: {
-        allOf: [
-          {
-            $ref: '#/components/schemas/JsonObject',
-          },
-          {
-            type: 'object',
-            properties: {
-              code: {
-                type: 'string',
-                description:
-                  'A custom code (not necessarily the same as an HTTP response code); may not be present',
-              },
-              stack: {
-                type: 'string',
-                description: 'A stringified stack trace; may not be present',
-              },
-              message: {
-                type: 'string',
-                description: 'The message of the exception that was thrown',
-              },
-              name: {
-                type: 'string',
-                description: 'The name of the exception that was thrown',
-              },
-            },
-            required: ['message', 'name'],
-          },
-        ],
-        description: 'The serialized form of an Error.',
-        additionalProperties: false,
-      },
       EntitiesQueryResponse: {
         type: 'object',
         properties: {
@@ -777,6 +802,12 @@ export const spec = {
           '200': {
             description: 'Refreshed',
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -829,6 +860,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -882,6 +919,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -902,6 +945,12 @@ export const spec = {
           '204': {
             description: 'Deleted successfully.',
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -931,6 +980,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -966,6 +1021,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1002,6 +1063,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1015,7 +1082,7 @@ export const spec = {
           },
         ],
         requestBody: {
-          required: true,
+          required: false,
           content: {
             'application/json': {
               schema: {
@@ -1072,6 +1139,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1139,6 +1212,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1203,6 +1282,44 @@ export const spec = {
               },
             },
           },
+          '201': {
+            description: '201 response',
+            content: {
+              'application/json': {
+                schema: {
+                  type: 'object',
+                  properties: {
+                    location: {
+                      type: 'object',
+                      properties: {
+                        id: {
+                          type: 'string',
+                        },
+                        type: {
+                          type: 'string',
+                        },
+                        target: {
+                          type: 'string',
+                        },
+                      },
+                      required: ['id', 'type', 'target'],
+                    },
+                    entities: {
+                      type: 'array',
+                      items: {},
+                    },
+                  },
+                  required: ['location', 'entities'],
+                },
+              },
+            },
+          },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1264,6 +1381,9 @@ export const spec = {
               },
             },
           },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1289,6 +1409,9 @@ export const spec = {
               },
             },
           },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1315,6 +1438,12 @@ export const spec = {
           '204': {
             description: 'No content',
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1350,6 +1479,12 @@ export const spec = {
               },
             },
           },
+          '400': {
+            $ref: '#/components/responses/ErrorResponse',
+          },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
+          },
         },
         security: [
           {},
@@ -1387,32 +1522,31 @@ export const spec = {
         responses: {
           '200': {
             description: 'Ok',
+          },
+          '400': {
+            description: '400 response',
             content: {
-              'application/json': {
+              'application/json; charset=utf-8': {
                 schema: {
-                  anyOf: [
-                    {
-                      type: 'object',
-                      properties: {
-                        errors: {
-                          $ref: '#/components/schemas/SerializedError',
-                        },
-                      },
-                      required: ['errors'],
-                    },
-                    {
-                      type: 'object',
-                      properties: {
-                        errors: {
-                          type: 'array',
-                          items: {
-                            $ref: '#/components/schemas/SerializedError',
+                  type: 'object',
+                  properties: {
+                    errors: {
+                      type: 'array',
+                      items: {
+                        type: 'object',
+                        properties: {
+                          name: {
+                            type: 'string',
+                          },
+                          message: {
+                            type: 'string',
                           },
                         },
+                        required: ['name', 'message'],
                       },
-                      required: ['errors'],
                     },
-                  ],
+                  },
+                  required: ['errors'],
                 },
               },
             },

plugins/catalog-backend/src/schema/openapi.yaml

@@ -1,5 +1,4 @@
 openapi: 3.0.3
-
 info:
   title: '@backstage/plugin-catalog-backend'
   version: '1'
@@ -8,10 +7,8 @@ info:
     name: Apache-2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html
   contact: {}
-
 servers:
   - url: /
-
 components:
   examples: {}
   headers: {}
@@ -73,10 +70,15 @@ components:
         items:
           type: string
       examples:
-        'Get name and the entire relations collection':
-          value: ['metadata.name', 'relations']
-        'Get kind, name and namespace':
-          value: ['kind', 'metadata.name', 'metadata.namespace']
+        Get name and the entire relations collection:
+          value:
+            - metadata.name
+            - relations
+        Get kind, name and namespace:
+          value:
+            - kind
+            - metadata.name
+            - metadata.namespace
     filter:
       name: filter
       in: query
@@ -88,11 +90,12 @@ components:
         items:
           type: string
       examples:
-        'Get groups':
-          value: ['kind=group']
-        'Get orphaned components':
+        Get groups:
           value:
-            ['kind=component,metadata.annotations.backstage.io/orphan=true']
+            - kind=group
+        Get orphaned components:
+          value:
+            - kind=component,metadata.annotations.backstage.io/orphan=true
     offset:
       name: offset
       in: query
@@ -125,17 +128,63 @@ components:
       explode: true
       style: form
       examples:
-        'Order ascending by name':
-          value: ['metadata.name,asc']
-        'Order descending by owner':
-          value: ['spec.owner,desc']
+        Order ascending by name:
+          value:
+            - metadata.name,asc
+        Order descending by owner:
+          value:
+            - spec.owner,desc
   requestBodies: {}
-  responses: {}
+  responses:
+    ErrorResponse:
+      description: An error response from the backend.
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
   schemas:
+    Error:
+      type: object
+      properties:
+        error:
+          type: object
+          properties:
+            name:
+              type: string
+            message:
+              type: string
+            stack:
+              type: string
+            code:
+              type: string
+          required:
+            - name
+            - message
+        request:
+          type: object
+          properties:
+            method:
+              type: string
+            url:
+              type: string
+          required:
+            - method
+            - url
+        response:
+          type: object
+          properties:
+            statusCode:
+              type: number
+          required:
+            - statusCode
+      required:
+        - error
+        - response
     JsonObject:
       type: object
       properties: {}
       description: A type representing all allowed JSON object values.
+      additionalProperties: true
     MapStringString:
       type: object
       properties: {}
@@ -350,7 +399,6 @@ components:
         count:
           type: number
       additionalProperties: false
-
     EntityFacetsResponse:
       type: object
       properties:
@@ -593,28 +641,6 @@ components:
         - type
         - target
       additionalProperties: false
-    SerializedError:
-      allOf:
-        - $ref: '#/components/schemas/JsonObject'
-        - type: object
-          properties:
-            code:
-              type: string
-              description: A custom code (not necessarily the same as an HTTP response code); may not be present
-            stack:
-              type: string
-              description: A stringified stack trace; may not be present
-            message:
-              type: string
-              description: The message of the exception that was thrown
-            name:
-              type: string
-              description: The name of the exception that was thrown
-          required:
-            - message
-            - name
-      description: The serialized form of an Error.
-      additionalProperties: false
     EntitiesQueryResponse:
       type: object
       properties:
@@ -622,8 +648,7 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Entity'
-          description: |-
-            The list of entities paginated by a specific filter.
+          description: The list of entities paginated by a specific filter.
         totalItems:
           type: number
         pageInfo:
@@ -631,12 +656,10 @@ components:
           properties:
             nextCursor:
               type: string
-              description: |-
-                The cursor for the next batch of entities.
+              description: The cursor for the next batch of entities.
             prevCursor:
               type: string
-              description: |-
-                The cursor for the previous batch of entities.
+              description: The cursor for the previous batch of entities.
       additionalProperties: false
   securitySchemes:
     JWT:
@@ -651,6 +674,10 @@ paths:
       responses:
         '200':
           description: Refreshed
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -684,6 +711,10 @@ paths:
                 type: array
                 items:
                   $ref: '#/components/schemas/Entity'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -712,6 +743,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Entity'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -723,6 +758,10 @@ paths:
       responses:
         '204':
           description: Deleted successfully.
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -739,6 +778,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Entity'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -757,6 +800,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/EntityAncestryResponse'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -775,13 +822,17 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/EntitiesBatchResponse'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
       parameters:
         - $ref: '#/components/parameters/fields'
       requestBody:
-        required: true
+        required: false
         content:
           application/json:
             schema:
@@ -798,14 +849,17 @@ paths:
                   items:
                     type: string
             examples:
-              'Fetch Backstage entities':
+              Fetch Backstage entities:
                 value:
                   entityRefs:
-                    ['component:default/backstage', 'api:default/backstage']
-              'Fetch annotations for backstage entity':
+                    - component:default/backstage
+                    - api:default/backstage
+              Fetch annotations for backstage entity:
                 value:
-                  entityRefs: ['component:default/backstage']
-                  fields: ['metadata.annotations']
+                  entityRefs:
+                    - component:default/backstage
+                  fields:
+                    - metadata.annotations
   /entities/by-query:
     get:
       operationId: GetEntitiesByQuery
@@ -817,6 +871,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/EntitiesQueryResponse'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -855,6 +913,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/EntityFacetsResponse'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -868,10 +930,12 @@ paths:
             items:
               type: string
           examples:
-            'Entities by kind':
-              value: ['kind']
-            'Entities by spec type':
-              value: ['spec.type']
+            Entities by kind:
+              value:
+                - kind
+            Entities by spec type:
+              value:
+                - spec.type
         - $ref: '#/components/parameters/filter'
   /locations:
     post:
@@ -896,6 +960,36 @@ paths:
                 required:
                   - entities
                   - location
+        '201':
+          description: 201 response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  location:
+                    type: object
+                    properties:
+                      id:
+                        type: string
+                      type:
+                        type: string
+                      target:
+                        type: string
+                    required:
+                      - id
+                      - type
+                      - target
+                  entities:
+                    type: array
+                    items: {}
+                required:
+                  - location
+                  - entities
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -937,6 +1031,8 @@ paths:
                       $ref: '#/components/schemas/Location'
                   required:
                     - data
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -952,6 +1048,8 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Location'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -968,6 +1066,10 @@ paths:
       responses:
         '204':
           description: No content
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -989,6 +1091,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/AnalyzeLocationResponse'
+        '400':
+          $ref: '#/components/responses/ErrorResponse'
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []
@@ -1013,24 +1119,27 @@ paths:
       responses:
         '200':
           description: Ok
+        '400':
+          description: 400 response
           content:
-            application/json:
+            application/json; charset=utf-8:
               schema:
-                anyOf:
-                  - type: object
-                    properties:
-                      errors:
-                        $ref: '#/components/schemas/SerializedError'
-                    required:
-                      - errors
-                  - type: object
-                    properties:
-                      errors:
-                        type: array
-                        items:
-                          $ref: '#/components/schemas/SerializedError'
-                    required:
-                      - errors
+                type: object
+                properties:
+                  errors:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        name:
+                          type: string
+                        message:
+                          type: string
+                      required:
+                        - name
+                        - message
+                required:
+                  - errors
       security:
         - {}
         - JWT: []

plugins/catalog-backend/src/service/createRouter.test.ts

@@ -22,6 +22,7 @@ import {
   ANNOTATION_LOCATION,
   ANNOTATION_ORIGIN_LOCATION,
   Entity,
+  stringifyEntityRef,
 } from '@backstage/catalog-model';
 import express from 'express';
 import request from 'supertest';
@@ -38,12 +39,14 @@ import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common/a
 import { CatalogProcessingOrchestrator } from '../processing/types';
 import { z } from 'zod';
 import { encodeCursor } from './util';
+import { wrapInOpenApiTestServer } from '@backstage/backend-openapi-utils';
+import { Server } from 'http';
 
 describe('createRouter readonly disabled', () => {
   let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
   let locationService: jest.Mocked<LocationService>;
   let orchestrator: jest.Mocked<CatalogProcessingOrchestrator>;
-  let app: express.Express;
+  let app: express.Express | Server;
   let refreshService: RefreshService;
 
   beforeAll(async () => {
@@ -72,7 +75,7 @@ describe('createRouter readonly disabled', () => {
       config: new ConfigReader(undefined),
       permissionIntegrationRouter: express.Router(),
     });
-    app = express().use(router);
+    app = wrapInOpenApiTestServer(express().use(router));
   });
 
   beforeEach(() => {
@@ -412,15 +415,27 @@ describe('createRouter readonly disabled', () => {
     });
 
     it('can fetch entities by refs', async () => {
-      const entity: Entity = {} as any;
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'component',
+        metadata: {
+          name: 'a',
+        },
+      };
+      const entityRef = stringifyEntityRef(entity);
       entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] });
       const response = await request(app)
         .post('/entities/by-refs')
         .set('Content-Type', 'application/json')
-        .send('{"entityRefs":["a"],"fields":["b"]}');
+        .send(
+          JSON.stringify({
+            entityRefs: [entityRef],
+            fields: ['metadata.name'],
+          }),
+        );
       expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledTimes(1);
       expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledWith({
-        entityRefs: ['a'],
+        entityRefs: [entityRef],
         fields: expect.any(Function),
       });
       expect(response.status).toEqual(200);

plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts

@@ -24,10 +24,10 @@ import { parseEntityFilterParams } from './parseEntityFilterParams';
 import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams';
 import { parseEntityTransformParams } from './parseEntityTransformParams';
 import { spec } from '../../schema/openapi.generated';
-import { QueryParameters } from '@backstage/backend-openapi-utils';
+import { internal } from '@backstage/backend-openapi-utils';
 
 export function parseQueryEntitiesParams(
-  params: QueryParameters<typeof spec, '/entities/by-query', 'get'>,
+  params: internal.QuerySchema<typeof spec, '/entities/by-query', 'get'>,
 ): Omit<QueryEntitiesRequest, 'authorizationToken' | 'limit'> {
   const fields = parseEntityTransformParams(params);
 

plugins/search-backend/optic.yml

@@ -0,0 +1,15 @@
+ruleset:
+  - breaking-changes
+capture:
+  src/schema/openapi.yaml:
+    # 🔧 Runnable example with simple get requests.
+    # Run with "PORT=3000 optic capture src/schema/openapi.yaml --update interactive" in 'plugins/search-backend'
+    # You can change the server and the 'requests' section to experiment
+    server:
+      # This will not be used by 'backstage-repo-tools schema openapi test', but may be useful for interactive updates.
+      url: http://localhost:3000
+    requests:
+      # ℹ️ Requests should be sent to the Optic proxy, the address of which is injected into 'run.command's env as OPTIC_PROXY (or the value of 'run.proxy_variable').
+      run:
+        # 🔧 Specify a command that will generate traffic
+        command: yarn backstage-cli package test --no-watch src/service/router.test.ts src/service/createRouter.test.ts

plugins/search-backend/src/schema/openapi.generated.ts

@@ -42,8 +42,58 @@ export const spec = {
     headers: {},
     parameters: {},
     requestBodies: {},
-    responses: {},
+    responses: {
+      ErrorResponse: {
+        description: 'An error response from the backend.',
+        content: {
+          'application/json; charset=utf-8': {
+            schema: {
+              $ref: '#/components/schemas/Error',
+            },
+          },
+        },
+      },
+    },
     schemas: {
+      Error: {
+        type: 'object',
+        properties: {
+          error: {
+            type: 'object',
+            properties: {
+              name: {
+                type: 'string',
+              },
+              message: {
+                type: 'string',
+              },
+            },
+            required: ['name', 'message'],
+          },
+          request: {
+            type: 'object',
+            properties: {
+              method: {
+                type: 'string',
+              },
+              url: {
+                type: 'string',
+              },
+            },
+            required: ['method', 'url'],
+          },
+          response: {
+            type: 'object',
+            properties: {
+              statusCode: {
+                type: 'number',
+              },
+            },
+            required: ['statusCode'],
+          },
+        },
+        required: ['error', 'request', 'response'],
+      },
       JsonObject: {
         type: 'object',
         properties: {},
@@ -132,25 +182,8 @@ export const spec = {
               },
             },
           },
-          '400': {
-            description: 'Bad request',
-            content: {
-              'application/json': {
-                schema: {
-                  type: 'object',
-                  properties: {
-                    error: {
-                      type: 'object',
-                      properties: {
-                        message: {
-                          type: 'string',
-                        },
-                      },
-                    },
-                  },
-                },
-              },
-            },
+          default: {
+            $ref: '#/components/responses/ErrorResponse',
           },
         },
         security: [

plugins/search-backend/src/schema/openapi.yaml

@@ -1,5 +1,4 @@
 openapi: 3.0.3
-
 info:
   title: '@backstage/plugin-search-backend'
   version: '1'
@@ -8,17 +7,56 @@ info:
     name: Apache-2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html
   contact: {}
-
 servers:
   - url: /
-
 components:
   examples: {}
   headers: {}
   parameters: {}
   requestBodies: {}
-  responses: {}
+  responses:
+    ErrorResponse:
+      description: An error response from the backend.
+      content:
+        application/json; charset=utf-8:
+          schema:
+            $ref: '#/components/schemas/Error'
   schemas:
+    Error:
+      type: object
+      properties:
+        error:
+          type: object
+          properties:
+            name:
+              type: string
+            message:
+              type: string
+          required:
+            - name
+            - message
+        request:
+          type: object
+          properties:
+            method:
+              type: string
+            url:
+              type: string
+          required:
+            - method
+            - url
+        response:
+          type: object
+          properties:
+            statusCode:
+              type: number
+          required:
+            - statusCode
+      required:
+        - error
+        - request
+        - response
+
     JsonObject:
       type: object
       properties: {}
@@ -85,18 +123,8 @@ paths:
                     type: integer
                 required:
                   - results
-        '400':
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  error:
-                    type: object
-                    properties:
-                      message:
-                        type: string
+        default:
+          $ref: '#/components/responses/ErrorResponse'
       security:
         - {}
         - JWT: []

plugins/search-backend/src/service/router.test.ts

@@ -22,6 +22,8 @@ import { SearchEngine } from '@backstage/plugin-search-common';
 import express from 'express';
 import request from 'supertest';
 import { createRouter } from './router';
+import { wrapInOpenApiTestServer } from '@backstage/backend-openapi-utils';
+import { Server } from 'http';
 
 const mockPermissionEvaluator: PermissionEvaluator = {
   authorize: () => {
@@ -33,7 +35,7 @@ const mockPermissionEvaluator: PermissionEvaluator = {
 };
 
 describe('createRouter', () => {
-  let app: express.Express;
+  let app: express.Express | Server;
   let mockSearchEngine: jest.Mocked<SearchEngine>;
 
   beforeAll(async () => {
@@ -65,7 +67,7 @@ describe('createRouter', () => {
       permissions: mockPermissionEvaluator,
       logger,
     });
-    app = express().use(router);
+    app = wrapInOpenApiTestServer(express().use(router));
   });
 
   beforeEach(() => {
@@ -78,6 +80,7 @@ describe('createRouter', () => {
       mockSearchEngine.query.mockRejectedValueOnce(error);
 
       const response = await request(app).get('/query');
+      console.log((response as any).text);
 
       expect(response.status).toEqual(500);
       expect(response.body).toMatchObject(