testing using the uffizzi workflow

Signed-off-by: aramissennyeydd <aramis.sennyey@doordash.com>

packages/repo-tools/package.json

@@ -49,6 +49,7 @@
     "@stoplight/spectral-rulesets": "^1.18.0",
     "@stoplight/spectral-runtime": "^1.1.2",
     "@stoplight/types": "^14.0.0",
+    "@useoptic/openapi-utilities": "^0.54.8",
     "chalk": "^4.0.0",
     "codeowners-utils": "^1.0.2",
     "command-exists": "^1.2.9",

packages/repo-tools/src/commands/index.ts

@@ -78,6 +78,14 @@ function registerPackageCommand(program: Command) {
     .action(
       lazy(() => import('./package/schema/openapi/fuzz').then(m => m.command)),
     );
+
+  openApiCommand
+    .command('check')
+    .option('--ignore', 'Ignore linting failures and only log the results.')
+    .option('--json', 'Output the results as JSON')
+    .action(
+      lazy(() => import('./package/schema/openapi/check').then(m => m.command)),
+    );
 }
 
 function registerRepoCommand(program: Command) {
@@ -96,11 +104,7 @@ function registerRepoCommand(program: Command) {
   openApiCommand
     .command('verify [paths...]')
     .description(
-      'Verify that all OpenAPI schemas are valid and set up correctly. This also verifies that your API has not changed in a breaking way.',
-    )
-    .option(
-      '--from <ref>',
-      'The base ref to compare against. Defaults to the fork point of the current branch.',
+      'Verify that all OpenAPI schemas are valid and set up correctly.',
     )
     .action(
       lazy(() =>
@@ -137,6 +141,20 @@ function registerRepoCommand(program: Command) {
     .action(
       lazy(() => import('./repo/schema/openapi/fuzz').then(m => m.command)),
     );
+
+  openApiCommand
+    .command('check')
+    .description(
+      'Check the repository against a specific ref, will run all package `check:api` scripts.',
+    )
+    .option(
+      '--since <ref>',
+      'Check the API against a specific ref',
+      'origin/master',
+    )
+    .action(
+      lazy(() => import('./repo/schema/openapi/check').then(m => m.command)),
+    );
 }
 
 export function registerCommands(program: Command) {

packages/repo-tools/src/commands/package/schema/openapi/check.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import chalk from 'chalk';
+import { exec } from '../../../../lib/exec';
+import { getPathToCurrentOpenApiSpec } from '../../../../lib/openapi/helpers';
+import { paths as cliPaths } from '../../../../lib/paths';
+import { OptionValues } from 'commander';
+import { env } from 'process';
+import { readFile, rm } from 'fs/promises';
+import { resolve } from 'path';
+
+const reduceOpticOutput = (output: string) => {
+  return output
+    .split('\n')
+    .filter(e => !e.startsWith('Rerun') && e.trim())
+    .join('\n');
+};
+
+async function check(opts: OptionValues) {
+  const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+
+  let baseRef = opts.since ?? process.env.GITHUB_BASE_REF;
+  if (!baseRef) {
+    const { stdout: branch } = await exec(
+      'git merge-base --fork-point origin/master',
+    );
+    baseRef = branch.toString().trim();
+  }
+
+  let failed = false;
+  let output = '';
+  try {
+    const { stdout } = await exec(
+      'yarn optic diff',
+      [
+        resolvedOpenapiPath,
+        '--check',
+        opts.json ? '--json' : '',
+        '--base',
+        baseRef,
+      ],
+      {
+        cwd: cliPaths.targetRoot,
+        env: { CI: opts.json ? '1' : undefined, ...env },
+      },
+    );
+    output = stdout.toString();
+  } catch (err) {
+    output = err.stdout;
+    failed = true;
+  }
+
+  if (opts.json) {
+    const file = (
+      await readFile(resolve(cliPaths.targetRoot, 'ci-run-details.json'))
+    ).toString();
+    const results = JSON.parse(file);
+    console.log(file);
+    if (!opts.ignore && results.failed) {
+      throw new Error('Some checks failed');
+    }
+
+    await rm(resolve(cliPaths.targetRoot, 'ci-run-details.json'));
+  } else {
+    console.log(reduceOpticOutput(output));
+    if (!opts.ignore && failed) {
+      throw new Error('Some checks failed');
+    }
+  }
+}
+
+export async function command(opts: OptionValues) {
+  try {
+    await check(opts);
+    if (!opts.json) console.log(chalk.green(`All checks passed.`));
+  } catch (err) {
+    if (!opts.json) console.log(chalk.red(err.message));
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/check.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PackageGraph } from '@backstage/cli-node';
+import { OptionValues } from 'commander';
+import { exec } from '../../../../lib/exec';
+import {
+  CiRunDetails,
+  generateCompareSummaryMarkdown,
+} from '../../../../lib/openapi/optic/helpers';
+
+export async function command(opts: OptionValues) {
+  let packages = await PackageGraph.listTargetPackages();
+  if (opts.since) {
+    const graph = PackageGraph.fromPackages(packages);
+    const changedPackages = await graph.listChangedPackages({
+      ref: opts.since,
+      analyzeLockfile: true,
+    });
+    const withDevDependents = graph.collectPackageNames(
+      changedPackages.map(pkg => pkg.name),
+      pkg => pkg.localDevDependents.keys(),
+    );
+    packages = Array.from(withDevDependents).map(name => graph.get(name)!);
+  }
+
+  const checkablePackages = packages.filter(
+    e => e.packageJson.scripts?.['check:api'],
+  );
+  try {
+    const outputs = {
+      completed: [],
+      failed: [],
+      noop: [],
+      severity: 0,
+    } as CiRunDetails;
+    for (const pkg of checkablePackages) {
+      const { stdout } = await exec(
+        'yarn',
+        ['check:api', '--ignore', '--json'],
+        {
+          cwd: pkg.dir,
+        },
+      );
+      const result = JSON.parse(stdout.toString());
+      outputs.completed.push(...(result.completed ?? []));
+      outputs.failed.push(...(result.failed ?? []));
+      outputs.noop.push(...(result.noop ?? []));
+    }
+
+    const { stdout: currentSha } = await exec('git', ['rev-parse', 'HEAD']);
+    console.log(
+      generateCompareSummaryMarkdown(
+        { sha: currentSha.toString().trim() },
+        outputs,
+        { verbose: true },
+      ),
+    );
+
+    const failed = outputs.failed.length > 0;
+    if (failed) {
+      throw new Error('Some checks failed');
+    }
+  } catch (err) {
+    console.error(err);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/verify.ts

@@ -29,17 +29,14 @@ import {
   YAML_SCHEMA_PATH,
 } from '../../../../lib/openapi/constants';
 import { getPathToOpenApiSpec } from '../../../../lib/openapi/helpers';
-import { exec } from '../../../../lib/exec';
-import { OptionValues } from 'commander';
 
-async function verify(directoryPath: string, options: OptionValues) {
-  let openapiPath = '';
-  try {
-    openapiPath = await getPathToOpenApiSpec(directoryPath);
-  } catch {
-    // Unable to find spec at path.
-    return;
-  }
+const verifySpecAndGeneratedSpecMatch = async (
+  openapiPath: string,
+  directoryPath: string,
+) => {
+  const openapiTempDirectory = resolvePath(cliPaths.targetDir, '.openapi');
+  await fs.mkdirp(openapiTempDirectory);
+  console.log(openapiTempDirectory);
 
   const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8'));
   await Parser.validate(cloneDeep(yaml) as any);
@@ -60,39 +57,22 @@ async function verify(directoryPath: string, options: OptionValues) {
       `\`${YAML_SCHEMA_PATH}\` and \`${TS_SCHEMA_PATH}\` do not match. Please run \`yarn backstage-repo-tools package schema openapi generate\` from '${path}' to regenerate \`${TS_SCHEMA_PATH}\`.`,
     );
   }
+};
 
-  let baseRef = options.from ?? process.env.GITHUB_BASE_REF;
-  if (!baseRef) {
-    const { stdout: branch } = await exec('git merge-base --fork-point HEAD');
-    baseRef = branch.toString().trim();
-  }
-
+async function verify(directoryPath: string) {
+  let openapiPath = '';
   try {
-    const { stdout } = await exec('optic diff', [
-      openapiPath,
-      '--check',
-      '--base',
-      baseRef,
-    ]);
-    // Log out the results as this still shows API changes that aren't breakages.
-    console.log(
-      stdout
-        .toString()
-        .split('\n')
-        .filter(e => !e.startsWith('Rerun') && e.trim())
-        .join('\n'),
-    );
-  } catch (err) {
-    err.message = err.stdout;
-    throw err;
+    openapiPath = await getPathToOpenApiSpec(directoryPath);
+  } catch {
+    // Unable to find spec at path.
+    return;
   }
+
+  await verifySpecAndGeneratedSpecMatch(openapiPath, directoryPath);
 }
 
-export async function bulkCommand(
-  paths: string[] = [],
-  options: OptionValues,
-): Promise<void> {
-  const resultsList = await runner(paths, dir => verify(dir, options));
+export async function bulkCommand(paths: string[] = []): Promise<void> {
+  const resultsList = await runner(paths, dir => verify(dir));
 
   let failed = false;
   for (const { relativeDir, resultText } of resultsList) {

packages/repo-tools/src/lib/openapi/optic/helpers.ts

@@ -0,0 +1,244 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* eslint-disable no-nested-ternary */
+
+import {
+  compareSpecs,
+  groupDiffsByEndpoint,
+  Severity,
+  getOperationsChangedLabel,
+  getOperationsChanged,
+} from '@useoptic/openapi-utilities';
+import { GroupedDiffs } from '@useoptic/openapi-utilities/build/openapi3/group-diff';
+import { relative } from 'path';
+import { paths as cliPaths } from '../../paths';
+
+type Comparison = {
+  groupedDiffs: ReturnType<typeof groupDiffsByEndpoint>;
+  results: Awaited<ReturnType<typeof compareSpecs>>['results'];
+};
+
+export type CiRunDetails = {
+  completed: {
+    warnings: string[];
+    apiName: string;
+    opticWebUrl?: string | null;
+    comparison: Comparison;
+    specUrl?: string | null;
+    capture?: any;
+  }[];
+  failed: { apiName: string; error: string }[];
+  noop: { apiName: string }[];
+  severity: Severity;
+};
+
+const getChecksLabel = (
+  results: CiRunDetails['completed'][number]['comparison']['results'],
+  severity: Severity,
+) => {
+  const totalChecks = results.length;
+  let failingChecks = 0;
+  let exemptedFailingChecks = 0;
+
+  for (const result of results) {
+    if (result.passed) continue;
+    if (result.severity < severity) continue;
+    if (result.exempted) exemptedFailingChecks += 1;
+    else failingChecks += 1;
+  }
+
+  const exemptedChunk =
+    exemptedFailingChecks > 0 ? `, ${exemptedFailingChecks} exempted` : '';
+
+  return failingChecks > 0
+    ? `⚠️ **${failingChecks}**/**${totalChecks}** failed${exemptedChunk}`
+    : totalChecks > 0
+    ? `✅ **${totalChecks}** passed${exemptedChunk}`
+    : `ℹ️ No automated checks have run`;
+};
+
+function getOperationsText(
+  groupedDiffs: GroupedDiffs,
+  options: { webUrl?: string | null; verbose: boolean; labelJoiner?: string },
+) {
+  const ops = getOperationsChanged(groupedDiffs);
+
+  const operationsText = options.verbose
+    ? [
+        ...[...ops.added].map(o => `\`${o}\` (added)`),
+        ...[...ops.changed].map(o => `\`${o}\` (changed)`),
+        ...[...ops.removed].map(o => `\`${o}\` (removed)`),
+      ].join('\n')
+    : '';
+  return `${getOperationsChangedLabel(groupedDiffs, {
+    joiner: options.labelJoiner,
+  })}
+    
+      ${operationsText}
+    `;
+}
+
+const getCaptureIssuesLabel = ({
+  unmatchedInteractions,
+  mismatchedEndpoints,
+}: {
+  unmatchedInteractions: number;
+  mismatchedEndpoints: number;
+}) => {
+  return [
+    ...(unmatchedInteractions
+      ? [
+          `🆕 ${unmatchedInteractions} undocumented path${
+            unmatchedInteractions > 1 ? 's' : ''
+          }`,
+        ]
+      : []),
+    ...(mismatchedEndpoints
+      ? [
+          `⚠️  ${mismatchedEndpoints} mismatch${
+            mismatchedEndpoints > 1 ? 'es' : ''
+          }`,
+        ]
+      : []),
+  ].join('\n');
+};
+
+export const generateCompareSummaryMarkdown = (
+  commit: { sha: string },
+  results: CiRunDetails,
+  options: { verbose: boolean },
+) => {
+  const anyCompletedHasWarning = results.completed.some(
+    s => s.warnings.length > 0,
+  );
+  const anyCompletedHasCapture = results.completed.some(s => s.capture);
+  return `
+    ${
+      results.completed.length > 0
+        ? `### APIs Changed
+    
+    <table>
+    <thead>
+    <tr>
+    <th>API</th>
+    <th>Changes</th>
+    <th>Rules</th>
+    ${anyCompletedHasWarning ? '<th>Warnings</th>' : ''}
+    ${anyCompletedHasCapture ? '<th>Tests</th>' : ''}
+    <th></th>
+    </tr>
+    </thead>
+    <tbody>
+    ${results.completed
+      .map(
+        s =>
+          `<tr>
+    <td>
+    
+    ${relative(cliPaths.targetDir, s.apiName)}
+    
+    </td>
+    <td>
+    
+    ${getOperationsText(s.comparison.groupedDiffs, {
+      webUrl: s.opticWebUrl,
+      verbose: options.verbose,
+      labelJoiner: ',\n',
+    })}
+    
+    </td>
+    <td>
+    
+    ${getChecksLabel(s.comparison.results, results.severity)}
+    
+    </td>
+    
+    ${anyCompletedHasWarning ? `<td>${s.warnings.join('\n')}</td>` : ''}
+    
+    ${
+      anyCompletedHasCapture
+        ? `
+    
+    <td>
+    
+    ${
+      s.capture
+        ? s.capture.success
+          ? s.capture.mismatchedEndpoints || s.capture.unmatchedInteractions
+            ? getCaptureIssuesLabel({
+                unmatchedInteractions: s.capture.unmatchedInteractions,
+                mismatchedEndpoints: s.capture.mismatchedEndpoints,
+              })
+            : `✅ ${s.capture.percentCovered}% coverage`
+          : '❌ Failed to run'
+        : ''
+    }
+    
+    </td>
+    
+    `
+        : ''
+    }
+    </tr>`,
+      )
+      .join('\n')}
+    </tbody>
+    </table>
+    `
+        : ''
+    }
+    ${
+      results.failed.length > 0
+        ? `### Errors running optic
+    
+    <table>
+    <thead>
+    <tr>
+    <th>API</th>
+    <th>Error</th>
+    </tr>
+    </thead>
+    <tbody>
+    ${results.failed
+      .map(
+        s => `<tr>
+    <td>${s.apiName}</td>
+    <td>
+    
+    ${'```'}
+    ${s.error}
+    ${'```'}
+    
+    </td>
+    </tr>`,
+      )
+      .join('\n')}
+    </tbody>
+    </table>
+    `
+        : ''
+    }
+    
+    Summary of API changes for commit (${commit.sha})
+    
+    ${
+      results.noop.length > 0
+        ? `${
+            results.noop.length === 1 ? '1 API' : `${results.noop.length} APIs`
+          } had no changes.`
+        : ''
+    }`;
+};