[catalog] Make unprocessed entities visible

Create an API route to fetch failed and pending entities, as well as a frontend plugin to view them.

Added a broken entity to the sample data.

Signed-off-by: Rickard Dybeck <dybeck@spotify.com>

.changeset/grumpy-carrots-sin.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-unprocessed-entities': minor
+'@backstage/catalog-model': minor
+---
+
+Expose unprocessed entities

package.json

@@ -56,6 +56,7 @@
     "@backstage/cli": "workspace:*",
     "@backstage/codemods": "workspace:*",
     "@backstage/create-app": "workspace:*",
+    "@backstage/plugin-catalog-unprocessed-entities": "workspace:*",
     "@backstage/repo-tools": "workspace:*",
     "@changesets/cli": "^2.14.0",
     "@octokit/rest": "^19.0.3",

packages/app/package.json

@@ -26,6 +26,7 @@
     "@backstage/plugin-catalog-graph": "workspace:^",
     "@backstage/plugin-catalog-import": "workspace:^",
     "@backstage/plugin-catalog-react": "workspace:^",
+    "@backstage/plugin-catalog-unprocessed-entities": "workspace:^",
     "@backstage/plugin-circleci": "workspace:^",
     "@backstage/plugin-cloudbuild": "workspace:^",
     "@backstage/plugin-code-coverage": "workspace:^",

packages/app/src/App.tsx

@@ -111,6 +111,7 @@ import { StackstormPage } from '@backstage/plugin-stackstorm';
 import { PuppetDbPage } from '@backstage/plugin-puppetdb';
 import { DevToolsPage } from '@backstage/plugin-devtools';
 import { customDevToolsPage } from './components/devtools/CustomDevToolsPage';
+import { CatalogUnprocessedEntitiesPage } from '@backstage/plugin-catalog-unprocessed-entities';
 
 const app = createApp({
   apis,
@@ -171,6 +172,10 @@ const routes = (
     >
       {entityPage}
     </Route>
+    <Route
+      path="/catalog-unprocessed-entities"
+      element={<CatalogUnprocessedEntitiesPage />}
+    />
     <Route
       path="/catalog-import"
       element={

packages/backend-next/package.json

@@ -29,6 +29,7 @@
     "@backstage/plugin-app-backend": "workspace:^",
     "@backstage/plugin-auth-node": "workspace:^",
     "@backstage/plugin-catalog-backend": "workspace:^",
+    "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^",
     "@backstage/plugin-kubernetes-backend": "workspace:^",
     "@backstage/plugin-permission-backend": "workspace:^",
     "@backstage/plugin-permission-common": "workspace:^",

packages/backend-next/src/index.ts

@@ -30,6 +30,7 @@ import { searchModuleTechDocsCollator } from '@backstage/plugin-search-backend-m
 import { searchPlugin } from '@backstage/plugin-search-backend/alpha';
 import { techdocsPlugin } from '@backstage/plugin-techdocs-backend/alpha';
 import { todoPlugin } from '@backstage/plugin-todo-backend';
+import { catalogModuleUnprocessedEntities } from '@backstage/plugin-catalog-backend-module-unprocessed';
 
 const backend = createBackend();
 
@@ -60,4 +61,6 @@ backend.add(kubernetesPlugin());
 backend.add(permissionPlugin());
 backend.add(permissionModuleAllowAllPolicy());
 
+backend.add(catalogModuleUnprocessedEntities());
+
 backend.start();

packages/backend/package.json

@@ -40,6 +40,7 @@
     "@backstage/plugin-azure-sites-backend": "workspace:^",
     "@backstage/plugin-badges-backend": "workspace:^",
     "@backstage/plugin-catalog-backend": "workspace:^",
+    "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^",
     "@backstage/plugin-catalog-node": "workspace:^",
     "@backstage/plugin-code-coverage-backend": "workspace:^",
     "@backstage/plugin-devtools-backend": "workspace:^",

packages/backend/src/plugins/catalog.ts

@@ -19,6 +19,7 @@ import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backen
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 import { DemoEventBasedEntityProvider } from './DemoEventBasedEntityProvider';
+import { UnprocessedEntitesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -34,6 +35,13 @@ export default async function createPlugin(
   builder.addEntityProvider(demoProvider);
 
   const { processingEngine, router } = await builder.build();
+
+  const unprocessed = new UnprocessedEntitesModule(
+    await env.database.getClient(),
+    router,
+    env.logger,
+  );
+  unprocessed.registerRoutes();
   await processingEngine.start();
   return router;
 }

packages/catalog-model/examples/all-components.yaml

@@ -18,3 +18,4 @@ spec:
     - ./components/wayback-archive-ingestion-component.yaml
     - ./components/wayback-archive-storage-component.yaml
     - ./components/wayback-search-component.yaml
+    - ./components/invalid-component.yaml

packages/catalog-model/examples/components/invalid-component.yaml

@@ -0,0 +1,16 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: invalid component
+  description: An Invalid Component with a name that doesn't comply to the restrictions
+  labels:
+    goVersion: go1.15.3
+    category: music
+  tags:
+    - go
+    - go 1.15.3
+spec:
+  type: service
+  lifecycle: production
+  owner: user:guest
+  system: audio-playback

plugins/catalog-backend-module-unprocessed/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/catalog-backend-module-unprocessed/README.md

@@ -0,0 +1,38 @@
+# @backstage/plugin-catalog-backend-module-unprocessed-node
+
+This catalog-backend module adds support for viewing unprocessed entities. An unprocessed entity is one that doesn't show up in the catalog.
+
+A distinction is made between `failed` and `pending` entities.
+
+A `failed` entity has validation error that breaks processing.
+
+A `pending` entity has not been processed yet.
+
+## Installation
+
+### backend
+
+In `packages/backend/src/plugins/catalog.ts` import the module and initialize it after invoking `CatalogBuilder.build()`:
+
+```ts
+import { UnprocessedEntitesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
+
+//...
+
+const unprocessed = new UnprocessedEntitesModule(
+  await env.database.getClient(),
+  router,
+  env.logger,
+);
+unprocessed.registerRoutes();
+```
+
+### backend-next
+
+In `packages/backend-next/src/index.ts` add the module:
+
+```ts
+backend.add(catalogModuleUnprocessedEntities());
+```
+
+_This plugin was created through the Backstage CLI_

plugins/catalog-backend-module-unprocessed/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-unprocessed",
+  "description": "Backstage Catalog module to view unprocessed entities",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "devDependencies": {
+    "@backstage/cli": "workspace:^"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@backstage/backend-plugin-api": "workspace:^",
+    "@backstage/catalog-model": "workspace:^",
+    "@backstage/plugin-auth-node": "workspace:^",
+    "express-promise-router": "^4.1.1",
+    "knex": "^2.4.2"
+  }
+}

plugins/catalog-backend-module-unprocessed/src/index.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Node.js library for the catalog-backend-module-unprocessed plugin.
+ *
+ * @packageDocumentation
+ */
+
+export * from './module';
+export * from './plugin';
+export * from './types';

plugins/catalog-backend-module-unprocessed/src/module.ts

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  HydratedRefreshState,
+  RefreshState,
+  UnprocessedEntitiesRequest,
+  UnprocessedEntitiesResponse,
+} from './types';
+import { Knex } from 'knex';
+import {
+  HttpRouterService,
+  LoggerService,
+} from '@backstage/backend-plugin-api';
+import Router from 'express-promise-router';
+import { getBearerTokenFromAuthorizationHeader } from '@backstage/plugin-auth-node';
+
+export class UnprocessedEntitesModule {
+  private readonly moduleRouter;
+
+  constructor(
+    private readonly database: Knex,
+    private readonly router: HttpRouterService,
+    private readonly logger: LoggerService,
+  ) {
+    this.moduleRouter = Router();
+    this.router.use(this.moduleRouter);
+  }
+
+  private async unprocessed(
+    request: UnprocessedEntitiesRequest,
+  ): Promise<UnprocessedEntitiesResponse> {
+    if (request.reason === 'pending') {
+      return {
+        type: 'pending',
+        entities: await this.pending(request.owner),
+      };
+    }
+    return {
+      type: 'failed',
+      entities: await this.failed(request.owner),
+    };
+  }
+
+  private hydrateRefreshState(r: RefreshState): HydratedRefreshState {
+    return {
+      ...r,
+      unprocessed_entity: JSON.parse(r.unprocessed_entity),
+      ...(r.processed_entity && {
+        processed_entity: JSON.parse(r.processed_entity),
+      }),
+      ...(r.errors && { errors: JSON.parse(r.errors) }),
+      ...(r.cache && { cache: JSON.parse(r.cache) }),
+    };
+  }
+
+  private async pending(owner?: string): Promise<HydratedRefreshState[]> {
+    const res = (
+      await this.database('refresh_state.*')
+        .from('refresh_state')
+        .leftJoin(
+          'final_entities',
+          'final_entities.entity_id',
+          'refresh_state.entity_id',
+        )
+        .whereNull('final_entities.entity_id')
+    ).map(this.hydrateRefreshState);
+    if (owner) {
+      return res.filter(r => r.unprocessed_entity.spec?.owner === owner);
+    }
+
+    return res;
+  }
+
+  private async failed(owner?: string): Promise<HydratedRefreshState[]> {
+    const res = (
+      await this.database('refresh_state.*')
+        .from('refresh_state')
+        .rightJoin(
+          'final_entities',
+          'final_entities.entity_id',
+          'refresh_state.entity_id',
+        )
+        .whereNull('final_entities.final_entity')
+    ).map(this.hydrateRefreshState);
+
+    if (owner) {
+      return res.filter(r => r.unprocessed_entity.spec?.owner === owner);
+    }
+
+    return res;
+  }
+
+  registerRoutes() {
+    this.moduleRouter
+      .get('/entities/unprocessed/failed', async (req, res) => {
+        return res.json(
+          await this.unprocessed({
+            reason: 'failed',
+            owner: req.query.owner as string,
+            authorizationToken: getBearerTokenFromAuthorizationHeader(
+              req.header('authorization'),
+            ),
+          }),
+        );
+      })
+      .get('/entities/unprocessed/pending', async (req, res) => {
+        return res.json(
+          await this.unprocessed({
+            reason: 'pending',
+            owner: req.query.owner as string,
+            authorizationToken: getBearerTokenFromAuthorizationHeader(
+              req.header('authorization'),
+            ),
+          }),
+        );
+      });
+  }
+}

plugins/catalog-backend-module-unprocessed/src/plugin.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { UnprocessedEntitesModule } from './module';
+
+export const catalogModuleUnprocessedEntities = createBackendModule({
+  pluginId: 'catalog',
+  moduleId: 'catalogModuleUnprocessedEntities',
+  register(env) {
+    env.registerInit({
+      deps: {
+        database: coreServices.database,
+        router: coreServices.httpRouter,
+        logger: coreServices.logger,
+      },
+      async init({ database, router, logger }) {
+        const module = new UnprocessedEntitesModule(
+          await database.getClient(),
+          router,
+          logger,
+        );
+
+        module.registerRoutes();
+        logger.info(
+          'registered additional routes for catalogModuleUnprocessedEntities',
+        );
+      },
+    });
+  },
+});

plugins/catalog-backend-module-unprocessed/src/setupTests.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export {};

plugins/catalog-backend-module-unprocessed/src/types.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity } from '@backstage/catalog-model';
+
+export type HydratedRefreshState = {
+  entity_id: string;
+  entity_ref: string;
+  unprocessed_entity: Entity;
+  unprocessed_hash?: string;
+  processed_entity?: Entity;
+  result_hash?: string;
+  cache?: RefreshStateCache;
+  next_update_at: string | Date;
+  last_discovery_at: string | Date; // remove?
+  errors?: RefreshStateError[];
+  location_key?: string;
+};
+
+export type RefreshState = {
+  entity_id: string;
+  entity_ref: string;
+  unprocessed_entity: string;
+  unprocessed_hash?: string;
+  processed_entity?: string;
+  result_hash?: string;
+  cache?: string;
+  next_update_at: string | Date;
+  last_discovery_at: string | Date; // remove?
+  errors?: string;
+  location_key?: string;
+};
+export type RefreshStateCache = {
+  ttl: number;
+  cache: object;
+};
+
+export type RefreshStateError = {
+  name: string;
+  message: string;
+  cause: {
+    name: string;
+    message: string;
+    stack: string;
+  };
+};
+
+export interface UnprocessedEntitiesRequest {
+  reason: 'failed' | 'pending';
+  owner?: string;
+  authorizationToken?: string;
+}
+
+export interface UnprocessedEntitiesResponse {
+  type: 'pending' | 'failed';
+  entities: HydratedRefreshState[];
+}

plugins/catalog-unprocessed-entities/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/catalog-unprocessed-entities/README.md

@@ -0,0 +1,13 @@
+# catalog-unprocessed-entities
+
+Welcome to the catalog-unprocessed-entities plugin!
+
+_This plugin was created through the Backstage CLI_
+
+## Getting started
+
+Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/catalog-unprocessed-entities](http://localhost:3000/catalog-unprocessed-entities).
+
+You can also serve the plugin in isolation by running `yarn start` in the plugin directory.
+This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads.
+It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory.

plugins/catalog-unprocessed-entities/api-report.md

@@ -0,0 +1,24 @@
+## API Report File for "@backstage/plugin-catalog-unprocessed-entities"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+/// <reference types="react" />
+
+import { BackstagePlugin } from '@backstage/core-plugin-api';
+import { RouteRef } from '@backstage/core-plugin-api';
+
+// @public
+export const CatalogUnprocessedEntitiesPage: () => JSX.Element;
+
+// @public
+export const catalogUnprocessedEntitiesPlugin: BackstagePlugin<
+  {
+    root: RouteRef<undefined>;
+  },
+  {},
+  {}
+>;
+
+// (No @packageDocumentation comment for this package)
+```

plugins/catalog-unprocessed-entities/dev/index.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+import { createDevApp } from '@backstage/dev-utils';
+import {
+  catalogUnprocessedEntitiesPlugin,
+  CatalogUnprocessedEntitiesPage,
+} from '../src/plugin';
+
+createDevApp()
+  .registerPlugin(catalogUnprocessedEntitiesPlugin)
+  .addPage({
+    element: <CatalogUnprocessedEntitiesPage />,
+    title: 'Root Page',
+    path: '/catalog-unprocessed-entities',
+  })
+  .render();

plugins/catalog-unprocessed-entities/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@backstage/plugin-catalog-unprocessed-entities",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.esm.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "frontend-plugin"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/catalog-model": "workspace:^",
+    "@backstage/core-components": "workspace:^",
+    "@backstage/core-plugin-api": "workspace:^",
+    "@backstage/errors": "workspace:^",
+    "@backstage/theme": "workspace:^",
+    "@material-ui/core": "^4.9.13",
+    "@material-ui/icons": "^4.9.1",
+    "@material-ui/lab": "^4.0.0-alpha.60",
+    "react-use": "^17.2.4"
+  },
+  "peerDependencies": {
+    "react": "^16.13.1 || ^17.0.0"
+  },
+  "devDependencies": {
+    "@backstage/cli": "workspace:^",
+    "@backstage/core-app-api": "workspace:^",
+    "@backstage/dev-utils": "workspace:^",
+    "@backstage/test-utils": "workspace:^",
+    "@testing-library/jest-dom": "^5.10.1",
+    "@testing-library/react": "^12.1.3",
+    "@testing-library/user-event": "^14.0.0",
+    "@types/node": "*",
+    "cross-fetch": "^3.1.5",
+    "msw": "^1.0.0"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/catalog-unprocessed-entities/src/api/index.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { DiscoveryApi, createApiRef } from '@backstage/core-plugin-api';
+import { ResponseError } from '@backstage/errors';
+import { UnprocessedEntity } from '../types';
+
+/**
+ * {@link @backstage/core-plugin-api#ApiRef} for the {@link CatalogUnprocessedEntitiesApi}
+ *
+ * @public
+ */
+export const catalogUnprocessedEntitiesApiRef =
+  createApiRef<CatalogUnprocessedEntitiesApi>({
+    id: 'plugin.catalog-unprocessed-entities.service',
+  });
+
+/**
+ * API client for the Catalog Unprocessed Entities plugin
+ *
+ * @public
+ */
+export class CatalogUnprocessedEntitiesApi {
+  url: string = '';
+
+  constructor(public discovery: DiscoveryApi) {}
+
+  private async fetch<T>(path: string, init?: RequestInit): Promise<T> {
+    if (!this.url) {
+      this.url = await this.discovery.getBaseUrl('catalog');
+    }
+    const resp = await fetch(`${this.url}/${path}`, init);
+    if (!resp.ok) {
+      throw await ResponseError.fromResponse(resp);
+    }
+
+    return await resp.json();
+  }
+
+  async pending(): Promise<{ entities: UnprocessedEntity[] }> {
+    return await this.fetch('entities/unprocessed/pending');
+  }
+
+  async failed(): Promise<{ entities: UnprocessedEntity[] }> {
+    return await this.fetch('entities/unprocessed/failed');
+  }
+}

plugins/catalog-unprocessed-entities/src/components/EntityDialog.tsx

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React, { useState } from 'react';
+
+import Dialog from '@material-ui/core/Dialog';
+import DialogContent from '@material-ui/core/DialogContent';
+import DialogTitle from '@material-ui/core/DialogTitle';
+import IconButton from '@material-ui/core/IconButton';
+import { makeStyles, createStyles, Theme } from '@material-ui/core/styles';
+import CloseIcon from '@material-ui/icons/Close';
+import DescriptionIcon from '@material-ui/icons/Description';
+
+import { UnprocessedEntity } from './../types';
+import { CodeSnippet } from '@backstage/core-components';
+
+const useStyles = makeStyles((theme: Theme) =>
+  createStyles({
+    closeButton: {
+      position: 'absolute',
+      right: theme.spacing(1),
+      top: theme.spacing(1),
+      color: theme.palette.grey[500],
+    },
+    entity: {
+      overflow: 'scroll',
+      width: '100%',
+    },
+    codeBox: {
+      border: '1px solid black',
+      padding: '1em',
+    },
+  }),
+);
+
+export const EntityDialog = ({ entity }: { entity: UnprocessedEntity }) => {
+  const [open, setOpen] = useState(false);
+  const classes = useStyles();
+
+  const openDialog = () => {
+    setOpen(true);
+  };
+
+  const closeDialog = () => {
+    setOpen(false);
+  };
+
+  const dialogContent = () => {
+    return (
+      <CodeSnippet
+        language="json"
+        showLineNumbers
+        text={JSON.stringify(entity, null, 4)}
+      />
+    );
+  };
+
+  return (
+    <>
+      <IconButton color="primary" onClick={openDialog}>
+        <DescriptionIcon />
+      </IconButton>
+      <Dialog fullWidth open={open} onClose={closeDialog}>
+        <DialogTitle id="dialog-title">
+          <IconButton
+            aria-label="close"
+            className={classes.closeButton}
+            onClick={closeDialog}
+          >
+            <CloseIcon />
+          </IconButton>
+        </DialogTitle>
+        <DialogContent>{dialogContent()}</DialogContent>
+      </Dialog>
+    </>
+  );
+};

plugins/catalog-unprocessed-entities/src/components/FailedEntities.tsx

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+
+import {
+  ErrorPanel,
+  MarkdownContent,
+  Progress,
+  Table,
+  TableColumn,
+} from '@backstage/core-components';
+import { useApi } from '@backstage/core-plugin-api';
+import { BackstageThemeOptions } from '@backstage/theme';
+import { Box, Typography, makeStyles } from '@material-ui/core';
+
+import { UnprocessedEntity } from '../types';
+import { EntityDialog } from './EntityDialog';
+import { catalogUnprocessedEntitiesApiRef } from '../api';
+import useAsync from 'react-use/lib/useAsync';
+
+const useStyles = makeStyles((theme: BackstageThemeOptions) => ({
+  errorBox: {
+    color: theme.palette.status.error,
+    backgroundColor: theme.palette.errorBackground,
+    padding: '1em',
+    margin: '1em',
+    border: `1px solid ${theme.palette.status.error}`,
+  },
+  errorTitle: {
+    width: '100%',
+    fontWeight: 'bold',
+  },
+  successMessage: {
+    background: theme.palette.infoBackground,
+    color: theme.palette.infoText,
+  },
+}));
+
+const RenderErrorContext = ({
+  error,
+  rowData,
+}: {
+  error: { message: string };
+  rowData: UnprocessedEntity;
+}) => {
+  if (error.message.includes('tags.')) {
+    return (
+      <>
+        <Typography>Tags</Typography>
+        <ul>
+          {rowData.unprocessed_entity.metadata.tags?.map(t => (
+            <li>{t}</li>
+          ))}
+        </ul>
+      </>
+    );
+  }
+
+  if (error.message.includes('metadata.name')) {
+    return (
+      <>
+        <Typography>Name</Typography>
+        <Typography variant="caption">
+          {rowData.unprocessed_entity.metadata.name}
+        </Typography>
+      </>
+    );
+  }
+
+  return null;
+};
+
+export const FailedEntities = () => {
+  const classes = useStyles();
+  const unprocessedApi = useApi(catalogUnprocessedEntitiesApiRef);
+  const {
+    loading,
+    error,
+    value: data,
+  } = useAsync(async () => await unprocessedApi.failed());
+
+  if (loading) {
+    return <Progress />;
+  }
+  if (error) {
+    return <ErrorPanel error={error} />;
+  }
+
+  const columns: TableColumn[] = [
+    {
+      title: <Typography>entityRef</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).entity_ref,
+    },
+    {
+      title: <Typography>Kind</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).unprocessed_entity.kind,
+    },
+    {
+      title: <Typography>Owner</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).unprocessed_entity.spec?.owner ||
+        'unknown',
+    },
+    {
+      title: <Typography>Raw</Typography>,
+      render: (rowData: UnprocessedEntity | {}) => (
+        <EntityDialog entity={rowData as UnprocessedEntity} />
+      ),
+    },
+  ];
+  return (
+    <>
+      <Table
+        options={{ pageSize: 40, search: true }}
+        columns={columns}
+        data={data?.entities || []}
+        emptyContent={
+          <Typography className={classes.successMessage}>
+            No failed entities found
+          </Typography>
+        }
+        detailPanel={({ rowData }) => {
+          const errors = (rowData as UnprocessedEntity).errors;
+          return (
+            <>
+              {errors?.map(e => {
+                return (
+                  <Box className={classes.errorBox}>
+                    <Typography className={classes.errorTitle}>
+                      {e.name}
+                    </Typography>
+                    <MarkdownContent content={e.message} />
+                    <RenderErrorContext
+                      error={e}
+                      rowData={rowData as UnprocessedEntity}
+                    />
+                  </Box>
+                );
+              })}
+            </>
+          );
+        }}
+      />
+    </>
+  );
+};

plugins/catalog-unprocessed-entities/src/components/PendingEntities.tsx

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+
+import {
+  ErrorPanel,
+  Progress,
+  TableColumn,
+  Table,
+} from '@backstage/core-components';
+import { Typography, makeStyles } from '@material-ui/core';
+
+import { UnprocessedEntity } from '../types';
+
+import { EntityDialog } from './EntityDialog';
+import { useApi } from '@backstage/core-plugin-api';
+import useAsync from 'react-use/lib/useAsync';
+import { catalogUnprocessedEntitiesApiRef } from '../api';
+import { BackstageTheme } from '@backstage/theme';
+
+const useStyles = makeStyles((theme: BackstageTheme) => ({
+  successMessage: {
+    background: theme.palette.infoBackground,
+    color: theme.palette.infoText,
+    padding: theme.spacing(2),
+  },
+}));
+
+export const PendingEntities = () => {
+  const classes = useStyles();
+  const unprocessedApi = useApi(catalogUnprocessedEntitiesApiRef);
+  const {
+    loading,
+    error,
+    value: data,
+  } = useAsync(async () => await unprocessedApi.pending());
+
+  if (loading) {
+    return <Progress />;
+  }
+  if (error) {
+    return <ErrorPanel error={error} />;
+  }
+
+  const columns: TableColumn[] = [
+    {
+      title: <Typography>entityRef</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).entity_ref,
+    },
+    {
+      title: <Typography>Kind</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).unprocessed_entity.kind,
+    },
+    {
+      title: <Typography>Owner</Typography>,
+      render: (rowData: UnprocessedEntity | {}) =>
+        (rowData as UnprocessedEntity).unprocessed_entity.spec?.owner ||
+        'unknown',
+    },
+    {
+      title: <Typography>Raw</Typography>,
+      render: (rowData: UnprocessedEntity | {}) => (
+        <EntityDialog entity={rowData as UnprocessedEntity} />
+      ),
+    },
+  ];
+  return (
+    <>
+      <Table
+        options={{ pageSize: 40 }}
+        columns={columns}
+        data={data?.entities || []}
+        emptyContent={
+          <Typography className={classes.successMessage}>
+            No pending entities found
+          </Typography>
+        }
+      />
+    </>
+  );
+};

plugins/catalog-unprocessed-entities/src/components/UnprocessedEntities.tsx

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React, { useState } from 'react';
+
+import { Page, Header, Content } from '@backstage/core-components';
+import { Tab } from '@material-ui/core';
+import { TabContext, TabList, TabPanel } from '@material-ui/lab';
+
+import { FailedEntities } from './FailedEntities';
+import { PendingEntities } from './PendingEntities';
+
+export const UnprocessedEntitiesContent = () => {
+  const [tab, setTab] = useState('failed');
+  const handleChange = (_event: React.ChangeEvent<{}>, tabValue: string) => {
+    setTab(tabValue);
+  };
+
+  return (
+    <Content>
+      <TabContext value={tab}>
+        <TabList onChange={handleChange}>
+          <Tab label="Failed" value="failed" />
+          <Tab label="Pending" value="pending" />
+        </TabList>
+        <TabPanel value="failed">
+          <FailedEntities />
+        </TabPanel>
+        <TabPanel value="pending">
+          <PendingEntities />
+        </TabPanel>
+      </TabContext>
+    </Content>
+  );
+};
+
+export const UnprocessedEntities = () => {
+  return (
+    <Page themeId="tool">
+      <Header title="Unprocessed Entitites" />
+      <UnprocessedEntitiesContent />
+    </Page>
+  );
+};

plugins/catalog-unprocessed-entities/src/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export {
+  catalogUnprocessedEntitiesPlugin,
+  CatalogUnprocessedEntitiesPage,
+} from './plugin';

plugins/catalog-unprocessed-entities/src/plugin.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { catalogUnprocessedEntitiesPlugin } from './plugin';
+
+describe('catalog-unprocessed-entities', () => {
+  it('should export plugin', () => {
+    expect(catalogUnprocessedEntitiesPlugin).toBeDefined();
+  });
+});

plugins/catalog-unprocessed-entities/src/plugin.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  createApiFactory,
+  createPlugin,
+  createRoutableExtension,
+  discoveryApiRef,
+} from '@backstage/core-plugin-api';
+
+import { rootRouteRef } from './routes';
+import {
+  CatalogUnprocessedEntitiesApi,
+  catalogUnprocessedEntitiesApiRef,
+} from './api';
+
+/**
+ * Plugin entry point
+ *
+ * @public
+ */
+export const catalogUnprocessedEntitiesPlugin = createPlugin({
+  id: 'catalog-unprocessed-entities',
+  routes: {
+    root: rootRouteRef,
+  },
+  apis: [
+    createApiFactory({
+      api: catalogUnprocessedEntitiesApiRef,
+      deps: { discoveryApi: discoveryApiRef },
+      factory: ({ discoveryApi }) =>
+        new CatalogUnprocessedEntitiesApi(discoveryApi),
+    }),
+  ],
+});
+
+/**
+ * Tool page for the Catalog Unprocessed Entities Plugin
+ *
+ * @public
+ */
+export const CatalogUnprocessedEntitiesPage =
+  catalogUnprocessedEntitiesPlugin.provide(
+    createRoutableExtension({
+      name: 'CatalogUnprocessedEntitiesPage',
+      component: () =>
+        import('./components/UnprocessedEntities').then(
+          m => m.UnprocessedEntities,
+        ),
+      mountPoint: rootRouteRef,
+    }),
+  );

plugins/catalog-unprocessed-entities/src/routes.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { createRouteRef } from '@backstage/core-plugin-api';
+
+export const rootRouteRef = createRouteRef({
+  id: 'catalog-unprocessed-entities',
+});

plugins/catalog-unprocessed-entities/src/setupTests.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import '@testing-library/jest-dom';
+import 'cross-fetch/polyfill';

plugins/catalog-unprocessed-entities/src/types.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity } from '@backstage/catalog-model';
+
+export type UnprocessedEntity = {
+  entity_id: string;
+  entity_ref: string;
+  unprocessed_entity: Entity;
+  unprocessed_hash?: string;
+  processed_entity?: Entity;
+  result_hash?: string;
+  cache?: UnprocessedEntityCache;
+  next_update_at: string | Date;
+  last_discovery_at: string | Date; // remove?
+  errors?: UnprocessedEntityError[];
+  location_key?: string;
+};
+
+export type UnprocessedEntityCache = {
+  ttl: number;
+  cache: object;
+};
+
+export type UnprocessedEntityError = {
+  name: string;
+  message: string;
+  cause: {
+    name: string;
+    message: string;
+    stack: string;
+  };
+};