TechDocs: Make requestUrl and storageUrl optional configs by using discovery APIs

closes #3715

Co-authored-by: Himanshu Mishra <himanshu@orkohunter.net>

.changeset/gentle-scissors-compare.md

@@ -0,0 +1,7 @@
+---
+'@backstage/techdocs-common': patch
+'@backstage/plugin-techdocs': patch
+'@backstage/plugin-techdocs-backend': patch
+---
+
+`techdocs.requestUrl` and `techdocs.storageUrl` are now optional configs and the discovery API will be used to get the URL where techdocs plugin is hosted.

docs/features/techdocs/configuration.md

@@ -13,12 +13,12 @@ configuration options for TechDocs.
 # File: app-config.yaml
 
 techdocs:
-  # TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc.
+  # TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. (Optional)
 
   requestUrl: http://localhost:7000/api/techdocs
 
   # Just another route in techdocs-backend where TechDocs requests the static files from. This URL uses an HTTP middleware
-  # to serve files from either a local directory or an External storage provider.
+  # to serve files from either a local directory or an External storage provider. (Optional)
 
   storageUrl: http://localhost:7000/api/techdocs/static/docs
 

packages/techdocs-common/src/stages/publish/local.ts

@@ -23,6 +23,7 @@ import { Entity, EntityName } from '@backstage/catalog-model';
 import {
   resolvePackagePath,
   PluginEndpointDiscovery,
+  SingleHostDiscovery,
 } from '@backstage/backend-common';
 import { Config } from '@backstage/config';
 import {
@@ -109,9 +110,9 @@ export class LocalPublish implements PublisherBase {
 
   fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
     return new Promise((resolve, reject) => {
-      this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => {
+      this.discovery.getBaseUrl('techdocs').then(async techdocsApiUrl => {
         const storageUrl = new URL(
-          new URL(this.config.getString('techdocs.storageUrl')).pathname,
+          new URL(await this.getStorageUrl()).pathname,
           techdocsApiUrl,
         ).toString();
 
@@ -141,12 +142,20 @@ export class LocalPublish implements PublisherBase {
     return express.static(staticDocsDir);
   }
 
+  async getStorageUrl() {
+    const discoveryApi = SingleHostDiscovery.fromConfig(this.config);
+    return (
+      this.config.getOptionalString('techdocs.storageUrl') ??
+      (await discoveryApi.getBaseUrl('techdocs'))
+    );
+  }
+
   async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
     const namespace = entity.metadata.namespace ?? 'default';
     return new Promise(resolve => {
-      this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => {
+      this.discovery.getBaseUrl('techdocs').then(async techdocsApiUrl => {
         const storageUrl = new URL(
-          new URL(this.config.getString('techdocs.storageUrl')).pathname,
+          new URL(await this.getStorageUrl()).pathname,
           techdocsApiUrl,
         ).toString();
 

plugins/techdocs-backend/config.d.ts

@@ -24,7 +24,7 @@ export interface Config {
      * attr: 'storageUrl' - accepts a string value
      * e.g. storageUrl: http://localhost:7000/api/techdocs/static/docs
      */
-    storageUrl: string;
+    storageUrl?: string;
     /**
      * documentation building process depends on the builder attr
      * attr: 'builder' - accepts a string value

plugins/techdocs-backend/src/service/router.ts

@@ -26,7 +26,10 @@ import {
   PublisherBase,
   getLocationForEntity,
 } from '@backstage/techdocs-common';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import {
+  PluginEndpointDiscovery,
+  SingleHostDiscovery,
+} from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { getEntityNameFromUrlPath } from './helpers';
 import { DocsBuilder } from '../DocsBuilder';
@@ -102,7 +105,10 @@ export async function createRouter({
 
   router.get('/docs/:namespace/:kind/:name/*', async (req, res) => {
     const { kind, namespace, name } = req.params;
-    const storageUrl = config.getString('techdocs.storageUrl');
+    const discoveryApi = SingleHostDiscovery.fromConfig(config);
+    const storageUrl =
+      config.getOptionalString('techdocs.storageUrl') ??
+      `${await discoveryApi.getBaseUrl('techdocs')}/static/docs`;
 
     const catalogUrl = await discovery.getBaseUrl('catalog');
     const triple = [kind, namespace, name].map(encodeURIComponent).join('/');

plugins/techdocs/config.d.ts

@@ -22,12 +22,12 @@ export interface Config {
      * e.g. requestUrl: http://localhost:7000/api/techdocs
      * @visibility frontend
      */
-    requestUrl: string;
+    requestUrl?: string;
     /**
      * attr: 'storageUrl' - accepts a string value
      * e.g. storageUrl: http://localhost:7000/api/techdocs/static/docs
      */
-    storageUrl: string;
+    storageUrl?: string;
     /**
      * documentation building process depends on the builder attr
      * attr: 'builder' - accepts a string value

plugins/techdocs/dev/api.ts

@@ -13,20 +13,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { DiscoveryApi } from '@backstage/core';
+import { Config } from '@backstage/config';
 import { EntityName } from '@backstage/catalog-model';
 import { TechDocsStorage } from '../src/api';
 
 export class TechDocsDevStorageApi implements TechDocsStorage {
-  public apiOrigin: string;
+  public configApi: Config;
+  public discoveryApi: DiscoveryApi;
 
-  constructor({ apiOrigin }: { apiOrigin: string }) {
-    this.apiOrigin = apiOrigin;
+  constructor({
+    configApi,
+    discoveryApi,
+  }: {
+    configApi: Config;
+    discoveryApi: DiscoveryApi;
+  }) {
+    this.configApi = configApi;
+    this.discoveryApi = discoveryApi;
+  }
+
+  async getApiOrigin() {
+    return (
+      this.configApi.getOptionalString('techdocs.requestUrl') ??
+      (await this.discoveryApi.getBaseUrl('techdocs'))
+    );
   }
 
   async getEntityDocs(entityId: EntityName, path: string) {
     const { name } = entityId;
 
-    const url = `${this.apiOrigin}/${name}/${path}`;
+    const apiOrigin = await this.getApiOrigin();
+    const url = `${apiOrigin}/${name}/${path}`;
 
     const request = await fetch(
       `${url.endsWith('/') ? url : `${url}/`}index.html`,
@@ -39,8 +57,13 @@ export class TechDocsDevStorageApi implements TechDocsStorage {
     return request.text();
   }
 
-  getBaseUrl(oldBaseUrl: string, entityId: EntityName, path: string): string {
+  async getBaseUrl(
+    oldBaseUrl: string,
+    entityId: EntityName,
+    path: string,
+  ): Promise<string> {
     const { name } = entityId;
-    return new URL(oldBaseUrl, `${this.apiOrigin}/${name}/${path}`).toString();
+    const apiOrigin = await this.getApiOrigin();
+    return new URL(oldBaseUrl, `${apiOrigin}/${name}/${path}`).toString();
   }
 }

plugins/techdocs/dev/index.tsx

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { configApiRef, discoveryApiRef } from '@backstage/core';
 import { createDevApp } from '@backstage/dev-utils';
 import { plugin } from '../src/plugin';
 import { TechDocsDevStorageApi } from './api';
@@ -22,10 +23,11 @@ import { techdocsStorageApiRef } from '../src';
 createDevApp()
   .registerApi({
     api: techdocsStorageApiRef,
-    deps: {},
-    factory: () =>
+    deps: { configApi: configApiRef, discoveryApi: discoveryApiRef },
+    factory: ({ configApi, discoveryApi }) =>
       new TechDocsDevStorageApi({
-        apiOrigin: 'http://localhost:3000/api',
+        configApi,
+        discoveryApi,
       }),
   })
   .registerPlugin(plugin)

plugins/techdocs/package.json

@@ -31,6 +31,7 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@backstage/config": "^0.1.2",
     "@backstage/catalog-model": "^0.7.0",
     "@backstage/core": "^0.5.0",
     "@backstage/plugin-catalog-react": "^0.0.1",

plugins/techdocs/src/api.test.ts

@@ -13,10 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { useApi, configApiRef, UrlPatternDiscovery } from '@backstage/core';
 import { TechDocsStorageApi } from './api';
 
-const DOC_STORAGE_URL = 'https://example-storage.com';
-
 const mockEntity = {
   kind: 'Component',
   namespace: 'default',
@@ -24,19 +23,23 @@ const mockEntity = {
 };
 
 describe('TechDocsStorageApi', () => {
+  const mockBaseUrl = 'http://backstage:9191/api/techdocs';
+  const configApi = useApi(configApiRef);
+  const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
+
   it('should return correct base url based on defined storage', () => {
-    const storageApi = new TechDocsStorageApi({ apiOrigin: DOC_STORAGE_URL });
+    const storageApi = new TechDocsStorageApi({ configApi, discoveryApi });
 
     expect(storageApi.getBaseUrl('test.js', mockEntity, '')).toEqual(
-      `${DOC_STORAGE_URL}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test.js`,
+      `${mockBaseUrl}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test.js`,
     );
   });
 
   it('should return base url with correct entity structure', () => {
-    const storageApi = new TechDocsStorageApi({ apiOrigin: DOC_STORAGE_URL });
+    const storageApi = new TechDocsStorageApi({ configApi, discoveryApi });
 
     expect(storageApi.getBaseUrl('test/', mockEntity, '')).toEqual(
-      `${DOC_STORAGE_URL}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`,
+      `${mockBaseUrl}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`,
     );
   });
 });

plugins/techdocs/src/api.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { createApiRef } from '@backstage/core';
+import { createApiRef, DiscoveryApi } from '@backstage/core';
+import { Config } from '@backstage/config';
 import { EntityName } from '@backstage/catalog-model';
 import { TechDocsMetadata } from './types';
 
@@ -30,7 +31,11 @@ export const techdocsApiRef = createApiRef<TechDocsApi>({
 
 export interface TechDocsStorage {
   getEntityDocs(entityId: EntityName, path: string): Promise<string>;
-  getBaseUrl(oldBaseUrl: string, entityId: EntityName, path: string): string;
+  getBaseUrl(
+    oldBaseUrl: string,
+    entityId: EntityName,
+    path: string,
+  ): Promise<string>;
 }
 
 export interface TechDocs {
@@ -44,10 +49,25 @@ export interface TechDocs {
  * @property {string} apiOrigin Set to techdocs.requestUrl as the URL for techdocs-backend API
  */
 export class TechDocsApi implements TechDocs {
-  public apiOrigin: string;
+  public configApi: Config;
+  public discoveryApi: DiscoveryApi;
 
-  constructor({ apiOrigin }: { apiOrigin: string }) {
-    this.apiOrigin = apiOrigin;
+  constructor({
+    configApi,
+    discoveryApi,
+  }: {
+    configApi: Config;
+    discoveryApi: DiscoveryApi;
+  }) {
+    this.configApi = configApi;
+    this.discoveryApi = discoveryApi;
+  }
+
+  async getApiOrigin() {
+    return (
+      this.configApi.getOptionalString('techdocs.requestUrl') ??
+      (await this.discoveryApi.getBaseUrl('techdocs'))
+    );
   }
 
   /**
@@ -62,7 +82,8 @@ export class TechDocsApi implements TechDocs {
   async getTechDocsMetadata(entityId: EntityName) {
     const { kind, namespace, name } = entityId;
 
-    const requestUrl = `${this.apiOrigin}/metadata/techdocs/${namespace}/${kind}/${name}`;
+    const apiOrigin = await this.getApiOrigin();
+    const requestUrl = `${apiOrigin}/metadata/techdocs/${namespace}/${kind}/${name}`;
 
     const request = await fetch(`${requestUrl}`);
     const res = await request.json();
@@ -81,7 +102,8 @@ export class TechDocsApi implements TechDocs {
   async getEntityMetadata(entityId: EntityName) {
     const { kind, namespace, name } = entityId;
 
-    const requestUrl = `${this.apiOrigin}/metadata/entity/${namespace}/${kind}/${name}`;
+    const apiOrigin = await this.getApiOrigin();
+    const requestUrl = `${apiOrigin}/metadata/entity/${namespace}/${kind}/${name}`;
 
     const request = await fetch(`${requestUrl}`);
     const res = await request.json();
@@ -96,10 +118,25 @@ export class TechDocsApi implements TechDocs {
  * @property {string} apiOrigin Set to techdocs.requestUrl as the URL for techdocs-backend API
  */
 export class TechDocsStorageApi implements TechDocsStorage {
-  public apiOrigin: string;
+  public configApi: Config;
+  public discoveryApi: DiscoveryApi;
 
-  constructor({ apiOrigin }: { apiOrigin: string }) {
-    this.apiOrigin = apiOrigin;
+  constructor({
+    configApi,
+    discoveryApi,
+  }: {
+    configApi: Config;
+    discoveryApi: DiscoveryApi;
+  }) {
+    this.configApi = configApi;
+    this.discoveryApi = discoveryApi;
+  }
+
+  async getApiOrigin() {
+    return (
+      this.configApi.getOptionalString('techdocs.requestUrl') ??
+      (await this.discoveryApi.getBaseUrl('techdocs'))
+    );
   }
 
   /**
@@ -113,7 +150,8 @@ export class TechDocsStorageApi implements TechDocsStorage {
   async getEntityDocs(entityId: EntityName, path: string) {
     const { kind, namespace, name } = entityId;
 
-    const url = `${this.apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`;
+    const apiOrigin = await this.getApiOrigin();
+    const url = `${apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`;
 
     const request = await fetch(
       `${url.endsWith('/') ? url : `${url}/`}index.html`,
@@ -132,12 +170,17 @@ export class TechDocsStorageApi implements TechDocsStorage {
     return request.text();
   }
 
-  getBaseUrl(oldBaseUrl: string, entityId: EntityName, path: string): string {
+  async getBaseUrl(
+    oldBaseUrl: string,
+    entityId: EntityName,
+    path: string,
+  ): Promise<string> {
     const { kind, namespace, name } = entityId;
 
+    const apiOrigin = await this.getApiOrigin();
     return new URL(
       oldBaseUrl,
-      `${this.apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`,
+      `${apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`,
     ).toString();
   }
 }

plugins/techdocs/src/plugin.ts

@@ -34,6 +34,7 @@ import {
   createRouteRef,
   createApiFactory,
   configApiRef,
+  discoveryApiRef,
 } from '@backstage/core';
 import {
   techdocsStorageApiRef,
@@ -57,24 +58,25 @@ export const rootCatalogDocsRouteRef = createRouteRef({
   title: 'Docs',
 });
 
-// TODO: Use discovery API for frontend to get URL for techdocs-backend instead of requestUrl
 export const plugin = createPlugin({
   id: 'techdocs',
   apis: [
     createApiFactory({
       api: techdocsStorageApiRef,
-      deps: { configApi: configApiRef },
-      factory: ({ configApi }) =>
+      deps: { configApi: configApiRef, discoveryApi: discoveryApiRef },
+      factory: ({ configApi, discoveryApi }) =>
         new TechDocsStorageApi({
-          apiOrigin: configApi.getString('techdocs.requestUrl'),
+          configApi,
+          discoveryApi,
         }),
     }),
     createApiFactory({
       api: techdocsApiRef,
-      deps: { configApi: configApiRef },
-      factory: ({ configApi }) =>
+      deps: { configApi: configApiRef, discoveryApi: discoveryApiRef },
+      factory: ({ configApi, discoveryApi }) =>
         new TechDocsApi({
-          apiOrigin: configApi.getString('techdocs.requestUrl'),
+          configApi,
+          discoveryApi,
         }),
     }),
   ],

plugins/techdocs/src/reader/components/Reader.tsx

@@ -129,7 +129,7 @@ export const Reader = ({ entityId, onReady }: Props) => {
         },
       }),
       onCssReady({
-        docStorageUrl: techdocsStorageApi.apiOrigin,
+        docStorageUrl: techdocsStorageApi.getApiOrigin(),
         onLoading: (dom: Element) => {
           (dom as HTMLElement).style.setProperty('opacity', '0');
         },

plugins/techdocs/src/reader/components/TechDocsPage.test.tsx

@@ -56,7 +56,7 @@ describe('<TechDocsPage />', () => {
     };
     const techdocsStorageApi: Partial<TechDocsStorageApi> = {
       getEntityDocs: (): Promise<string> => Promise.resolve('String'),
-      getBaseUrl: (): string => '',
+      getBaseUrl: (): Promise<string> => Promise.resolve('String'),
     };
 
     const apiRegistry = ApiRegistry.from([

plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts

@@ -21,7 +21,7 @@ import { TechDocsStorage } from '../../api';
 const DOC_STORAGE_URL = 'https://example-host.storage.googleapis.com';
 
 const techdocsStorageApi: TechDocsStorage = {
-  getBaseUrl: jest.fn(() => DOC_STORAGE_URL),
+  getBaseUrl: () => new Promise(resolve => resolve(DOC_STORAGE_URL)),
   getEntityDocs: () => new Promise(resolve => resolve('yes!')),
 };
 

plugins/techdocs/src/reader/transformers/addBaseUrl.ts

@@ -35,12 +35,12 @@ export const addBaseUrl = ({
     ): void => {
       Array.from(list)
         .filter(elem => !!elem.getAttribute(attributeName))
-        .forEach((elem: T) => {
+        .forEach(async (elem: T) => {
           const elemAttribute = elem.getAttribute(attributeName);
           if (!elemAttribute) return;
           elem.setAttribute(
             attributeName,
-            techdocsStorageApi.getBaseUrl(elemAttribute, entityId, path),
+            await techdocsStorageApi.getBaseUrl(elemAttribute, entityId, path),
           );
         });
     };

plugins/techdocs/src/reader/transformers/onCssReady.test.ts

@@ -22,8 +22,9 @@ import {
 } from '../../test-utils';
 import { onCssReady } from '../transformers';
 
-const docStorageUrl: string =
-  'https://techdocs-mock-sites.storage.googleapis.com';
+const docStorageUrl: Promise<string> = Promise.resolve(
+  'https://techdocs-mock-sites.storage.googleapis.com',
+);
 
 const fixture = `
   <link rel="stylesheet" href="${docStorageUrl}/test.css" />

plugins/techdocs/src/reader/transformers/onCssReady.ts

@@ -17,7 +17,7 @@
 import type { Transformer } from './index';
 
 type OnCssReadyOptions = {
-  docStorageUrl: string;
+  docStorageUrl: Promise<string>;
   onLoading: (dom: Element) => void;
   onLoaded: (dom: Element) => void;
 };
@@ -30,7 +30,9 @@ export const onCssReady = ({
   return dom => {
     const cssPages = Array.from(
       dom.querySelectorAll('head > link[rel="stylesheet"]'),
-    ).filter(elem => elem.getAttribute('href')?.startsWith(docStorageUrl));
+    ).filter(async elem =>
+      elem.getAttribute('href')?.startsWith(await docStorageUrl),
+    );
 
     let count = cssPages.length;