add support for readTree

Signed-off-by: Calvin Lee <cjlee@ualberta.ca>

.changeset/orange-pianos-eat.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': minor
+'@backstage/integration': minor
+---
+
+Implemented `readTree` for Harness provider to support TechDocs functionality

docs/features/techdocs/README.md

@@ -70,6 +70,7 @@ See [TechDocs Architecture](architecture.md) to get an overview of where the bel
 | GitLab Enterprise            | Yes ✅         |
 | Gitea                        | Yes ✅         |
 | AWS CodeCommit               | Yes ✅         |
+| Harness Code                 | Yes ✅         |
 
 ### File storage providers
 

packages/backend-common/api-report.md

@@ -526,13 +526,18 @@ export class GitlabUrlReader implements UrlReader {
 
 // @public
 export class HarnessUrlReader implements UrlReader {
-  constructor(integration: HarnessIntegration);
+  constructor(
+    integration: HarnessIntegration,
+    deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+  );
   // (undocumented)
   static factory: ReaderFactory;
   // (undocumented)
   read(url: string): Promise<Buffer>;
   // (undocumented)
-  readTree(): Promise<ReadTreeResponse>;
+  readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
   // (undocumented)
   readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
   // (undocumented)

packages/backend-common/src/reading/HarnessUrlReader.test.ts

@@ -27,6 +27,9 @@ import { UrlReaderPredicateTuple } from './types';
 import { DefaultReadTreeResponseFactory } from './tree';
 import getRawBody from 'raw-body';
 import { HarnessUrlReader } from './HarnessUrlReader';
+import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import fs from 'fs-extra';
+import path from 'path';
 
 const treeResponseFactory = DefaultReadTreeResponseFactory.create({
   config: new ConfigReader({}),
@@ -49,6 +52,7 @@ const harnessProcessor = new HarnessUrlReader(
       }),
     ),
   ),
+  { treeResponseFactory },
 );
 
 const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
@@ -62,6 +66,7 @@ const responseBuffer = Buffer.from('Apache License');
 const harnessApiResponse = (content: any) => {
   return content;
 };
+const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2';
 
 const handlers = [
   rest.get(
@@ -95,6 +100,22 @@ const handlers = [
       );
     },
   ),
+  rest.get(
+    'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName',
+    (_req, res, ctx) => {
+      return res(
+        ctx.status(200),
+        ctx.set('Content-Type', 'application/json'),
+        ctx.json({ latest_commit: { sha: commitHash } }),
+      );
+    },
+  ),
+  rest.get(
+    'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName3/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName',
+    (_, res, ctx) => {
+      return res(ctx.status(404));
+    },
+  ),
 ];
 
 describe('HarnessUrlReader', () => {
@@ -170,7 +191,7 @@ describe('HarnessUrlReader', () => {
           'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml',
         ),
       ).rejects.toThrow(
-        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/404error.yaml?routingId=accountId&git_ref=refMain, 404 Not Found',
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/404error.yaml?routingId=accountId&git_ref=refs/heads/refMain, 404 Not Found',
       );
     });
 
@@ -180,8 +201,60 @@ describe('HarnessUrlReader', () => {
           'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml',
         ),
       ).rejects.toThrow(
-        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refMain, 500 Internal Server Error',
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refs/heads/refMain, 500 Internal Server Error',
       );
     });
   });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve(__dirname, '__fixtures__/mock-main.zip'),
+    );
+
+    it('should be able to get archive', async () => {
+      worker.use(
+        rest.get(
+          'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/archive/branchName.zip',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/gzip'),
+              ctx.set(
+                'content-disposition',
+                'attachment; filename=backstage-mock.zip',
+              ),
+              ctx.body(repoBuffer),
+            );
+          },
+        ),
+      );
+
+      const response = await harnessProcessor.readTree(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName',
+      );
+      expect(response.etag).toBe(commitHash);
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+    });
+
+    it('should return not modified', async () => {
+      await expect(
+        harnessProcessor.readTree(
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName2',
+          {
+            etag: commitHash,
+          },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return not found', async () => {
+      await expect(
+        harnessProcessor.readTree(
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName3/projects/projectName/repos/repoName/files/branchName3',
+        ),
+      ).rejects.toThrow(NotFoundError);
+    });
+  });
 });

packages/backend-common/src/reading/HarnessUrlReader.ts

@@ -18,12 +18,18 @@ import {
   getHarnessFileContentsUrl,
   HarnessIntegration,
   ScmIntegrations,
+  getHarnessLatestCommitUrl,
+  getHarnessArchiveUrl,
+  parseHarnessUrl,
 } from '@backstage/integration';
-import { ReadUrlOptions, ReadUrlResponse } from './types';
 import {
   ReaderFactory,
+  ReadTreeOptions,
   ReadTreeResponse,
   SearchResponse,
+  ReadTreeResponseFactory,
+  ReadUrlOptions,
+  ReadUrlResponse,
   UrlReader,
 } from './types';
 import fetch, { Response } from 'node-fetch';
@@ -42,11 +48,13 @@ import { Readable } from 'stream';
  * @public
  */
 export class HarnessUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     return ScmIntegrations.fromConfig(config)
       .harness.list()
       .map(integration => {
-        const reader = new HarnessUrlReader(integration);
+        const reader = new HarnessUrlReader(integration, {
+          treeResponseFactory,
+        });
         const predicate = (url: URL) => {
           return url.host === integration.config.host;
         };
@@ -54,8 +62,12 @@ export class HarnessUrlReader implements UrlReader {
       });
   };
 
-  constructor(private readonly integration: HarnessIntegration) {}
-
+  constructor(
+    private readonly integration: HarnessIntegration,
+    private readonly deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+  ) {}
   async read(url: string): Promise<Buffer> {
     const response = await this.readUrl(url);
     return response.buffer();
@@ -109,8 +121,37 @@ export class HarnessUrlReader implements UrlReader {
     throw new Error(message);
   }
 
-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('HarnessUrlReader readTree not implemented.');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const lastCommitHash = await this.getLastCommitHash(url);
+
+    if (options?.etag && options.etag === lastCommitHash) {
+      throw new NotModifiedError();
+    }
+
+    const archiveUri = getHarnessArchiveUrl(this.integration.config, url);
+
+    let response: Response;
+    try {
+      response = await fetch(archiveUri, {
+        method: 'GET',
+        ...getHarnessRequestOptions(this.integration.config),
+        signal: options?.signal as any,
+      });
+    } catch (e) {
+      throw new Error(`Unable to read ${archiveUri}, ${e}`);
+    }
+
+    const parsedUri = parseHarnessUrl(this.integration.config, url);
+
+    return this.deps.treeResponseFactory.fromZipArchive({
+      stream: Readable.from(response.body),
+      subpath: parsedUri.path,
+      etag: lastCommitHash,
+      filter: options?.filter,
+    });
   }
   search(): Promise<SearchResponse> {
     throw new Error('HarnessUrlReader search not implemented.');
@@ -122,4 +163,21 @@ export class HarnessUrlReader implements UrlReader {
       this.integration.config.token || this.integration.config.apiKey,
     )}}`;
   }
+  private async getLastCommitHash(url: string): Promise<string> {
+    const commitUri = getHarnessLatestCommitUrl(this.integration.config, url);
+
+    const response = await fetch(
+      commitUri,
+      getHarnessRequestOptions(this.integration.config),
+    );
+    if (!response.ok) {
+      const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    return (await response.json()).latest_commit.sha;
+  }
 }

packages/integration/api-report.md

@@ -513,12 +513,24 @@ export function getGitLabRequestOptions(config: GitLabIntegrationConfig): {
   headers: Record<string, string>;
 };
 
+// @public
+export function getHarnessArchiveUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+): string;
+
 // @public
 export function getHarnessFileContentsUrl(
   config: HarnessIntegrationConfig,
   url: string,
 ): string;
 
+// @public
+export function getHarnessLatestCommitUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+): string;
+
 // @public
 export function getHarnessRequestOptions(config: HarnessIntegrationConfig): {
   headers?: Record<string, string>;
@@ -764,6 +776,21 @@ export function parseGiteaUrl(
   path: string;
 };
 
+// @public
+export function parseHarnessUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+): {
+  baseUrl: string;
+  accountId: string;
+  orgName: string;
+  projectName: string;
+  refString: string;
+  repoName: string;
+  path: string;
+  refDashStr: string;
+};
+
 // @public
 export type PersonalAccessTokenCredential = AzureCredentialBase & {
   kind: 'PersonalAccessToken';

packages/integration/src/harness/core.test.ts

@@ -18,9 +18,12 @@ import { setupServer } from 'msw/node';
 import { setupRequestMockHandlers } from '../helpers';
 import { HarnessIntegrationConfig } from './config';
 import {
+  getHarnessArchiveUrl,
   getHarnessEditContentsUrl,
   getHarnessFileContentsUrl,
+  getHarnessLatestCommitUrl,
   getHarnessRequestOptions,
+  parseHarnessUrl,
 } from './core';
 
 describe('Harness code core', () => {
@@ -38,7 +41,7 @@ describe('Harness code core', () => {
           'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml',
         ),
       ).toEqual(
-        'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refMain',
+        'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refs/heads/refMain',
       );
     });
   });
@@ -59,6 +62,38 @@ describe('Harness code core', () => {
     });
   });
 
+  describe('getHarnessArchiveUrl', () => {
+    it('can create an url from arguments', () => {
+      const config: HarnessIntegrationConfig = {
+        host: 'app.harness.io',
+      };
+      expect(
+        getHarnessArchiveUrl(
+          config,
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName',
+        ),
+      ).toEqual(
+        'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/archive/branchName.zip',
+      );
+    });
+  });
+
+  describe('getGiteaLatestCommitUrl', () => {
+    it('can create an url from arguments', () => {
+      const config: HarnessIntegrationConfig = {
+        host: 'app.harness.io',
+      };
+      expect(
+        getHarnessLatestCommitUrl(
+          config,
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName',
+        ),
+      ).toEqual(
+        'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName',
+      );
+    });
+  });
+
   describe('getHarnessRequestOptions', () => {
     it('adds token header when only a token is specified', () => {
       const authRequest: HarnessIntegrationConfig = {
@@ -88,4 +123,50 @@ describe('Harness code core', () => {
       ).toEqual('a');
     });
   });
+
+  describe('parseHarnessUrl', () => {
+    it('can fetch harness url', () => {
+      const config: HarnessIntegrationConfig = {
+        host: 'app.harness.io',
+      };
+      expect(
+        parseHarnessUrl(
+          config,
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName',
+        ),
+      ).toEqual({
+        accountId: 'accountId',
+        baseUrl: 'https://app.harness.io',
+        orgName: 'orgName',
+        path: '',
+        projectName: 'projectName',
+        refDashStr: '',
+        refString: '',
+        repoName: 'repoName',
+        branch: 'branchName',
+      });
+    });
+
+    it('provide path without starting slash', () => {
+      const config: HarnessIntegrationConfig = {
+        host: 'app.harness.io',
+      };
+      expect(
+        parseHarnessUrl(
+          config,
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName/simple/path',
+        ),
+      ).toEqual({
+        accountId: 'accountId',
+        baseUrl: 'https://app.harness.io',
+        orgName: 'orgName',
+        path: 'path',
+        projectName: 'projectName',
+        refDashStr: 'branchName-simple',
+        refString: 'branchName/simple',
+        repoName: 'repoName',
+        branch: 'branchName/simple/path',
+      });
+    });
+  });
 });

packages/integration/src/harness/core.ts

@@ -33,31 +33,8 @@ export function getHarnessEditContentsUrl(
   config: HarnessIntegrationConfig,
   url: string,
 ) {
-  try {
-    const baseUrl = `https://${config.host}`;
-    const [
-      _blank,
-      _ng,
-      _account,
-      accountId,
-      _module,
-      _moduleName,
-      _org,
-      orgName,
-      _projects,
-      projectName,
-      _repos,
-      repoName,
-      _files,
-      _ref,
-      _branch,
-      ...path
-    ] = url.replace(baseUrl, '').split('/');
-    const pathWithoutSlash = path.join('/').replace(/^\//, '');
-    return `${baseUrl}/gateway/code/api/v1/repos/${accountId}/${orgName}/${projectName}/${repoName}/+/edit/${pathWithoutSlash}`;
-  } catch (e) {
-    throw new Error(`Incorrect URL: ${url}, ${e}`);
-  }
+  const parsedUrl = parseHarnessUrl(config, url);
+  return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/edit/${parsedUrl.path}`;
 }
 
 /**
@@ -77,35 +54,52 @@ export function getHarnessFileContentsUrl(
   config: HarnessIntegrationConfig,
   url: string,
 ) {
-  try {
-    const baseUrl = `https://${config.host}`;
-    const [
-      _blank,
-      _ng,
-      _account,
-      accountId,
-      _module,
-      _moduleName,
-      _org,
-      orgName,
-      _projects,
-      projectName,
-      _repos,
-      repoName,
-      _files,
-      _ref,
-      _branch,
-      ...path
-    ] = url.replace(baseUrl, '').split('/');
-    const urlParts = url.replace(baseUrl, '').split('/');
-    const refAndPath = urlParts.slice(13);
-    const refIndex = refAndPath.findIndex(item => item === '~');
-    const refString = refAndPath.slice(0, refIndex);
-    const pathWithoutSlash = path.join('/').replace(/^\//, '');
-    return `${baseUrl}/gateway/code/api/v1/repos/${accountId}/${orgName}/${projectName}/${repoName}/+/raw/${pathWithoutSlash}?routingId=${accountId}&git_ref=${refString}`;
-  } catch (e) {
-    throw new Error(`Incorrect URL: ${url}, ${e}`);
-  }
+  const parsedUrl = parseHarnessUrl(config, url);
+  return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/raw/${parsedUrl.path}?routingId=${parsedUrl.accountId}&git_ref=refs/heads/${parsedUrl.refString}`;
+}
+
+/**
+ * Given a URL pointing to a repository/path, returns a URL
+ * for archive contents of the repository.
+ *
+ * @remarks
+ *
+ * Converts
+ * from: https://gitea.com/a/b/src/branchname
+ * to:   https://gitea.com/api/v1/repos/a/b/archive/branchname.zip
+ *
+ * @param url - A URL pointing to a repository/path
+ * @param config - The relevant provider config
+ * @public
+ */
+export function getHarnessArchiveUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+) {
+  const parsedUrl = parseHarnessUrl(config, url);
+  return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/archive/${parsedUrl.branch}.zip`;
+}
+
+/**
+ * Given a URL pointing to a repository branch, returns a URL
+ * for latest commit information.
+ *
+ * @remarks
+ *
+ * Converts
+ * from: https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName
+ * to:   https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName
+ *
+ * @param url - A URL pointing to a repository branch
+ * @param config - The relevant provider config
+ * @public
+ */
+export function getHarnessLatestCommitUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+) {
+  const parsedUrl = parseHarnessUrl(config, url);
+  return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/content?routingId=${parsedUrl.accountId}&include_commit=true&git_ref=refs/heads/${parsedUrl.refString}`;
 }
 
 /**
@@ -130,3 +124,73 @@ export function getHarnessRequestOptions(config: HarnessIntegrationConfig): {
     headers,
   };
 }
+
+/**
+ * Return parsed git url properties.
+ *
+ * @param config - A Gitea provider config
+ * @param url - A URL pointing to a repository
+ * @public
+ */
+export function parseHarnessUrl(
+  config: HarnessIntegrationConfig,
+  url: string,
+): {
+  baseUrl: string;
+  accountId: string;
+  orgName: string;
+  projectName: string;
+  refString: string;
+  repoName: string;
+  path: string;
+  refDashStr: string;
+  branch: string;
+} {
+  const baseUrl = `https://${config.host}`;
+  try {
+    const pathUrl = new URL(url);
+    const pathSegments = pathUrl.pathname
+      .split('/')
+      .filter(segment => segment !== '');
+    const urlParts = pathUrl.pathname.split('/');
+    const [
+      _ng,
+      _account,
+      accountId,
+      _module,
+      _moduleName,
+      _org,
+      orgName,
+      _projects,
+      projectName,
+      _repos,
+      repoName,
+      _files,
+      _ref,
+      _branch,
+      ..._path
+    ] = pathSegments;
+    const refAndPath = urlParts.slice(
+      urlParts.findIndex(i => i === 'files') + 1,
+    );
+    const refIndex = refAndPath.findIndex(item => item === '~');
+    const refString = refAndPath.slice(0, refIndex).join('/');
+    const pathWithoutSlash = refAndPath
+      .slice(refIndex + 1)
+      .join('/')
+      .replace(/^\//, '');
+    return {
+      baseUrl: baseUrl,
+      accountId: accountId,
+      orgName: orgName,
+      projectName: projectName,
+      refString: refString,
+      path: pathWithoutSlash,
+      repoName: repoName,
+      refDashStr: refAndPath.slice(0, refIndex).join('-'),
+      branch: refAndPath.join('/'),
+    };
+  } catch (e) {
+    throw new Error(`Incorrect URL: ${url}, ${e}`);
+  }
+}

packages/integration/src/harness/index.ts

@@ -14,6 +14,12 @@
  * limitations under the License.
  */
 export { HarnessIntegration } from './HarnessIntegration';
-export { getHarnessRequestOptions, getHarnessFileContentsUrl } from './core';
+export {
+  getHarnessRequestOptions,
+  getHarnessFileContentsUrl,
+  getHarnessArchiveUrl,
+  getHarnessLatestCommitUrl,
+  parseHarnessUrl,
+} from './core';
 export { readHarnessConfig } from './config';
 export type { HarnessIntegrationConfig } from './config';