feat(gerrit-urlreader): support fetching content from a commit

Signed-off-by: Niklas Aronsson <niklasar@axis.com>

.changeset/light-sloths-report.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+The `GerritUrlReader` can now read content from a commit and not only from the top of a branch. The
+Gitiles URL must contain the full commit `SHA` hash like: `https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/catalog-info.yaml`.

.changeset/old-waves-camp.md

@@ -0,0 +1,13 @@
+---
+'@backstage/integration': patch
+---
+
+A new Gerrit helper function (`buildGerritGitilesArchiveUrlFromLocation`) has been added. This
+constructs a Gitiles URL to download an archive. It is similar to the existing
+`buildGerritGitilesArchiveUrl` but also support content referenced by a full commit `SHA`.
+
+**DEPRECATIONS**: The function `buildGerritGitilesArchiveUrl` is deprecated, use the
+`buildGerritGitilesArchiveUrlFromLocation` function instead.
+
+**DEPRECATIONS**: The function `parseGerritGitilesUrl` is deprecated, use the
+`parseGitilesUrlRef` function instead.

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts

@@ -20,7 +20,7 @@ import {
   registerMswTestHooks,
 } from '@backstage/backend-test-utils';
 import { ConfigReader } from '@backstage/config';
-import { NotModifiedError, NotFoundError } from '@backstage/errors';
+import { NotModifiedError } from '@backstage/errors';
 import {
   GerritIntegration,
   readGerritIntegrationConfig,
@@ -36,22 +36,11 @@ import { GerritUrlReader } from './GerritUrlReader';
 import getRawBody from 'raw-body';
 
 const mockDir = createMockDirectory({ mockOsTmpDir: true });
-const env = process.env;
-process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' };
 
 const treeResponseFactory = DefaultReadTreeResponseFactory.create({
   config: new ConfigReader({}),
 });
 
-const cloneMock = jest.fn(() => Promise.resolve());
-jest.mock('./git', () => ({
-  Git: {
-    fromAuth: () => ({
-      clone: cloneMock,
-    }),
-  },
-}));
-
 // Gerrit processor without a gitilesBaseUrl configured
 const gerritProcessor = new GerritUrlReader(
   new GerritIntegration(
@@ -96,12 +85,10 @@ describe.skip('GerritUrlReader', () => {
 
   beforeEach(() => {
     mockDir.clear();
-    process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' };
   });
 
   afterAll(() => {
     jest.clearAllMocks();
-    process.env = env;
   });
 
   describe('reader factory', () => {
@@ -186,7 +173,24 @@ describe.skip('GerritUrlReader', () => {
       const buffer = await result.buffer();
       expect(buffer.toString()).toBe(responseBuffer.toString());
     });
-
+    it('should be able to read file contents of a commit as buffer', async () => {
+      worker.use(
+        rest.get(
+          'https://gerrit.com/projects/web%2Fproject/commits/f775f9119c313c7ffc890d7908a45997273434d5/files/LICENSE/content',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.body(responseBuffer.toString('base64')),
+            );
+          },
+        ),
+      );
+      const result = await gerritProcessor.readUrl(
+        'https://gerrit.com/web/project/+/f775f9119c313c7ffc890d7908a45997273434d5/LICENSE',
+      );
+      const buffer = await result.buffer();
+      expect(buffer.toString()).toBe(responseBuffer.toString());
+    });
     it('should be able to read file contents as stream', async () => {
       worker.use(
         rest.get(
@@ -258,6 +262,7 @@ describe.skip('GerritUrlReader', () => {
     const treeUrlGitiles =
       'https://gerrit.com/gitiles/app/web/+/refs/heads/master/';
     const etag = '52432507a70b677b5674b019c9a46b2e9f29d0a1';
+    const sha = 'f775f9119c313c7ffc890d7908a45997273434d5';
     const mkdocsContent = 'a repo fetched using git clone';
     const mdContent = 'doc';
     const repoArchiveBuffer = fs.readFileSync(
@@ -302,6 +307,19 @@ describe.skip('GerritUrlReader', () => {
               ctx.body(repoArchiveDocsBuffer),
             ),
         ),
+        rest.get(
+          `https://gerrit.com/gitiles/app/web/\\+archive/${sha}.tar.gz`,
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.set(
+                'content-disposition',
+                'attachment; filename=web-refs/heads/master.tar.gz',
+              ),
+              ctx.body(repoArchiveBuffer),
+            ),
+        ),
       );
     });
 
@@ -330,8 +348,6 @@ describe.skip('GerritUrlReader', () => {
 
       const mdFile = await files[1].content();
       expect(mdFile.toString()).toBe('site_name: Test\n');
-
-      expect(cloneMock).not.toHaveBeenCalled();
     });
 
     it('throws NotModifiedError for matching etags.', async () => {
@@ -346,15 +362,17 @@ describe.skip('GerritUrlReader', () => {
       );
     });
 
-    it('throws NotFoundError if branch info not found.', async () => {
+    it('throws ResponseError if branch info not found.', async () => {
       worker.use(
         rest.get(branchAPIUrl, (_, res, ctx) => {
           return res(ctx.status(404, 'Not found.'));
         }),
       );
 
-      await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow(
-        NotFoundError,
+      await expect(
+        gerritProcessor.readTree(treeUrl),
+      ).rejects.toMatchInlineSnapshot(
+        `[ResponseError: Request failed with 404 Not found.]`,
       );
     });
 
@@ -386,8 +404,20 @@ describe.skip('GerritUrlReader', () => {
 
       const mdFile = await files[0].content();
       expect(mdFile.toString()).toBe('# Test\n');
+    });
+    it('throws NotModifiedError for a known commit.', async () => {
+      const shaTreeUrl = `https://gerrit.com/app/web/+/${sha}/`;
 
-      expect(cloneMock).not.toHaveBeenCalled();
+      await expect(
+        gerritProcessor.readTree(shaTreeUrl, { etag: sha }),
+      ).rejects.toThrow(NotModifiedError);
+    });
+    it('can fetch files for a specifc sha.', async () => {
+      const response = await gerritProcessorWithGitiles.readTree(
+        `https://gerrit.com/gitiles/app/web/+/${sha}/`,
+      );
+
+      expect(response.etag).toBe(sha);
     });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts

@@ -28,14 +28,18 @@ import { Readable } from 'stream';
 import {
   GerritIntegration,
   ScmIntegrations,
-  buildGerritGitilesArchiveUrl,
+  buildGerritGitilesArchiveUrlFromLocation,
   getGerritBranchApiUrl,
   getGerritFileContentsApiUrl,
   getGerritRequestOptions,
-  parseGerritGitilesUrl,
   parseGerritJsonResponse,
+  parseGitilesUrlRef,
 } from '@backstage/integration';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  NotFoundError,
+  NotModifiedError,
+  ResponseError,
+} from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 
 /**
@@ -135,34 +139,9 @@ export class GerritUrlReader implements UrlReaderService {
     url: string,
     options?: UrlReaderServiceReadTreeOptions,
   ): Promise<UrlReaderServiceReadTreeResponse> {
-    const apiUrl = getGerritBranchApiUrl(this.integration.config, url);
-    let response: Response;
-    try {
-      response = await fetch(apiUrl, {
-        method: 'GET',
-        ...getGerritRequestOptions(this.integration.config),
-      });
-    } catch (e) {
-      throw new Error(`Unable to read branch state ${url}, ${e}`);
-    }
+    const urlRevision = await this.getRevisionForUrl(url, options);
 
-    if (response.status === 404) {
-      throw new NotFoundError(`Not found: ${url}`);
-    }
-
-    if (!response.ok) {
-      throw new Error(
-        `${url} could not be read as ${apiUrl}, ${response.status} ${response.statusText}`,
-      );
-    }
-    const branchInfo = (await parseGerritJsonResponse(response as any)) as {
-      revision: string;
-    };
-    if (options?.etag === branchInfo.revision) {
-      throw new NotModifiedError();
-    }
-
-    return this.readTreeFromGitiles(url, branchInfo.revision, options);
+    return this.readTreeFromGitiles(url, urlRevision, options);
   }
 
   async search(): Promise<UrlReaderServiceSearchResponse> {
@@ -179,16 +158,10 @@ export class GerritUrlReader implements UrlReaderService {
     revision: string,
     options?: UrlReaderServiceReadTreeOptions,
   ) {
-    const { branch, filePath, project } = parseGerritGitilesUrl(
+    const archiveUrl = buildGerritGitilesArchiveUrlFromLocation(
       this.integration.config,
       url,
     );
-    const archiveUrl = buildGerritGitilesArchiveUrl(
-      this.integration.config,
-      project,
-      branch,
-      filePath,
-    );
     const archiveResponse = await fetch(archiveUrl, {
       ...getGerritRequestOptions(this.integration.config),
       // TODO(freben): The signal cast is there because pre-3.x versions of
@@ -217,4 +190,41 @@ export class GerritUrlReader implements UrlReaderService {
       stripFirstDirectory: false,
     });
   }
+
+  private async getRevisionForUrl(
+    url: string,
+    options?: UrlReaderServiceReadTreeOptions,
+  ): Promise<string> {
+    const { ref, refType } = parseGitilesUrlRef(this.integration.config, url);
+    // The url points to a static revision.
+    if (refType === 'sha') {
+      if (options?.etag === ref) {
+        throw new NotModifiedError();
+      }
+      return ref;
+    }
+
+    const apiUrl = getGerritBranchApiUrl(this.integration.config, url);
+    let response: Response;
+    try {
+      response = await fetch(apiUrl, {
+        method: 'GET',
+        ...getGerritRequestOptions(this.integration.config),
+      });
+    } catch (e) {
+      throw new Error(`Unable to read branch state ${url}, ${e}`);
+    }
+
+    if (!response.ok) {
+      throw await ResponseError.fromResponse(response);
+    }
+
+    const branchInfo = (await parseGerritJsonResponse(response as any)) as {
+      revision: string;
+    };
+    if (options?.etag === branchInfo.revision) {
+      throw new NotModifiedError();
+    }
+    return branchInfo.revision;
+  }
 }

packages/integration/report.api.md

@@ -299,7 +299,7 @@ export type BitbucketServerIntegrationConfig = {
   password?: string;
 };
 
-// @public
+// @public @deprecated
 export function buildGerritGitilesArchiveUrl(
   config: GerritIntegrationConfig,
   project: string,
@@ -307,6 +307,12 @@ export function buildGerritGitilesArchiveUrl(
   filePath: string,
 ): string;
 
+// @public
+export function buildGerritGitilesArchiveUrlFromLocation(
+  config: GerritIntegrationConfig,
+  url: string,
+): string;
+
 // @public
 export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
   static fromIntegrations(
@@ -802,7 +808,7 @@ export interface IntegrationsByType {
   harness: ScmIntegrationsGroup<HarnessIntegration>;
 }
 
-// @public
+// @public @deprecated
 export function parseGerritGitilesUrl(
   config: GerritIntegrationConfig,
   url: string,

packages/integration/src/gerrit/core.test.ts

@@ -21,6 +21,7 @@ import { registerMswTestHooks } from '../helpers';
 import { GerritIntegrationConfig } from './config';
 import {
   buildGerritGitilesArchiveUrl,
+  buildGerritGitilesArchiveUrlFromLocation,
   buildGerritGitilesUrl,
   getGerritBranchApiUrl,
   getGerritCloneRepoUrl,
@@ -115,6 +116,94 @@ describe('gerrit core', () => {
     });
   });
 
+  describe('buildGerritGitilesArchiveUrlFromLocation', () => {
+    const config: GerritIntegrationConfig = {
+      host: 'gerrit.com',
+      baseUrl: 'https://gerrit.com',
+      gitilesBaseUrl: 'https://gerrit.com/gitiles',
+    };
+    const configWithPath: GerritIntegrationConfig = {
+      host: 'gerrit.com',
+      baseUrl: 'https://gerrit.com/gerrit',
+      gitilesBaseUrl: 'https://gerrit.com/gerrit/plugins/gitiles',
+    };
+    const configWithDedicatedGitiles: GerritIntegrationConfig = {
+      host: 'gerrit.com',
+      baseUrl: 'https://gerrit.com/gerrit',
+      gitilesBaseUrl: 'https://dedicated-gitiles-server.com/gerrit/gitiles',
+    };
+    it('can create an archive url for a branch', () => {
+      expect(
+        buildGerritGitilesArchiveUrlFromLocation(
+          config,
+          'https://gerrit.com/gitiles/repo/+/refs/heads/dev/',
+        ),
+      ).toEqual(
+        'https://gerrit.com/gitiles/repo/+archive/refs/heads/dev.tar.gz',
+      );
+    });
+    it('can create an archive url for a sha', () => {
+      expect(
+        buildGerritGitilesArchiveUrlFromLocation(
+          config,
+          'https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/',
+        ),
+      ).toEqual(
+        'https://gerrit.com/gitiles/repo/+archive/2846e8dc327ae2f60249983b1c3b96f42f205bae.tar.gz',
+      );
+    });
+    it('can create an archive url for a sha with a specific directory', () => {
+      expect(
+        buildGerritGitilesArchiveUrlFromLocation(
+          config,
+          'https://gerrit.com/gitiles/repo/+/2846e8dc327ae2f60249983b1c3b96f42f205bae/docs',
+        ),
+      ).toEqual(
+        'https://gerrit.com/gitiles/repo/+archive/2846e8dc327ae2f60249983b1c3b96f42f205bae/docs.tar.gz',
+      );
+    });
+    it('can create an archive url for a specific directory', () => {
+      expect(
+        buildGerritGitilesArchiveUrlFromLocation(
+          config,
+          'https://gerrit.com/gitiles/repo/+/refs/heads/dev/docs/',
+        ),
+      ).toEqual(
+        'https://gerrit.com/gitiles/repo/+archive/refs/heads/dev/docs.tar.gz',
+      );
+    });
+    it('can create an authenticated url when auth is enabled and an url-path is used', () => {
+      const authConfig = {
+        ...configWithPath,
+        username: 'username',
+        password: 'password',
+      };
+      expect(
+        buildGerritGitilesArchiveUrlFromLocation(
+          authConfig,
+          'https://gerrit.com/gerrit/plugins/gitiles/repo/+/refs/heads/dev/docs/',
+        ),
+      ).toEqual(
+        'https://gerrit.com/gerrit/a/plugins/gitiles/repo/+archive/refs/heads/dev/docs.tar.gz',
+      );
+    });
+    it('Cannot build an authenticated url when a dedicated Gitiles server is used', () => {
+      const authConfig = {
+        ...configWithDedicatedGitiles,
+        username: 'username',
+        password: 'password',
+      };
+      expect(() =>
+        buildGerritGitilesArchiveUrlFromLocation(
+          authConfig,
+          'https://gerrit.com/gitiles/repo/+/refs/heads/dev/',
+        ),
+      ).toThrow(
+        'Since the baseUrl (Gerrit) is not part of the gitilesBaseUrl, an authentication URL could not be constructed.',
+      );
+    });
+  });
+
   describe('buildGerritGitilesUrl', () => {
     it('can create an url from arguments', () => {
       const config: GerritIntegrationConfig = {
@@ -426,6 +515,39 @@ describe('gerrit core', () => {
         'https://gerrit.com/a/projects/web%2Fproject/branches/master/files/README.md/content',
       );
     });
+    it('can create an authenticated url for a commit.', () => {
+      const authConfig: GerritIntegrationConfig = {
+        host: 'gerrit.com',
+        baseUrl: 'https://gerrit.com',
+        gitilesBaseUrl: 'https://gerrit.com',
+        username: 'u',
+        password: 'u',
+      };
+
+      const authFileContentUrl = getGerritFileContentsApiUrl(
+        authConfig,
+        'https://gerrit.com/web/project/+/157f862803d45b9d269f0e390f88aece1ded51e8/README.md',
+      );
+      expect(authFileContentUrl).toEqual(
+        'https://gerrit.com/a/projects/web%2Fproject/commits/157f862803d45b9d269f0e390f88aece1ded51e8/files/README.md/content',
+      );
+    });
+    it('will throw for unsupported ref types (tag).', () => {
+      const authConfig: GerritIntegrationConfig = {
+        host: 'gerrit.com',
+        baseUrl: 'https://gerrit.com',
+        gitilesBaseUrl: 'https://gerrit.com',
+        username: 'u',
+        password: 'u',
+      };
+
+      expect(() =>
+        getGerritFileContentsApiUrl(
+          authConfig,
+          'https://gerrit.com/modules/events-broker/+/refs/tags/v3.5.6/src/main/java/com/gerritforge/gerrit/eventbroker/BrokerApi.java',
+        ),
+      ).toThrow(/gitiles ref type/);
+    });
   });
 
   describe('parseGerritJsonResponse', () => {

packages/integration/src/gerrit/core.ts

@@ -41,8 +41,9 @@ const GERRIT_BODY_PREFIX = ")]}'";
  *
  * @param url - An URL pointing to a file stored in git.
  * @public
+ * @deprecated `parseGerritGitilesUrl` is deprecated. Use
+ *  {@link parseGitilesUrlRef} instead.
  */
-
 export function parseGerritGitilesUrl(
   config: GerritIntegrationConfig,
   url: string,
@@ -215,6 +216,8 @@ export function buildGerritGitilesUrl(
  * @param branch - The branch we will target.
  * @param filePath - The absolute file path.
  * @public
+ * @deprecated `buildGerritGitilesArchiveUrl` is deprecated. Use
+ *  {@link buildGerritGitilesArchiveUrlFromLocation} instead.
  */
 export function buildGerritGitilesArchiveUrl(
   config: GerritIntegrationConfig,
@@ -229,6 +232,38 @@ export function buildGerritGitilesArchiveUrl(
   )}/${project}/+archive/refs/heads/${branch}${archiveName}`;
 }
 
+/**
+ * Build a Gerrit Gitiles archive url from a Gitiles url.
+ *
+ * @param config - A Gerrit provider config.
+ * @param url - The gitiles url
+ * @public
+ */
+export function buildGerritGitilesArchiveUrlFromLocation(
+  config: GerritIntegrationConfig,
+  url: string,
+): string {
+  const {
+    path: filePath,
+    ref,
+    project,
+    refType,
+  } = parseGitilesUrlRef(config, url);
+  const archiveName =
+    filePath === '/' || filePath === '' ? '.tar.gz' : `/${filePath}.tar.gz`;
+  if (refType === 'branch') {
+    return `${getGitilesAuthenticationUrl(
+      config,
+    )}/${project}/+archive/refs/heads/${ref}${archiveName}`;
+  }
+  if (refType === 'sha') {
+    return `${getGitilesAuthenticationUrl(
+      config,
+    )}/${project}/+archive/${ref}${archiveName}`;
+  }
+  throw new Error(`Unsupported gitiles ref type: ${refType}`);
+}
+
 /**
  * Return the authentication prefix.
  *
@@ -324,13 +359,25 @@ export function getGerritFileContentsApiUrl(
   config: GerritIntegrationConfig,
   url: string,
 ) {
-  const { branch, filePath, project } = parseGerritGitilesUrl(config, url);
+  const { ref, refType, path, project } = parseGitilesUrlRef(config, url);
 
-  return `${config.baseUrl}${getAuthenticationPrefix(
-    config,
-  )}projects/${encodeURIComponent(
-    project,
-  )}/branches/${branch}/files/${encodeURIComponent(filePath)}/content`;
+  // https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content
+  if (refType === 'branch') {
+    return `${config.baseUrl}${getAuthenticationPrefix(
+      config,
+    )}projects/${encodeURIComponent(
+      project,
+    )}/branches/${ref}/files/${encodeURIComponent(path)}/content`;
+  }
+  // https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content-from-commit
+  if (refType === 'sha') {
+    return `${config.baseUrl}${getAuthenticationPrefix(
+      config,
+    )}projects/${encodeURIComponent(
+      project,
+    )}/commits/${ref}/files/${encodeURIComponent(path)}/content`;
+  }
+  throw new Error(`Unsupported gitiles ref type: ${refType}`);
 }
 
 /**

packages/integration/src/gerrit/index.ts

@@ -20,6 +20,7 @@ export {
 } from './config';
 export {
   buildGerritGitilesArchiveUrl,
+  buildGerritGitilesArchiveUrlFromLocation,
   getGerritBranchApiUrl,
   getGerritCloneRepoUrl,
   getGerritFileContentsApiUrl,