feat(catalog): always use search for UrlReaderProcessor

Signed-off-by: Thomas Cardonne <thomas.cardonne@adevinta.com>

.changeset/tiny-spoons-repeat.md

@@ -0,0 +1,11 @@
+---
+'@backstage/backend-defaults': patch
+'@backstage/plugin-catalog-backend': minor
+---
+
+**BREAKING:** The `UrlReaderProccessor` now always calls the `search` method of the `UrlReaders`. Previous behavior was to call the `search` method only if the parsed Git URL's filename contained a wildcard and use `readUrl` otherwise. `UrlReaderService` must implement this logic in the `search` method instead.
+
+This allows each `UrlReaderService` implementation to check whether it's a search URL (that contains a wildcard pattern) or not using logic that is specific to each provider.
+
+In the different `UrlReadersService`, the `search` method have been updated to use the `readUrl` if the given URL doesn't contain a pattern.
+For `UrlReaders` that didn't implement the `search` method, `readUrl` is called internally.

packages/backend-defaults/report-urlReader.api.md

@@ -57,7 +57,10 @@ export class AwsS3UrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -230,7 +233,10 @@ export class FetchUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -265,7 +271,10 @@ export class GerritUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -293,7 +302,10 @@ export class GiteaUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -384,7 +396,10 @@ export class HarnessUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts

@@ -608,4 +608,48 @@ describe('AwsCodeCommitUrlReader', () => {
       expect(bodySubfolderFile.toString().trim()).toBe('site_name: Test2');
     });
   });
+
+  describe('search', () => {
+    const [{ reader }] = createReader({
+      integrations: {
+        awsCodeCommit: [
+          {
+            host: AMAZON_AWS_CODECOMMIT_HOST,
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+            region: 'fakeregion',
+          },
+        ],
+      },
+    });
+
+    beforeEach(() => {
+      codeCommitClient.reset();
+
+      codeCommitClient.on(GetFileCommand).resolves({
+        fileContent: fs.readFileSync(
+          path.resolve(
+            __dirname,
+            '__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
+          ),
+        ),
+        commitId: `123abc`,
+      });
+    });
+
+    it('should return a file when given an exact valid url', async () => {
+      const data = await reader.search(
+        'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
+      );
+
+      expect(data.etag).toBe('123abc');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'site_name: Test\n',
+      );
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts

@@ -21,6 +21,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import {
@@ -31,7 +32,11 @@ import {
   AwsCodeCommitIntegration,
   ScmIntegrations,
 } from '@backstage/integration';
-import { ForwardedError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  ForwardedError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import {
   CodeCommitClient,
@@ -257,7 +262,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
       }
 
       return ReadUrlResponseFactory.fromReadable(
-        Readable.from([response?.fileContent] || []),
+        Readable.from([response?.fileContent]),
         {
           etag: response.commitId,
         },
@@ -357,7 +362,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
           commitSpecifier: commitSpecifier,
         });
         const response = await codeCommitClient.send(getFileCommand);
-        const objectData = await Readable.from([response?.fileContent] || []);
+        const objectData = await Readable.from([response?.fileContent]);
 
         responses.push({
           data: objectData,
@@ -380,8 +385,33 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
     }
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('AwsCodeCommitReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts

@@ -527,4 +527,50 @@ describe('AwsS3UrlReader', () => {
       expect(body.toString().trim()).toBe('site_name: Test');
     });
   });
+
+  describe('search', () => {
+    const [{ reader }] = createReader({
+      integrations: {
+        awsS3: [
+          {
+            host: 'amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          },
+        ],
+      },
+    });
+
+    beforeEach(() => {
+      s3Client.reset();
+
+      s3Client.on(GetObjectCommand).resolves({
+        Body: sdkStreamMixin(
+          fs.createReadStream(
+            path.resolve(
+              __dirname,
+              '__fixtures__/awsS3/awsS3-mock-object.yaml',
+            ),
+          ),
+        ),
+        ETag: '123abc',
+        LastModified: new Date('2020-01-01T00:00:00Z'),
+      });
+    });
+
+    it('should return a file when given an exact valid url', async () => {
+      const data = await reader.search(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+
+      expect(data.etag).toBe('123abc');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'site_name: Test\n',
+      );
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
@@ -32,7 +33,11 @@ import {
   ScmIntegrations,
   AwsS3IntegrationConfig,
 } from '@backstage/integration';
-import { ForwardedError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  ForwardedError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
 import {
@@ -357,8 +362,33 @@ export class AwsS3UrlReader implements UrlReaderService {
     }
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('AwsS3Reader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts

@@ -351,6 +351,17 @@ describe('AzureUrlReader', () => {
       );
     });
 
+    it('works for exact urls', async () => {
+      const result = await processor.search(
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+      );
+      expect(result.etag).toBe('');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+      );
+    });
+
     it('works for the naive case', async () => {
       const result = await processor.search(
         'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster',

packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts

@@ -32,8 +32,13 @@ import {
   ScmIntegrations,
   AzureIntegration,
 } from '@backstage/integration';
+import parseGitUrl from 'git-url-parse';
 import { Minimatch } from 'minimatch';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 
@@ -181,6 +186,35 @@ export class AzureUrlReader implements UrlReaderService {
     url: string,
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
+    const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const treeUrl = new URL(url);
 
     const path = treeUrl.searchParams.get('path');

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts

@@ -451,5 +451,31 @@ describe('BitbucketCloudUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs', async () => {
+      worker.use(
+        rest.get(
+          'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
+          (req, res, ctx) => {
+            expect(req.headers.get('If-None-Match')).toBeNull();
+            return res(
+              ctx.status(200),
+              ctx.body('foo'),
+              ctx.set('ETag', 'etag-value'),
+            );
+          },
+        ),
+      );
+
+      const result = await reader.search(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      expect(result.etag).toBe('etag-value');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('foo');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketCloudIntegration,
   getBitbucketCloudDefaultBranch,
@@ -163,6 +167,34 @@ export class BitbucketCloudUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts

@@ -30,6 +30,7 @@ import path from 'path';
 import { NotModifiedError } from '@backstage/errors';
 import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 createMockDirectory({ mockOsTmpDir: true });
 
@@ -277,5 +278,23 @@ describe('BitbucketServerUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs by using readUrl directly', async () => {
+      reader.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+
+      const result = await reader.search(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml',
+      );
+      expect(reader.readUrl).toHaveBeenCalledTimes(1);
+      expect(result.etag).toBe('etag');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketServerIntegration,
   getBitbucketServerDownloadUrl,
@@ -168,6 +172,34 @@ export class BitbucketServerUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts

@@ -32,6 +32,7 @@ import { NotModifiedError } from '@backstage/errors';
 import { BitbucketUrlReader } from './BitbucketUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
 import getRawBody from 'raw-body';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const logger = mockServices.logger.mock();
 
@@ -613,5 +614,22 @@ describe('BitbucketUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs', async () => {
+      hostedBitbucketProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+
+      const result = await hostedBitbucketProcessor.search(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
+      );
+      expect(result.etag).toBe('etag');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketIntegration,
   getBitbucketDefaultBranch,
@@ -174,6 +178,34 @@ export class BitbucketUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts

@@ -266,4 +266,26 @@ describe('FetchUrlReader', () => {
       ).rejects.toThrow(Error);
     });
   });
+
+  describe('search', () => {
+    it('should return a file', async () => {
+      const data = await fetchUrlReader.search(
+        `https://backstage.io/some-resource`,
+        { etag: 'etag' },
+      );
+      expect(data.etag).toBe('foo');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(`https://backstage.io/some-resource`);
+      expect((await data.files[0].content()).toString()).toEqual('content foo');
+    });
+
+    it('should return an empty list of file if not found', async () => {
+      const data = await fetchUrlReader.search(
+        `https://backstage.io/not-exists`,
+        { etag: 'etag' },
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts

@@ -19,9 +19,14 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReaderFactory } from './types';
 import path from 'path';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
@@ -162,8 +167,33 @@ export class FetchUrlReader implements UrlReaderService {
     throw new Error('FetchUrlReader does not implement readTree');
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('FetchUrlReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts

@@ -420,4 +420,51 @@ describe.skip('GerritUrlReader', () => {
       expect(response.etag).toBe(sha);
     });
   });
+
+  describe('search', () => {
+    const responseBuffer = Buffer.from('Apache License');
+
+    it('should return a single file when given an exact URL', async () => {
+      worker.use(
+        rest.get(
+          'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.body(responseBuffer.toString('base64')),
+            );
+          },
+        ),
+      );
+
+      const data = await gerritProcessor.search(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      worker.use(
+        rest.get(
+          'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
+          (_, res, ctx) => {
+            return res(ctx.status(404, 'File not found.'));
+          },
+        ),
+      );
+
+      const data = await gerritProcessor.search(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { Base64Decode } from 'base64-stream';
@@ -39,6 +40,7 @@ import {
   NotFoundError,
   NotModifiedError,
   ResponseError,
+  assertError,
 } from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 
@@ -144,8 +146,33 @@ export class GerritUrlReader implements UrlReaderService {
     return this.readTreeFromGitiles(url, urlRevision, options);
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('GerritReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts

@@ -330,4 +330,62 @@ describe('GiteaUrlReader', () => {
       ).rejects.toThrow(NotFoundError);
     });
   });
+
+  describe('search', () => {
+    const responseBuffer = Buffer.from('Apache License');
+    const giteaApiResponse = (content: any) => {
+      return JSON.stringify({
+        encoding: 'base64',
+        content: Buffer.from(content).toString('base64'),
+      });
+    };
+
+    it('should return a single file when given an exact URL', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
+          (req, res, ctx) => {
+            // Test utils prefers matching URL directly but it is part of Gitea's API
+            if (req.url.searchParams.get('ref') === 'branch2') {
+              return res(
+                ctx.status(200),
+                ctx.body(giteaApiResponse(responseBuffer.toString())),
+              );
+            }
+
+            return res(ctx.status(500));
+          },
+        ),
+      );
+
+      const data = await giteaProcessor.search(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
+          (_, res, ctx) => {
+            return res(ctx.status(404, 'File not found.'));
+          },
+        ),
+      );
+
+      const data = await giteaProcessor.search(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import {
@@ -34,6 +35,7 @@ import {
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import {
+  assertError,
   AuthenticationError,
   NotFoundError,
   NotModifiedError,
@@ -155,8 +157,33 @@ export class GiteaUrlReader implements UrlReaderService {
     });
   }
 
-  search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('GiteaUrlReader search not implemented.');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts

@@ -37,6 +37,7 @@ import {
   GithubUrlReader,
 } from './GithubUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const mockDir = createMockDirectory({ mockOsTmpDir: true });
 
@@ -887,9 +888,6 @@ describe('GithubUrlReader', () => {
       expect(r2.etag).toBe('etag123abc');
       expect(r2.files.length).toBe(2);
 
-      const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`);
-      expect(r3.files.length).toBe(0);
-
       const r4 = await reader.search(
         `${baseUrl}/backstage/mock/tree/main/*docs*`,
       );
@@ -1017,6 +1015,23 @@ describe('GithubUrlReader', () => {
       await runTests(gheProcessor, 'https://ghe.github.com');
     });
 
+    it('uses readUrl when searching for an exact file', async () => {
+      githubProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await githubProcessor.search(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect(githubProcessor.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
+
     it('throws NotModifiedError when same etag', async () => {
       await expect(
         githubProcessor.search(

packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts

@@ -37,7 +37,11 @@ import fetch, { RequestInit, Response } from 'node-fetch';
 import parseGitUrl from 'git-url-parse';
 import { Minimatch } from 'minimatch';
 import { Readable } from 'stream';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import { parseLastModified } from './util';
@@ -179,6 +183,35 @@ export class GithubUrlReader implements UrlReaderService {
     url: string,
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
+    const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const repoDetails = await this.getRepoDetails(url);
     const commitSha = repoDetails.commitSha;
 
@@ -186,7 +219,6 @@ export class GithubUrlReader implements UrlReaderService {
       throw new NotModifiedError();
     }
 
-    const { filepath } = parseGitUrl(url);
     const { headers } = await this.getCredentials(url, options);
 
     const files = await this.doSearch(

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts

@@ -31,6 +31,7 @@ import {
   GitLabIntegration,
   readGitLabIntegrationConfig,
 } from '@backstage/integration';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const logger = mockServices.logger.mock();
 
@@ -663,6 +664,23 @@ describe('GitlabUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('returns a single file for exact urls', async () => {
+      gitlabProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await gitlabProcessor.search(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect(gitlabProcessor.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
   });
 
   describe('getGitlabFetchUrl', () => {

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts

@@ -26,7 +26,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   GitLabIntegration,
   ScmIntegrations,
@@ -242,6 +246,34 @@ export class GitlabUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const staticPart = this.getStaticPart(filepath);
     const matcher = new Minimatch(filepath);
     const treeUrl = trimEnd(url.replace(filepath, staticPart), `/`);

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts

@@ -22,6 +22,7 @@ import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 import { UrlReaderPredicateTuple } from './types';
 import packageinfo from '../../../../package.json';
 import { mockServices } from '@backstage/backend-test-utils';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const bucketGetFilesMock = jest.fn();
 class Bucket {
@@ -130,7 +131,7 @@ describe('GcsUrlReader', () => {
     });
 
     it('throws if search url does not end with *', async () => {
-      const glob = 'https://storage.cloud.google.com/bucket/no-asterisk';
+      const glob = 'https://storage.cloud.google.com/bucket/*no-asterisk';
       await expect(() => reader.search(glob)).rejects.toThrow(
         'GcsUrlReader only supports prefix-based searches',
       );
@@ -167,5 +168,22 @@ describe('GcsUrlReader', () => {
         'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
       );
     });
+
+    it('returns single file if there is no wildcard', async () => {
+      reader.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await reader.search(
+        'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
+      );
+      expect(reader.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { ReaderFactory } from './types';
@@ -31,6 +32,7 @@ import {
 import { Readable } from 'stream';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import packageinfo from '../../../../package.json';
+import { assertError } from '@backstage/errors';
 
 const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
 
@@ -117,9 +119,39 @@ export class GoogleGcsUrlReader implements UrlReaderService {
     throw new Error('GcsUrlReader does not implement readTree');
   }
 
-  async search(url: string): Promise<UrlReaderServiceSearchResponse> {
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
     const { bucket, key: pattern } = parseURL(url);
 
+    // If it's a direct URL we use readUrl instead
+    if (!pattern?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) {
       throw new Error('GcsUrlReader only supports prefix-based searches');
     }

packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts

@@ -257,4 +257,28 @@ describe('HarnessUrlReader', () => {
       ).rejects.toThrow(NotFoundError);
     });
   });
+
+  describe('search', () => {
+    it('should return a single file when given an exact URL', async () => {
+      const data = await harnessProcessor.search(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      const data = await harnessProcessor.search(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts

@@ -21,6 +21,7 @@ import {
   UrlReaderServiceReadUrlResponse,
   UrlReaderServiceSearchResponse,
   UrlReaderServiceReadTreeOptions,
+  UrlReaderServiceSearchOptions,
 } from '@backstage/backend-plugin-api';
 import {
   getHarnessRequestOptions,
@@ -35,6 +36,7 @@ import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import fetch, { Response } from 'node-fetch';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import {
+  assertError,
   AuthenticationError,
   NotFoundError,
   NotModifiedError,
@@ -154,8 +156,33 @@ export class HarnessUrlReader implements UrlReaderService {
     });
   }
 
-  search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('HarnessUrlReader search not implemented.');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts

@@ -180,12 +180,11 @@ describe('UrlReaderProcessor', () => {
         mockCache,
       ),
     )) as CatalogProcessorErrorResult;
-
     expect(generated.type).toBe('error');
     expect(generated.location).toBe(spec);
     expect(generated.error.name).toBe('NotFoundError');
     expect(generated.error.message).toBe(
-      `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`,
+      `Unable to read url, no matching files found for ${mockApiOrigin}/component-notfound.yaml`,
     );
   });
 

plugins/catalog-backend/src/processors/UrlReaderProcessor.ts

@@ -18,7 +18,6 @@ import { Entity } from '@backstage/catalog-model';
 import { assertError } from '@backstage/errors';
 import limiterFactory, { Limit } from 'p-limit';
 import { LocationSpec } from '@backstage/plugin-catalog-common';
-import parseGitUrl from 'git-url-parse';
 import {
   CatalogProcessor,
   CatalogProcessorCache,
@@ -80,6 +79,15 @@ export class UrlReaderProcessor implements CatalogProcessor {
         cacheItem?.etag,
       );
 
+      if (response.length === 0 && !optional) {
+        emit(
+          processingResult.notFoundError(
+            location,
+            `Unable to read ${location.type}, no matching files found for ${location.target}`,
+          ),
+        );
+      }
+
       const parseResults: CatalogProcessorResult[] = [];
       for (const item of response) {
         for await (const parseResult of parser({
@@ -112,10 +120,6 @@ export class UrlReaderProcessor implements CatalogProcessor {
         }
         emit(processingResult.refresh(`${location.type}:${location.target}`));
         await cache.set(CACHE_KEY, cacheItem);
-      } else if (error.name === 'NotFoundError') {
-        if (!optional) {
-          emit(processingResult.notFoundError(location, message));
-        }
       } else {
         emit(processingResult.generalError(location, message));
       }
@@ -128,23 +132,13 @@ export class UrlReaderProcessor implements CatalogProcessor {
     location: string,
     etag?: string,
   ): Promise<{ response: { data: Buffer; url: string }[]; etag?: string }> {
-    // Does it contain globs? I.e. does it contain asterisks or question marks
-    // (no curly braces for now)
+    const response = await this.options.reader.search(location, { etag });
 
-    const { filepath } = parseGitUrl(location);
-    if (filepath?.match(/[*?]/)) {
-      const response = await this.options.reader.search(location, { etag });
-      const output = response.files.map(async file => ({
-        url: file.url,
-        data: await this.#limiter(file.content),
-      }));
-      return { response: await Promise.all(output), etag: response.etag };
-    }
+    const output = response.files.map(async file => ({
+      url: file.url,
+      data: await this.#limiter(file.content),
+    }));
 
-    const data = await this.options.reader.readUrl(location, { etag });
-    return {
-      response: [{ url: location, data: await data.buffer() }],
-      etag: data.etag,
-    };
+    return { response: await Promise.all(output), etag: response.etag };
   }
 }