Added support for github-discovery locations

.changeset/moody-mice-cheer.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added an option to scan github for repositories using a new location type 'github-discovery'

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
+import { getOrganizationRepositories } from './github';
+
+jest.mock('./github');
+const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction<
+  typeof getOrganizationRepositories
+>;
+
+describe('GithubOrgReaderProcessor', () => {
+  describe('parseUrl', () => {
+    it('parses well formed URLs', () => {
+      expect(
+        parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'),
+      ).toEqual({
+        org: 'foo',
+        repoSearchPath: /proj/,
+        catalogPath: 'blob/master/catalog.yaml',
+      });
+      expect(
+        parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'),
+      ).toEqual({
+        org: 'foo',
+        repoSearchPath: /proj.*/,
+        catalogPath: 'blob/master/catalog.yaml',
+      });
+    });
+
+    it('throws on incorrectly formed URLs', () => {
+      expect(() => parseUrl('https://github.com')).toThrow();
+      expect(() => parseUrl('https://github.com//')).toThrow();
+      expect(() => parseUrl('https://github.com/foo')).toThrow();
+      expect(() => parseUrl('https://github.com//foo')).toThrow();
+      expect(() => parseUrl('https://github.com/org/teams')).toThrow();
+      expect(() => parseUrl('https://github.com/org//teams')).toThrow();
+    });
+  });
+
+  describe('reject unrelated entries', () => {
+    it('rejects unknown types', async () => {
+      const processor = new GithubDiscoveryProcessor({
+        providers: [
+          {
+            target: 'https://github.com',
+            apiBaseUrl: 'https://api.github.com',
+          },
+        ],
+        logger: getVoidLogger(),
+      });
+      const location: LocationSpec = {
+        type: 'not-github-discovery',
+        target: 'https://github.com',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).resolves.toBeFalsy();
+    });
+
+    it('rejects unknown targets', async () => {
+      const processor = new GithubDiscoveryProcessor({
+        providers: [
+          {
+            target: 'https://github.com',
+            apiBaseUrl: 'https://api.github.com',
+          },
+        ],
+        logger: getVoidLogger(),
+      });
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target: 'https://not.github.com/apa',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).rejects.toThrow(
+        /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/,
+      );
+    });
+  });
+
+  describe('handles repositories', () => {
+    const processor = new GithubDiscoveryProcessor({
+      providers: [
+        {
+          target: 'https://github.com',
+          apiBaseUrl: 'https://api.github.com',
+        },
+      ],
+      logger: getVoidLogger(),
+    });
+
+    beforeEach(() => {
+      mockGetOrganizationRepositories.mockClear();
+    });
+
+    it('output all repositories', async () => {
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target: 'https://github.com/backstage/*/blob/master/catalog.yaml',
+      };
+      mockGetOrganizationRepositories.mockResolvedValueOnce({
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          { name: 'demo', url: 'https://github.com/backstage/demo' },
+        ],
+      });
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/backstage/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target: 'https://github.com/backstage/demo/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+    });
+
+    it('filter unrelated repositories', async () => {
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target:
+          'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml',
+      };
+      mockGetOrganizationRepositories.mockResolvedValueOnce({
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'techdocs-cli',
+            url: 'https://github.com/backstage/techdocs-cli',
+          },
+          {
+            name: 'techdocs-container',
+            url: 'https://github.com/backstage/techdocs-container',
+          },
+        ],
+      });
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { graphql } from '@octokit/graphql';
+import { Logger } from 'winston';
+import {
+  getOrganizationRepositories,
+  ProviderConfig,
+  readGithubConfig,
+} from './github';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ */
+export class GithubDiscoveryProcessor implements CatalogProcessor {
+  private readonly providers: ProviderConfig[];
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    return new GithubDiscoveryProcessor({
+      ...options,
+      providers: readGithubConfig(config),
+    });
+  }
+
+  constructor(options: { providers: ProviderConfig[]; logger: Logger }) {
+    this.providers = options.providers;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-discovery') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(`${p.target}/`),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`,
+      );
+    }
+
+    const { org, repoSearchPath, catalogPath } = parseUrl(location.target);
+    const client = !provider.token
+      ? graphql
+      : graphql.defaults({
+          baseUrl: provider.apiBaseUrl,
+          headers: {
+            authorization: `token ${provider.token}`,
+          },
+        });
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info('Reading GitHub repositories');
+
+    const { repositories } = await getOrganizationRepositories(client, org);
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${repositories.length} GitHub repositories in ${duration} seconds`,
+    );
+
+    for (const repository of repositories) {
+      if (!repoSearchPath.test(repository.name)) {
+        continue;
+      }
+      emit(
+        results.location(
+          {
+            type: 'url',
+            target: `${repository.url}/${catalogPath}`,
+          },
+          false,
+        ),
+      );
+    }
+
+    return true;
+  }
+}
+
+/*
+ * Helpers
+ */
+
+export function parseUrl(
+  urlString: string,
+): { org: string; repoSearchPath: RegExp; catalogPath: string } {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // /backstage/techdocs-*/blob/master/catalog-info.yaml
+  if (path.length > 2 && path[0].length && path[1].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      catalogPath: decodeURIComponent(path.slice(2).join('/')),
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
+export function escapeRegExp(str: string): RegExp {
+  return new RegExp(str.replace(/\*/g, '.*'));
+}

plugins/catalog-backend/src/ingestion/processors/github/github.test.ts

@@ -22,6 +22,7 @@ import {
   getOrganizationTeams,
   getOrganizationUsers,
   getTeamMembers,
+  getOrganizationRepositories,
   QueryResponse,
 } from './github';
 
@@ -151,4 +152,48 @@ describe('github', () => {
       await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output);
     });
   });
+
+  describe('getOrganizationRepositories', () => {
+    it('read repositories', async () => {
+      const input: QueryResponse = {
+        organization: {
+          repositories: {
+            nodes: [
+              {
+                name: 'backstage',
+                url: 'https://github.com/backstage/backstage',
+              },
+              {
+                name: 'demo',
+                url: 'https://github.com/backstage/demo',
+              },
+            ],
+            pageInfo: {
+              hasNextPage: false,
+            },
+          },
+        },
+      };
+
+      const output = {
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'demo',
+            url: 'https://github.com/backstage/demo',
+          },
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('repositories', (_req, res, ctx) =>
+          res(ctx.data(input)),
+        ),
+      );
+
+      await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual(
+        output,
+      );
+    });
+  });
 });

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -14,8 +14,13 @@
  * limitations under the License.
  */
 
-import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import {
+  GroupEntity,
+  LocationEntity,
+  UserEntity,
+} from '@backstage/catalog-model';
 import { graphql } from '@octokit/graphql';
+import { xorWith } from 'lodash/fp';
 
 // Graphql types
 
@@ -27,6 +32,7 @@ export type Organization = {
   membersWithRole?: Connection<User>;
   team?: Team;
   teams?: Connection<Team>;
+  repositories?: Connection<Repository>;
 };
 
 export type PageInfo = {
@@ -52,6 +58,11 @@ export type Team = {
   members: Connection<User>;
 };
 
+export type Repository = {
+  name: string;
+  url: string;
+};
+
 export type Connection<T> = {
   pageInfo: PageInfo;
   nodes: T[];
@@ -216,6 +227,39 @@ export async function getOrganizationTeams(
   return { groups, groupMemberUsers };
 }
 
+export async function getOrganizationRepositories(
+  client: typeof graphql,
+  org: string,
+): Promise<{ repositories: Repository[] }> {
+  const query = `
+  query repositories($org: String!, $cursor: String) {
+    organization(login: $org) {
+      name
+      repositories(first: 100, after: $cursor) {
+        nodes {
+          name
+          url
+        }
+        pageInfo {
+          hasNextPage
+          endCursor
+        }
+      }
+    }
+  }
+  `;
+
+  const repositories = await queryWithPaging(
+    client,
+    query,
+    r => r.organization?.repositories,
+    x => x,
+    { org },
+  );
+
+  return { repositories };
+}
+
 /**
  * Gets all the users out of a GitHub organization.
  *

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -16,4 +16,8 @@
 
 export { readGithubConfig } from './config';
 export type { ProviderConfig } from './config';
-export { getOrganizationTeams, getOrganizationUsers } from './github';
+export {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  getOrganizationRepositories,
+} from './github';