feat(GithubMultiOrgReaderProcessor): implement processor to handle multi-org ingestion

Signed-off-by: Phil Kuang <pkuang@factset.com>

.changeset/orange-news-sparkle.md

@@ -0,0 +1,46 @@
+---
+'@backstage/integration': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Support ingesting multiple GitHub organizations via a new `GithubMultiOrgReaderProcessor`.
+
+This new processor handles namespacing created groups according to the org of the associated GitHub team to prevent potential name clashes between organizations. Be aware that this processor is considered alpha and may not be compatible with future org structures in the catalog.
+
+NOTE: This processor only fully supports auth via GitHub Apps
+
+To install this processor, import and add it as follows:
+
+```typescript
+// Typically in packages/backend/src/plugins/catalog.ts
+import { GithubMultiOrgReaderProcessor } from '@backstage/plugin-catalog-backend';
+// ...
+export default async function createPlugin(env: PluginEnvironment) {
+  const builder = new CatalogBuilder(env);
+  builder.addProcessor(
+    GithubMultiOrgReaderProcessor.fromConfig(env.config, {
+      logger: env.logger,
+    }),
+  );
+  // ...
+}
+```
+
+Configure in your `app-config.yaml` by pointing to your GitHub instance and optionally list which GitHub organizations you wish to import. You can also configure what namespace you want to set for teams from each org. If unspecified, the org name will be used as the namespace. If no organizations are listed, by default this processor will import from all organizations accessible by all configured GitHub Apps:
+
+```yaml
+catalog:
+  locations:
+    - type: github-multi-org
+      target: https://github.myorg.com
+
+  processors:
+    githubMultiOrg:
+      orgs:
+        - name: fooOrg
+          groupNamespace: foo
+        - name: barOrg
+          groupNamespace: bar
+        - name: awesomeOrg
+        - name: anotherOrg
+```

packages/integration/api-report.md

@@ -5,6 +5,7 @@
 ```ts
 
 import { Config } from '@backstage/config';
+import { RestEndpointMethodTypes } from '@octokit/rest';
 
 // @public (undocumented)
 export class AzureIntegration implements ScmIntegration {
@@ -106,6 +107,15 @@ export function getGitLabFileFetchUrl(url: string, config: GitLabIntegrationConf
 // @public
 export function getGitLabRequestOptions(config: GitLabIntegrationConfig): RequestInit;
 
+// @public (undocumented)
+export class GithubAppCredentialsMux {
+    constructor(config: GitHubIntegrationConfig);
+    // (undocumented)
+    getAllInstallations(): Promise<RestEndpointMethodTypes['apps']['listInstallations']['response']['data']>;
+    // (undocumented)
+    getAppToken(owner: string, repo?: string): Promise<string | undefined>;
+}
+
 // @public (undocumented)
 export class GithubCredentialsProvider {
     // (undocumented)

packages/integration/src/github/GithubCredentialsProvider.test.ts

@@ -15,6 +15,7 @@
  */
 
 const octokit = {
+  paginate: async (fn: any) => (await fn()).data,
   apps: {
     listInstallations: jest.fn(),
     createInstallationAccessToken: jest.fn(),
@@ -53,7 +54,7 @@ describe('GithubCredentialsProvider tests', () => {
     jest.resetAllMocks();
   });
   it('create repository specific tokens', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -72,7 +73,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -84,12 +84,8 @@ describe('GithubCredentialsProvider tests', () => {
     const { token, headers, type } = await github.getCredentials({
       url: 'https://github.com/backstage/foobar',
     });
-    const { token: accessToken2 } = await github.getCredentials({
-      url: 'https://github.com/backstage/foobar',
-    });
     expect(type).toEqual('app');
     expect(token).toEqual('secret_token');
-    expect(token).toEqual(accessToken2);
     expect(headers).toEqual({ Authorization: 'Bearer secret_token' });
 
     // fallback to the configured token if no application is matching
@@ -107,7 +103,7 @@ describe('GithubCredentialsProvider tests', () => {
   });
 
   it('creates tokens for an organization', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -121,7 +117,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -133,17 +128,13 @@ describe('GithubCredentialsProvider tests', () => {
     const { token, headers } = await github.getCredentials({
       url: 'https://github.com/backstage',
     });
-    const { token: accessToken2 } = await github.getCredentials({
-      url: 'https://github.com/backstage',
-    });
 
     expect(headers).toEqual({ Authorization: 'Bearer secret_token' });
     expect(token).toEqual('secret_token');
-    expect(token).toEqual(accessToken2);
   });
 
   it('should fail to issue tokens for an organization when the app is installed for a single repo', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -157,7 +148,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -176,7 +166,7 @@ describe('GithubCredentialsProvider tests', () => {
   });
 
   it('should throw if the app is suspended', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -193,7 +183,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     await expect(
       github.getCredentials({
@@ -229,7 +218,7 @@ describe('GithubCredentialsProvider tests', () => {
     ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' }));
   });
 
-  it('should return the configured token if listing installations throws', async () => {
+  it('should return the configured token if there are no installations', async () => {
     const githubProvider = GithubCredentialsProvider.create({
       host: 'github.com',
       apps: [
@@ -243,7 +232,9 @@ describe('GithubCredentialsProvider tests', () => {
       ],
       token: 'hardcoded_token',
     });
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
+    octokit.apps.listInstallations.mockResolvedValue(({
+      data: [],
+    } as unknown) as RestEndpointMethodTypes['apps']['listInstallations']['response']);
 
     await expect(
       githubProvider.getCredentials({

packages/integration/src/github/GithubCredentialsProvider.ts

@@ -66,7 +66,6 @@ const HEADERS = {
 class GithubAppManager {
   private readonly appClient: Octokit;
   private readonly baseAuthConfig: { appId: number; privateKey: string };
-  private installations?: RestEndpointMethodTypes['apps']['listInstallations']['response'];
   private readonly cache = new Cache();
 
   constructor(config: GithubAppConfig, baseUrl?: string) {
@@ -121,22 +120,15 @@ class GithubAppManager {
     });
   }
 
+  getInstallations(): Promise<
+    RestEndpointMethodTypes['apps']['listInstallations']['response']['data']
+  > {
+    return this.appClient.paginate(this.appClient.apps.listInstallations);
+  }
+
   private async getInstallationData(owner: string): Promise<InstallationData> {
-    // List all installations using the last used etag.
-    // Return cached InstallationData if error with status 304 is thrown.
-    try {
-      this.installations = await this.appClient.apps.listInstallations({
-        headers: {
-          'If-None-Match': this.installations?.headers.etag,
-          Accept: HEADERS.Accept,
-        },
-      });
-    } catch (error) {
-      if (error.status !== 304) {
-        throw error;
-      }
-    }
-    const installation = this.installations?.data.find(
+    const allInstallations = await this.getInstallations();
+    const installation = allInstallations.find(
       inst => inst.account?.login === owner,
     );
     if (installation) {
@@ -163,6 +155,20 @@ export class GithubAppCredentialsMux {
       config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
   }
 
+  async getAllInstallations(): Promise<
+    RestEndpointMethodTypes['apps']['listInstallations']['response']['data']
+  > {
+    if (!this.apps.length) {
+      return [];
+    }
+
+    const installs = await Promise.all(
+      this.apps.map(app => app.getInstallations()),
+    );
+
+    return installs.flat();
+  }
+
   async getAppToken(owner: string, repo?: string): Promise<string | undefined> {
     if (this.apps.length === 0) {
       return undefined;

packages/integration/src/github/index.ts

@@ -20,6 +20,9 @@ export {
 } from './config';
 export type { GitHubIntegrationConfig } from './config';
 export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
-export { GithubCredentialsProvider } from './GithubCredentialsProvider';
+export {
+  GithubAppCredentialsMux,
+  GithubCredentialsProvider,
+} from './GithubCredentialsProvider';
 export type { GithubCredentialType } from './GithubCredentialsProvider';
 export { GitHubIntegration } from './GitHubIntegration';

plugins/catalog-backend/config.d.ts

@@ -145,6 +145,27 @@ export interface Config {
         }>;
       };
 
+      /**
+       * GithubMultiOrgReaderProcessor configuration
+       */
+      githubMultiOrg?: {
+        /**
+         * The configuration parameters for each GitHub org to process.
+         */
+        orgs: Array<{
+          /**
+           * The name of the GitHub org to process.
+           */
+          name: string;
+          /**
+           * The namespace of the group created for this org.
+           *
+           * Defaults to org name if omitted.
+           */
+          groupNamespace?: string;
+        }>;
+      };
+
       /**
        * LdapOrgReaderProcessor configuration
        */

plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts

@@ -0,0 +1,181 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  GithubAppCredentialsMux,
+  GithubCredentialsProvider,
+  GitHubIntegrationConfig,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { Logger } from 'winston';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  GithubMultiOrgConfig,
+  readGithubMultiOrgConfig,
+} from './github';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { buildOrgHierarchy } from './util/org';
+
+/**
+ * @alpha
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ */
+export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly orgs: GithubMultiOrgConfig;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const c = config.getOptionalConfig('catalog.processors.githubMultiOrg');
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new GithubMultiOrgReaderProcessor({
+      ...options,
+      integrations,
+      orgs: c ? readGithubMultiOrgConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    integrations: ScmIntegrations;
+    logger: Logger;
+    orgs: GithubMultiOrgConfig;
+  }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.orgs = options.orgs;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-multi-org') {
+      return false;
+    }
+
+    const gitHubConfig = this.integrations.github.byUrl(location.target)
+      ?.config;
+    if (!gitHubConfig) {
+      throw new Error(
+        `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`,
+      );
+    }
+
+    const allUsersMap = new Map();
+    const baseUrl = new URL(location.target).origin;
+    const credentialsProvider = GithubCredentialsProvider.create(gitHubConfig);
+
+    const orgsToProcess = this.orgs.length
+      ? this.orgs
+      : await this.getAllOrgs(gitHubConfig);
+
+    for (const orgConfig of orgsToProcess) {
+      try {
+        const {
+          headers,
+          type: tokenType,
+        } = await credentialsProvider.getCredentials({
+          url: `${baseUrl}/${orgConfig.name}`,
+        });
+        const client = graphql.defaults({
+          baseUrl: gitHubConfig.apiBaseUrl,
+          headers,
+        });
+
+        const startTimestamp = Date.now();
+        this.logger.info(
+          `Reading GitHub users and teams for org: ${orgConfig.name}`,
+        );
+        const { users } = await getOrganizationUsers(
+          client,
+          orgConfig.name,
+          tokenType,
+        );
+        const { groups, groupMemberUsers } = await getOrganizationTeams(
+          client,
+          orgConfig.name,
+          orgConfig.groupNamespace,
+        );
+
+        const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+        this.logger.debug(
+          `Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`,
+        );
+
+        users.forEach(u => {
+          if (!allUsersMap.has(u.metadata.name)) {
+            allUsersMap.set(u.metadata.name, u);
+          }
+        });
+
+        for (const [groupName, userNames] of groupMemberUsers.entries()) {
+          for (const userName of userNames) {
+            const user = allUsersMap.get(userName);
+            if (user && !user.spec.memberOf.includes(groupName)) {
+              user.spec.memberOf.push(groupName);
+            }
+          }
+        }
+        buildOrgHierarchy(groups);
+
+        for (const group of groups) {
+          emit(results.entity(location, group));
+        }
+      } catch (e) {
+        this.logger.error(
+          `Failed to read GitHub org data for ${orgConfig.name}: ${e}`,
+        );
+      }
+    }
+
+    const allUsers = Array.from(allUsersMap.values());
+    for (const user of allUsers) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+
+  // Note: Does not support usage of PATs
+  private async getAllOrgs(
+    gitHubConfig: GitHubIntegrationConfig,
+  ): Promise<GithubMultiOrgConfig> {
+    const githubAppMux = new GithubAppCredentialsMux(gitHubConfig);
+    const installs = await githubAppMux.getAllInstallations();
+
+    return installs
+      .map(install =>
+        install.target_type === 'Organization' &&
+        install.account &&
+        install.account.login
+          ? {
+              name: install.account.login,
+              groupNamespace: install.account.login.toLowerCase(),
+            }
+          : undefined,
+      )
+      .filter(Boolean) as GithubMultiOrgConfig;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/github/config.test.ts

@@ -15,7 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
-import { readGithubConfig } from './config';
+import { readGithubConfig, readGithubMultiOrgConfig } from './config';
 
 describe('config', () => {
   describe('readGithubConfig', () => {
@@ -76,4 +76,35 @@ describe('config', () => {
       ).toThrow(/token/);
     });
   });
+
+  describe('readGithubMultiOrgConfig', () => {
+    function config(orgs: { name: string; groupNamespace?: string }[]) {
+      return new ConfigReader({ orgs });
+    }
+
+    it('reads org configs', () => {
+      const output = readGithubMultiOrgConfig(
+        config([
+          { name: 'foo', groupNamespace: 'apple' },
+          { name: 'bar', groupNamespace: 'Orange' },
+        ]),
+      );
+
+      expect(output).toEqual([
+        { name: 'foo', groupNamespace: 'apple' },
+        { name: 'bar', groupNamespace: 'orange' },
+      ]);
+    });
+
+    it('defaults groupNamespace to org name if undefined', () => {
+      const output = readGithubMultiOrgConfig(
+        config([{ name: 'foo' }, { name: 'bar' }]),
+      );
+
+      expect(output).toEqual([
+        { name: 'foo', groupNamespace: 'foo' },
+        { name: 'bar', groupNamespace: 'bar' },
+      ]);
+    });
+  });
 });

plugins/catalog-backend/src/ingestion/processors/github/config.ts

@@ -82,3 +82,27 @@ export function readGithubConfig(config: Config): ProviderConfig[] {
 
   return providers;
 }
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ */
+export type GithubMultiOrgConfig = Array<{
+  /**
+   * The name of the GitHub org to process.
+   */
+  name: string;
+  /**
+   * The namespace of the group created for this org.
+   */
+  groupNamespace: string;
+}>;
+
+export function readGithubMultiOrgConfig(config: Config): GithubMultiOrgConfig {
+  const orgConfigs = config.getOptionalConfigArray('orgs') ?? [];
+  return orgConfigs.map(c => ({
+    name: c.getString('name'),
+    groupNamespace: (
+      c.getOptionalString('groupNamespace') ?? c.getString('name')
+    ).toLowerCase(),
+  }));
+}

plugins/catalog-backend/src/ingestion/processors/github/github.test.ts

@@ -72,8 +72,10 @@ describe('github', () => {
   });
 
   describe('getOrganizationTeams', () => {
-    it('reads teams', async () => {
-      const input: QueryResponse = {
+    let input: QueryResponse;
+
+    beforeEach(() => {
+      input = {
         organization: {
           teams: {
             pageInfo: { hasNextPage: false },
@@ -98,7 +100,9 @@ describe('github', () => {
           },
         },
       };
+    });
 
+    it('reads teams', async () => {
       const output = {
         groups: [
           expect.objectContaining({
@@ -126,6 +130,38 @@ describe('github', () => {
 
       await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
     });
+
+    it('applies namespaces', async () => {
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'team',
+              namespace: 'foo',
+              description: 'The one and only team',
+            }),
+            spec: {
+              type: 'team',
+              profile: {
+                displayName: 'Team',
+                picture: 'http://example.com/team.jpeg',
+              },
+              parent: 'parent',
+              children: [],
+            },
+          }),
+        ],
+        groupMemberUsers: new Map([['foo/team', ['user']]]),
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual(
+        output,
+      );
+    });
   });
 
   describe('getTeamMembers', () => {

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -144,6 +144,7 @@ export async function getOrganizationUsers(
 export async function getOrganizationTeams(
   client: typeof graphql,
   org: string,
+  orgNamespace?: string,
 ): Promise<{
   groups: GroupEntity[];
   groupMemberUsers: Map<string, string[]>;
@@ -189,6 +190,10 @@ export async function getOrganizationTeams(
       },
     };
 
+    if (orgNamespace) {
+      entity.metadata.namespace = orgNamespace;
+    }
+
     if (team.description) {
       entity.metadata.description = team.description;
     }
@@ -203,7 +208,8 @@ export async function getOrganizationTeams(
     }
 
     const memberNames: string[] = [];
-    groupMemberUsers.set(team.slug, memberNames);
+    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
+    groupMemberUsers.set(groupKey, memberNames);
 
     if (!team.members.pageInfo.hasNextPage) {
       // We got all the members in one go, run the fast path

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-export { readGithubConfig } from './config';
-export type { ProviderConfig } from './config';
+export { readGithubConfig, readGithubMultiOrgConfig } from './config';
+export type { GithubMultiOrgConfig, ProviderConfig } from './config';
 export {
   getOrganizationTeams,
   getOrganizationUsers,

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -25,6 +25,7 @@ export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
+export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
 export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor';
 export { LocationEntityProcessor } from './LocationEntityProcessor';
 export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';