diff --git a/.changeset/fuzzy-phones-own.md b/.changeset/fuzzy-phones-own.md new file mode 100644 index 0000000000..61725d1f1d --- /dev/null +++ b/.changeset/fuzzy-phones-own.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend-module-github-org': patch +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Introduce new configuration option to exclude suspended users from GitHub Enterprise instances. + +When it’s set to true, suspended users won’t be returned when querying the organization users for GitHub Enterprise instances. +Note that this option should be used only against GitHub Enterprise instances, the property does not exist in the github.com GraphQL schema, setting it will cause a schema validation error and the syncing of users will fail. diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 6526183f2d..284451af8c 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -90,6 +90,7 @@ catalog: initialDelay: { seconds: 30 } frequency: { hours: 1 } timeout: { minutes: 50 } + excludeSuspendedUsers: true ``` Directly under the `githubOrg` is a list of configurations, each entry is a structure with the following elements: @@ -106,6 +107,8 @@ Directly under the `githubOrg` is a list of configurations, each entry is a stru Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent API resource limits for organizations with large number of teams and members. +- `excludeSuspendedUsers` (optional): Whether to exclude suspended users when querying organization users. Only for GitHub Enterprise instances. Will error if used against GitHub.com API. + ### Events Support The catalog module for GitHub Org comes with events support enabled. diff --git a/plugins/catalog-backend-module-github-org/src/module.ts b/plugins/catalog-backend-module-github-org/src/module.ts index 2d349cb873..26abede133 100644 --- a/plugins/catalog-backend-module-github-org/src/module.ts +++ b/plugins/catalog-backend-module-github-org/src/module.ts @@ -121,6 +121,7 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({ alwaysUseDefaultNamespace: definitions.length === 1 && definition.orgs?.length === 1, pageSizes: definition.pageSizes, + excludeSuspendedUsers: definition.excludeSuspendedUsers, }), ); } @@ -139,6 +140,7 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{ teamMembers?: number; organizationMembers?: number; }; + excludeSuspendedUsers?: boolean; }> { const baseKey = 'catalog.providers.githubOrg'; const baseConfig = rootConfig.getOptional(baseKey); @@ -166,5 +168,7 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{ ), } : undefined, + excludeSuspendedUsers: + c.getOptionalBoolean('excludeSuspendedUsers') ?? false, })); } diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index f3cbb15e06..5425cae9d0 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -264,6 +264,12 @@ export interface Config { */ orgs?: string[]; + /** + * (Optional) Only for GitHub Enterprise. Whether to exclude suspended users when querying organization users. + * Default: `false`. + */ + excludeSuspendedUsers?: boolean; + /** * The refresh schedule to use. */ @@ -315,6 +321,12 @@ export interface Config { */ orgs?: string[]; + /** + * (Optional) Only for GitHub Enterprise. Whether to exclude suspended users when querying organization users. + * Default: `false`. + */ + excludeSuspendedUsers?: boolean; + /** * The refresh schedule to use. */ diff --git a/plugins/catalog-backend-module-github/report.api.md b/plugins/catalog-backend-module-github/report.api.md index 016bb6da65..5f140a0d5e 100644 --- a/plugins/catalog-backend-module-github/report.api.md +++ b/plugins/catalog-backend-module-github/report.api.md @@ -151,6 +151,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { teamTransformer?: TeamTransformer; alwaysUseDefaultNamespace?: boolean; pageSizes?: Partial; + excludeSuspendedUsers?: boolean; }); connect(connection: EntityProviderConnection): Promise; // (undocumented) @@ -166,6 +167,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { export interface GithubMultiOrgEntityProviderOptions { alwaysUseDefaultNamespace?: boolean; events?: EventsService; + excludeSuspendedUsers?: boolean; githubCredentialsProvider?: GithubCredentialsProvider; githubUrl: string; id: string; @@ -227,6 +229,7 @@ export class GithubOrgEntityProvider implements EntityProvider { githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + excludeSuspendedUsers?: boolean; }); connect(connection: EntityProviderConnection): Promise; // (undocumented) @@ -244,6 +247,7 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions; // @public export interface GithubOrgEntityProviderOptions { events?: EventsService; + excludeSuspendedUsers?: boolean; githubCredentialsProvider?: GithubCredentialsProvider; id: string; logger: LoggerService; @@ -306,6 +310,7 @@ export type GithubUser = { email?: string; name?: string; organizationVerifiedDomainEmails?: string[]; + suspendedAt?: string; }; // @public diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index ca9b3eb692..6933c9a245 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -166,6 +166,54 @@ describe('github', () => { getOrganizationUsers(graphql, 'a', 'token'), ).resolves.toEqual(output); }); + + it('reads members excluding suspended users', async () => { + const input: QueryResponse = { + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + suspendedAt: '2025-01-01', + }, + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + suspendedAt: undefined, + }, + ], + }, + }, + }; + + const output = { + users: [ + expect.objectContaining({ + metadata: expect.objectContaining({ name: 'a', description: 'c' }), + spec: { + profile: { displayName: 'b', email: 'd', picture: 'e' }, + memberOf: [], + }, + }), + ], + }; + + server.use( + graphqlMsw.query('users', () => HttpResponse.json({ data: input })), + ); + + await expect( + getOrganizationUsers(graphql, 'a', 'token', undefined, undefined, true), + ).resolves.toEqual(output); + }); }); describe('getOrganizationUsers using custom UserTransformer', () => { @@ -279,6 +327,63 @@ describe('github', () => { expect(users.users).toHaveLength(1); expect(users).toEqual(output); }); + + it('reads members including suspended users', async () => { + const input: QueryResponse = { + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'ab', + name: 'bb', + bio: 'cc', + email: 'dd', + avatarUrl: 'ee', + suspendedAt: '2025-01-01', + }, + ], + }, + }, + }; + + const output = { + users: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'a-custom', + }), + }), + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'ab-custom', + }), + }), + ], + }; + + server.use( + graphqlMsw.query('users', () => HttpResponse.json({ data: input })), + ); + + await expect( + getOrganizationUsers( + graphql, + 'a', + 'token', + customUserTransformer, + undefined, + false, + ), + ).resolves.toEqual(output); + }); }); describe('getOrganizationTeams using default TeamTransformer', () => { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 625fb57bba..bd77df4da7 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -117,6 +117,7 @@ export type GithubUser = { email?: string; name?: string; organizationVerifiedDomainEmails?: string[]; + suspendedAt?: string; }; /** @@ -181,6 +182,7 @@ export type Connection = { * @param tokenType - The type of GitHub credential * @param userTransformer - Optional transformer for user entities * @param pageSizes - Optional page sizes configuration + * @param excludeSuspendedUsers - Optional flag to exclude suspended users (only for GitHub Enterprise instances) */ export async function getOrganizationUsers( client: typeof graphql, @@ -188,7 +190,9 @@ export async function getOrganizationUsers( tokenType: GithubCredentialType, userTransformer: UserTransformer = defaultUserTransformer, pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES, + excludeSuspendedUsers: boolean = false, ): Promise<{ users: Entity[] }> { + const suspendedAtField = excludeSuspendedUsers ? 'suspendedAt,' : ''; const query = ` query users($org: String!, $email: Boolean!, $cursor: String, $organizationMembersPageSize: Int!) { organization(login: $org) { @@ -200,6 +204,7 @@ export async function getOrganizationUsers( email @include(if: $email), login, name, + ${suspendedAtField} organizationVerifiedDomainEmails(login: $org) } } @@ -209,18 +214,19 @@ export async function getOrganizationUsers( // There is no user -> teams edge, so we leave the memberships empty for // now and let the team iteration handle it instead - const users = await queryWithPaging( + const users = await queryWithPaging({ client, query, org, - r => r.organization?.membersWithRole, - userTransformer, - { + connection: r => r.organization?.membersWithRole, + transformer: userTransformer, + variables: { org, email: tokenType === 'token', organizationMembersPageSize: pageSizes.organizationMembers, }, - ); + filter: u => (excludeSuspendedUsers ? !u.suspendedAt : true), + }); return { users }; } @@ -305,18 +311,18 @@ export async function getOrganizationTeams( return await teamTransformer(team, ctx); }; - const teams = await queryWithPaging( + const teams = await queryWithPaging({ client, query, org, - r => r.organization?.teams, - materialisedTeams, - { + connection: r => r.organization?.teams, + transformer: materialisedTeams, + variables: { org, teamsPageSize: pageSizes.teams, membersPageSize: pageSizes.teamMembers, }, - ); + }); return { teams }; } @@ -399,19 +405,19 @@ export async function getOrganizationTeamsFromUsers( return await teamTransformer(team, ctx); }; - const teams = await queryWithPaging( + const teams = await queryWithPaging({ client, query, org, - r => r.organization?.teams, - materialisedTeams, - { + connection: r => r.organization?.teams, + transformer: materialisedTeams, + variables: { org, userLogins, teamsPageSize: pageSizes.teams, membersPageSize: pageSizes.teamMembers, }, - ); + }); return { teams }; } @@ -458,14 +464,14 @@ export async function getOrganizationTeamsForUser( return await teamTransformer(team, ctx); }; - const teams = await queryWithPaging( + const teams = await queryWithPaging({ client, query, org, - r => r.organization?.teams, - materialisedTeams, - { org, userLogins: [userLogin], teamsPageSize: pageSizes.teams }, - ); + connection: r => r.organization?.teams, + transformer: materialisedTeams, + variables: { org, userLogins: [userLogin], teamsPageSize: pageSizes.teams }, + }); return { teams }; } @@ -486,14 +492,14 @@ export async function getOrganizationsFromUser( } }`; - const orgs = await queryWithPaging( + const orgs = await queryWithPaging({ client, query, - '', - r => r.user?.organizations, - async o => o.login, - { user }, - ); + org: '', + connection: r => r.user?.organizations, + transformer: async o => o.login, + variables: { user }, + }); return { orgs }; } @@ -632,14 +638,18 @@ export async function getOrganizationRepositories( } }`; - const repositories = await queryWithPaging( + const repositories = await queryWithPaging({ client, query, org, - r => r.repositoryOwner?.repositories, - async x => x, - { org, catalogPathRef, repositoriesPageSize: pageSizes.repositories }, - ); + connection: r => r.repositoryOwner?.repositories, + transformer: async x => x, + variables: { + org, + catalogPathRef, + repositoriesPageSize: pageSizes.repositories, + }, + }); return { repositories }; } @@ -727,14 +737,14 @@ export async function getTeamMembers( } }`; - const members = await queryWithPaging( + const members = await queryWithPaging({ client, query, org, - r => r.organization?.team?.members, - async user => user, - { org, teamSlug, membersPageSize: pageSizes.teamMembers }, - ); + connection: r => r.organization?.team?.members, + transformer: async user => user, + variables: { org, teamSlug, membersPageSize: pageSizes.teamMembers }, + }); return { members }; } @@ -748,31 +758,36 @@ export async function getTeamMembers( * * Requires that the query accepts a $cursor variable. * - * @param client - The octokit client - * @param query - The query to execute - * @param org - The slug of the org to read - * @param connection - A function that, given the response, picks out the actual + * @param params - Object containing all parameters + * @param params.client - The octokit client + * @param params.query - The query to execute + * @param params.org - The slug of the org to read + * @param params.connection - A function that, given the response, picks out the actual * Connection object that's being iterated - * @param transformer - A function that, given one of the nodes in the Connection, + * @param params.transformer - A function that, given one of the nodes in the Connection, * returns the model mapped form of it - * @param variables - The variable values that the query needs, minus the cursor + * @param params.variables - The variable values that the query needs, minus the cursor + * @param params.filter - An optional filter function to filter the nodes before transforming them */ export async function queryWithPaging< GraphqlType, OutputType, Variables extends {}, Response = QueryResponse, ->( - client: typeof graphql, - query: string, - org: string, - connection: (response: Response) => Connection | undefined, +>(params: { + client: typeof graphql; + query: string; + org: string; + connection: (response: Response) => Connection | undefined; transformer: ( item: GraphqlType, ctx: TransformerContext, - ) => Promise, - variables: Variables, -): Promise { + ) => Promise; + variables: Variables; + filter?: (item: GraphqlType) => boolean; +}): Promise { + const { client, query, org, connection, transformer, variables, filter } = + params; const result: OutputType[] = []; const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); @@ -789,6 +804,9 @@ export async function queryWithPaging< } for (const node of conn.nodes) { + if (filter && !filter(node)) { + continue; + } const transformedNode = await transformer(node, { client, query, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index 839ca66e40..173da84f58 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -174,6 +174,14 @@ export interface GithubMultiOrgEntityProviderOptions { * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: Partial; + + /** + * Optionally exclude suspended users when querying organization users. + * @defaultValue false + * @remarks + * Only for GitHub Enterprise instances. Will error if used against GitHub.com API. + */ + excludeSuspendedUsers?: boolean; } type CreateDeltaOperation = (entities: Entity[]) => { @@ -221,6 +229,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { events: options.events, alwaysUseDefaultNamespace: options.alwaysUseDefaultNamespace, pageSizes: options.pageSizes, + excludeSuspendedUsers: options.excludeSuspendedUsers, }); provider.schedule(options.schedule); @@ -241,6 +250,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { teamTransformer?: TeamTransformer; alwaysUseDefaultNamespace?: boolean; pageSizes?: Partial; + excludeSuspendedUsers?: boolean; }, ) {} @@ -306,6 +316,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { tokenType, this.options.userTransformer, pageSizes, + this.options.excludeSuspendedUsers, ); const { teams } = await getOrganizationTeams( @@ -458,6 +469,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { tokenType, this.options.userTransformer, pageSizes, + this.options.excludeSuspendedUsers, ); const { teams } = await getOrganizationTeams( @@ -692,6 +704,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { tokenType, this.options.userTransformer, pageSizes, + this.options.excludeSuspendedUsers, ); const usersFromChangedGroup = isGroupEntity(team) diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 6dfa8a07f8..4758311e83 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -53,6 +53,7 @@ import { createGraphqlClient, createRemoveEntitiesOperation, createReplaceEntitiesOperation, + DEFAULT_PAGE_SIZES, DeferredEntitiesBuilder, getOrganizationTeam, getOrganizationTeams, @@ -130,6 +131,14 @@ export interface GithubOrgEntityProviderOptions { * Optionally include a team transformer for transforming from GitHub teams to Group Entities */ teamTransformer?: TeamTransformer; + + /** + * Optionally exclude suspended users when querying organization users. + * @defaultValue false + * @remarks + * Only for GitHub Enterprise instances. Will error if used against GitHub.com API. + */ + excludeSuspendedUsers?: boolean; } /** @@ -167,6 +176,7 @@ export class GithubOrgEntityProvider implements EntityProvider { userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, events: options.events, + excludeSuspendedUsers: options.excludeSuspendedUsers, }); provider.schedule(options.schedule); @@ -184,6 +194,7 @@ export class GithubOrgEntityProvider implements EntityProvider { githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + excludeSuspendedUsers?: boolean; }, ) { this.credentialsProvider = @@ -236,6 +247,8 @@ export class GithubOrgEntityProvider implements EntityProvider { org, tokenType, this.options.userTransformer, + DEFAULT_PAGE_SIZES, + this.options.excludeSuspendedUsers, ); const { teams } = await getOrganizationTeams( client, @@ -364,6 +377,8 @@ export class GithubOrgEntityProvider implements EntityProvider { org, tokenType, this.options.userTransformer, + DEFAULT_PAGE_SIZES, + this.options.excludeSuspendedUsers, ); if (!isGroupEntity(team)) { @@ -455,6 +470,8 @@ export class GithubOrgEntityProvider implements EntityProvider { org, tokenType, this.options.userTransformer, + DEFAULT_PAGE_SIZES, + this.options.excludeSuspendedUsers, ); const usersToRebuild = users.filter(u => u.metadata.name === userLogin);