feat(catalog,github): support repository events

The provider adds a subscription to the topic `github.repository`.

Hereby, it supports events of type `repository` with actions

- `archived`
- `deleted`
- `edited`
- `renamed`
- `transferred`
- `unarchived`

Actions skipped as they don't require entity changes:

- `created`
- `privatized`
- `publicized`

If the config option `validateLocationsExist` is enabled, an API request
is necessary and will be executed.
This affects the actions `renamed`, `transferred`, and `unarchive`
of event type `repository`.

Catalog entities related to the `GithubEntityProvider` instance will be adjusted
according to action and its meaning for them.

Closes: #21906
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2024-06-22 03:02:42 +02:00
parent 89145d7efc
commit 9112efc364
5 changed files with 971 additions and 63 deletions
+30
View File
@@ -0,0 +1,30 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Adds support for `repository` events.
The provider adds a subscription to the topic `github.repository`.
Hereby, it supports events of type `repository` with actions
- `archived`
- `deleted`
- `edited`
- `renamed`
- `transferred`
- `unarchived`
Actions skipped as they don't require entity changes:
- `created`
- `privatized`
- `publicized`
If the config option `validateLocationsExist` is enabled, an API request
is necessary and will be executed.
This affects the actions `renamed`, `transferred`, and `unarchive`
of event type `repository`.
Catalog entities related to the `GithubEntityProvider` instance will be adjusted
according to action and its meaning for them.
+12 -3
View File
@@ -40,8 +40,9 @@ backend.add(import('@backstage/plugin-catalog-backend-module-github/alpha'));
## Events Support
The catalog module for GitHub comes with events support enabled.
This will make it subscribe to its relevant topics (`github.push`)
and expects these events to be published via the `EventsService`.
This will make it subscribe to its relevant topics (`github.push`,
`github.repository`) and expects these events to be published
via the `EventsService`.
Additionally, you should install the
[event router by `events-backend-module-github`](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md)
@@ -55,7 +56,15 @@ You can decide between the following options (extensible):
- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `push` events.
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks).
The webhook(s) will need to be configured to react to `push` and
`repository` events.
Certain actions like `transferred` by the `repository` event type
will not be supported when you use repository webhooks.
Please check the GitHubs documentation for these event types and
its actions.
## Configuration
@@ -38,6 +38,7 @@ export type QueryResponse = {
type RepositoryOwnerResponse = {
repositories?: Connection<RepositoryResponse>;
repository?: RepositoryResponse;
};
export type OrganizationResponse = {
@@ -509,6 +510,61 @@ export async function getOrganizationRepositories(
return { repositories };
}
export async function getOrganizationRepository(
client: typeof graphql,
org: string,
repoName: string,
catalogPath: string,
): Promise<RepositoryResponse | null> {
let relativeCatalogPathRef: string;
// We must strip the leading slash or the query for objects does not work
if (catalogPath.startsWith('/')) {
relativeCatalogPathRef = catalogPath.substring(1);
} else {
relativeCatalogPathRef = catalogPath;
}
const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
const query = `
query repository($org: String!, $repoName: String!, $catalogPathRef: String!) {
repositoryOwner(login: $org) {
repository(name: $repoName) {
name
catalogInfoFile: object(expression: $catalogPathRef) {
__typename
... on Blob {
id
text
}
}
url
isArchived
isFork
visibility
repositoryTopics(first: 100) {
nodes {
... on RepositoryTopic {
topic {
name
}
}
}
}
defaultBranchRef {
name
}
}
}
}`;
const response: QueryResponse = await client(query, {
org,
repoName,
catalogPathRef,
});
return response.repositoryOwner?.repository || null;
}
/**
* Gets all the users out of a Github organization.
*
@@ -29,7 +29,12 @@ import { GithubEntityProvider } from './GithubEntityProvider';
import * as helpers from '../lib/github';
import { EventParams } from '@backstage/plugin-events-node';
import { mockServices } from '@backstage/backend-test-utils';
import { Commit, PushEvent } from '@octokit/webhooks-types';
import {
Commit,
PushEvent,
RepositoryEvent,
RepositoryRenamedEvent,
} from '@octokit/webhooks-types';
jest.mock('../lib/github', () => {
return {
@@ -657,7 +662,7 @@ describe('GithubEntityProvider', () => {
describe('on event', () => {
const createExpectedEntitiesForEvent = (
event: EventParams<PushEvent>,
event: EventParams<PushEvent | RepositoryEvent>,
options?: { branch?: string; catalogFilePath?: string },
): DeferredEntity[] => {
const url = `${event.eventPayload.repository.url}/blob/${
@@ -948,5 +953,543 @@ describe('GithubEntityProvider', () => {
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
});
describe('on repository event', () => {
const createRepoEvent = (
action: RepositoryEvent['action'],
): EventParams<RepositoryEvent> => {
const repo = {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
master_branch: 'main',
organization: 'test-org',
topics: [],
archived: action === 'archived',
private: action !== 'publicized',
} as Partial<RepositoryEvent['repository']>;
const event = {
action,
repository: repo as RepositoryEvent['repository'],
} as RepositoryEvent;
if (action === 'renamed') {
(event as RepositoryRenamedEvent).changes = {
repository: {
name: {
from: `old-${event.repository.name}`,
},
},
};
}
return {
topic: 'github.repository',
metadata: {
'x-github-event': 'repository',
},
eventPayload: event,
};
};
describe('on repository archived event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('archived');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update removing entities', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('archived');
const expectedEntities = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntities,
});
});
});
describe('on repository created event', () => {
it('skip', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('created');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
});
describe('on repository deleted event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('deleted');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update removing entities', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('deleted');
const expectedEntities = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntities,
});
});
});
describe('on repository edited event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('edited');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update removing entities with non-matching filters', async () => {
const config = createSingleProviderConfig({
providerConfig: {
filters: {
topic: {
include: ['backstage-include'],
},
},
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('edited');
const expectedEntities = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntities,
});
});
it('apply no delta update with matching filters', async () => {
const config = createSingleProviderConfig({
providerConfig: {
filters: {
topic: {
include: ['backstage-include'],
},
},
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('edited');
event.eventPayload.repository.topics = ['backstage-include'];
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
});
describe('on repository privatized event', () => {
it('skip', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('privatized');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
});
describe('on repository publicized event', () => {
it('skip', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('publicized');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
});
describe('on repository renamed event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('renamed');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update removing entities with non-matching filters', async () => {
const config = createSingleProviderConfig({
providerConfig: {
filters: {
repository: 'other-.*',
},
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent(
'renamed',
) as EventParams<RepositoryRenamedEvent>;
const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`;
const expectedEntitiesRemoved =
createExpectedEntitiesForUrl(urlOldRepo);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntitiesRemoved,
});
});
it('apply delta update removing and adding entities with matching filters', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent(
'renamed',
) as EventParams<RepositoryRenamedEvent>;
const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`;
const expectedEntitiesRemoved =
createExpectedEntitiesForUrl(urlOldRepo);
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
2,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntitiesRemoved,
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: expectedEntitiesAdded,
removed: [],
});
});
});
describe('on repository transferred event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('transferred');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply no delta update with non-matching filters', async () => {
const config = createSingleProviderConfig({
providerConfig: {
filters: {
topic: {
include: ['backstage-include'],
},
},
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('transferred');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update adding entities with matching filters', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('transferred');
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: expectedEntitiesAdded,
removed: [],
});
});
});
describe('on repository unarchived event', () => {
it('skip on non-matching org', async () => {
const config = createSingleProviderConfig({
providerConfig: {
organization: 'other-org',
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('unarchived');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply no delta update with non-matching filters', async () => {
const config = createSingleProviderConfig({
providerConfig: {
filters: {
topic: {
include: ['backstage-include'],
},
},
},
});
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('unarchived');
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
0,
);
});
it('apply delta update adding entities with matching filters', async () => {
const config = createSingleProviderConfig();
const provider = createProviders(config)[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event = createRepoEvent('unarchived');
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
1,
);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: expectedEntitiesAdded,
removed: [],
});
});
});
});
});
});
@@ -38,7 +38,11 @@ import {
GithubEntityProviderConfig,
readProviderConfigs,
} from './GithubEntityProviderConfig';
import { getOrganizationRepositories } from '../lib/github';
import {
getOrganizationRepositories,
getOrganizationRepository,
RepositoryResponse,
} from '../lib/github';
import {
satisfiesForkFilter,
satisfiesTopicFilter,
@@ -50,11 +54,21 @@ import {
EventsService,
EventSubscriber,
} from '@backstage/plugin-events-node';
import { Commit, PushEvent } from '@octokit/webhooks-types';
import {
Commit,
PushEvent,
RepositoryArchivedEvent,
RepositoryDeletedEvent,
RepositoryEditedEvent,
RepositoryEvent,
RepositoryRenamedEvent,
RepositoryTransferredEvent,
RepositoryUnarchivedEvent,
} from '@octokit/webhooks-types';
import { Minimatch } from 'minimatch';
import { LoggerService } from '@backstage/backend-plugin-api';
const TOPIC_REPO_PUSH = 'github.push';
const EVENT_TOPICS = ['github.push', 'github.repository'];
type Repository = {
name: string;
@@ -157,7 +171,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
this.connection = connection;
await this.events?.subscribe({
id: this.getProviderName(),
topics: [TOPIC_REPO_PUSH],
topics: EVENT_TOPICS,
onEvent: params => this.onEvent(params),
});
return await this.scheduleFn();
@@ -194,15 +208,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
const targets = await this.findCatalogFiles();
const matchingTargets = this.matchesFilters(targets);
const entities = matchingTargets
.map(repository => this.createLocationUrl(repository))
.map(GithubEntityProvider.toLocationSpec)
.map(location => {
return {
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
};
});
const entities = this.toDeferredEntitiesFromRepos(matchingTargets);
await this.connection.applyMutation({
type: 'full',
@@ -214,38 +220,32 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
);
}
// go to the server and get all of the repositories
private async findCatalogFiles(): Promise<Repository[]> {
private async createGraphqlClient() {
const organization = this.config.organization;
const host = this.integration.host;
const catalogPath = this.config.catalogPath;
const orgUrl = `https://${host}/${organization}`;
const { headers } = await this.githubCredentialsProvider.getCredentials({
url: orgUrl,
});
const client = graphql.defaults({
return graphql.defaults({
baseUrl: this.integration.apiBaseUrl,
headers,
});
}
// go to the server and get all repositories
private async findCatalogFiles(): Promise<Repository[]> {
const organization = this.config.organization;
const catalogPath = this.config.catalogPath;
const client = await this.createGraphqlClient();
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
const repositories = repositoriesFromGithub.map(r => {
return {
url: r.url,
name: r.name,
defaultBranchRef: r.defaultBranchRef?.name,
repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name),
isArchived: r.isArchived,
isFork: r.isFork,
isCatalogInfoFilePresent:
r.catalogInfoFile?.__typename === 'Blob' &&
r.catalogInfoFile.text !== '',
visibility: r.visibility,
};
});
const repositories = repositoriesFromGithub.map(
this.createRepoFromGithubResponse,
);
if (this.config.validateLocationsExist) {
return repositories.filter(
@@ -256,13 +256,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
return repositories;
}
private matchesFilters(repositories: Repository[]) {
private matchesFilters(repositories: Repository[]): Repository[] {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const allowForks = this.config.filters?.allowForks ?? true;
const visibilities = this.config.filters?.visibility ?? [];
const matchingRepositories = repositories.filter(r => {
return repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics;
return (
!r.isArchived &&
@@ -273,7 +273,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
r.defaultBranchRef
);
});
return matchingRepositories;
}
private createLocationUrl(repository: Repository): string {
@@ -296,23 +295,34 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
async onEvent(params: EventParams): Promise<void> {
this.logger.debug(`Received event from ${params.topic}`);
if (params.topic !== TOPIC_REPO_PUSH) {
return;
}
if (EVENT_TOPICS.some(topic => topic === params.topic)) {
if (!this.connection) {
throw new Error('Not initialized');
}
await this.onRepoPush(params.eventPayload as PushEvent);
switch (params.topic) {
case 'github.push':
await this.onPush(params.eventPayload as PushEvent);
return;
case 'github.repository':
await this.onRepoChange(params.eventPayload as RepositoryEvent);
return;
default: // should never be reached
this.logger.warn(
`Missing implementation for event of topic ${params.topic}`,
);
}
}
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
supportsEventTopics(): string[] {
return [TOPIC_REPO_PUSH];
return EVENT_TOPICS;
}
private async onRepoPush(event: PushEvent) {
if (!this.connection) {
throw new Error('Not initialized');
}
private async onPush(event: PushEvent) {
if (this.config.organization !== event.repository.organization) {
this.logger.debug(
`skipping push event from organization ${event.repository.organization}`,
@@ -332,18 +342,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
return;
}
const repository: Repository = {
url: event.repository.url,
name: event.repository.name,
defaultBranchRef: event.repository.default_branch,
repositoryTopics: event.repository.topics,
isArchived: event.repository.archived,
isFork: event.repository.fork,
// we can consider this file present because
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
visibility: event.repository.visibility,
};
const repository = this.createRepoFromEvent(event);
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
@@ -378,7 +377,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
? this.config.catalogPath.substring(1)
: this.config.catalogPath;
await this.connection.refresh({
await this.connection!.refresh({
keys: [
...new Set([
...modified.map(
@@ -396,7 +395,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
}
if (added.length > 0 || removed.length > 0) {
await this.connection.applyMutation({
await this.connection!.applyMutation({
type: 'delta',
added: added,
removed: removed,
@@ -408,6 +407,263 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
);
}
private async onRepoChange(event: RepositoryEvent) {
if (this.config.organization !== event.repository.organization) {
this.logger.debug(
`skipping repository event from organization ${event.repository.organization}`,
);
return;
}
const action = event.action;
switch (action) {
case 'archived':
await this.onRepoArchived(event as RepositoryArchivedEvent);
return;
// A repository was created.
case 'created':
// skip these events
return;
case 'deleted':
await this.onRepoDeleted(event as RepositoryDeletedEvent);
return;
case 'edited':
await this.onRepoEdited(event as RepositoryEditedEvent);
return;
// The visibility of a repository was changed to `private`.
case 'privatized':
// skip these events
return;
// The visibility of a repository was changed to `public`.
case 'publicized':
// skip these events
return;
case 'renamed':
await this.onRepoRenamed(event as RepositoryRenamedEvent);
return;
case 'transferred':
await this.onRepoTransferred(event as RepositoryTransferredEvent);
return;
case 'unarchived':
await this.onRepoUnarchived(event as RepositoryUnarchivedEvent);
return;
default: // should never be reached
this.logger.warn(
`Missing implementation for event of topic repository with action ${action}`,
);
}
}
/**
* A repository was archived.
*
* Removes all entities associated with the repository.
*
* @param event - The repository archived event.
* @private
*/
private async onRepoArchived(event: RepositoryArchivedEvent) {
const repository = this.createRepoFromEvent(event);
await this.removeEntitiesForRepo(repository);
this.logger.debug(
`Removed entities for archived repository ${repository.name}`,
);
}
/**
* A repository was deleted.
*
* Removes all entities associated with the repository.
*
* @param event - The repository deleted event.
* @private
*/
private async onRepoDeleted(event: RepositoryDeletedEvent) {
const repository = this.createRepoFromEvent(event);
await this.removeEntitiesForRepo(repository);
this.logger.debug(
`Removed entities for deleted repository ${repository.name}`,
);
}
/**
* The topics, default branch, description, or homepage of a repository was changed.
*
* We are interested in potential topic changes as these can be used as part of the filters.
*
* Removes all entities associated with the repository if the repository no longer matches the filters.
*
* @param event - The repository edited event.
* @private
*/
private async onRepoEdited(event: RepositoryEditedEvent) {
const repository = this.createRepoFromEvent(event);
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
await this.removeEntitiesForRepo(repository);
}
// else: repository is (still) matching the filters, so we don't need to do anything
}
/**
* The name of a repository was changed.
*
* Removes all entities associated with the repository's old name.
* Creates new entities for the repository's new name if it still matches the filters.
*
* @param event - The repository renamed event.
* @private
*/
private async onRepoRenamed(event: RepositoryRenamedEvent) {
const repository = this.createRepoFromEvent(event);
const oldRepoName = event.changes.repository.name.from;
const urlParts = event.repository.url.split('/');
urlParts[urlParts.length - 1] = oldRepoName;
const oldRepoUrl = urlParts.join('/');
const oldRepository: Repository = {
...repository,
name: oldRepoName,
url: oldRepoUrl,
};
await this.removeEntitiesForRepo(oldRepository);
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
);
return;
}
await this.addEntitiesForRepo(repository);
}
/**
* Ownership of the repository was transferred to a user or organization account.
* This event is only sent to the account where the ownership is transferred.
* To receive the `repository.transferred` event, the new owner account must have the GitHub App installed,
* and the App must be subscribed to "Repository" events.
*
* Creates new entities for the repository if it matches the filters.
*
* @param event - The repository unarchived event.
* @private
*/
private async onRepoTransferred(event: RepositoryTransferredEvent) {
const repository = this.createRepoFromEvent(event);
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
);
return;
}
await this.addEntitiesForRepo(repository);
}
/**
* A previously archived repository was unarchived.
*
* Creates new entities for the repository if it matches the filters.
*
* @param event - The repository unarchived event.
* @private
*/
private async onRepoUnarchived(event: RepositoryUnarchivedEvent) {
const repository = this.createRepoFromEvent(event);
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
);
return;
}
await this.addEntitiesForRepo(repository);
}
private async removeEntitiesForRepo(repository: Repository) {
const removed = this.toDeferredEntitiesFromRepos([repository]);
await this.connection!.applyMutation({
type: 'delta',
added: [],
removed: removed,
});
}
private async addEntitiesForRepo(repository: Repository) {
if (this.config.validateLocationsExist) {
const organization = this.config.organization;
const catalogPath = this.config.catalogPath;
const client = await this.createGraphqlClient();
const repositoryFromGithub = await getOrganizationRepository(
client,
organization,
repository.name,
catalogPath,
).then(r => (r ? this.createRepoFromGithubResponse(r) : null));
if (!repositoryFromGithub?.isCatalogInfoFilePresent) {
return;
}
}
const added = this.toDeferredEntitiesFromRepos([repository]);
await this.connection!.applyMutation({
type: 'delta',
added: added,
removed: [],
});
}
private createRepoFromEvent(event: RepositoryEvent | PushEvent): Repository {
return {
url: event.repository.url,
name: event.repository.name,
defaultBranchRef: event.repository.default_branch,
repositoryTopics: event.repository.topics,
isArchived: event.repository.archived,
isFork: event.repository.fork,
// we can consider this file present because
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
visibility: event.repository.visibility,
};
}
private createRepoFromGithubResponse(
repositoryResponse: RepositoryResponse,
): Repository {
return {
url: repositoryResponse.url,
name: repositoryResponse.name,
defaultBranchRef: repositoryResponse.defaultBranchRef?.name,
repositoryTopics: repositoryResponse.repositoryTopics.nodes.map(
t => t.topic.name,
),
isArchived: repositoryResponse.isArchived,
isFork: repositoryResponse.isFork,
isCatalogInfoFilePresent:
repositoryResponse.catalogInfoFile?.__typename === 'Blob' &&
repositoryResponse.catalogInfoFile.text !== '',
visibility: repositoryResponse.visibility,
};
}
private collectDeferredEntitiesFromCommit(
repositoryUrl: string,
branch: string,
@@ -454,6 +710,20 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
};
});
}
private toDeferredEntitiesFromRepos(
repositories: Repository[],
): DeferredEntity[] {
return repositories
.map(repository => this.createLocationUrl(repository))
.map(GithubEntityProvider.toLocationSpec)
.map(location => {
return {
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
};
});
}
}
/*