paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat(catalog-backend-module-gitlab): create url location only if file exists (#12)

Browse Source 
Signed-off-by: Ruslan.Nasyrov <goququ@ya.ru>
This commit is contained in:
qu
2022-05-11 17:36:30 +05:00
committed by Ruslan.Nasyrov
parent da3bdb192e
commit 3ac4522537
4 changed files with 175 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/real-beers-type.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': patch
---
do not creating url location object if file with component definition do not exists in project, that decrease count of request to gitlab with 404 status code
plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts
+119 -23
View File
@@ -24,26 +24,36 @@ import { GitLabProject } from './lib';
const server = setupServer();
const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects';
const GROUP_PROJECTS_URL =
'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects';
const DOMAIN = 'gitlab.fake';
const SERVER_URL = `https://${DOMAIN}`;
const API_URL = `${SERVER_URL}/api/v4`;
const PROJECTS_URL = `${API_URL}/projects`;
const GROUP_PROJECTS_URL = `${API_URL}/groups/group%2Fsubgroup/projects`;
const PROJECT_LOCATION: LocationSpec = {
type: 'gitlab-discovery',
target: 'https://gitlab.fake/blob/*/catalog-info.yaml',
target: `${SERVER_URL}/blob/*/catalog-info.yaml`,
};
const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = {
type: 'gitlab-discovery',
target: 'https://gitlab.fake/blob/master/catalog-info.yaml',
target: `${SERVER_URL}/blob/master/catalog-info.yaml`,
};
const GROUP_LOCATION: LocationSpec = {
type: 'gitlab-discovery',
target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml',
target: `${SERVER_URL}/group/subgroup/blob/*/catalog-info.yaml`,
};
const GROUP_LOCATION_CUSTOM_BRANCH: LocationSpec = {
type: 'gitlab-discovery',
target: `${SERVER_URL}/group/subgroup/blob/test/catalog-info.yaml`,
};
function setupFakeServer(
url: string,
callback: (request: { page: number; include_subgroups: boolean }) => {
list_projects_callback: (request: {
page: number;
include_subgroups: boolean;
}) => {
data: GitLabProject[];
nextPage?: number;
},
@@ -55,7 +65,7 @@ function setupFakeServer(
}
const page = req.url.searchParams.get('page');
const include_subgroups = req.url.searchParams.get('include_subgroups');
const response = callback({
const response = list_projects_callback({
page: parseInt(page!, 10),
include_subgroups: include_subgroups === 'true',
});
@@ -75,6 +85,20 @@ function setupFakeServer(
ctx.json(filteredData),
);
}),
rest.head(
`${API_URL}/projects/:project_path/repository/files/:file_path`,
(req, res, ctx) => {
if (req.headers.get('private-token') !== 'test-token') {
return res(ctx.status(401), ctx.json({}));
}
const ref = req.url.searchParams.get('ref');
if (ref === 'main' || ref === 'master') {
return res(ctx.status(200));
}
return res(ctx.status(404));
},
),
);
}
@@ -86,8 +110,8 @@ function getConfig(): any {
integrations: {
gitlab: [
{
host: 'gitlab.fake',
apiBaseUrl: 'https://gitlab.fake/api/v4',
host: DOMAIN,
apiBaseUrl: API_URL,
token: 'test-token',
},
],
@@ -95,11 +119,18 @@ function getConfig(): any {
};
}
function getProcessor(config?: any): GitLabDiscoveryProcessor {
function getProcessor({
config,
options,
}: {
config?: any;
options?: Partial<Parameters<typeof GitLabDiscoveryProcessor.fromConfig>[1]>;
} = {}): GitLabDiscoveryProcessor {
return GitLabDiscoveryProcessor.fromConfig(
new ConfigReader(config || getConfig()),
{
logger: getVoidLogger(),
...options,
},
);
}
@@ -163,6 +194,15 @@ describe('GitlabDiscoveryProcessor', () => {
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/1',
path_with_namespace: '1',
},
{
id: 2,
archived: false,
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/g/2',
path_with_namespace: 'g/2',
},
],
nextPage: 2,
@@ -170,26 +210,29 @@ describe('GitlabDiscoveryProcessor', () => {
case 2:
return {
data: [
{
id: 2,
archived: false,
default_branch: 'master',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/2',
},
{
id: 3,
archived: true, // ARCHIVED
archived: false,
default_branch: 'master',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/3',
path_with_namespace: '3',
},
{
id: 4,
archived: true, // ARCHIVED
default_branch: 'master',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/4',
path_with_namespace: '4',
},
{
id: 5,
archived: false,
default_branch: undefined, // MISSING DEFAULT BRANCH
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/4',
web_url: 'https://gitlab.fake/g/5',
path_with_namespace: 'g/5',
},
],
};
@@ -215,7 +258,15 @@ describe('GitlabDiscoveryProcessor', () => {
type: 'location',
location: {
type: 'url',
target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml',
target: 'https://gitlab.fake/g/2/-/blob/main/catalog-info.yaml',
presence: 'optional',
},
},
{
type: 'location',
location: {
type: 'url',
target: 'https://gitlab.fake/3/-/blob/master/catalog-info.yaml',
presence: 'optional',
},
},
@@ -235,6 +286,7 @@ describe('GitlabDiscoveryProcessor', () => {
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/1',
path_with_namespace: '1',
},
],
};
@@ -275,6 +327,7 @@ describe('GitlabDiscoveryProcessor', () => {
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/1',
path_with_namespace: '1',
},
],
};
@@ -291,6 +344,47 @@ describe('GitlabDiscoveryProcessor', () => {
expect(result).toHaveLength(1);
});
it('can filter based on file existing', async () => {
const processor = getProcessor({ options: { checkFileExistence: true } });
setupFakeServer(GROUP_PROJECTS_URL, request => {
if (!request.include_subgroups) {
throw new Error('include_subgroups should be set');
}
switch (request.page) {
case 1:
return {
data: [
{
id: 1,
archived: false,
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/1',
path_with_namespace: '1',
},
{
id: 1,
archived: false,
default_branch: 'main',
last_activity_at: '2021-08-05T11:03:05.774Z',
web_url: 'https://gitlab.fake/g/2',
path_with_namespace: 'g/2',
},
],
};
default:
throw new Error('Invalid request');
}
});
const result: any[] = [];
await processor.readLocation(GROUP_LOCATION_CUSTOM_BRANCH, false, e => {
result.push(e);
});
// If everything was set up correctly, we should have received the fake repo specified above
expect(result).toHaveLength(0);
});
it('uses the previous scan timestamp to filter', async () => {
const processor = getProcessor();
setupFakeServer(PROJECTS_URL, request => {
@@ -304,6 +398,7 @@ describe('GitlabDiscoveryProcessor', () => {
default_branch: 'main',
last_activity_at: '2000-01-01T00:00:00Z',
web_url: 'https://gitlab.fake/1',
path_with_namespace: '1',
},
{
id: 2,
@@ -311,6 +406,7 @@ describe('GitlabDiscoveryProcessor', () => {
default_branch: 'main',
last_activity_at: '2002-01-01T00:00:00Z',
web_url: 'https://gitlab.fake/2',
path_with_namespace: '2',
},
],
};
@@ -349,7 +445,7 @@ describe('GitlabDiscoveryProcessor', () => {
const config = getConfig();
config.integrations.gitlab[0].token = 'invalid';
await expect(
getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}),
getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}),
).rejects.toThrow(/Unauthorized/);
});
@@ -357,7 +453,7 @@ describe('GitlabDiscoveryProcessor', () => {
const config = getConfig();
delete config.integrations;
await expect(
getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}),
getProcessor({ config }).readLocation(PROJECT_LOCATION, false, _ => {}),
).rejects.toThrow(/no GitLab integration/);
});
plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts
+21 -1
View File
@@ -41,8 +41,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrationRegistry;
private readonly logger: Logger;
private readonly cache: CacheClient;
private readonly checkFileExistence: boolean;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: { logger: Logger; checkFileExistence?: boolean },
): GitLabDiscoveryProcessor {
const integrations = ScmIntegrations.fromConfig(config);
const pluginCache =
CacheManager.fromConfig(config).forPlugin('gitlab-discovery');
@@ -58,10 +62,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
integrations: ScmIntegrationRegistry;
pluginCache: PluginCacheManager;
logger: Logger;
checkFileExistence?: boolean;
}) {
this.integrations = options.integrations;
this.cache = options.pluginCache.getClient();
this.logger = options.logger;
this.checkFileExistence = options.checkFileExistence || false;
}
getProcessorName(): string {
@@ -114,6 +120,20 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
continue;
}
if (this.checkFileExistence) {
const project_branch = branch === '*' ? project.default_branch : branch;
const projectHasFile: boolean = await client.hasFile(
project.path_with_namespace,
project_branch,
catalogPath,
);
if (!projectHasFile) {
continue;
}
}
res.matches.push(project);
}
plugins/catalog-backend-module-gitlab/src/lib/client.ts
+30
View File
@@ -63,6 +63,36 @@ export class GitLabClient {
return this.pagedRequest(`/projects`, options);
}
async hasFile(
projectPath: string,
branch: string,
filePath: string,
): Promise<boolean> {
const endpoint: string = `/projects/${encodeURIComponent(
projectPath,
)}/repository/files/${encodeURIComponent(filePath)}`;
const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
request.searchParams.append('ref', branch);
this.logger.debug(`Fetching: ${request.toString()}`);
const response = await fetch(request.toString(), {
headers: getGitLabRequestOptions(this.config).headers,
method: 'HEAD',
});
if (!response.ok) {
this.logger.debug(
`Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
response.status
} - ${response.statusText}`,
);
return false;
}
return true;
}
/**
* Performs a request against a given paginated GitLab endpoint.
*