paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend-common: implement UrlReader.search for the other providers too

Browse Source
This commit is contained in:
Fredrik Adelöw
2021-02-08 15:06:47 +01:00
parent d9ce84c732
commit 491f3a0ecf
26 changed files with 822 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/angry-walls-mate.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': minor
---
Make `ScmIntegration.resolveUrl` mandatory.
.changeset/dry-llamas-wave.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/backend-common': patch
---
Implement `UrlReader.search` for the other providers (Azure, Bitbucket, GitLab) as well.
The `UrlReader` subclasses now are implemented in terms of the respective `Integration` class.
packages/backend-common/src/reading/AzureUrlReader.test.ts
+101 -16
View File
@@ -14,18 +14,22 @@
* limitations under the License.
*/
import * as os from 'os';
import { ConfigReader } from '@backstage/config';
import {
AzureIntegration,
readAzureIntegrationConfig,
} from '@backstage/integration';
import { msw } from '@backstage/test-utils';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import path from 'path';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { ConfigReader } from '@backstage/config';
import * as os from 'os';
import path from 'path';
import { NotModifiedError } from '../errors';
import { getVoidLogger } from '../logging';
import { AzureUrlReader } from './AzureUrlReader';
import { msw } from '@backstage/test-utils';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError } from '../errors';
const logger = getVoidLogger();
@@ -36,6 +40,16 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('AzureUrlReader', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -143,22 +157,18 @@ describe('AzureUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
);
const processor = new AzureUrlReader(
{ host: 'dev.azure.com' },
new AzureIntegration(
readAzureIntegrationConfig(
new ConfigReader({
host: 'dev.azure.com',
}),
),
),
{ treeResponseFactory },
);
@@ -257,4 +267,79 @@ describe('AzureUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
});
describe('search', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
);
const processor = new AzureUrlReader(
new AzureIntegration(
readAzureIntegrationConfig(
new ConfigReader({
host: 'dev.azure.com',
}),
),
),
{ treeResponseFactory },
);
beforeEach(() => {
worker.use(
rest.get(
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
),
),
rest.get(
// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
count: 2,
value: [
{
commitId: '123abc2',
comment: 'second commit',
},
{
commitId: '123abc1',
comment: 'first commit',
},
],
}),
),
),
);
});
it('works for the naive case', async () => {
const result = await processor.search(
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster',
);
expect(result.etag).toBe('123abc2');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2Fdocs%2Findex.md&version=GBmaster',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
processor.search(
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=**/index.*&version=GBmaster',
{ etag: '123abc2' },
),
).rejects.toThrow(NotModifiedError);
});
});
});
packages/backend-common/src/reading/AzureUrlReader.ts
+52 -19
View File
@@ -15,39 +15,41 @@
*/
import {
AzureIntegrationConfig,
readAzureIntegrationConfigs,
getAzureFileFetchUrl,
getAzureDownloadUrl,
getAzureRequestOptions,
AzureIntegration,
getAzureCommitsUrl,
getAzureDownloadUrl,
getAzureFileFetchUrl,
getAzureRequestOptions,
ScmIntegrations,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import { stripFirstDirectoryFromPath } from './tree/util';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
SearchOptions,
SearchResponse,
UrlReader,
} from './types';
import { ReadTreeResponseFactory } from './tree';
export class AzureUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const configs = readAzureIntegrationConfigs(
config.getOptionalConfigArray('integrations.azure') ?? [],
);
return configs.map(options => {
const reader = new AzureUrlReader(options, { treeResponseFactory });
const predicate = (url: URL) => url.host === options.host;
const integrations = ScmIntegrations.fromConfig(config);
return integrations.azure.list().map(integration => {
const reader = new AzureUrlReader(integration, { treeResponseFactory });
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly options: AzureIntegrationConfig,
private readonly integration: AzureIntegration,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {}
@@ -56,7 +58,10 @@ export class AzureUrlReader implements UrlReader {
let response: Response;
try {
response = await fetch(builtUrl, getAzureRequestOptions(this.options));
response = await fetch(
builtUrl,
getAzureRequestOptions(this.integration.config),
);
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
@@ -83,7 +88,7 @@ export class AzureUrlReader implements UrlReader {
const commitsAzureResponse = await fetch(
getAzureCommitsUrl(url),
getAzureRequestOptions(this.options),
getAzureRequestOptions(this.integration.config),
);
if (!commitsAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`;
@@ -100,7 +105,9 @@ export class AzureUrlReader implements UrlReader {
const archiveAzureResponse = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
getAzureRequestOptions(this.integration.config, {
Accept: 'application/zip',
}),
);
if (!archiveAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`;
@@ -117,12 +124,38 @@ export class AzureUrlReader implements UrlReader {
});
}
async search(): Promise<SearchResponse> {
throw new Error('AzureUrlReader does not implement search');
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
const { filepath } = parseGitUrl(url);
const matcher = new Minimatch(filepath);
// TODO(freben): For now, read the entire repo and filter through that. In
// a future improvement, we could be smart and try to deduce that non-glob
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
// to get just that part of the repo.
const treeUrl = new URL(url);
treeUrl.searchParams.delete('path');
treeUrl.pathname = treeUrl.pathname.replace(/\/+$/, '');
const tree = await this.readTree(treeUrl.toString(), {
etag: options?.etag,
filter: path => matcher.match(stripFirstDirectoryFromPath(path)),
});
const files = await tree.files();
return {
etag: tree.etag,
files: files.map(file => ({
url: this.integration.resolveUrl({
url: `/${file.path}`,
base: url,
}),
content: file.content,
})),
};
}
toString() {
const { host, token } = this.options;
const { host, token } = this.integration.config;
return `azure{host=${host},authed=${Boolean(token)}}`;
}
}
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+180 -21
View File
@@ -15,11 +15,16 @@
*/
import { ConfigReader } from '@backstage/config';
import {
BitbucketIntegration,
readBitbucketIntegrationConfig,
} from '@backstage/integration';
import { msw } from '@backstage/test-utils';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import os from 'os';
import path from 'path';
import { NotModifiedError } from '../errors';
import { BitbucketUrlReader } from './BitbucketUrlReader';
@@ -30,19 +35,45 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
});
const bitbucketProcessor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
}),
),
),
{ treeResponseFactory },
);
const hostedBitbucketProcessor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
}),
),
),
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('BitbucketUrlReader', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
msw.setupDefaultHandlers(worker);
describe('implementation', () => {
it('rejects unknown targets', async () => {
await expect(
@@ -54,19 +85,6 @@ describe('BitbucketUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
msw.setupDefaultHandlers(worker);
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
@@ -247,12 +265,153 @@ describe('BitbucketUrlReader', () => {
expect(() => {
/* eslint-disable no-new */
new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
},
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.mycompany.net',
}),
),
),
{ treeResponseFactory },
);
}).toThrowError('must configure an explicit apiBaseUrl');
});
});
describe('search hosted', () => {
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-repo-with-commit-hash.zip',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
mainbranch: {
type: 'branch',
name: 'master',
},
}),
),
),
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock-12ab34cd56ef.zip',
),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('works for the naive case', async () => {
const result = await bitbucketProcessor.search(
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
bitbucketProcessor.search(
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
{ etag: '12ab34cd56ef' },
),
).rejects.toThrow(NotModifiedError);
});
});
describe('search private', () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock.zip',
),
ctx.body(privateBitbucketRepoBuffer),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('works for the naive case', async () => {
const result = await hostedBitbucketProcessor.search(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
hostedBitbucketProcessor.search(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
{ etag: '12ab34cd56ef' },
),
).rejects.toThrow(NotModifiedError);
});
});
});
packages/backend-common/src/reading/BitbucketUrlReader.ts
+60 -32
View File
@@ -15,22 +15,25 @@
*/
import {
BitbucketIntegrationConfig,
BitbucketIntegration,
getBitbucketDefaultBranch,
getBitbucketDownloadUrl,
getBitbucketFileFetchUrl,
getBitbucketRequestOptions,
readBitbucketIntegrationConfigs,
ScmIntegrations,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import { stripFirstDirectoryFromPath } from './tree/util';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
SearchOptions,
SearchResponse,
UrlReader,
} from './types';
@@ -40,45 +43,43 @@ import {
* the one exposed by Bitbucket Cloud itself.
*/
export class BitbucketUrlReader implements UrlReader {
private readonly config: BitbucketIntegrationConfig;
private readonly treeResponseFactory: ReadTreeResponseFactory;
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const configs = readBitbucketIntegrationConfigs(
config.getOptionalConfigArray('integrations.bitbucket') ?? [],
);
return configs.map(provider => {
const reader = new BitbucketUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
const integrations = ScmIntegrations.fromConfig(config);
return integrations.bitbucket.list().map(integration => {
const reader = new BitbucketUrlReader(integration, {
treeResponseFactory,
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
config: BitbucketIntegrationConfig,
deps: { treeResponseFactory: ReadTreeResponseFactory },
private readonly integration: BitbucketIntegration,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
const { host, apiBaseUrl, token, username, appPassword } = config;
const {
host,
apiBaseUrl,
token,
username,
appPassword,
} = integration.config;
if (!apiBaseUrl) {
throw new Error(
`Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`,
);
}
if (!token && username && !appPassword) {
} else if (!token && username && !appPassword) {
throw new Error(
`Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`,
);
}
this.config = config;
this.treeResponseFactory = deps.treeResponseFactory;
}
async read(url: string): Promise<Buffer> {
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.config);
const options = getBitbucketRequestOptions(this.config);
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.integration.config);
const options = getBitbucketRequestOptions(this.integration.config);
let response: Response;
try {
@@ -109,10 +110,13 @@ export class BitbucketUrlReader implements UrlReader {
throw new NotModifiedError();
}
const downloadUrl = await getBitbucketDownloadUrl(url, this.config);
const downloadUrl = await getBitbucketDownloadUrl(
url,
this.integration.config,
);
const archiveBitbucketResponse = await fetch(
downloadUrl,
getBitbucketRequestOptions(this.config),
getBitbucketRequestOptions(this.integration.config),
);
if (!archiveBitbucketResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`;
@@ -122,7 +126,7 @@ export class BitbucketUrlReader implements UrlReader {
throw new Error(message);
}
return await this.treeResponseFactory.fromZipArchive({
return await this.deps.treeResponseFactory.fromZipArchive({
stream: (archiveBitbucketResponse.body as unknown) as Readable,
subpath: filepath,
etag: lastCommitShortHash,
@@ -130,12 +134,36 @@ export class BitbucketUrlReader implements UrlReader {
});
}
async search(): Promise<SearchResponse> {
throw new Error('BitbucketUrlReader does not implement search');
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
const { filepath } = parseGitUrl(url);
const matcher = new Minimatch(filepath);
// TODO(freben): For now, read the entire repo and filter through that. In
// a future improvement, we could be smart and try to deduce that non-glob
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
// to get just that part of the repo.
const treeUrl = url.replace(filepath, '').replace(/\/+$/, '');
const tree = await this.readTree(treeUrl, {
etag: options?.etag,
filter: path => matcher.match(stripFirstDirectoryFromPath(path)),
});
const files = await tree.files();
return {
etag: tree.etag,
files: files.map(file => ({
url: this.integration.resolveUrl({
url: `/${file.path}`,
base: url,
}),
content: file.content,
})),
};
}
toString() {
const { host, token, username, appPassword } = this.config;
const { host, token, username, appPassword } = this.integration.config;
let authed = Boolean(token);
if (!authed) {
authed = Boolean(username && appPassword);
@@ -148,18 +176,18 @@ export class BitbucketUrlReader implements UrlReader {
let branch = ref;
if (!branch) {
branch = await getBitbucketDefaultBranch(url, this.config);
branch = await getBitbucketDefaultBranch(url, this.integration.config);
}
const isHosted = resource === 'bitbucket.org';
// Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222
const commitsApiUrl = isHosted
? `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`
: `${this.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`;
? `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`
: `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`;
const commitsResponse = await fetch(
commitsApiUrl,
getBitbucketRequestOptions(this.config),
getBitbucketRequestOptions(this.integration.config),
);
if (!commitsResponse.ok) {
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
packages/backend-common/src/reading/GithubUrlReader.test.ts
+43 -33
View File
@@ -15,12 +15,17 @@
*/
import { ConfigReader } from '@backstage/config';
import { GithubCredentialsProvider } from '@backstage/integration';
import {
GithubCredentialsProvider,
GitHubIntegration,
readGitHubIntegrationConfig,
} from '@backstage/integration';
import { msw } from '@backstage/test-utils';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import os from 'os';
import path from 'path';
import { NotFoundError, NotModifiedError } from '../errors';
import {
@@ -41,26 +46,45 @@ const mockCredentialsProvider = ({
} as unknown) as GithubCredentialsProvider;
const githubProcessor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
new GitHubIntegration(
readGitHubIntegrationConfig(
new ConfigReader({
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
}),
),
),
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const gheProcessor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://ghe.github.com/api/v3',
},
new GitHubIntegration(
readGitHubIntegrationConfig(
new ConfigReader({
host: 'ghe.github.com',
apiBaseUrl: 'https://ghe.github.com/api/v3',
}),
),
),
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('GithubUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
beforeEach(() => {
jest.clearAllMocks();
});
@@ -94,7 +118,7 @@ describe('GithubUrlReader', () => {
worker.use(
rest.get(
'https://api.github.com/repos/backstage/mock/tree/contents/?ref=main',
'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/?ref=main',
(req, res, ctx) => {
expect(req.headers.get('authorization')).toBe(
mockHeaders.Authorization,
@@ -111,7 +135,7 @@ describe('GithubUrlReader', () => {
),
);
await githubProcessor.read(
await gheProcessor.read(
'https://github.com/backstage/mock/tree/blob/main',
);
});
@@ -122,16 +146,6 @@ describe('GithubUrlReader', () => {
*/
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
@@ -397,9 +411,13 @@ describe('GithubUrlReader', () => {
expect(() => {
/* eslint-disable no-new */
new GithubUrlReader(
{
host: 'ghe.mycompany.net',
},
new GitHubIntegration(
readGitHubIntegrationConfig(
new ConfigReader({
host: 'ghe.mycompany.net',
}),
),
),
{
treeResponseFactory,
credentialsProvider: mockCredentialsProvider,
@@ -414,14 +432,6 @@ describe('GithubUrlReader', () => {
*/
describe('search', () => {
beforeEach(() => {
mockFs({ '/tmp': mockFs.directory() });
});
afterEach(() => {
mockFs.restore();
});
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
packages/backend-common/src/reading/GithubUrlReader.ts
+15 -15
View File
@@ -17,8 +17,8 @@
import {
getGitHubFileFetchUrl,
GithubCredentialsProvider,
GitHubIntegrationConfig,
readGitHubIntegrationConfigs,
GitHubIntegration,
ScmIntegrations,
} from '@backstage/integration';
import { RestEndpointMethodTypes } from '@octokit/rest';
import fetch from 'cross-fetch';
@@ -48,36 +48,36 @@ export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response
*/
export class GithubUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const configs = readGitHubIntegrationConfigs(
config.getOptionalConfigArray('integrations.github') ?? [],
);
return configs.map(provider => {
const credentialsProvider = GithubCredentialsProvider.create(provider);
const reader = new GithubUrlReader(provider, {
const integrations = ScmIntegrations.fromConfig(config);
return integrations.github.list().map(integration => {
const credentialsProvider = GithubCredentialsProvider.create(
integration.config,
);
const reader = new GithubUrlReader(integration, {
treeResponseFactory,
credentialsProvider,
});
const predicate = (url: URL) => url.host === provider.host;
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly config: GitHubIntegrationConfig,
private readonly integration: GitHubIntegration,
private readonly deps: {
treeResponseFactory: ReadTreeResponseFactory;
credentialsProvider: GithubCredentialsProvider;
},
) {
if (!config.apiBaseUrl && !config.rawBaseUrl) {
if (!integration.config.apiBaseUrl && !integration.config.rawBaseUrl) {
throw new Error(
`GitHub integration for '${config.host}' must configure an explicit apiBaseUrl and rawBaseUrl`,
`GitHub integration '${integration.title}' must configure an explicit apiBaseUrl or rawBaseUrl`,
);
}
}
async read(url: string): Promise<Buffer> {
const ghUrl = getGitHubFileFetchUrl(url, this.config);
const ghUrl = getGitHubFileFetchUrl(url, this.integration.config);
const { headers } = await this.deps.credentialsProvider.getCredentials({
url,
});
@@ -155,7 +155,7 @@ export class GithubUrlReader implements UrlReader {
}
toString() {
const { host, token } = this.config;
const { host, token } = this.integration.config;
return `github{host=${host},authed=${Boolean(token)}}`;
}
@@ -258,7 +258,7 @@ export class GithubUrlReader implements UrlReader {
});
const repo: GhRepoResponse = await this.fetchJson(
`${this.config.apiBaseUrl}/repos/${full_name}`,
`${this.integration.config.apiBaseUrl}/repos/${full_name}`,
{ headers },
);
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+111 -20
View File
@@ -20,11 +20,16 @@ import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import os from 'os';
import path from 'path';
import { getVoidLogger } from '../logging';
import { GitlabUrlReader } from './GitlabUrlReader';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError, NotFoundError } from '../errors';
import {
GitLabIntegration,
readGitLabIntegrationConfig,
} from '@backstage/integration';
const logger = getVoidLogger();
@@ -33,24 +38,44 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
});
const gitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
baseUrl: 'https://gitlab.com',
},
new GitLabIntegration(
readGitLabIntegrationConfig(
new ConfigReader({
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
baseUrl: 'https://gitlab.com',
}),
),
),
{ treeResponseFactory },
);
const hostedGitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.mycompany.com',
apiBaseUrl: 'https://gitlab.mycompany.com/api/v4',
baseUrl: 'https://gitlab.mycompany.com',
},
new GitLabIntegration(
readGitLabIntegrationConfig(
new ConfigReader({
host: 'gitlab.mycompany.com',
apiBaseUrl: 'https://gitlab.mycompany.com/api/v4',
baseUrl: 'https://gitlab.mycompany.com',
}),
),
),
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('GitlabUrlReader', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -156,16 +181,6 @@ describe('GitlabUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const archiveBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
);
@@ -382,4 +397,80 @@ describe('GitlabUrlReader', () => {
await expect(fnGithub).rejects.toThrow(NotFoundError);
});
});
describe('search', () => {
const archiveBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
);
const projectGitlabApiResponse = {
id: 11111111,
default_branch: 'main',
};
const branchGitlabApiResponse = {
commit: {
id: 'sha123abc',
},
};
beforeEach(() => {
worker.use(
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename="mock-main-sha123abc.zip"',
),
ctx.body(archiveBuffer),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(projectGitlabApiResponse),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchGitlabApiResponse),
),
),
);
});
it('works for the naive case', async () => {
const result = await gitlabProcessor.search(
'https://gitlab.com/backstage/mock/tree/main/**/index.*',
);
expect(result.etag).toBe('sha123abc');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://gitlab.com/backstage/mock/tree/main/docs/index.md',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
gitlabProcessor.search(
'https://gitlab.com/backstage/mock/tree/main/**/index.*',
{ etag: 'sha123abc' },
),
).rejects.toThrow(NotModifiedError);
});
});
});
packages/backend-common/src/reading/GitlabUrlReader.ts
+52 -27
View File
@@ -17,49 +17,51 @@
import {
getGitLabFileFetchUrl,
getGitLabRequestOptions,
GitLabIntegrationConfig,
readGitLabIntegrationConfigs,
GitLabIntegration,
ScmIntegrations,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import { stripFirstDirectoryFromPath } from './tree/util';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
SearchOptions,
SearchResponse,
UrlReader,
} from './types';
export class GitlabUrlReader implements UrlReader {
private readonly treeResponseFactory: ReadTreeResponseFactory;
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const configs = readGitLabIntegrationConfigs(
config.getOptionalConfigArray('integrations.gitlab') ?? [],
);
return configs.map(provider => {
const reader = new GitlabUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
const integrations = ScmIntegrations.fromConfig(config);
return integrations.gitlab.list().map(integration => {
const reader = new GitlabUrlReader(integration, {
treeResponseFactory,
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly config: GitLabIntegrationConfig,
deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
this.treeResponseFactory = deps.treeResponseFactory;
}
private readonly integration: GitLabIntegration,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {}
async read(url: string): Promise<Buffer> {
const builtUrl = await getGitLabFileFetchUrl(url, this.config);
const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config);
let response: Response;
try {
response = await fetch(builtUrl, getGitLabRequestOptions(this.config));
response = await fetch(
builtUrl,
getGitLabRequestOptions(this.integration.config),
);
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
@@ -86,9 +88,11 @@ export class GitlabUrlReader implements UrlReader {
// https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
const projectGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(full_name)}`,
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}`,
).toString(),
getGitLabRequestOptions(this.config),
getGitLabRequestOptions(this.integration.config),
);
if (!projectGitlabResponse.ok) {
const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`;
@@ -106,11 +110,11 @@ export class GitlabUrlReader implements UrlReader {
// the provided sha.
const branchGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/branches/${branch}`,
).toString(),
getGitLabRequestOptions(this.config),
getGitLabRequestOptions(this.integration.config),
);
if (!branchGitlabResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${branchGitlabResponse.status} ${branchGitlabResponse.statusText}`;
@@ -128,10 +132,10 @@ export class GitlabUrlReader implements UrlReader {
// https://docs.gitlab.com/ee/api/repositories.html#get-file-archive
const archiveGitLabResponse = await fetch(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/archive.zip?sha=${branch}`,
getGitLabRequestOptions(this.config),
getGitLabRequestOptions(this.integration.config),
);
if (!archiveGitLabResponse.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`;
@@ -141,7 +145,7 @@ export class GitlabUrlReader implements UrlReader {
throw new Error(message);
}
return await this.treeResponseFactory.fromZipArchive({
return await this.deps.treeResponseFactory.fromZipArchive({
stream: (archiveGitLabResponse.body as unknown) as Readable,
subpath: filepath,
etag: commitSha,
@@ -149,12 +153,33 @@ export class GitlabUrlReader implements UrlReader {
});
}
async search(): Promise<SearchResponse> {
throw new Error('GitlabUrlReader does not implement search');
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
const { filepath } = parseGitUrl(url);
const matcher = new Minimatch(filepath);
// TODO(freben): For now, read the entire repo and filter through that. In
// a future improvement, we could be smart and try to deduce that non-glob
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
// to get just that part of the repo.
const treeUrl = url.replace(filepath, '').replace(/\/+$/, '');
const tree = await this.readTree(treeUrl, {
etag: options?.etag,
filter: path => matcher.match(stripFirstDirectoryFromPath(path)),
});
const files = await tree.files();
return {
etag: tree.etag,
files: files.map(file => ({
url: this.integration.resolveUrl({ url: `/${file.path}`, base: url }),
content: file.content,
})),
};
}
toString() {
const { host, token } = this.config;
const { host, token } = this.integration.config;
return `gitlab{host=${host},authed=${Boolean(token)}}`;
}
}
packages/backend-common/src/reading/tree/TarArchiveResponse.ts
+3 -6
View File
@@ -25,15 +25,12 @@ import {
ReadTreeResponseDirOptions,
ReadTreeResponseFile,
} from '../types';
import { stripFirstDirectoryFromPath } from './util';
// Tar types for `Parse` is not a proper constructor, but it should be
const TarParseStream = (Parse as unknown) as { new (): ParseStream };
const pipeline = promisify(pipelineCb);
// Matches a directory name + one `/` at the start of any string,
// containing any character except `/` one or more times, and ending with a `/`
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
const directoryNameRegex = /^[^\/]+\//;
/**
* Wraps a tar archive stream into a tree response reader.
@@ -84,7 +81,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
// File path relative to the root extracted directory. Will remove the
// top level dir name from the path since its name is hard to predetermine.
const relativePath = entry.path.replace(directoryNameRegex, '');
const relativePath = stripFirstDirectoryFromPath(entry.path);
if (this.subPath) {
if (!relativePath.startsWith(this.subPath)) {
@@ -161,7 +158,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
filter: path => {
// File path relative to the root extracted directory. Will remove the
// top level dir name from the path since its name is hard to predetermine.
const relativePath = path.replace(directoryNameRegex, '');
const relativePath = stripFirstDirectoryFromPath(path);
if (this.subPath && !relativePath.startsWith(this.subPath)) {
return false;
}
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+4 -13
View File
@@ -24,11 +24,7 @@ import {
ReadTreeResponseDirOptions,
ReadTreeResponseFile,
} from '../types';
// Matches a directory name + one `/` at the start of any string,
// containing any character except / one or more times, and ending with a `/`
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
const directoryNameRegex = /^[^\/]+\//;
import { stripFirstDirectoryFromPath } from './util';
/**
* Wraps a zip archive stream into a tree response reader.
@@ -65,18 +61,13 @@ export class ZipArchiveResponse implements ReadTreeResponse {
this.read = true;
}
// Will remove the top level dir name from the path since its name is hard to predetermine.
private stripTopDirectory(path: string): string {
return path.replace(directoryNameRegex, '');
}
// File path relative to the root extracted directory or a sub directory if subpath is set.
private getInnerPath(path: string): string {
return path.slice(this.subPath.length);
}
private shouldBeIncluded(entry: Entry): boolean {
const strippedPath = this.stripTopDirectory(entry.path);
const strippedPath = stripFirstDirectoryFromPath(entry.path);
if (this.subPath) {
if (!strippedPath.startsWith(this.subPath)) {
@@ -104,7 +95,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
if (this.shouldBeIncluded(entry)) {
files.push({
path: this.getInnerPath(this.stripTopDirectory(entry.path)),
path: this.getInnerPath(stripFirstDirectoryFromPath(entry.path)),
content: () => entry.buffer(),
});
} else {
@@ -153,7 +144,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
// as a zip can have files with directories without directory entries
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
const entryPath = this.getInnerPath(
this.stripTopDirectory(entry.path),
stripFirstDirectoryFromPath(entry.path),
);
const dirname = platformPath.dirname(entryPath);
if (dirname) {
packages/backend-common/src/reading/tree/util.ts
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Matches a directory name + one `/` at the start of any string,
// containing any character except `/` one or more times, and ending with a `/`
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
const directoryNameRegex = /^[^\/]+\//;
// Removes the first segment of a forward-slash-separated path
export function stripFirstDirectoryFromPath(path: string): string {
return path.replace(directoryNameRegex, '');
}
packages/integration/src/ScmIntegrations.ts
+5 -4
View File
@@ -19,6 +19,7 @@ import { AzureIntegration } from './azure/AzureIntegration';
import { BitbucketIntegration } from './bitbucket/BitbucketIntegration';
import { GitHubIntegration } from './github/GitHubIntegration';
import { GitLabIntegration } from './gitlab/GitLabIntegration';
import { defaultScmResolveUrl } from './helpers';
import {
ScmIntegration,
ScmIntegrationRegistry,
@@ -83,11 +84,11 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
}
resolveUrl(options: { url: string; base: string }): string {
const resolve = this.byUrl(options.base)?.resolveUrl;
if (!resolve) {
return new URL(options.url, options.base).toString();
const integration = this.byUrl(options.base);
if (!integration) {
return defaultScmResolveUrl(options);
}
return resolve(options);
return integration.resolveUrl(options);
}
}
packages/integration/src/azure/AzureIntegration.test.ts
+10
View File
@@ -58,6 +58,16 @@ describe('AzureIntegration', () => {
'https://dev.azure.com/organization/project/_git/repository?path=%2Fa.yaml',
);
expect(
integration.resolveUrl({
url: '/a.yaml',
base:
'https://dev.azure.com/organization/project/_git/repository?path=%2Ffolder%2Fcatalog-info.yaml',
}),
).toBe(
'https://dev.azure.com/organization/project/_git/repository?path=%2Fa.yaml',
);
expect(
integration.resolveUrl({
url: './a.yaml',
packages/integration/src/azure/index.ts
+2 -1
View File
@@ -14,14 +14,15 @@
* limitations under the License.
*/
export { AzureIntegration } from './AzureIntegration';
export {
readAzureIntegrationConfig,
readAzureIntegrationConfigs,
} from './config';
export type { AzureIntegrationConfig } from './config';
export {
getAzureCommitsUrl,
getAzureDownloadUrl,
getAzureFileFetchUrl,
getAzureRequestOptions,
getAzureCommitsUrl,
} from './core';
packages/integration/src/bitbucket/BitbucketIntegration.ts
+5 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { basicIntegrations } from '../helpers';
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
import {
BitbucketIntegrationConfig,
@@ -47,4 +47,8 @@ export class BitbucketIntegration implements ScmIntegration {
get config(): BitbucketIntegrationConfig {
return this.integrationConfig;
}
resolveUrl(options: { url: string; base: string }): string {
return defaultScmResolveUrl(options);
}
}
packages/integration/src/bitbucket/index.ts
+1
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export { BitbucketIntegration } from './BitbucketIntegration';
export {
readBitbucketIntegrationConfig,
readBitbucketIntegrationConfigs,
packages/integration/src/github/GitHubIntegration.ts
+5 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { basicIntegrations } from '../helpers';
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
import {
GitHubIntegrationConfig,
@@ -45,4 +45,8 @@ export class GitHubIntegration implements ScmIntegration {
get config(): GitHubIntegrationConfig {
return this.integrationConfig;
}
resolveUrl(options: { url: string; base: string }): string {
return defaultScmResolveUrl(options);
}
}
packages/integration/src/github/index.ts
+1
View File
@@ -21,3 +21,4 @@ export {
export type { GitHubIntegrationConfig } from './config';
export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
export { GithubCredentialsProvider } from './GithubCredentialsProvider';
export { GitHubIntegration } from './GitHubIntegration';
packages/integration/src/gitlab/GitLabIntegration.ts
+5 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { basicIntegrations } from '../helpers';
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
import {
GitLabIntegrationConfig,
@@ -45,4 +45,8 @@ export class GitLabIntegration implements ScmIntegration {
get config(): GitLabIntegrationConfig {
return this.integrationConfig;
}
resolveUrl(options: { url: string; base: string }): string {
return defaultScmResolveUrl(options);
}
}
packages/integration/src/gitlab/index.ts
+1
View File
@@ -20,3 +20,4 @@ export {
} from './config';
export type { GitLabIntegrationConfig } from './config';
export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core';
export { GitLabIntegration } from './GitLabIntegration';
packages/integration/src/helpers.test.ts
+67 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { isValidHost } from './helpers';
import { defaultScmResolveUrl, isValidHost } from './helpers';
describe('isValidHost', () => {
it.each([
@@ -51,3 +51,69 @@ describe('isValidHost', () => {
expect(isValidHost(str)).toBe(expected);
});
});
describe('defaultScmResolveUrl', () => {
it('works for relative paths and retains query params', () => {
expect(
defaultScmResolveUrl({
url: './b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml',
}),
).toBe(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml',
);
expect(
defaultScmResolveUrl({
url: './b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master',
}),
).toBe(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml?at=master',
);
expect(
defaultScmResolveUrl({
url: 'b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml',
}),
).toBe(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml',
);
});
it('works for absolute paths and retains query params', () => {
expect(
defaultScmResolveUrl({
url: '/other/b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml',
}),
).toBe(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/other/b.yaml',
);
expect(
defaultScmResolveUrl({
url: '/other/b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master',
}),
).toBe(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/other/b.yaml?at=master',
);
});
it('works for full urls and throws away query params', () => {
expect(
defaultScmResolveUrl({
url: 'https://b.com/b.yaml',
base:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master',
}),
).toBe('https://b.com/b.yaml');
});
});
packages/integration/src/helpers.ts
+41
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import parseGitUrl from 'git-url-parse';
import { ScmIntegration, ScmIntegrationsGroup } from './types';
/** Checks whether the given argument is a valid URL hostname */
@@ -51,3 +52,43 @@ export function basicIntegrations<T extends ScmIntegration>(
},
};
}
/**
* Default implementation of ScmIntegration.resolveUrl, that only works with
* URL pathname based providers.
*/
export function defaultScmResolveUrl(options: {
url: string;
base: string;
}): string {
const { url, base } = options;
// If it is a fully qualified URL - then return it verbatim
try {
// eslint-disable-next-line no-new
new URL(url);
return url;
} catch {
// ignore intentionally
}
let updated: URL;
if (url.startsWith('/')) {
// If it is an absolute path, move relative to the repo root
const { filepath } = parseGitUrl(base);
updated = new URL(base);
const repoRootPath = updated.pathname
.substring(0, updated.pathname.length - filepath.length)
.replace(/\/+$/, '');
updated.pathname = `${repoRootPath}${url}`;
} else {
// For relative URLs, just let the default URL constructor handle the
// resolving. Note that this essentially will treat the last segment of the
// base as a file - NOT a folder - unless the url ends in a slash.
updated = new URL(url, base);
}
updated.search = new URL(base).search;
return updated.toString();
}
packages/integration/src/index.ts
+1
View File
@@ -18,5 +18,6 @@ export * from './azure';
export * from './bitbucket';
export * from './github';
export * from './gitlab';
export { defaultScmResolveUrl } from './helpers';
export { ScmIntegrations } from './ScmIntegrations';
export type { ScmIntegration, ScmIntegrationRegistry } from './types';
packages/integration/src/types.ts
+20 -7
View File
@@ -36,16 +36,21 @@ export interface ScmIntegration {
title: string;
/**
* Works like the two-argument form of the URL constructor, resolving an
* absolute or relative URL in relation to a base URL.
* Resolves an absolute or relative URL in relation to a base URL.
*
* If this method is not implemented, the URL constructor is used instead for
* URLs that match this integration.
* This method is adapted for use within SCM systems, so relative URLs are
* within the context of the root of the hierarchy pointed to by the base
* URL.
*
* For example, if the base URL is `<repo root url>/folder/a.yaml`, i.e.
* within the file tree of a certain repo, an absolute path of `/b.yaml` does
* not resolve to `https://hostname/b.yaml` but rather to
* `<repo root url>/b.yaml` inside the file tree of that same repo.
*
* @param options.url The (absolute or relative) URL or path to resolve
* @param options.base The base URL onto which this resolution happens
*/
resolveUrl?(options: { url: string; base: string }): string;
resolveUrl(options: { url: string; base: string }): string;
}
/**
@@ -83,8 +88,16 @@ export interface ScmIntegrationRegistry
gitlab: ScmIntegrationsGroup<GitLabIntegration>;
/**
* Works like the two-argument form of the URL constructor, resolving an
* absolute or relative URL in relation to a base URL.
* Resolves an absolute or relative URL in relation to a base URL.
*
* This method is adapted for use within SCM systems, so relative URLs are
* within the context of the root of the hierarchy pointed to by the base
* URL.
*
* For example, if the base URL is `<repo root url>/folder/a.yaml`, i.e.
* within the file tree of a certain repo, an absolute path of `/b.yaml` does
* not resolve to `https://hostname/b.yaml` but rather to
* `<repo root url>/b.yaml` inside the file tree of that same repo.
*
* @param options.url The (absolute or relative) URL or path to resolve
* @param options.base The base URL onto which this resolution happens