paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: add lastModifiedAt to UrlReader methods

Browse Source 
Signed-off-by: Andrew Thauer <athauer@wealthsimple.com>
This commit is contained in:
Andrew Thauer
2023-03-13 16:06:08 -04:00
parent 80698c1de8
commit c1ee073a82
27 changed files with 477 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/sharp-rings-cry.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/backend-common': minor
'@backstage/backend-plugin-api': minor
---
Added `lastModifiedAt` field on `UrlReaderService` responses and a `lastModifiedAfter` option to `UrlReaderService.readUrl`.
packages/backend-common/api-report.md
+2
View File
@@ -301,6 +301,7 @@ export class FetchUrlReader implements UrlReader {
export type FromReadableArrayOptions = Array<{
data: Readable;
path: string;
lastModifiedAt?: Date;
}>;
// @public
@@ -635,6 +636,7 @@ export class ReadUrlResponseFactory {
// @public
export type ReadUrlResponseFactoryFromStreamOptions = {
etag?: string;
lastModifiedAt?: Date;
};
// @public
packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+40 -2
View File
@@ -297,23 +297,26 @@ describe('AwsS3UrlReader', () => {
),
),
ETag: '123abc',
LastModified: new Date('2020-01-01T00:00:00Z'),
});
});
it('returns contents of an object in a bucket via buffer', async () => {
const { buffer, etag } = await reader.readUrl(
const { buffer, etag, lastModifiedAt } = await reader.readUrl(
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
);
expect(etag).toBe('123abc');
expect(lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
const response = await buffer();
expect(response.toString().trim()).toBe('site_name: Test');
});
it('returns contents of an object in a bucket via stream', async () => {
const { buffer, etag } = await reader.readUrl(
const { buffer, etag, lastModifiedAt } = await reader.readUrl(
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
);
expect(etag).toBe('123abc');
expect(lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
const response = await buffer();
expect(response.toString().trim()).toBe('site_name: Test');
});
@@ -403,6 +406,41 @@ describe('AwsS3UrlReader', () => {
});
});
describe('readUrl with lastModifiedAfter', () => {
const [{ reader }] = createReader({
integrations: {
awsS3: [
{
host: 'amazonaws.com',
accessKeyId: 'fake-access-key',
secretAccessKey: 'fake-secret-key',
},
],
},
});
beforeEach(() => {
s3Client.reset();
const t = new S3ServiceException({
name: '304',
$fault: 'client',
$metadata: { httpStatusCode: 304 },
});
s3Client.on(GetObjectCommand).rejects(t);
});
it('returns contents of an object in a bucket', async () => {
await expect(
reader.readUrl!(
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
{
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
},
),
).rejects.toThrow(NotModifiedError);
});
});
describe('readTree', () => {
let awsS3UrlReader: AwsS3UrlReader;
packages/backend-common/src/reading/AwsS3UrlReader.ts
+15 -15
View File
@@ -41,6 +41,7 @@ import {
ListObjectsV2Command,
ListObjectsV2CommandOutput,
GetObjectCommand,
GetObjectCommandInput,
} from '@aws-sdk/client-s3';
import { AbortController } from '@aws-sdk/abort-controller';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
@@ -253,6 +254,8 @@ export class AwsS3UrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, lastModifiedAfter } = options ?? {};
try {
const { path, bucket, region } = parseUrl(url, this.integration.config);
const s3Client = await this.buildS3Client(
@@ -262,19 +265,15 @@ export class AwsS3UrlReader implements UrlReader {
);
const abortController = new AbortController();
let params;
if (options?.etag) {
params = {
Bucket: bucket,
Key: path,
IfNoneMatch: options.etag,
};
} else {
params = {
Bucket: bucket,
Key: path,
};
}
const params: GetObjectCommandInput = {
Bucket: bucket,
Key: path,
...(etag && { IfNoneMatch: etag }),
...(lastModifiedAfter && {
IfModifiedSince: lastModifiedAfter,
}),
};
options?.signal?.addEventListener('abort', () => abortController.abort());
const getObjectCommand = new GetObjectCommand(params);
const response = await s3Client.send(getObjectCommand, {
@@ -284,10 +283,10 @@ export class AwsS3UrlReader implements UrlReader {
const s3ObjectData = await this.retrieveS3ObjectData(
response.Body as Readable,
);
const etag = response.ETag;
return ReadUrlResponseFactory.fromReadable(s3ObjectData, {
etag: etag,
etag: response.ETag,
lastModifiedAt: response.LastModified,
});
} catch (e) {
if (e.$metadata && e.$metadata.httpStatusCode === 304) {
@@ -347,6 +346,7 @@ export class AwsS3UrlReader implements UrlReader {
responses.push({
data: s3ObjectData,
path: String(allObjects[i]),
lastModifiedAt: response?.LastModified ?? undefined,
});
}
packages/backend-common/src/reading/AzureUrlReader.ts
+1
View File
@@ -192,6 +192,7 @@ export class AzureUrlReader implements UrlReader {
base: url,
}),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
})),
};
}
packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts
+77
View File
@@ -156,6 +156,83 @@ describe('BitbucketCloudUrlReader', () => {
expect(buffer.toString()).toBe('foo');
expect(result.etag).toBe('new-etag-value');
});
it('should be able to readUrl via buffer without If-Modified-Since', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-None-Match')).toBeNull();
return res(
ctx.status(200),
ctx.body('foo'),
ctx.set('ETag', 'etag-value'),
ctx.set(
'Last-Modified',
new Date('2020-01-01T00:00:00Z').toUTCString(),
),
);
},
),
);
const result = await reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
);
const buffer = await result.buffer();
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
expect(buffer.toString()).toBe('foo');
});
it('should be throw not modified when If-Modified-Since returns a 304', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-Modified-Since')).toBe(
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
);
return res(ctx.status(304));
},
),
);
await expect(
reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
),
).rejects.toThrow(NotModifiedError);
});
it('should be able to readUrl when If-Modified-Since is before Last-Modified', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-Modified-Since')).toBe(
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
);
return res(
ctx.status(200),
ctx.set(
'Last-Modified',
new Date('2020-01-01T00:00:00Z').toUTCString(),
),
ctx.body('foo'),
);
},
),
);
const result = await reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
);
const buffer = await result.buffer();
expect(buffer.toString()).toBe('foo');
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
});
});
describe('read', () => {
packages/backend-common/src/reading/BitbucketCloudUrlReader.ts
+9 -1
View File
@@ -40,6 +40,7 @@ import {
UrlReader,
} from './types';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
/**
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Cloud.
@@ -80,7 +81,7 @@ export class BitbucketCloudUrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const { etag, lastModifiedAfter, signal } = options ?? {};
const bitbucketUrl = getBitbucketCloudFileFetchUrl(
url,
this.integration.config,
@@ -95,6 +96,9 @@ export class BitbucketCloudUrlReader implements UrlReader {
headers: {
...requestOptions.headers,
...(etag && { 'If-None-Match': etag }),
...(lastModifiedAfter && {
'If-Modified-Since': lastModifiedAfter.toUTCString(),
}),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
@@ -115,6 +119,9 @@ export class BitbucketCloudUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
@@ -184,6 +191,7 @@ export class BitbucketCloudUrlReader implements UrlReader {
base: url,
}),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
})),
};
}
packages/backend-common/src/reading/BitbucketServerUrlReader.ts
+9 -1
View File
@@ -39,6 +39,7 @@ import {
UrlReader,
} from './types';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
/**
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs.
@@ -71,7 +72,7 @@ export class BitbucketServerUrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const { etag, lastModifiedAfter, signal } = options ?? {};
const bitbucketUrl = getBitbucketServerFileFetchUrl(
url,
this.integration.config,
@@ -86,6 +87,9 @@ export class BitbucketServerUrlReader implements UrlReader {
headers: {
...requestOptions.headers,
...(etag && { 'If-None-Match': etag }),
...(lastModifiedAfter && {
'If-Modified-Since': lastModifiedAfter.toUTCString(),
}),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
@@ -106,6 +110,9 @@ export class BitbucketServerUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
@@ -175,6 +182,7 @@ export class BitbucketServerUrlReader implements UrlReader {
base: url,
}),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
})),
};
}
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+77
View File
@@ -204,6 +204,83 @@ describe('BitbucketUrlReader', () => {
expect(buffer.toString()).toBe('foo');
expect(result.etag).toBe('new-etag-value');
});
it('should be able to readUrl via buffer without If-Modified-Since', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-None-Match')).toBeNull();
return res(
ctx.status(200),
ctx.body('foo'),
ctx.set('ETag', 'etag-value'),
ctx.set(
'Last-Modified',
new Date('2020-01-01T00:00:00Z').toUTCString(),
),
);
},
),
);
const result = await bitbucketProcessor.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
);
const buffer = await result.buffer();
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
expect(buffer.toString()).toBe('foo');
});
it('should be throw not modified when If-Modified-Since returns a 304', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-Modified-Since')).toBe(
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
);
return res(ctx.status(304));
},
),
);
await expect(
bitbucketProcessor.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
),
).rejects.toThrow(NotModifiedError);
});
it('should be able to readUrl when If-Modified-Since is before Last-Modified', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-Modified-Since')).toBe(
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
);
return res(
ctx.status(200),
ctx.set(
'Last-Modified',
new Date('2020-01-01T00:00:00Z').toUTCString(),
),
ctx.body('foo'),
);
},
),
);
const result = await bitbucketProcessor.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
);
const buffer = await result.buffer();
expect(buffer.toString()).toBe('foo');
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
});
});
describe('read', () => {
packages/backend-common/src/reading/BitbucketUrlReader.ts
+9 -1
View File
@@ -41,6 +41,7 @@ import {
UrlReader,
} from './types';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
/**
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket v1 and v2 APIs, such
@@ -96,7 +97,7 @@ export class BitbucketUrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const { etag, lastModifiedAfter, signal } = options ?? {};
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.integration.config);
const requestOptions = getBitbucketRequestOptions(this.integration.config);
@@ -106,6 +107,9 @@ export class BitbucketUrlReader implements UrlReader {
headers: {
...requestOptions.headers,
...(etag && { 'If-None-Match': etag }),
...(lastModifiedAfter && {
'If-Modified-Since': lastModifiedAfter.toUTCString(),
}),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
@@ -126,6 +130,9 @@ export class BitbucketUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
@@ -195,6 +202,7 @@ export class BitbucketUrlReader implements UrlReader {
base: url,
}),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
})),
};
}
packages/backend-common/src/reading/FetchUrlReader.test.ts
+24 -1
View File
@@ -45,6 +45,21 @@ describe('FetchUrlReader', () => {
);
}
if (
req.headers.get('if-modified-since') &&
new Date(req.headers.get('if-modified-since') ?? '') <
new Date('2021-01-01T00:00:00Z')
) {
return res(
ctx.status(304),
ctx.set('Content-Type', 'text/plain'),
ctx.set(
'last-modified',
new Date('2021-01-01T00:00:00Z').toUTCString(),
),
);
}
return res(
ctx.status(200),
ctx.set('Content-Type', 'text/plain'),
@@ -192,7 +207,7 @@ describe('FetchUrlReader', () => {
});
describe('readUrl', () => {
it('should throw NotModified if server responds with 304', async () => {
it('should throw NotModified if server responds with 304 from etag', async () => {
await expect(
fetchUrlReader.readUrl('https://backstage.io/some-resource', {
etag: 'foo',
@@ -200,6 +215,14 @@ describe('FetchUrlReader', () => {
).rejects.toThrow(NotModifiedError);
});
it('should throw NotModified if server responds with 304 from lastModifiedAfter', async () => {
await expect(
fetchUrlReader.readUrl('https://backstage.io/some-resource', {
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
}),
).rejects.toThrow(NotModifiedError);
});
it('should return etag from the response', async () => {
const response = await fetchUrlReader.readUrl(
'https://backstage.io/some-resource',
packages/backend-common/src/reading/FetchUrlReader.ts
+7
View File
@@ -26,6 +26,7 @@ import {
} from './types';
import path from 'path';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
const isInRange = (num: number, [start, end]: [number, number]) => {
return num >= start && num <= end;
@@ -127,6 +128,9 @@ export class FetchUrlReader implements UrlReader {
response = await fetch(url, {
headers: {
...(options?.etag && { 'If-None-Match': options.etag }),
...(options?.lastModifiedAfter && {
'If-Modified-Since': options.lastModifiedAfter.toUTCString(),
}),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
@@ -147,6 +151,9 @@ export class FetchUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
packages/backend-common/src/reading/GiteaUrlReader.test.ts
+47 -1
View File
@@ -25,7 +25,7 @@ import { UrlReaderPredicateTuple } from './types';
import { DefaultReadTreeResponseFactory } from './tree';
import getRawBody from 'raw-body';
import { GiteaUrlReader } from './GiteaUrlReader';
import { NotFoundError } from '@backstage/errors';
import { NotFoundError, NotModifiedError } from '@backstage/errors';
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: new ConfigReader({}),
@@ -200,5 +200,51 @@ describe('GiteaUrlReader', () => {
'https://gitea.com/owner/project/src/branch/branch2/LICENSE could not be read as https://gitea.com/api/v1/repos/owner/project/contents/LICENSE?ref=branch2, 500 Error!!!',
);
});
it('should throw NotModified if server responds with 304 from etag', async () => {
worker.use(
rest.get(
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
(_, res, ctx) => {
return res(ctx.set('ETag', 'foo'), ctx.status(304, 'Error!!!'));
},
),
);
await expect(
giteaProcessor.readUrl(
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
{
etag: 'foo',
},
),
).rejects.toThrow(NotModifiedError);
});
it('should throw NotModified if server responds with 304 from lastModifiedAfter', async () => {
worker.use(
rest.get(
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
(_, res, ctx) => {
return res(
ctx.set(
'Last-Modified',
new Date('2020-01-01T00:00:00Z').toUTCString(),
),
ctx.status(304, 'Error!!!'),
);
},
),
);
await expect(
giteaProcessor.readUrl(
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
{
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
},
),
).rejects.toThrow(NotModifiedError);
});
});
});
packages/backend-common/src/reading/GiteaUrlReader.ts
+4
View File
@@ -34,6 +34,7 @@ import {
NotModifiedError,
} from '@backstage/errors';
import { Readable } from 'stream';
import { parseLastModified } from './util';
/**
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api.
@@ -86,6 +87,9 @@ export class GiteaUrlReader implements UrlReader {
Readable.from(Buffer.from(content, 'base64')),
{
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
},
);
}
packages/backend-common/src/reading/GithubUrlReader.test.ts
+7 -1
View File
@@ -247,7 +247,7 @@ describe('GithubUrlReader', () => {
).rejects.toThrow(/rate limit exceeded/);
});
it('should return etag from the response', async () => {
it('should return etag and last-modified from the response', async () => {
(mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
headers: {
Authorization: 'bearer blah',
@@ -261,6 +261,11 @@ describe('GithubUrlReader', () => {
return res(
ctx.status(200),
ctx.set('Etag', 'foo'),
ctx.set(
'Last-Modified',
new Date('2021-01-01T00:00:00Z').toUTCString(),
),
ctx.body('bar'),
);
},
@@ -271,6 +276,7 @@ describe('GithubUrlReader', () => {
'https://github.com/backstage/mock/tree/blob/main',
);
expect(response.etag).toBe('foo');
expect(response.lastModifiedAt).toEqual(new Date('2021-01-01T00:00:00Z'));
});
});
packages/backend-common/src/reading/GithubUrlReader.ts
+8
View File
@@ -40,6 +40,7 @@ import {
ReadUrlResponse,
} from './types';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
export type GhRepoResponse =
RestEndpointMethodTypes['repos']['get']['response']['data'];
@@ -109,6 +110,9 @@ export class GithubUrlReader implements UrlReader {
headers: {
...credentials?.headers,
...(options?.etag && { 'If-None-Match': options.etag }),
...(options?.lastModifiedAfter && {
'If-Modified-Since': options.lastModifiedAfter.toUTCString(),
}),
Accept: 'application/vnd.github.v3.raw',
},
// TODO(freben): The signal cast is there because pre-3.x versions of
@@ -130,6 +134,9 @@ export class GithubUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
@@ -290,6 +297,7 @@ export class GithubUrlReader implements UrlReader {
return files.map(file => ({
url: pathToUrl(file.path),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
}));
}
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+35 -3
View File
@@ -184,7 +184,7 @@ describe('GitlabUrlReader', () => {
treeResponseFactory,
});
it('should throw NotModified on HTTP 304', async () => {
it('should throw NotModified on HTTP 304 from etag', async () => {
worker.use(
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ id: 12345 })),
@@ -205,13 +205,44 @@ describe('GitlabUrlReader', () => {
).rejects.toThrow(NotModifiedError);
});
it('should return etag in response', async () => {
it('should throw NotModified on HTTP 304 from lastModifiedAt', async () => {
worker.use(
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ id: 12345 })),
),
rest.get('*', (req, res, ctx) => {
expect(req.headers.get('If-Modified-Since')).toBe(
new Date('2019 12 31 23:59:59 GMT').toUTCString(),
);
return res(ctx.status(304));
}),
);
await expect(
reader.readUrl!(
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
{
lastModifiedAfter: new Date('2019 12 31 23:59:59 GMT'),
},
),
).rejects.toThrow(NotModifiedError);
});
it('should return etag and last-modified in response', async () => {
worker.use(
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ id: 12345 })),
),
rest.get('*', (_req, res, ctx) => {
return res(ctx.status(200), ctx.set('ETag', '999'), ctx.body('foo'));
return res(
ctx.status(200),
ctx.set('ETag', '999'),
ctx.set(
'Last-Modified',
new Date('2020 01 01 00:0:00 GMT').toUTCString(),
),
ctx.body('foo'),
);
}),
);
@@ -219,6 +250,7 @@ describe('GitlabUrlReader', () => {
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
);
expect(result.etag).toBe('999');
expect(result.lastModifiedAt).toEqual(new Date('2020 01 01 00:0:00 GMT'));
const content = await result.buffer();
expect(content.toString()).toBe('foo');
});
packages/backend-common/src/reading/GitlabUrlReader.ts
+9 -1
View File
@@ -40,6 +40,7 @@ import {
} from './types';
import { trimEnd, trimStart } from 'lodash';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import { parseLastModified } from './util';
/**
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on GitLab.
@@ -72,7 +73,7 @@ export class GitlabUrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const { etag, lastModifiedAfter, signal } = options ?? {};
const builtUrl = await this.getGitlabFetchUrl(url);
let response: Response;
@@ -81,6 +82,9 @@ export class GitlabUrlReader implements UrlReader {
headers: {
...getGitLabRequestOptions(this.integration.config).headers,
...(etag && { 'If-None-Match': etag }),
...(lastModifiedAfter && {
'If-Modified-Since': lastModifiedAfter.toUTCString(),
}),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
@@ -101,6 +105,9 @@ export class GitlabUrlReader implements UrlReader {
if (response.ok) {
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
etag: response.headers.get('ETag') ?? undefined,
lastModifiedAt: parseLastModified(
response.headers.get('Last-Modified'),
),
});
}
@@ -247,6 +254,7 @@ export class GitlabUrlReader implements UrlReader {
files: files.map(file => ({
url: this.integration.resolveUrl({ url: `/${file.path}`, base: url }),
content: file.content,
lastModifiedAt: file.lastModifiedAt,
})),
};
}
packages/backend-common/src/reading/ReadUrlResponseFactory.ts
+1
View File
@@ -61,6 +61,7 @@ export class ReadUrlResponseFactory {
return stream;
},
etag: options?.etag,
lastModifiedAt: options?.lastModifiedAt,
};
}
packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
+1
View File
@@ -63,6 +63,7 @@ export class ReadableArrayResponse implements ReadTreeResponse {
files.push({
path: this.stream[i].path,
content: () => getRawBody(this.stream[i].data),
lastModifiedAt: this.stream[i]?.lastModifiedAt,
});
}
}
packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts
+5
View File
@@ -45,10 +45,12 @@ describe('TarArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: undefined,
},
{
path: 'docs/index.md',
content: expect.any(Function),
lastModifiedAt: undefined,
},
]);
const contents = await Promise.all(files.map(f => f.content()));
@@ -70,6 +72,7 @@ describe('TarArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: undefined,
},
]);
const content = await files[0].content();
@@ -93,10 +96,12 @@ describe('TarArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: undefined,
},
{
path: 'docs/index.md',
content: expect.any(Function),
lastModifiedAt: undefined,
},
]);
const contents = await Promise.all(files.map(f => f.content()));
packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts
+5
View File
@@ -59,10 +59,12 @@ describe('ZipArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: expect.any(Date),
},
{
path: 'docs/index.md',
content: expect.any(Function),
lastModifiedAt: expect.any(Date),
},
]);
@@ -85,6 +87,7 @@ describe('ZipArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: expect.any(Date),
},
]);
const content = await files[0].content();
@@ -108,10 +111,12 @@ describe('ZipArchiveResponse', () => {
{
path: 'mkdocs.yml',
content: expect.any(Function),
lastModifiedAt: expect.any(Date),
},
{
path: 'docs/index.md',
content: expect.any(Function),
lastModifiedAt: expect.any(Date),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+3
View File
@@ -146,6 +146,9 @@ export class ZipArchiveResponse implements ReadTreeResponse {
files.push({
path: this.getInnerPath(entry.fileName),
content: async () => await streamToBuffer(content),
lastModifiedAt: entry.lastModFileTime
? new Date(entry.lastModFileTime)
: undefined,
});
});
packages/backend-common/src/reading/types.ts
+7
View File
@@ -65,6 +65,7 @@ export type ReaderFactory = (options: {
*/
export type ReadUrlResponseFactoryFromStreamOptions = {
etag?: string;
lastModifiedAt?: Date;
};
/**
@@ -95,10 +96,16 @@ export type FromReadableArrayOptions = Array<{
* The raw data itself.
*/
data: Readable;
/**
* The filepath of the data.
*/
path: string;
/**
* Last modified date of the file contents.
*/
lastModifiedAt?: Date;
}>;
/**
packages/backend-common/src/reading/util.ts
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function parseLastModified(value: string | null | undefined) {
if (!value) {
return undefined;
}
return new Date(value);
}
packages/backend-plugin-api/api-report.md
+4
View File
@@ -359,11 +359,13 @@ export type ReadTreeResponseDirOptions = {
export type ReadTreeResponseFile = {
path: string;
content(): Promise<Buffer>;
lastModifiedAt?: Date;
};
// @public
export type ReadUrlOptions = {
etag?: string;
lastModifiedAfter?: Date;
signal?: AbortSignal;
};
@@ -372,6 +374,7 @@ export type ReadUrlResponse = {
buffer(): Promise<Buffer>;
stream?(): Readable;
etag?: string;
lastModifiedAt?: Date;
};
// @public (undocumented)
@@ -420,6 +423,7 @@ export type SearchResponse = {
export type SearchResponseFile = {
url: string;
content(): Promise<Buffer>;
lastModifiedAt?: Date;
};
// @public (undocumented)
packages/backend-plugin-api/src/services/definitions/UrlReaderService.ts
+42
View File
@@ -64,6 +64,26 @@ export type ReadUrlOptions = {
*/
etag?: string;
/**
* A date which can be provided to check whether a
* {@link UrlReaderService.readUrl} response has changed since the lastModifiedAt.
*
* @remarks
*
* In the {@link UrlReaderService.readUrl} response, an lastModifiedAt is returned
* along with data. The lastModifiedAt date represents the last time the data
* was modified.
*
* When an lastModifiedAfter is given in ReadUrlOptions, {@link UrlReaderService.readUrl}
* will compare the lastModifiedAfter against the lastModifiedAt of the target. If
* the data has not been modified since this date, the {@link UrlReaderService.readUrl}
* will throw a {@link @backstage/errors#NotModifiedError} indicating that the
* response does not contain any new data. If they do not match,
* {@link UrlReaderService.readUrl} will return the rest of the response along with new
* lastModifiedAt date.
*/
lastModifiedAfter?: Date;
/**
* An abort signal to pass down to the underlying request.
*
@@ -102,6 +122,11 @@ export type ReadUrlResponse = {
* Can be used to compare and cache responses when doing subsequent calls.
*/
etag?: string;
/**
* Last modified date of the file contents.
*/
lastModifiedAt?: Date;
};
/**
@@ -213,8 +238,20 @@ export type ReadTreeResponse = {
* @public
*/
export type ReadTreeResponseFile = {
/**
* The filepath of the data.
*/
path: string;
/**
* The binary contents of the file.
*/
content(): Promise<Buffer>;
/**
* The last modified timestamp of the data.
*/
lastModifiedAt?: Date;
};
/**
@@ -278,4 +315,9 @@ export type SearchResponseFile = {
* The binary contents of the file.
*/
content(): Promise<Buffer>;
/**
* The last modified timestamp of the data.
*/
lastModifiedAt?: Date;
};