paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

Merge pull request #17678 from aochsner/feature/remove-aws-v2

Browse Source 
start to replace aws-sdk v2
This commit is contained in:
Fredrik Adelöw
2023-05-11 18:20:44 +02:00
committed by GitHub
parent 79ca6079f4 1a3b5f1e39
commit 6abcba3e2d
18 changed files with 1109 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/itchy-impalas-fold.md
+9
View File
@@ -0,0 +1,9 @@
---
'@backstage/plugin-events-backend-module-aws-sqs': patch
'@backstage/integration-aws-node': patch
'@backstage/plugin-kubernetes-backend': patch
'@backstage/backend-common': patch
'@backstage/plugin-techdocs-node': patch
---
Standardize @aws-sdk v3 versions
.changeset/nice-ants-type.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-aws': minor
---
**BREAKING**: AwsOrganizationCloudAccountProcessor.fromConfig now returns a promise instead of the instance directly.
packages/backend-common/package.json
+5 -5
View File
@@ -46,10 +46,10 @@
"test:kubernetes": "backstage-cli package test -t KubernetesContainerRunner --no-watch"
},
"dependencies": {
"@aws-sdk/abort-controller": "^3.208.0",
"@aws-sdk/client-s3": "^3.208.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/types": "^3.208.0",
"@aws-sdk/abort-controller": "^3.310.0",
"@aws-sdk/client-s3": "^3.310.0",
"@aws-sdk/credential-providers": "^3.310.0",
"@aws-sdk/types": "^3.310.0",
"@backstage/backend-app-api": "workspace:^",
"@backstage/backend-dev-utils": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
@@ -114,7 +114,7 @@
}
},
"devDependencies": {
"@aws-sdk/util-stream-node": "^3.208.0",
"@aws-sdk/util-stream-node": "^3.310.0",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/archiver": "^5.1.0",
packages/integration-aws-node/package.json
+5 -5
View File
@@ -32,11 +32,11 @@
"clean": "backstage-cli package clean"
},
"dependencies": {
"@aws-sdk/client-sts": "^3.208.0",
"@aws-sdk/credential-provider-node": "^3.208.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/types": "^3.208.0",
"@aws-sdk/util-arn-parser": "^3.208.0",
"@aws-sdk/client-sts": "^3.310.0",
"@aws-sdk/credential-provider-node": "^3.310.0",
"@aws-sdk/credential-providers": "^3.310.0",
"@aws-sdk/types": "^3.310.0",
"@aws-sdk/util-arn-parser": "^3.310.0",
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^"
},
plugins/catalog-backend-module-aws/api-report.md
+10 -4
View File
@@ -3,11 +3,12 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { AwsCredentialIdentity } from '@aws-sdk/types';
import { AwsCredentialsManager } from '@backstage/integration-aws-node';
import { CatalogProcessor } from '@backstage/plugin-catalog-node';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
import { CatalogProcessorParser } from '@backstage/plugin-catalog-node';
import { Config } from '@backstage/config';
import { Credentials } from 'aws-sdk';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { LocationSpec } from '@backstage/plugin-catalog-common';
@@ -19,11 +20,16 @@ import { UrlReader } from '@backstage/backend-common';
// @public
export type AWSCredentialFactory = (
awsAccountId: string,
) => Promise<Credentials>;
) => Promise<AwsCredentialIdentity>;
// @public
export class AwsEKSClusterProcessor implements CatalogProcessor {
constructor(options: { credentialsFactory?: AWSCredentialFactory });
constructor(options: {
credentialsFactory?: AWSCredentialFactory;
credentialsManager?: AwsCredentialsManager;
});
// (undocumented)
static fromConfig(configRoot: Config): AwsEKSClusterProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
@@ -44,7 +50,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
options: {
logger: Logger;
},
): AwsOrganizationCloudAccountProcessor;
): Promise<AwsOrganizationCloudAccountProcessor>;
// (undocumented)
getProcessorName(): string;
// (undocumented)
plugins/catalog-backend-module-aws/package.json
+9 -4
View File
@@ -45,7 +45,12 @@
"clean": "backstage-cli package clean"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.208.0",
"@aws-sdk/client-eks": "^3.310.0",
"@aws-sdk/client-organizations": "^3.310.0",
"@aws-sdk/client-s3": "^3.310.0",
"@aws-sdk/credential-providers": "^3.310.0",
"@aws-sdk/middleware-endpoint": "^3.310.0",
"@aws-sdk/types": "^3.310.0",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/backend-tasks": "workspace:^",
@@ -53,23 +58,23 @@
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/integration-aws-node": "workspace:^",
"@backstage/plugin-catalog-common": "workspace:^",
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/plugin-kubernetes-common": "workspace:^",
"@backstage/types": "workspace:^",
"aws-sdk": "^2.840.0",
"lodash": "^4.17.21",
"p-limit": "^3.0.2",
"uuid": "^8.0.0",
"winston": "^3.2.1"
},
"devDependencies": {
"@aws-sdk/util-stream-node": "^3.208.0",
"@aws-sdk/util-stream-node": "^3.310.0",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151",
"aws-sdk-client-mock": "^2.0.0",
"aws-sdk-mock": "^5.2.1",
"aws-sdk-client-mock-jest": "^2.0.0",
"luxon": "^3.0.0",
"yaml": "^2.0.0"
},
plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts
-63
View File
@@ -1,63 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import aws, { Credentials } from 'aws-sdk';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
export class AwsCredentials {
/**
* If accessKeyId and secretAccessKey are missing, the DefaultAWSCredentialsProviderChain will be used:
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
*/
static create(
config: {
accessKeyId?: string;
secretAccessKey?: string;
roleArn?: string;
externalId?: string;
},
roleSessionName: string,
): Credentials | CredentialsOptions | undefined {
if (!config) {
return undefined;
}
const accessKeyId = config.accessKeyId;
const secretAccessKey = config.secretAccessKey;
let explicitCredentials: Credentials | undefined;
if (accessKeyId && secretAccessKey) {
explicitCredentials = new Credentials({
accessKeyId,
secretAccessKey,
});
}
const roleArn = config.roleArn;
if (roleArn) {
return new aws.ChainableTemporaryCredentials({
masterCredentials: explicitCredentials,
params: {
RoleArn: roleArn,
RoleSessionName: roleSessionName,
ExternalId: config.externalId,
},
});
}
return explicitCredentials;
}
}
plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.test.ts
+13 -8
View File
@@ -15,24 +15,28 @@
*/
import { AwsEKSClusterProcessor } from './AwsEKSClusterProcessor';
import AWSMock from 'aws-sdk-mock';
import aws from 'aws-sdk';
import { mockClient } from 'aws-sdk-client-mock';
import {
ListClustersResponse,
DescribeClusterResponse,
ListClustersCommand,
DescribeClusterCommand,
EKSClient,
} from '@aws-sdk/client-eks';
describe('AwsEKSClusterProcessor', () => {
AWSMock.setSDKInstance(aws);
describe('readLocation', () => {
const processor = new (AwsEKSClusterProcessor as any)({});
const location = { type: 'aws-eks', target: '957140518395/us-west-2' };
const emit = jest.fn();
it('generates cluster correctly', async () => {
const clusters: aws.EKS.Types.ListClustersResponse = {
const clusters: ListClustersResponse = {
clusters: ['backstage-test'],
nextToken: undefined,
};
const cluster: aws.EKS.Types.DescribeClusterResponse = {
const cluster: DescribeClusterResponse = {
cluster: {
name: 'backstage-test',
arn: 'arn:aws:1',
@@ -42,9 +46,10 @@ describe('AwsEKSClusterProcessor', () => {
},
},
};
const mock = mockClient(EKSClient);
AWSMock.mock('EKS', 'listClusters', clusters);
AWSMock.mock('EKS', 'describeCluster', cluster);
mock.on(ListClustersCommand).resolves(clusters);
mock.on(DescribeClusterCommand).resolves(cluster);
await processor.readLocation(location, false, emit);
plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts
+35 -6
View File
@@ -24,8 +24,14 @@ import {
ANNOTATION_KUBERNETES_API_SERVER_CA,
ANNOTATION_KUBERNETES_AUTH_PROVIDER,
} from '@backstage/plugin-kubernetes-common';
import { Credentials, EKS } from 'aws-sdk';
import { EKS } from '@aws-sdk/client-eks';
import { AWSCredentialFactory } from '../types';
import { AwsCredentialIdentity, Provider } from '@aws-sdk/types';
import {
AwsCredentialsManager,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
import { Config } from '@backstage/config';
const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id';
const ARN_ANNOTATION: string = 'amazonaws.com/arn';
@@ -39,9 +45,22 @@ const ARN_ANNOTATION: string = 'amazonaws.com/arn';
*/
export class AwsEKSClusterProcessor implements CatalogProcessor {
private credentialsFactory?: AWSCredentialFactory;
private credentialsManager?: AwsCredentialsManager;
constructor(options: { credentialsFactory?: AWSCredentialFactory }) {
static fromConfig(configRoot: Config): AwsEKSClusterProcessor {
const awsCredentaislManager =
DefaultAwsCredentialsManager.fromConfig(configRoot);
return new AwsEKSClusterProcessor({
credentialsManager: awsCredentaislManager,
});
}
constructor(options: {
credentialsFactory?: AWSCredentialFactory;
credentialsManager?: AwsCredentialsManager;
}) {
this.credentialsFactory = options.credentialsFactory;
this.credentialsManager = options.credentialsManager;
}
getProcessorName(): string {
@@ -73,20 +92,30 @@ export class AwsEKSClusterProcessor implements CatalogProcessor {
);
}
let credentials: Credentials | undefined;
let credentials: AwsCredentialIdentity | undefined;
if (this.credentialsFactory) {
credentials = await this.credentialsFactory(accountId);
}
const eksClient = new EKS({ credentials, region });
const clusters = await eksClient.listClusters({}).promise();
let providerFunction: (() => Provider<AwsCredentialIdentity>) | undefined;
if (this.credentialsManager) {
const credentialsProvider =
await this.credentialsManager.getCredentialProvider({ accountId });
providerFunction = () => credentialsProvider.sdkCredentialProvider;
}
const eksClient = new EKS({
credentials,
credentialDefaultProvider: providerFunction,
});
const clusters = await eksClient.listClusters({});
if (clusters.clusters === undefined) {
return true;
}
const results = clusters.clusters
.map(cluster => eksClient.describeCluster({ name: cluster }).promise())
.map(cluster => eksClient.describeCluster({ name: cluster }))
.map(async describedClusterPromise => {
const describedCluster = await describedClusterPromise;
if (describedCluster.cluster) {
plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts
+28 -33
View File
@@ -14,7 +14,13 @@
* limitations under the License.
*/
import { mockClient } from 'aws-sdk-client-mock';
import 'aws-sdk-client-mock-jest';
import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
import {
ListAccountsCommand,
OrganizationsClient,
} from '@aws-sdk/client-organizations';
describe('AwsOrganizationCloudAccountProcessor', () => {
describe('readLocation', () => {
@@ -23,26 +29,21 @@ describe('AwsOrganizationCloudAccountProcessor', () => {
});
const location = { type: 'aws-cloud-accounts', target: '' };
const emit = jest.fn();
const listAccounts = jest.fn();
const mock = mockClient(OrganizationsClient);
processor.organizations.listAccounts = listAccounts;
afterEach(() => jest.resetAllMocks());
afterEach(() => {
jest.resetAllMocks();
});
it('generates component entities for accounts', async () => {
listAccounts.mockImplementation(() => {
return {
async promise() {
return {
Accounts: [
{
Arn: 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
],
NextToken: undefined,
};
mock.on(ListAccountsCommand).resolves({
Accounts: [
{
Arn: 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
};
],
NextToken: undefined,
});
await processor.readLocation(location, false, emit);
expect(emit).toHaveBeenCalledWith({
@@ -74,24 +75,18 @@ describe('AwsOrganizationCloudAccountProcessor', () => {
type: 'aws-cloud-accounts',
target: 'o-1vl18kc5a3',
};
listAccounts.mockImplementation(() => {
return {
async promise() {
return {
Accounts: [
{
Arn: 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
{
Arn: 'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395',
Name: 'testaccount2',
},
],
NextToken: undefined,
};
mock.on(ListAccountsCommand).resolves({
Accounts: [
{
Arn: 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
};
{
Arn: 'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395',
Name: 'testaccount2',
},
],
NextToken: undefined,
});
await processor.readLocation(locationTest, false, emit);
expect(emit).toHaveBeenCalledTimes(1);
plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts
+35 -38
View File
@@ -22,13 +22,17 @@ import {
processingResult,
} from '@backstage/plugin-catalog-node';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import AWS, { Credentials, Organizations } from 'aws-sdk';
import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';
import { Logger } from 'winston';
import {
AwsOrganizationProviderConfig,
readAwsOrganizationConfig,
} from '../awsOrganization/config';
Account,
ListAccountsResponse,
Organizations,
} from '@aws-sdk/client-organizations';
import { Logger } from 'winston';
import { readAwsOrganizationConfig } from '../awsOrganization/config';
import {
AwsCredentialProvider,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
const AWS_ORGANIZATION_REGION = 'us-east-1';
const LOCATION_TYPE = 'aws-cloud-accounts';
@@ -47,39 +51,31 @@ const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';
*/
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
private readonly organizations: Organizations;
private readonly provider: AwsOrganizationProviderConfig;
private readonly logger: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
static async fromConfig(config: Config, options: { logger: Logger }) {
const c = config.getOptionalConfig('catalog.processors.awsOrganization');
return new AwsOrganizationCloudAccountProcessor({
...options,
provider: c ? readAwsOrganizationConfig(c) : {},
const orgConfig = c ? readAwsOrganizationConfig(c) : undefined;
const awsCredentialsManager =
DefaultAwsCredentialsManager.fromConfig(config);
const credProvider = await awsCredentialsManager.getCredentialProvider({
arn: orgConfig?.roleArn,
});
}
private static buildCredentials(
config: AwsOrganizationProviderConfig,
): Credentials | undefined {
const roleArn = config.roleArn;
if (!roleArn) {
return undefined;
}
return new AWS.ChainableTemporaryCredentials({
params: {
RoleSessionName: 'backstage-aws-organization-processor',
RoleArn: roleArn,
},
});
}
private constructor(options: { provider: AwsOrganizationProviderConfig }) {
this.provider = options.provider;
const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(
this.provider,
return new AwsOrganizationCloudAccountProcessor(
credProvider,
options.logger,
);
this.organizations = new AWS.Organizations({
credentials,
}
private constructor(
private readonly credProvider: AwsCredentialProvider,
logger: Logger,
) {
this.logger = logger?.child({
target: this.getProcessorName(),
});
this.organizations = new Organizations({
credentialDefaultProvider: () => this.credProvider.sdkCredentialProvider,
region: AWS_ORGANIZATION_REGION,
}); // Only available in us-east-1
}
@@ -97,6 +93,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
return false;
}
this.logger?.info('Discovering AWS Organization Account objects');
(await this.getAwsAccounts())
.map(account => this.mapAccountToComponent(account))
.filter(entity => {
@@ -143,9 +141,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
let nextToken = undefined;
while (isInitialAttempt || nextToken) {
isInitialAttempt = false;
const orgAccounts: ListAccountsResponse = await this.organizations
.listAccounts({ NextToken: nextToken })
.promise();
const orgAccounts: ListAccountsResponse =
await this.organizations.listAccounts({ NextToken: nextToken });
if (orgAccounts.Accounts) {
awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
}
plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts
+168 -41
View File
@@ -23,8 +23,9 @@ import {
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { AwsS3EntityProvider } from './AwsS3EntityProvider';
import aws from 'aws-sdk';
import AWSMock from 'aws-sdk-mock';
import { mockClient } from 'aws-sdk-client-mock';
import 'aws-sdk-client-mock-jest';
import { ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
@@ -42,38 +43,39 @@ class PersistingTaskRunner implements TaskRunner {
const logger = getVoidLogger();
describe('AwsS3EntityProvider', () => {
AWSMock.setSDKInstance(aws);
const createObjectList = (keys: string[]): aws.S3.ObjectList => {
const createObjectList = (keys: string[]) => {
const objects = keys.map(key => {
return {
Key: key,
} as aws.S3.Types.Object;
};
});
return objects as aws.S3.ObjectList;
return objects;
};
const keys = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key 4.yaml'];
const mock = mockClient(S3Client);
AWSMock.mock('S3', 'listObjectsV2', async req => {
const prefix = req.Prefix ?? '';
beforeEach(() => {
mock.on(ListObjectsV2Command).callsFake(async req => {
const prefix = req.Prefix ?? '';
if (!req.ContinuationToken) {
return {
Contents: createObjectList(
keys
.slice(0, Math.ceil(keys.length / 2))
.map(key => `${prefix}${key}`),
),
NextContinuationToken: 'next-token',
};
}
if (!req.ContinuationToken) {
return {
Contents: createObjectList(
keys
.slice(0, Math.ceil(keys.length / 2))
.map(key => `${prefix}${key}`),
keys.slice(Math.ceil(keys.length / 2)).map(key => `${prefix}${key}`),
),
NextContinuationToken: 'next-token',
} as aws.S3.Types.ListObjectsV2Output;
}
return {
Contents: createObjectList(
keys.slice(Math.ceil(keys.length / 2)).map(key => `${prefix}${key}`),
),
} as aws.S3.Types.ListObjectsV2Output;
};
});
});
afterEach(() => {
@@ -103,7 +105,9 @@ describe('AwsS3EntityProvider', () => {
});
const schedulingConfig: Record<string, any> = {};
const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/')
? expectedBaseUrl
: `${expectedBaseUrl}/`;
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
@@ -131,7 +135,7 @@ describe('AwsS3EntityProvider', () => {
await (taskDef.fn as () => Promise<void>)();
const expectedEntities = keys.map(key => {
const url = encodeURI(`${expectedBaseUrl}${key}`);
const url = encodeURI(`${normalizedExpectedBaseUrl}${key}`);
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
@@ -161,6 +165,25 @@ describe('AwsS3EntityProvider', () => {
// eslint-disable-next-line jest/expect-expect
it('apply full update on scheduled execution', async () => {
return expectMutation(
'regionalStatic',
{
bucketName: 'bucket-1',
prefix: 'sub/dir/',
region: 'eu-west-1',
},
'https://bucket-1.s3.eu-west-1.amazonaws.com/sub/dir/',
{
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
'key 4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('apply full update on scheduled execution force path style', async () => {
return expectMutation(
'regionalStatic',
{
@@ -175,11 +198,72 @@ describe('AwsS3EntityProvider', () => {
'key3.yaml': 'generated-8d75f78ed9fa618ce433b226dc24eeab441f3a2d',
'key 4.yaml': 'generated-1e0249dcb5805fc2ce6ac2d3c4d2a3ef4f1270c0',
},
{
s3ForcePathStyle: true,
},
);
});
// eslint-disable-next-line jest/expect-expect
it('us-east-1 has region-less URLs', async () => {
it('apply full update on scheduled execution no prefix', async () => {
return expectMutation(
'regionalStatic',
{
bucketName: 'bucket-1',
region: 'eu-west-1',
},
'https://bucket-1.s3.eu-west-1.amazonaws.com/',
{
'key1.yaml': 'generated-ac6d99e56e0fbd6bf3719a5182f37bfa84b1cddc',
'key2.yaml': 'generated-cc3e1145496cd95bf3898687bb5b7d2991a7b7fd',
'key3.yaml': 'generated-a197557b0cb89d14ac1297231ed930bd4c40b664',
'key 4.yaml': 'generated-a774e6c1d1c7ee511b7d79ebb99b08a39cf439c4',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('apply full update on scheduled execution force path style no prefix', async () => {
return expectMutation(
'regionalStatic',
{
bucketName: 'bucket-1',
region: 'eu-west-1',
},
'https://s3.eu-west-1.amazonaws.com/bucket-1',
{
'key1.yaml': 'generated-b390cd4de7ff00f6d51426870fc0e685029fbf09',
'key2.yaml': 'generated-15ebeb1e53ffe71ea93f037076aefcdc5cc5e617',
'key3.yaml': 'generated-fd0a33e70bafd707a60139decbd0383361ab3b4e',
'key 4.yaml': 'generated-ac574fb266d9c8bd983b25a208c538e4ad2c1d06',
},
{
s3ForcePathStyle: true,
},
);
});
// eslint-disable-next-line jest/expect-expect
it('apply full update on scheduled execution bucket with dot in it', async () => {
return expectMutation(
'regionalStatic',
{
bucketName: 'bucket.1',
prefix: 'sub/dir/',
region: 'eu-west-1',
},
'https://s3.eu-west-1.amazonaws.com/bucket.1/sub/dir/',
{
'key1.yaml': 'generated-651b5851d4670b7d5bb9a319d8f6eb295092e4ad',
'key2.yaml': 'generated-ccddcdc3ad814aea072883118f3a9d0ac39a161e',
'key3.yaml': 'generated-b2b8548903aa764544ab79dc81ab5093e6e4ba83',
'key 4.yaml': 'generated-aafa9cc0422f5e8ccce1616dc0c74fa94f692af3',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('us-east-1 does not have region-less URLs', async () => {
return expectMutation(
'usEast1',
{
@@ -187,12 +271,12 @@ describe('AwsS3EntityProvider', () => {
prefix: 'sub/dir/',
region: 'us-east-1',
},
'https://s3.amazonaws.com/bucket-1/sub/dir/',
'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/',
{
'key1.yaml': 'generated-f7e3f1c89f62cbf0d82db16452faaa7c040fc331',
'key2.yaml': 'generated-925173f7dba1acaa73cac5ef4d10c3ed7660aa25',
'key3.yaml': 'generated-d94cf017911ed7fb3be4a62a3dae1f5202879de3',
'key 4.yaml': 'generated-f917fca0cfacc2be478ca2a3cff94d00917a1cde',
'key1.yaml': 'generated-980e6ad47fbfbfeead708a9c7c87331b7540296a',
'key2.yaml': 'generated-266794d8e789089dddba2b42cd79e70b149aa61c',
'key3.yaml': 'generated-96f0cdcd7e33aa687c19d160ec7d5b1975cb9ea1',
'key 4.yaml': 'generated-4f6d3ffb232e4a5b82c3a52549e4c36dab1c2f8d',
},
);
});
@@ -209,18 +293,60 @@ describe('AwsS3EntityProvider', () => {
bucketName: 'bucket-1',
prefix: 'sub/dir/',
},
'https://s3.eu-central-1.amazonaws.com/bucket-1/sub/dir/',
'https://bucket-1.s3.eu-central-1.amazonaws.com/sub/dir/',
{
'key1.yaml': 'generated-285d144b5c1e24e801b97f61f794d566dabc7236',
'key2.yaml': 'generated-36b78d7fa69690059797d4cc4f40c2a5eaed1d6d',
'key3.yaml': 'generated-a39ad804bf65993cbc0f3810ff128a3c91daf768',
'key 4.yaml': 'generated-bd7622ac86c313cb812d90e335b2f5f6bff14670',
'key1.yaml': 'generated-bb17a9f9b420edd93137692f000b2d443a29239c',
'key2.yaml': 'generated-b4559ecc8b615407e421dbaa3fcb9338e977b8ac',
'key3.yaml': 'generated-bb3d500a6f8b351b0eed1aec77ebce65c1ab41fb',
'key 4.yaml': 'generated-686de87f5882f6558b43280c5b6047e485199e88',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('custom endpoint', async () => {
return expectMutation(
'customEndpoint',
{
bucketName: 'bucket-1',
prefix: 'sub/dir/',
},
'http://bucket-1.localhost:1234/sub/dir/',
{
'key1.yaml': 'generated-79625971ead9c39c91b0bfef4d7c748325e31234',
'key2.yaml': 'generated-eeae95f16605d3afdc615436c18c9fbd35781299',
'key3.yaml': 'generated-692976561563c3fa55889ad090363716c7b7e915',
'key 4.yaml': 'generated-c8f92ea0eb96c355b7e56cbc461cef29f830747e',
},
{
endpoint: 'http://localhost:1234/',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('custom endpoint bucket dot', async () => {
return expectMutation(
'customEndpoint',
{
bucketName: 'bucket.1',
prefix: 'sub/dir/',
},
'http://localhost:1234/bucket.1/sub/dir/',
{
'key1.yaml': 'generated-07be77ac8626ac76a924414e0edf5ce66d3e0b27',
'key2.yaml': 'generated-47295ada3c894cf9bac0d570a34d909244ea2f74',
'key3.yaml': 'generated-30e98f3ebd350f3b618acd024199b5f31545495d',
'key 4.yaml': 'generated-b71bc7661b92cef0db62c9652912fd5e4b245aa7',
},
{
endpoint: 'http://localhost:1234/',
},
);
});
// eslint-disable-next-line jest/expect-expect
it('custom endpoint force path style', async () => {
return expectMutation(
'customEndpoint',
{
@@ -235,7 +361,8 @@ describe('AwsS3EntityProvider', () => {
'key 4.yaml': 'generated-659f17a2429f107db04850f9081e3e9c56084aeb',
},
{
endpoint: 'http://localhost:1234',
endpoint: 'http://localhost:1234/',
s3ForcePathStyle: true,
},
);
});
@@ -305,12 +432,12 @@ describe('AwsS3EntityProvider', () => {
},
},
},
'https://s3.eu-west-1.amazonaws.com/bucket-1/sub/dir/',
'https://bucket-1.s3.eu-west-1.amazonaws.com/sub/dir/',
{
'key1.yaml': 'generated-7f6d5861b0b3401a38b5fe62e6c7ca11da5fd6d8',
'key2.yaml': 'generated-a290be145586042af7d80715626399c9d661718d',
'key3.yaml': 'generated-8d75f78ed9fa618ce433b226dc24eeab441f3a2d',
'key 4.yaml': 'generated-1e0249dcb5805fc2ce6ac2d3c4d2a3ef4f1270c0',
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
'key 4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
},
undefined,
true,
plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts
+45 -23
View File
@@ -23,13 +23,20 @@ import {
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-node';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import { AwsCredentials } from '../credentials/AwsCredentials';
import { readAwsS3Configs } from './config';
import { AwsS3Config } from './types';
import { S3 } from 'aws-sdk';
import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
import {
ListObjectsV2Command,
ListObjectsV2Output,
S3,
} from '@aws-sdk/client-s3';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint';
import {
AwsCredentialsManager,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
// TODO: event-based updates using S3 events (+ queue like SQS)?
/**
@@ -41,9 +48,10 @@ import { Logger } from 'winston';
*/
export class AwsS3EntityProvider implements EntityProvider {
private readonly logger: Logger;
private readonly s3: S3;
private s3?: S3;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
private endpoint?: string;
static fromConfig(
configRoot: Config,
@@ -78,7 +86,8 @@ export class AwsS3EntityProvider implements EntityProvider {
`No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`,
);
}
const awsCredentialsManager =
DefaultAwsCredentialsManager.fromConfig(configRoot);
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
@@ -86,6 +95,7 @@ export class AwsS3EntityProvider implements EntityProvider {
return new AwsS3EntityProvider(
providerConfig,
integration,
awsCredentialsManager,
options.logger,
taskRunner,
);
@@ -94,7 +104,8 @@ export class AwsS3EntityProvider implements EntityProvider {
private constructor(
private readonly config: AwsS3Config,
integration: AwsS3Integration,
private readonly integration: AwsS3Integration,
private readonly awsCredentialsManager: AwsCredentialsManager,
logger: Logger,
taskRunner: TaskRunner,
) {
@@ -102,17 +113,6 @@ export class AwsS3EntityProvider implements EntityProvider {
target: this.getProviderName(),
});
this.s3 = new S3({
apiVersion: '2006-03-01',
credentials: AwsCredentials.create(
integration.config,
'backstage-aws-s3-provider',
),
endpoint: integration.config.endpoint,
region: this.config.region,
s3ForcePathStyle: integration.config.s3ForcePathStyle,
});
this.scheduleFn = this.createScheduleFn(taskRunner);
}
@@ -146,6 +146,28 @@ export class AwsS3EntityProvider implements EntityProvider {
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
const credProvider =
await this.awsCredentialsManager.getCredentialProvider();
this.s3 = new S3({
apiVersion: '2006-03-01',
credentialDefaultProvider: () => credProvider.sdkCredentialProvider,
endpoint: this.integration.config.endpoint,
region: this.config.region,
forcePathStyle: this.integration.config.s3ForcePathStyle,
});
// https://github.com/aws/aws-sdk-js-v3/issues/4122#issuecomment-1298968804
const endpoint = await getEndpointFromInstructions(
{
Bucket: this.config.bucketName,
},
ListObjectsV2Command,
this.s3.config as unknown as Record<string, unknown>,
);
if (endpoint?.url)
this.endpoint = endpoint.url.href.endsWith('/')
? endpoint.url.href
: `${endpoint.url.href}/`;
await this.scheduleFn();
}
@@ -175,18 +197,21 @@ export class AwsS3EntityProvider implements EntityProvider {
}
private async listAllObjectKeys(): Promise<string[]> {
if (!this.s3) {
throw new Error('Not initialized');
}
const keys: string[] = [];
let continuationToken: string | undefined = undefined;
let output: ListObjectsV2Output;
do {
const request = this.s3.listObjectsV2({
output = await this.s3.listObjectsV2({
Bucket: this.config.bucketName,
ContinuationToken: continuationToken,
Prefix: this.config.prefix,
});
output = await request.promise();
if (output.Contents) {
output.Contents.forEach(item => {
if (item.Key && !item.Key.endsWith('/')) {
@@ -209,9 +234,6 @@ export class AwsS3EntityProvider implements EntityProvider {
}
private createObjectUrl(key: string): string {
const bucketName = this.config.bucketName;
const endpoint = this.s3.endpoint.href;
return encodeURI(`${endpoint}${bucketName}/${key}`);
return new URL(key, this.endpoint).href;
}
}
plugins/catalog-backend-module-aws/src/types.ts
+2 -2
View File
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Credentials } from 'aws-sdk';
import { AwsCredentialIdentity } from '@aws-sdk/types';
/**
* A factory for providing user-specified AWS credentials for a given AWS account.
@@ -22,4 +22,4 @@ import { Credentials } from 'aws-sdk';
*/
export type AWSCredentialFactory = (
awsAccountId: string,
) => Promise<Credentials>;
) => Promise<AwsCredentialIdentity>;
plugins/events-backend-module-aws-sqs/package.json
+2 -2
View File
@@ -35,7 +35,7 @@
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@aws-sdk/client-sqs": "^3.208.0",
"@aws-sdk/client-sqs": "^3.310.0",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/backend-tasks": "workspace:^",
@@ -46,7 +46,7 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@aws-sdk/types": "^3.208.0",
"@aws-sdk/types": "^3.310.0",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
plugins/kubernetes-backend/package.json
+1 -2
View File
@@ -50,7 +50,7 @@
},
"dependencies": {
"@aws-crypto/sha256-js": "^3.0.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/credential-providers": "^3.310.0",
"@aws-sdk/signature-v4": "^3.310.0",
"@azure/identity": "^2.0.4",
"@backstage/backend-common": "workspace:^",
@@ -70,7 +70,6 @@
"@kubernetes/client-node": "0.18.1",
"@types/express": "^4.17.6",
"@types/luxon": "^3.0.0",
"aws4": "^1.11.0",
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
plugins/techdocs-node/package.json
+5 -5
View File
@@ -39,11 +39,11 @@
"url": "https://github.com/backstage/backstage/issues"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.208.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/lib-storage": "^3.208.0",
"@aws-sdk/node-http-handler": "^3.208.0",
"@aws-sdk/types": "^3.208.0",
"@aws-sdk/client-s3": "^3.310.0",
"@aws-sdk/credential-providers": "^3.310.0",
"@aws-sdk/lib-storage": "^3.310.0",
"@aws-sdk/node-http-handler": "^3.310.0",
"@aws-sdk/types": "^3.310.0",
"@azure/identity": "^2.1.0",
"@azure/storage-blob": "^12.5.0",
"@backstage/backend-common": "workspace:^",
yarn.lock
+732 -854
View File
File diff suppressed because it is too large Load Diff