paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 44 Packages Projects Releases Wiki Activity

migrate to

Browse Source 
Signed-off-by: Andrew Ochsner <andrew.ochsner@cognizant.com>
This commit is contained in:
Andrew Ochsner
2023-05-08 11:08:31 -05:00
parent 00f6ccee10
commit df35616118
9 changed files with 96 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend-module-aws/api-report.md
+9 -3
View File
@@ -4,6 +4,7 @@
```ts
import { AwsCredentialIdentity } from '@aws-sdk/types';
import { AwsCredentialsManager } from '@backstage/integration-aws-node';
import { CatalogProcessor } from '@backstage/plugin-catalog-node';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
import { CatalogProcessorParser } from '@backstage/plugin-catalog-node';
@@ -23,7 +24,12 @@ export type AWSCredentialFactory = (
// @public
export class AwsEKSClusterProcessor implements CatalogProcessor {
constructor(options: { credentialsFactory?: AWSCredentialFactory });
constructor(options: {
credentialsFactory?: AWSCredentialFactory;
credentialsManager?: AwsCredentialsManager;
});
// (undocumented)
static fromConfig(configRoot: Config): AwsEKSClusterProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
@@ -42,9 +48,9 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
static fromConfig(
config: Config,
options: {
logger: Logger;
logger?: Logger;
},
): AwsOrganizationCloudAccountProcessor;
): Promise<AwsOrganizationCloudAccountProcessor>;
// (undocumented)
getProcessorName(): string;
// (undocumented)
plugins/catalog-backend-module-aws/package.json
+2
View File
@@ -58,6 +58,7 @@
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/integration-aws-node": "workspace:^",
"@backstage/plugin-catalog-common": "workspace:^",
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/plugin-kubernetes-common": "workspace:^",
@@ -73,6 +74,7 @@
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151",
"aws-sdk-client-mock": "^2.0.0",
"aws-sdk-client-mock-jest": "^2.0.0",
"luxon": "^3.0.0",
"yaml": "^2.0.0"
},
plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts
-66
View File
@@ -1,66 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
AwsCredentialIdentity,
AwsCredentialIdentityProvider,
} from '@aws-sdk/types';
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
export class AwsCredentials {
/**
* If accessKeyId and secretAccessKey are missing, the DefaultAWSCredentialsProviderChain will be used:
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
*/
static create(
config: {
accessKeyId?: string;
secretAccessKey?: string;
roleArn?: string;
externalId?: string;
},
roleSessionName: string,
): AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined {
if (!config) {
return undefined;
}
const accessKeyId = config.accessKeyId;
const secretAccessKey = config.secretAccessKey;
let explicitCredentials: AwsCredentialIdentity | undefined;
if (accessKeyId && secretAccessKey) {
explicitCredentials = {
accessKeyId,
secretAccessKey,
};
}
const roleArn = config.roleArn;
if (roleArn) {
return fromTemporaryCredentials({
masterCredentials: explicitCredentials,
params: {
RoleArn: roleArn,
RoleSessionName: roleSessionName,
ExternalId: config.externalId,
},
});
}
return explicitCredentials;
}
}
plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts
+31 -3
View File
@@ -24,9 +24,14 @@ import {
ANNOTATION_KUBERNETES_API_SERVER_CA,
ANNOTATION_KUBERNETES_AUTH_PROVIDER,
} from '@backstage/plugin-kubernetes-common';
import { AwsCredentialIdentity } from '@aws-sdk/types';
import { EKS } from '@aws-sdk/client-eks';
import { AWSCredentialFactory } from '../types';
import { AwsCredentialIdentity, Provider } from '@aws-sdk/types';
import {
AwsCredentialsManager,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
import { Config } from '@backstage/config';
const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id';
const ARN_ANNOTATION: string = 'amazonaws.com/arn';
@@ -40,9 +45,22 @@ const ARN_ANNOTATION: string = 'amazonaws.com/arn';
*/
export class AwsEKSClusterProcessor implements CatalogProcessor {
private credentialsFactory?: AWSCredentialFactory;
private credentialsManager?: AwsCredentialsManager;
constructor(options: { credentialsFactory?: AWSCredentialFactory }) {
static fromConfig(configRoot: Config): AwsEKSClusterProcessor {
const awsCredentaislManager =
DefaultAwsCredentialsManager.fromConfig(configRoot);
return new AwsEKSClusterProcessor({
credentialsManager: awsCredentaislManager,
});
}
constructor(options: {
credentialsFactory?: AWSCredentialFactory;
credentialsManager?: AwsCredentialsManager;
}) {
this.credentialsFactory = options.credentialsFactory;
this.credentialsManager = options.credentialsManager;
}
getProcessorName(): string {
@@ -80,7 +98,17 @@ export class AwsEKSClusterProcessor implements CatalogProcessor {
credentials = await this.credentialsFactory(accountId);
}
const eksClient = new EKS({ credentials, region });
let providerFunction: (() => Provider<AwsCredentialIdentity>) | undefined;
if (this.credentialsManager) {
const credentialsProvider =
await this.credentialsManager.getCredentialProvider({ accountId });
providerFunction = () => credentialsProvider.sdkCredentialProvider;
}
const eksClient = new EKS({
credentials,
credentialDefaultProvider: providerFunction,
});
const clusters = await eksClient.listClusters({});
if (clusters.clusters === undefined) {
return true;
plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts
+1 -1
View File
@@ -15,6 +15,7 @@
*/
import { mockClient } from 'aws-sdk-client-mock';
import 'aws-sdk-client-mock-jest';
import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
import {
ListAccountsCommand,
@@ -32,7 +33,6 @@ describe('AwsOrganizationCloudAccountProcessor', () => {
afterEach(() => {
jest.resetAllMocks();
mock.reset();
});
it('generates component entities for accounts', async () => {
plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts
+26 -33
View File
@@ -27,13 +27,12 @@ import {
ListAccountsResponse,
Organizations,
} from '@aws-sdk/client-organizations';
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
import { Logger } from 'winston';
import { readAwsOrganizationConfig } from '../awsOrganization/config';
import {
AwsOrganizationProviderConfig,
readAwsOrganizationConfig,
} from '../awsOrganization/config';
AwsCredentialProvider,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
const AWS_ORGANIZATION_REGION = 'us-east-1';
const LOCATION_TYPE = 'aws-cloud-accounts';
@@ -52,39 +51,31 @@ const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';
*/
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
private readonly organizations: Organizations;
private readonly provider: AwsOrganizationProviderConfig;
private readonly logger?: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
static async fromConfig(config: Config, options: { logger?: Logger }) {
const c = config.getOptionalConfig('catalog.processors.awsOrganization');
return new AwsOrganizationCloudAccountProcessor({
...options,
provider: c ? readAwsOrganizationConfig(c) : {},
const orgConfig = c ? readAwsOrganizationConfig(c) : undefined;
const awsCredentialsManager =
DefaultAwsCredentialsManager.fromConfig(config);
const credProvider = await awsCredentialsManager.getCredentialProvider({
arn: orgConfig?.roleArn,
});
}
private static buildCredentials(
config: AwsOrganizationProviderConfig,
): AwsCredentialIdentityProvider | undefined {
const roleArn = config.roleArn;
if (!roleArn) {
return undefined;
}
return fromTemporaryCredentials({
params: {
RoleSessionName: 'backstage-aws-organization-processor',
RoleArn: roleArn,
},
});
}
private constructor(options: { provider: AwsOrganizationProviderConfig }) {
this.provider = options.provider;
const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(
this.provider,
return new AwsOrganizationCloudAccountProcessor(
credProvider,
options.logger,
);
}
private constructor(
private readonly credProvider: AwsCredentialProvider,
logger?: Logger,
) {
this.logger = logger?.child({
target: this.getProcessorName(),
});
this.organizations = new Organizations({
credentials,
credentialDefaultProvider: () => this.credProvider.sdkCredentialProvider,
region: AWS_ORGANIZATION_REGION,
}); // Only available in us-east-1
}
@@ -102,6 +93,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
return false;
}
this.logger?.info('Discovering AWS Organization Account objects');
(await this.getAwsAccounts())
.map(account => this.mapAccountToComponent(account))
.filter(entity => {
plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts
+1 -2
View File
@@ -24,6 +24,7 @@ import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { AwsS3EntityProvider } from './AwsS3EntityProvider';
import { mockClient } from 'aws-sdk-client-mock';
import 'aws-sdk-client-mock-jest';
import { ListObjectsV2Command, S3 } from '@aws-sdk/client-s3';
class PersistingTaskRunner implements TaskRunner {
@@ -55,8 +56,6 @@ describe('AwsS3EntityProvider', () => {
const mock = mockClient(S3);
beforeEach(() => {
mock.reset();
mock.on(ListObjectsV2Command).callsFake(async req => {
const prefix = req.Prefix ?? '';
plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts
+24 -14
View File
@@ -23,7 +23,6 @@ import {
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-node';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import { AwsCredentials } from '../credentials/AwsCredentials';
import { readAwsS3Configs } from './config';
import { AwsS3Config } from './types';
import {
@@ -34,6 +33,10 @@ import {
import * as uuid from 'uuid';
import { Logger } from 'winston';
import { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint';
import {
AwsCredentialsManager,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
// TODO: event-based updates using S3 events (+ queue like SQS)?
/**
@@ -45,7 +48,7 @@ import { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint';
*/
export class AwsS3EntityProvider implements EntityProvider {
private readonly logger: Logger;
private readonly s3: S3;
private s3?: S3;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
private endpoint?: string;
@@ -83,7 +86,8 @@ export class AwsS3EntityProvider implements EntityProvider {
`No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`,
);
}
const awsCredentialsManager =
DefaultAwsCredentialsManager.fromConfig(configRoot);
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
@@ -91,6 +95,7 @@ export class AwsS3EntityProvider implements EntityProvider {
return new AwsS3EntityProvider(
providerConfig,
integration,
awsCredentialsManager,
options.logger,
taskRunner,
);
@@ -99,7 +104,8 @@ export class AwsS3EntityProvider implements EntityProvider {
private constructor(
private readonly config: AwsS3Config,
integration: AwsS3Integration,
private readonly integration: AwsS3Integration,
private readonly awsCredentialsManager: AwsCredentialsManager,
logger: Logger,
taskRunner: TaskRunner,
) {
@@ -107,16 +113,6 @@ export class AwsS3EntityProvider implements EntityProvider {
target: this.getProviderName(),
});
this.s3 = new S3({
apiVersion: '2006-03-01',
credentials: AwsCredentials.create(
integration.config,
'backstage-aws-s3-provider',
),
endpoint: integration.config.endpoint,
region: this.config.region,
forcePathStyle: integration.config.s3ForcePathStyle,
});
this.scheduleFn = this.createScheduleFn(taskRunner);
}
@@ -150,6 +146,16 @@ export class AwsS3EntityProvider implements EntityProvider {
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
const credProvider =
await this.awsCredentialsManager.getCredentialProvider();
this.s3 = new S3({
apiVersion: '2006-03-01',
credentialDefaultProvider: () => credProvider.sdkCredentialProvider,
endpoint: this.integration.config.endpoint,
region: this.config.region,
forcePathStyle: this.integration.config.s3ForcePathStyle,
});
// https://github.com/aws/aws-sdk-js-v3/issues/4122#issuecomment-1298968804
const endpoint = await getEndpointFromInstructions(
{
@@ -191,6 +197,10 @@ export class AwsS3EntityProvider implements EntityProvider {
}
private async listAllObjectKeys(): Promise<string[]> {
if (!this.s3) {
throw new Error('Not initialized');
}
const keys: string[] = [];
let continuationToken: string | undefined = undefined;
yarn.lock
+2
View File
@@ -5840,12 +5840,14 @@ __metadata:
"@backstage/config": "workspace:^"
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/integration-aws-node": "workspace:^"
"@backstage/plugin-catalog-common": "workspace:^"
"@backstage/plugin-catalog-node": "workspace:^"
"@backstage/plugin-kubernetes-common": "workspace:^"
"@backstage/types": "workspace:^"
"@types/lodash": ^4.14.151
aws-sdk-client-mock: ^2.0.0
aws-sdk-client-mock-jest: ^2.0.0
lodash: ^4.17.21
luxon: ^3.0.0
p-limit: ^3.0.2