feat: azure blob storage entity provider for catalog implemented
Signed-off-by: NIKUNJ LALITKUMAR HUDKA <nk856850@dal.ca>
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/integration': major
|
||||
---
|
||||
|
||||
Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/backend-defaults': minor
|
||||
---
|
||||
|
||||
Implemented `AzureBlobStorageUrlReader` to read from the url of committed location from the entity provider
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-azure': minor
|
||||
---
|
||||
|
||||
Added the Azure Blob Storage as catalog entity provider to import all the desired entities from storage account provided in app-config.yaml
|
||||
@@ -221,6 +221,17 @@ integrations:
|
||||
# apiBaseUrl: server.bitbucket.com
|
||||
# username: ${BITBUCKET_SERVER_USERNAME}
|
||||
# appPassword: ${BITBUCKET_SERVER_APP_PASSWORD}
|
||||
|
||||
azureBlobStorage:
|
||||
- accountName: ${ACCOUNT_NAME} # required
|
||||
endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken
|
||||
sasToken: ${SAS_TOKEN}
|
||||
aadCredential:
|
||||
clientId: ${CLIENT_ID}
|
||||
tenantId: ${TENANT_ID}
|
||||
clientSecret: ${CLIENT_SECRET}
|
||||
accountKey: ${ACCOUNT_KEY}
|
||||
|
||||
azure:
|
||||
- host: dev.azure.com
|
||||
token: ${AZURE_TOKEN}
|
||||
@@ -245,6 +256,13 @@ catalog:
|
||||
- Domain
|
||||
- Location
|
||||
providers:
|
||||
azureBlob:
|
||||
containerName: ${CONTAINER_NAME}
|
||||
schedule: # same options as in TaskScheduleDefinition
|
||||
# supports cron, ISO duration, "human duration" as used in code
|
||||
frequency: { minutes: 30 }
|
||||
# supports ISO duration, "human duration" as used in code
|
||||
timeout: { minutes: 3 }
|
||||
backstageOpenapi:
|
||||
plugins:
|
||||
- catalog
|
||||
|
||||
@@ -119,6 +119,8 @@
|
||||
"@aws-sdk/client-s3": "^3.350.0",
|
||||
"@aws-sdk/credential-providers": "^3.350.0",
|
||||
"@aws-sdk/types": "^3.347.0",
|
||||
"@azure/identity": "^4.0.0",
|
||||
"@azure/storage-blob": "^12.5.0",
|
||||
"@backstage/backend-app-api": "workspace:^",
|
||||
"@backstage/backend-common": "^0.25.0",
|
||||
"@backstage/backend-dev-utils": "workspace:^",
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
BlobServiceClient,
|
||||
ContainerClient,
|
||||
StorageSharedKeyCredential,
|
||||
} from '@azure/storage-blob';
|
||||
import { ReaderFactory, ReadTreeResponseFactory } from './types';
|
||||
import { ForwardedError, NotModifiedError } from '@backstage/errors';
|
||||
import { Readable } from 'stream';
|
||||
import { relative } from 'path/posix';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import {
|
||||
AzureBlobStorageIntergation,
|
||||
AzureCredentialsManager,
|
||||
DefaultAzureCredentialsManager,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
UrlReaderService,
|
||||
UrlReaderServiceReadTreeOptions,
|
||||
UrlReaderServiceReadTreeResponse,
|
||||
UrlReaderServiceReadUrlOptions,
|
||||
UrlReaderServiceReadUrlResponse,
|
||||
UrlReaderServiceSearchResponse,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
|
||||
export function parseUrl(
|
||||
url: string,
|
||||
config: AzureBlobStorageIntergation,
|
||||
): { path: string; container: string } {
|
||||
const parsedUrl = new URL(url);
|
||||
const pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
|
||||
|
||||
if (pathSegments.length < 2) {
|
||||
throw new Error(`Invalid Azure Blob Storage URL format: ${url}`);
|
||||
}
|
||||
|
||||
// First segment is the container name, rest is the blob path
|
||||
const container = pathSegments[0];
|
||||
const path = pathSegments.slice(1).join('/');
|
||||
|
||||
return { path, container };
|
||||
}
|
||||
export class AzureBlobStorageUrlReader implements UrlReaderService {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
|
||||
const credsManager =
|
||||
DefaultAzureCredentialsManager.fromIntegrations(integrations);
|
||||
|
||||
return integrations.azureBlobStorage.list().map(integrationConfig => {
|
||||
const reader = new AzureBlobStorageUrlReader(
|
||||
credsManager,
|
||||
integrationConfig,
|
||||
{
|
||||
treeResponseFactory,
|
||||
},
|
||||
);
|
||||
|
||||
const predicate = (url: URL) =>
|
||||
url.host.endsWith(integrationConfig.config.host);
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
// private readonly blobServiceClient: BlobServiceClient;
|
||||
|
||||
constructor(
|
||||
private readonly credsManager: AzureCredentialsManager,
|
||||
private readonly integration: AzureBlobStorageIntergation,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
) {}
|
||||
|
||||
private async createContainerClient(
|
||||
containerName: string,
|
||||
): Promise<ContainerClient> {
|
||||
const accountName = this.integration.config.accountName; // Use the account name from the integration config
|
||||
const accountKey = this.integration.config.accountKey; // Get the account key if it exists
|
||||
|
||||
if (accountKey && accountName) {
|
||||
const creds = new StorageSharedKeyCredential(accountName, accountKey);
|
||||
const blobServiceClient = new BlobServiceClient(
|
||||
`https://${accountName}.${this.integration.config.host}`,
|
||||
creds,
|
||||
);
|
||||
return blobServiceClient.getContainerClient(containerName);
|
||||
}
|
||||
// Use the credentials manager to get the correct credentials
|
||||
const credential = await this.credsManager.getCredentials(
|
||||
accountName as string,
|
||||
);
|
||||
|
||||
let blobServiceClientUrl: string;
|
||||
|
||||
if (this.integration.config.endpoint) {
|
||||
if (this.integration.config.sasToken) {
|
||||
blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`;
|
||||
} else {
|
||||
blobServiceClientUrl = `${this.integration.config.endpoint}`;
|
||||
}
|
||||
} else {
|
||||
blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`;
|
||||
}
|
||||
|
||||
const blobServiceClient = new BlobServiceClient(
|
||||
blobServiceClientUrl,
|
||||
credential,
|
||||
);
|
||||
return blobServiceClient.getContainerClient(containerName);
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: UrlReaderServiceReadUrlOptions,
|
||||
): Promise<UrlReaderServiceReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter } = options ?? {};
|
||||
|
||||
try {
|
||||
const { path, container } = parseUrl(url, this.integration);
|
||||
|
||||
const containerClient = await this.createContainerClient(container);
|
||||
const blobClient = containerClient.getBlobClient(path);
|
||||
|
||||
const abortController = new AbortController();
|
||||
|
||||
const getBlobOptions = {
|
||||
conditions: {
|
||||
...(etag && { ifNoneMatch: etag }),
|
||||
...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }),
|
||||
},
|
||||
};
|
||||
options?.signal?.addEventListener('abort', () => abortController.abort());
|
||||
|
||||
const downloadBlockBlobResponse = await blobClient.download(
|
||||
0,
|
||||
undefined,
|
||||
getBlobOptions,
|
||||
);
|
||||
|
||||
const data = await this.retrieveAzureBlobData(
|
||||
downloadBlockBlobResponse.readableStreamBody as Readable,
|
||||
);
|
||||
|
||||
return ReadUrlResponseFactory.fromReadable(data, {
|
||||
etag: downloadBlockBlobResponse.etag,
|
||||
lastModifiedAt: downloadBlockBlobResponse.lastModified,
|
||||
});
|
||||
} catch (e) {
|
||||
if (e.$metadata && e.$metadata.httpStatusCode === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
throw new ForwardedError(
|
||||
'Could not retrieve file from Azure Blob Storage',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: UrlReaderServiceReadTreeOptions,
|
||||
): Promise<UrlReaderServiceReadTreeResponse> {
|
||||
try {
|
||||
const { path, container } = parseUrl(url, this.integration);
|
||||
|
||||
const containerClient = await this.createContainerClient(container);
|
||||
|
||||
const blobs = containerClient.listBlobsFlat({ prefix: path });
|
||||
|
||||
const responses = [];
|
||||
const abortController = new AbortController();
|
||||
for await (const blob of blobs) {
|
||||
const blobClient = containerClient.getBlobClient(blob.name);
|
||||
options?.signal?.addEventListener('abort', () =>
|
||||
abortController.abort(),
|
||||
);
|
||||
const downloadBlockBlobResponse = await blobClient.download();
|
||||
const data = await this.retrieveAzureBlobData(
|
||||
downloadBlockBlobResponse.readableStreamBody as Readable,
|
||||
);
|
||||
|
||||
responses.push({
|
||||
data: Readable.from(data),
|
||||
path: relative(path, blob.name),
|
||||
lastModifiedAt: blob.properties.lastModified,
|
||||
});
|
||||
}
|
||||
|
||||
return this.deps.treeResponseFactory.fromReadableArray(responses);
|
||||
} catch (e) {
|
||||
throw new ForwardedError(
|
||||
'Could not retrieve file tree from Azure Blob Storage',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async search(): Promise<UrlReaderServiceSearchResponse> {
|
||||
throw new Error('AzureBlobStorageUrlReader does not implement search');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const accountName = this.integration.config.accountName;
|
||||
const accountKey = this.integration.config.accountKey;
|
||||
return `azureBlobStorage{accountName=${accountName},authed=${Boolean(
|
||||
accountKey,
|
||||
)}}`;
|
||||
}
|
||||
|
||||
private parseUrl(url: string): { path: string } {
|
||||
const parsedUrl = new URL(url);
|
||||
const path = parsedUrl.pathname.substring(
|
||||
parsedUrl.pathname.lastIndexOf('/') + 1,
|
||||
);
|
||||
|
||||
return { path };
|
||||
}
|
||||
|
||||
private async retrieveAzureBlobData(stream: Readable): Promise<Readable> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const chunks: any[] = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
stream.on('error', (e: Error) =>
|
||||
reject(new ForwardedError('Unable to read stream', e)),
|
||||
);
|
||||
stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks))));
|
||||
} catch (e) {
|
||||
throw new ForwardedError('Unable to parse the response data', e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader';
|
||||
import { GiteaUrlReader } from './GiteaUrlReader';
|
||||
import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader';
|
||||
import { HarnessUrlReader } from './HarnessUrlReader';
|
||||
import { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader';
|
||||
|
||||
/**
|
||||
* Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}.
|
||||
@@ -99,6 +100,7 @@ export class UrlReaders {
|
||||
GoogleGcsUrlReader.factory,
|
||||
HarnessUrlReader.factory,
|
||||
AwsS3UrlReader.factory,
|
||||
AzureBlobStorageUrlReader.factory,
|
||||
AwsCodeCommitUrlReader.factory,
|
||||
FetchUrlReader.factory,
|
||||
]),
|
||||
|
||||
@@ -24,6 +24,7 @@ export { GitlabUrlReader } from './GitlabUrlReader';
|
||||
export { GiteaUrlReader } from './GiteaUrlReader';
|
||||
export { HarnessUrlReader } from './HarnessUrlReader';
|
||||
export { AwsS3UrlReader } from './AwsS3UrlReader';
|
||||
export { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader';
|
||||
export { FetchUrlReader } from './FetchUrlReader';
|
||||
export { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
export type {
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@azure/identity": "^4.0.0",
|
||||
"@azure/storage-blob": "^12.5.0",
|
||||
"@backstage/config": "workspace:^",
|
||||
"@backstage/errors": "workspace:^",
|
||||
"@octokit/auth-app": "^4.0.0",
|
||||
|
||||
@@ -39,6 +39,10 @@ import { GiteaIntegration, GiteaIntegrationConfig } from './gitea';
|
||||
import { AwsCodeCommitIntegration } from './awsCodeCommit/AwsCodeCommitIntegration';
|
||||
import { AwsCodeCommitIntegrationConfig } from './awsCodeCommit';
|
||||
import { HarnessIntegration, HarnessIntegrationConfig } from './harness';
|
||||
import {
|
||||
AzureBlobStorageIntegrationConfig,
|
||||
AzureBlobStorageIntergation,
|
||||
} from './azureBlobStorage';
|
||||
|
||||
describe('ScmIntegrations', () => {
|
||||
const awsS3 = new AwsS3Integration({
|
||||
@@ -53,6 +57,10 @@ describe('ScmIntegrations', () => {
|
||||
host: 'azure.local',
|
||||
} as AzureIntegrationConfig);
|
||||
|
||||
const azureBlob = new AzureBlobStorageIntergation({
|
||||
host: 'azureblobstorage.local',
|
||||
} as AzureBlobStorageIntegrationConfig);
|
||||
|
||||
const bitbucket = new BitbucketIntegration({
|
||||
host: 'bitbucket.local',
|
||||
} as BitbucketIntegrationConfig);
|
||||
@@ -89,6 +97,7 @@ describe('ScmIntegrations', () => {
|
||||
awsS3: basicIntegrations([awsS3], item => item.config.host),
|
||||
awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host),
|
||||
azure: basicIntegrations([azure], item => item.config.host),
|
||||
azureBlobStorage: basicIntegrations([azureBlob], item => item.config.host),
|
||||
bitbucket: basicIntegrations([bitbucket], item => item.config.host),
|
||||
bitbucketCloud: basicIntegrations([bitbucketCloud], item => item.title),
|
||||
bitbucketServer: basicIntegrations(
|
||||
@@ -108,6 +117,9 @@ describe('ScmIntegrations', () => {
|
||||
awsCodeCommit,
|
||||
);
|
||||
expect(i.azure.byUrl('https://azure.local')).toBe(azure);
|
||||
expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe(
|
||||
azureBlob,
|
||||
);
|
||||
expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket);
|
||||
expect(i.bitbucketCloud.byUrl('https://bitbucket.org')).toBe(
|
||||
bitbucketCloud,
|
||||
@@ -128,6 +140,7 @@ describe('ScmIntegrations', () => {
|
||||
awsS3,
|
||||
awsCodeCommit,
|
||||
azure,
|
||||
azureBlob,
|
||||
bitbucket,
|
||||
bitbucketCloud,
|
||||
bitbucketServer,
|
||||
@@ -144,6 +157,9 @@ describe('ScmIntegrations', () => {
|
||||
expect(i.byUrl('https://awss3.local')).toBe(awsS3);
|
||||
expect(i.byUrl('https://awscodecommit.local')).toBe(awsCodeCommit);
|
||||
expect(i.byUrl('https://azure.local')).toBe(azure);
|
||||
expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe(
|
||||
azureBlob,
|
||||
);
|
||||
expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket);
|
||||
expect(i.byUrl('https://bitbucket.org')).toBe(bitbucketCloud);
|
||||
expect(i.byUrl('https://bitbucket-server.local')).toBe(bitbucketServer);
|
||||
@@ -156,6 +172,7 @@ describe('ScmIntegrations', () => {
|
||||
expect(i.byHost('awss3.local')).toBe(awsS3);
|
||||
expect(i.byHost('awscodecommit.local')).toBe(awsCodeCommit);
|
||||
expect(i.byHost('azure.local')).toBe(azure);
|
||||
expect(i.byHost('azureblobstorage.local')).toBe(azureBlob);
|
||||
expect(i.byHost('bitbucket.local')).toBe(bitbucket);
|
||||
expect(i.byHost('bitbucket.org')).toBe(bitbucketCloud);
|
||||
expect(i.byHost('bitbucket-server.local')).toBe(bitbucketServer);
|
||||
|
||||
@@ -29,6 +29,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types';
|
||||
import { ScmIntegrationRegistry } from './registry';
|
||||
import { GiteaIntegration } from './gitea';
|
||||
import { HarnessIntegration } from './harness/HarnessIntegration';
|
||||
import { AzureBlobStorageIntergation } from './azureBlobStorage';
|
||||
|
||||
/**
|
||||
* The set of supported integrations.
|
||||
@@ -38,6 +39,7 @@ import { HarnessIntegration } from './harness/HarnessIntegration';
|
||||
export interface IntegrationsByType {
|
||||
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
|
||||
awsCodeCommit: ScmIntegrationsGroup<AwsCodeCommitIntegration>;
|
||||
azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
|
||||
azure: ScmIntegrationsGroup<AzureIntegration>;
|
||||
/**
|
||||
* @deprecated in favor of `bitbucketCloud` and `bitbucketServer`
|
||||
@@ -64,6 +66,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
|
||||
return new ScmIntegrations({
|
||||
awsS3: AwsS3Integration.factory({ config }),
|
||||
awsCodeCommit: AwsCodeCommitIntegration.factory({ config }),
|
||||
azureBlobStorage: AzureBlobStorageIntergation.factory({ config }),
|
||||
azure: AzureIntegration.factory({ config }),
|
||||
bitbucket: BitbucketIntegration.factory({ config }),
|
||||
bitbucketCloud: BitbucketCloudIntegration.factory({ config }),
|
||||
@@ -88,6 +91,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
|
||||
return this.byType.awsCodeCommit;
|
||||
}
|
||||
|
||||
get azureBlobStorage(): ScmIntegrationsGroup<AzureBlobStorageIntergation> {
|
||||
return this.byType.azureBlobStorage;
|
||||
}
|
||||
|
||||
get azure(): ScmIntegrationsGroup<AzureIntegration> {
|
||||
return this.byType.azure;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration';
|
||||
|
||||
describe('AzureBlobStorageIntegration', () => {
|
||||
it('has a working factory', () => {
|
||||
const integrations = AzureBlobStorageIntergation.factory({
|
||||
config: new ConfigReader({
|
||||
integrations: {
|
||||
azureBlobStorage: [
|
||||
{
|
||||
endpoint: 'https://myaccount.blob.core.windows.net',
|
||||
accountName: 'myaccount',
|
||||
accountKey: 'someAccountKey',
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(integrations.list().length).toBe(2); // including default
|
||||
expect(integrations.list()[0].config.host).toBe(
|
||||
'myaccount.blob.core.windows.net',
|
||||
);
|
||||
expect(integrations.list()[1].config.host).toBe('blob.core.windows.net'); // default integration
|
||||
});
|
||||
|
||||
it('returns the basics', () => {
|
||||
const integration = new AzureBlobStorageIntergation({
|
||||
host: 'myaccount.blob.core.windows.net',
|
||||
} as any);
|
||||
expect(integration.type).toBe('azureBlobStorage');
|
||||
expect(integration.title).toBe('myaccount.blob.core.windows.net');
|
||||
});
|
||||
|
||||
describe('resolveUrl', () => {
|
||||
it('works for valid URLs', () => {
|
||||
const integration = new AzureBlobStorageIntergation({
|
||||
host: 'blob.core.windows.net',
|
||||
} as any);
|
||||
|
||||
expect(
|
||||
integration.resolveUrl({
|
||||
url: 'https://myaccount.blob.core.windows.net/container/file.yaml',
|
||||
base: 'https://myaccount.blob.core.windows.net/container/file.yaml',
|
||||
}),
|
||||
).toBe('https://myaccount.blob.core.windows.net/container/file.yaml');
|
||||
});
|
||||
});
|
||||
|
||||
it('resolve edit URL', () => {
|
||||
const integration = new AzureBlobStorageIntergation({
|
||||
host: 'myaccount.blob.core.windows.net',
|
||||
} as any);
|
||||
|
||||
// TODO: The Azure Blob Storage integration doesn't support resolving an edit URL,
|
||||
// instead we keep the input URL.
|
||||
expect(
|
||||
integration.resolveEditUrl(
|
||||
'https://myaccount.blob.core.windows.net/container/file.yaml',
|
||||
),
|
||||
).toBe('https://myaccount.blob.core.windows.net/container/file.yaml');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
|
||||
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
|
||||
import {
|
||||
AzureBlobStorageIntegrationConfig,
|
||||
readAzureBlobStorageIntegrationConfigs,
|
||||
} from './config';
|
||||
|
||||
export class AzureBlobStorageIntergation implements ScmIntegration {
|
||||
static factory: ScmIntegrationsFactory<AzureBlobStorageIntergation> = ({
|
||||
config,
|
||||
}) => {
|
||||
const configs = readAzureBlobStorageIntegrationConfigs(
|
||||
config.getOptionalConfigArray('integrations.azureBlobStorage') ?? [],
|
||||
);
|
||||
return basicIntegrations(
|
||||
configs.map(c => new AzureBlobStorageIntergation(c)),
|
||||
i => i.config.host,
|
||||
);
|
||||
};
|
||||
|
||||
get type(): string {
|
||||
return 'azureBlobStorage';
|
||||
}
|
||||
|
||||
get title(): string {
|
||||
return this.integrationConfig.host;
|
||||
}
|
||||
|
||||
get config(): AzureBlobStorageIntegrationConfig {
|
||||
return this.integrationConfig;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly integrationConfig: AzureBlobStorageIntegrationConfig,
|
||||
) {}
|
||||
|
||||
resolveUrl(options: {
|
||||
url: string;
|
||||
base: string;
|
||||
lineNumber?: number | undefined;
|
||||
}): string {
|
||||
const resolved = defaultScmResolveUrl(options);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
resolveEditUrl(url: string): string {
|
||||
// TODO: Implement edit URL for azureBlobStorage
|
||||
return url;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
AccessToken,
|
||||
ClientSecretCredential,
|
||||
DefaultAzureCredential,
|
||||
TokenCredential,
|
||||
} from '@azure/identity';
|
||||
import { AzureBlobStorageIntegrationConfig } from './config';
|
||||
import { ScmIntegrationRegistry } from '../registry';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider';
|
||||
import { ScmIntegrations } from '../ScmIntegrations';
|
||||
import { DateTime } from 'luxon';
|
||||
|
||||
const MockedClientSecretCredential = ClientSecretCredential as jest.MockedClass<
|
||||
typeof ClientSecretCredential
|
||||
>;
|
||||
|
||||
jest.mock('@azure/identity');
|
||||
|
||||
describe('DefaultAzureCredentialsManager', () => {
|
||||
let mockIntegration: ScmIntegrationRegistry;
|
||||
|
||||
const buildProvider = (azureIntegrations: any[]) =>
|
||||
DefaultAzureCredentialsManager.fromIntegrations(
|
||||
ScmIntegrations.fromConfig(
|
||||
new ConfigReader({
|
||||
integrations: {
|
||||
azureBlobStorage: azureIntegrations,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
mockIntegration = {
|
||||
azureBlobStorage: {
|
||||
list: jest.fn().mockReturnValue([
|
||||
{
|
||||
config: {
|
||||
accountName: 'testaccount',
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
},
|
||||
},
|
||||
]),
|
||||
},
|
||||
} as unknown as ScmIntegrationRegistry;
|
||||
|
||||
MockedClientSecretCredential.prototype.getToken.mockImplementation(() =>
|
||||
Promise.resolve({
|
||||
expiresOnTimestamp: DateTime.local().plus({ days: 1 }).toSeconds(),
|
||||
token: 'fake-client-secret-token',
|
||||
} as AccessToken),
|
||||
);
|
||||
});
|
||||
|
||||
it('should create an instance from ScmIntegrationRegistry', () => {
|
||||
const manager =
|
||||
DefaultAzureCredentialsManager.fromIntegrations(mockIntegration);
|
||||
expect(manager).toBeInstanceOf(DefaultAzureCredentialsManager);
|
||||
});
|
||||
|
||||
it('should return cached credentials if available', async () => {
|
||||
const manager = buildProvider([
|
||||
{
|
||||
accountName: 'testaccount',
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const mockCredential = new MockedClientSecretCredential(
|
||||
'someTenantId',
|
||||
'someClientId',
|
||||
'someClientSecret',
|
||||
);
|
||||
|
||||
const credential = await manager.getCredentials('testaccount');
|
||||
|
||||
const scopes = ['https://storage.azure.com/.default'];
|
||||
|
||||
const expectedToken = await mockCredential.getToken(scopes);
|
||||
const receivedToken = await credential.getToken(scopes);
|
||||
|
||||
expect(receivedToken?.token).toEqual(expectedToken.token);
|
||||
});
|
||||
|
||||
it('should use Azure AD credentials if aadCredential is provided', async () => {
|
||||
const manager = buildProvider([
|
||||
{
|
||||
accountName: 'testaccount',
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const credential = await manager.getCredentials('testaccount');
|
||||
|
||||
expect(credential).toBeInstanceOf(ClientSecretCredential);
|
||||
});
|
||||
|
||||
it('should use DefaultAzureCredential if no aadCredential is provided', async () => {
|
||||
const manager = buildProvider([
|
||||
{
|
||||
accountName: 'testaccount',
|
||||
},
|
||||
]);
|
||||
|
||||
const credential = await manager.getCredentials('testaccount');
|
||||
|
||||
expect(credential).toBeInstanceOf(DefaultAzureCredential);
|
||||
});
|
||||
|
||||
it('should cache credentials after first retrieval', async () => {
|
||||
const manager = buildProvider([
|
||||
{
|
||||
accountName: 'testaccount',
|
||||
},
|
||||
]);
|
||||
|
||||
const credential = await manager.getCredentials('testaccount');
|
||||
|
||||
const cachedCredential = await manager.getCredentials('testaccount');
|
||||
expect(cachedCredential).toBe(credential);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
DefaultAzureCredential,
|
||||
ClientSecretCredential,
|
||||
TokenCredential,
|
||||
} from '@azure/identity';
|
||||
import { AzureBlobStorageIntegrationConfig } from './config';
|
||||
import { AzureCredentialsManager } from './types';
|
||||
import { ScmIntegrationRegistry } from '../registry';
|
||||
|
||||
/**
|
||||
* Default Azure Credentials Manager to dynamically select and manage Azure credentials.
|
||||
* It supports Service Principal, Managed Identity, SAS Token, Connection String, Account Key, and Anonymous access.
|
||||
*/
|
||||
export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
|
||||
private config: AzureBlobStorageIntegrationConfig;
|
||||
private cachedCredentials: Map<string, TokenCredential>;
|
||||
|
||||
constructor(config: AzureBlobStorageIntegrationConfig) {
|
||||
this.config = config;
|
||||
this.cachedCredentials = new Map<string, TokenCredential>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance of DefaultAzureCredentialsManager from a Backstage Config.
|
||||
*/
|
||||
static fromIntegrations(
|
||||
integration: ScmIntegrationRegistry,
|
||||
): DefaultAzureCredentialsManager {
|
||||
const azureConfig = integration.azureBlobStorage.list().length
|
||||
? integration.azureBlobStorage.list()[0].config
|
||||
: { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found
|
||||
|
||||
return new DefaultAzureCredentialsManager(azureConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the appropriate credential method and returns credentials for BlobServiceClient.
|
||||
* Supports:
|
||||
* - Service Principal
|
||||
* - Managed Identity
|
||||
* - SAS Token
|
||||
* - Connection String
|
||||
* - Account Key
|
||||
* - Anonymous access
|
||||
*/
|
||||
async getCredentials(accountName: string): Promise<TokenCredential> {
|
||||
// Check if the credentials are already cached
|
||||
if (this.cachedCredentials.has(accountName)) {
|
||||
return this.cachedCredentials.get(accountName)!;
|
||||
}
|
||||
|
||||
let credential: TokenCredential;
|
||||
|
||||
// Check for SAS Token
|
||||
// if (this.config.sasToken) {
|
||||
// // console.log('Using SAS Token for Azure Blob Storage authentication');
|
||||
// // SAS Token does not return a credential but can be used directly in BlobServiceClient
|
||||
// // Here we can simply return undefined or keep a placeholder if needed
|
||||
// return this.config.sasToken; // Or return a string for the URL using the SAS token
|
||||
// }
|
||||
// // Check for Connection String
|
||||
// else if (this.config.connectionString) {
|
||||
// // console.log(
|
||||
// // 'Using Connection String for Azure Blob Storage authentication',
|
||||
// // );
|
||||
// // return undefined; // Connection string will also not return a specific credential object
|
||||
// }
|
||||
// Check for Account Key
|
||||
// if (this.config.accountKey) {
|
||||
// // console.log('Using Account Key for Azure Blob Storage authentication');
|
||||
// credential = new StorageSharedKeyCredential(
|
||||
// accountName,
|
||||
// this.config.accountKey,
|
||||
// );
|
||||
// }
|
||||
// Check for AAD credentials
|
||||
|
||||
if (
|
||||
this.config.aadCredential &&
|
||||
this.config.aadCredential.clientId &&
|
||||
this.config.aadCredential.clientSecret &&
|
||||
this.config.aadCredential.tenantId
|
||||
) {
|
||||
credential = new ClientSecretCredential(
|
||||
this.config.aadCredential.tenantId,
|
||||
this.config.aadCredential.clientId,
|
||||
this.config.aadCredential.clientSecret,
|
||||
);
|
||||
}
|
||||
// Check for Anonymous access
|
||||
// else if (this.config.anonymousAccess) {
|
||||
// console.log('Using Anonymous Credential for Azure Blob Storage access');
|
||||
// credential = new AnonymousCredential();
|
||||
// }
|
||||
// Fallback to Managed Identity
|
||||
else {
|
||||
credential = new DefaultAzureCredential();
|
||||
}
|
||||
|
||||
// Cache the credentials for future use
|
||||
this.cachedCredentials.set(accountName, credential);
|
||||
|
||||
return credential;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Config, ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
AzureBlobStorageIntegrationConfig,
|
||||
readAzureBlobStorageIntegrationConfig,
|
||||
readAzureBlobStorageIntegrationConfigs,
|
||||
} from './config';
|
||||
|
||||
describe('readAzureBlobStorageIntegrationConfig', () => {
|
||||
function buildConfig(
|
||||
data: Partial<AzureBlobStorageIntegrationConfig>,
|
||||
): Config {
|
||||
return new ConfigReader(data);
|
||||
}
|
||||
|
||||
it('reads valid configuration with accountKey', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfig(
|
||||
buildConfig({
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: 'someAccountKey',
|
||||
}),
|
||||
);
|
||||
expect(output).toEqual({
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: undefined,
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: 'someAccountKey',
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads valid configuration with sasToken', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfig(
|
||||
buildConfig({
|
||||
endpoint: 'https://blob.core.windows.net',
|
||||
accountName: 'mystorageaccount',
|
||||
sasToken: 'someSASToken',
|
||||
}),
|
||||
);
|
||||
expect(output).toEqual({
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: 'https://blob.core.windows.net',
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: undefined,
|
||||
sasToken: 'someSASToken',
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads valid configuration with Azure AD credentials', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfig(
|
||||
buildConfig({
|
||||
accountName: 'mystorageaccount',
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(output).toEqual({
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: undefined,
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('reads valid configuration with a custom endpoint', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfig(
|
||||
buildConfig({
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'customaccount',
|
||||
}),
|
||||
);
|
||||
expect(output).toEqual({
|
||||
host: 'custom.blob.core.windows.net',
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'customaccount',
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('throws error for invalid endpoint URL', () => {
|
||||
const config = buildConfig({
|
||||
endpoint: 'invalid-url',
|
||||
accountName: 'invalidaccount',
|
||||
});
|
||||
|
||||
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
|
||||
`invalid azureBlobStorage integration config, endpoint 'invalid-url' is not a valid URL`,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws error if endpoint has a path', () => {
|
||||
const config = buildConfig({
|
||||
endpoint: 'https://blob.core.windows.net/path',
|
||||
accountName: 'accountwithpath',
|
||||
});
|
||||
|
||||
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
|
||||
`invalid azureBlobStorage integration config, endpoints cannot contain path, got 'https://blob.core.windows.net/path'`,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws error if both accountKey and sasToken are provided', () => {
|
||||
const config = buildConfig({
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: 'someAccountKey',
|
||||
sasToken: 'someSASToken',
|
||||
});
|
||||
|
||||
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
|
||||
`Invalid Azure Blob Storage config for mystorageaccount: Both account key and SAS token cannot be used simultaneously.`,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws error if both aadCredential and accountKey/sasToken are provided', () => {
|
||||
const config = buildConfig({
|
||||
accountName: 'mystorageaccount',
|
||||
accountKey: 'someAccountKey',
|
||||
aadCredential: {
|
||||
clientId: 'someClientId',
|
||||
tenantId: 'someTenantId',
|
||||
clientSecret: 'someClientSecret',
|
||||
},
|
||||
});
|
||||
|
||||
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
|
||||
`Invalid Azure Blob Storage config for mystorageaccount: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readAzureBlobStorageIntegrationConfigs', () => {
|
||||
function buildConfigs(
|
||||
data: Partial<AzureBlobStorageIntegrationConfig>[],
|
||||
): Config[] {
|
||||
return data.map(item => new ConfigReader(item));
|
||||
}
|
||||
|
||||
it('reads all provided configurations', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfigs(
|
||||
buildConfigs([
|
||||
{
|
||||
host: 'blob.core.windows.net',
|
||||
accountName: 'account1',
|
||||
accountKey: 'someAccountKey',
|
||||
},
|
||||
{
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'account2',
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(output).toEqual([
|
||||
{
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: undefined,
|
||||
accountName: 'account1',
|
||||
accountKey: 'someAccountKey',
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
},
|
||||
{
|
||||
host: 'custom.blob.core.windows.net',
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'account2',
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('adds default integration for blob.core.windows.net when missing', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfigs(
|
||||
buildConfigs([
|
||||
{
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'account2',
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
expect(output).toEqual([
|
||||
{
|
||||
host: 'custom.blob.core.windows.net',
|
||||
endpoint: 'https://custom.blob.core.windows.net',
|
||||
accountName: 'account2',
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
},
|
||||
{
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: undefined,
|
||||
accountName: undefined,
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not add default integration if blob.core.windows.net already exists', () => {
|
||||
const output = readAzureBlobStorageIntegrationConfigs(
|
||||
buildConfigs([
|
||||
{ host: 'blob.core.windows.net', accountName: 'account1' },
|
||||
]),
|
||||
);
|
||||
expect(output).toEqual([
|
||||
{
|
||||
host: 'blob.core.windows.net',
|
||||
endpoint: undefined,
|
||||
accountName: 'account1',
|
||||
accountKey: undefined,
|
||||
sasToken: undefined,
|
||||
connectionString: undefined,
|
||||
endpointSuffix: undefined,
|
||||
aadCredential: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
const AZURE_HOST = 'blob.core.windows.net';
|
||||
|
||||
/**
|
||||
* The configuration parameters for a single Azure Blob Storage account.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type AzureBlobStorageIntegrationConfig = {
|
||||
/**
|
||||
* The name of the Azure Storage Account, e.g., "mystorageaccount".
|
||||
*/
|
||||
accountName?: string;
|
||||
|
||||
/**
|
||||
* The primary or secondary key for the Azure Storage Account.
|
||||
* Only required if connectionString or SAS token are not specified.
|
||||
*/
|
||||
accountKey?: string;
|
||||
|
||||
/**
|
||||
* A Shared Access Signature (SAS) token for limited access to resources.
|
||||
*/
|
||||
sasToken?: string;
|
||||
|
||||
/**
|
||||
* A full connection string for the Azure Storage Account.
|
||||
* This includes the account name, key, and endpoint details.
|
||||
*/
|
||||
connectionString?: string;
|
||||
|
||||
/**
|
||||
* Optional endpoint suffix for custom domains or sovereign clouds.
|
||||
* e.g., "core.windows.net" for public Azure or "core.usgovcloudapi.net" for US Government cloud.
|
||||
*/
|
||||
endpointSuffix?: string;
|
||||
|
||||
/**
|
||||
* The host of the target that this matches on, e.g., "blob.core.windows.net".
|
||||
*/
|
||||
host: string;
|
||||
|
||||
endpoint?: string;
|
||||
/**
|
||||
* Optional credential to use for Azure Active Directory authentication.
|
||||
*/
|
||||
aadCredential?: {
|
||||
/**
|
||||
* The client ID of the Azure AD application.
|
||||
*/
|
||||
clientId: string;
|
||||
|
||||
/**
|
||||
* The tenant ID for Azure AD.
|
||||
*/
|
||||
tenantId: string;
|
||||
|
||||
/**
|
||||
* The client secret for the Azure AD application.
|
||||
*/
|
||||
clientSecret: string;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads a single Azure Blob Storage integration config.
|
||||
*
|
||||
* @param config - The config object of a single integration.
|
||||
* @public
|
||||
*/
|
||||
export function readAzureBlobStorageIntegrationConfig(
|
||||
config: Config,
|
||||
): AzureBlobStorageIntegrationConfig {
|
||||
const endpoint = config.getOptionalString('endpoint');
|
||||
const accountName = config.getString('accountName');
|
||||
const accountKey = config.getOptionalString('accountKey')?.trim();
|
||||
const sasToken = config.getOptionalString('sasToken')?.trim();
|
||||
const connectionString = config.getOptionalString('connectionString')?.trim();
|
||||
const endpointSuffix = config.getOptionalString('endpointSuffix')?.trim();
|
||||
|
||||
let host;
|
||||
let pathname;
|
||||
if (endpoint) {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
host = url.host;
|
||||
pathname = url.pathname;
|
||||
} catch {
|
||||
throw new Error(
|
||||
`invalid azureBlobStorage integration config, endpoint '${endpoint}' is not a valid URL`,
|
||||
);
|
||||
}
|
||||
if (pathname !== '/') {
|
||||
throw new Error(
|
||||
`invalid azureBlobStorage integration config, endpoints cannot contain path, got '${endpoint}'`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
host = AZURE_HOST;
|
||||
}
|
||||
const aadCredential = config.has('aadCredential')
|
||||
? {
|
||||
clientId: config.getString('aadCredential.clientId'),
|
||||
tenantId: config.getString('aadCredential.tenantId'),
|
||||
clientSecret: config.getString('aadCredential.clientSecret')?.trim(),
|
||||
}
|
||||
: undefined;
|
||||
|
||||
if (accountKey && sasToken) {
|
||||
throw new Error(
|
||||
`Invalid Azure Blob Storage config for ${accountName}: Both account key and SAS token cannot be used simultaneously.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (aadCredential && (accountKey || sasToken)) {
|
||||
throw new Error(
|
||||
`Invalid Azure Blob Storage config for ${accountName}: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
host,
|
||||
endpoint,
|
||||
accountName,
|
||||
accountKey,
|
||||
sasToken,
|
||||
connectionString,
|
||||
endpointSuffix,
|
||||
aadCredential,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a set of Azure Blob Storage integration configs.
|
||||
*
|
||||
* @param configs - All of the integration config objects.
|
||||
* @public
|
||||
*/
|
||||
export function readAzureBlobStorageIntegrationConfigs(
|
||||
configs: Config[],
|
||||
): AzureBlobStorageIntegrationConfig[] {
|
||||
// First read all the explicit integrations
|
||||
const result = configs.map(readAzureBlobStorageIntegrationConfig);
|
||||
|
||||
// If no explicit blob.core.windows.net integration was added, put one in the list as
|
||||
// a convenience
|
||||
if (!result.some(c => c.host === AZURE_HOST)) {
|
||||
result.push({
|
||||
host: AZURE_HOST,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration';
|
||||
export {
|
||||
readAzureBlobStorageIntegrationConfig,
|
||||
readAzureBlobStorageIntegrationConfigs,
|
||||
} from './config';
|
||||
export type { AzureBlobStorageIntegrationConfig } from './config';
|
||||
export { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider';
|
||||
export type { AzureCredentialsManager } from './types';
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TokenCredential } from '@azure/identity';
|
||||
import {
|
||||
StorageSharedKeyCredential,
|
||||
AnonymousCredential,
|
||||
} from '@azure/storage-blob';
|
||||
|
||||
export interface AzureCredentialsManager {
|
||||
getCredentials(
|
||||
accountName: string,
|
||||
): Promise<
|
||||
TokenCredential | StorageSharedKeyCredential | AnonymousCredential
|
||||
>;
|
||||
}
|
||||
@@ -22,6 +22,7 @@
|
||||
|
||||
export * from './awsS3';
|
||||
export * from './awsCodeCommit';
|
||||
export * from './azureBlobStorage';
|
||||
export * from './azure';
|
||||
export * from './bitbucket';
|
||||
export * from './bitbucketCloud';
|
||||
|
||||
@@ -26,6 +26,7 @@ import { GithubIntegration } from './github/GithubIntegration';
|
||||
import { GitLabIntegration } from './gitlab/GitLabIntegration';
|
||||
import { GiteaIntegration } from './gitea/GiteaIntegration';
|
||||
import { HarnessIntegration } from './harness/HarnessIntegration';
|
||||
import { AzureBlobStorageIntergation } from './azureBlobStorage';
|
||||
|
||||
/**
|
||||
* Holds all registered SCM integrations, of all types.
|
||||
@@ -36,6 +37,7 @@ export interface ScmIntegrationRegistry
|
||||
extends ScmIntegrationsGroup<ScmIntegration> {
|
||||
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
|
||||
awsCodeCommit: ScmIntegrationsGroup<AwsCodeCommitIntegration>;
|
||||
azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
|
||||
azure: ScmIntegrationsGroup<AzureIntegration>;
|
||||
/**
|
||||
* @deprecated in favor of `bitbucketCloud` and `bitbucketServer`
|
||||
|
||||
@@ -51,6 +51,8 @@
|
||||
"test": "backstage-cli package test"
|
||||
},
|
||||
"dependencies": {
|
||||
"@azure/identity": "^4.0.0",
|
||||
"@azure/storage-blob": "^12.5.0",
|
||||
"@backstage/backend-plugin-api": "workspace:^",
|
||||
"@backstage/config": "workspace:^",
|
||||
"@backstage/integration": "workspace:^",
|
||||
|
||||
@@ -23,3 +23,4 @@
|
||||
export { default } from './module';
|
||||
export { AzureDevOpsDiscoveryProcessor } from './processors';
|
||||
export { AzureDevOpsEntityProvider } from './providers';
|
||||
export { AzureBlobStorageEntityProvider } from './providers';
|
||||
|
||||
+56
-3
@@ -17,8 +17,11 @@
|
||||
import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api';
|
||||
import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
|
||||
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
|
||||
import { catalogModuleAzureDevOpsEntityProvider } from './catalogModuleAzureDevOpsEntityProvider';
|
||||
import { AzureDevOpsEntityProvider } from '../providers';
|
||||
import { catalogModuleAzureEntityProvider } from './catalogModuleAzureDevOpsEntityProvider';
|
||||
import {
|
||||
AzureBlobStorageEntityProvider,
|
||||
AzureDevOpsEntityProvider,
|
||||
} from '../providers';
|
||||
|
||||
describe('catalogModuleAzureDevOpsEntityProvider', () => {
|
||||
it('should register provider at the catalog extension point', async () => {
|
||||
@@ -58,7 +61,7 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => {
|
||||
await startTestBackend({
|
||||
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
|
||||
features: [
|
||||
catalogModuleAzureDevOpsEntityProvider,
|
||||
catalogModuleAzureEntityProvider,
|
||||
mockServices.rootConfig.factory({ data: config }),
|
||||
mockServices.logger.factory(),
|
||||
scheduler.factory,
|
||||
@@ -73,4 +76,54 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => {
|
||||
);
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should register azure blob storage provider at the catalog extension point', async () => {
|
||||
let addedProviders: Array<AzureBlobStorageEntityProvider> | undefined;
|
||||
let usedSchedule: SchedulerServiceTaskScheduleDefinition | undefined;
|
||||
|
||||
const extensionPoint = {
|
||||
addEntityProvider: (providers: any) => {
|
||||
addedProviders = providers;
|
||||
},
|
||||
};
|
||||
const runner = jest.fn();
|
||||
const scheduler = mockServices.scheduler.mock({
|
||||
createScheduledTaskRunner(schedule) {
|
||||
usedSchedule = schedule;
|
||||
return { run: runner };
|
||||
},
|
||||
});
|
||||
|
||||
const config = {
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: {
|
||||
containerName: 'test',
|
||||
schedule: {
|
||||
frequency: 'P1M',
|
||||
timeout: 'PT3M',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await startTestBackend({
|
||||
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
|
||||
features: [
|
||||
catalogModuleAzureEntityProvider,
|
||||
mockServices.rootConfig.factory({ data: config }),
|
||||
mockServices.logger.factory(),
|
||||
scheduler.factory,
|
||||
],
|
||||
});
|
||||
|
||||
expect(usedSchedule?.frequency).toEqual({ months: 1 });
|
||||
expect(usedSchedule?.timeout).toEqual({ minutes: 3 });
|
||||
expect(addedProviders?.length).toEqual(1);
|
||||
expect(addedProviders?.pop()?.getProviderName()).toEqual(
|
||||
'azureBlobStorage-provider:default',
|
||||
);
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
+25
-9
@@ -19,16 +19,19 @@ import {
|
||||
createBackendModule,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
|
||||
import { AzureDevOpsEntityProvider } from '../providers';
|
||||
import {
|
||||
AzureBlobStorageEntityProvider,
|
||||
AzureDevOpsEntityProvider,
|
||||
} from '../providers';
|
||||
|
||||
/**
|
||||
* Registers the AzureDevOpsEntityProvider with the catalog processing extension point.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({
|
||||
export const catalogModuleAzureEntityProvider = createBackendModule({
|
||||
pluginId: 'catalog',
|
||||
moduleId: 'azure-dev-ops-entity-provider',
|
||||
moduleId: 'azure-providers',
|
||||
register(env) {
|
||||
env.registerInit({
|
||||
deps: {
|
||||
@@ -38,12 +41,25 @@ export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({
|
||||
scheduler: coreServices.scheduler,
|
||||
},
|
||||
async init({ config, catalog, logger, scheduler }) {
|
||||
catalog.addEntityProvider(
|
||||
AzureDevOpsEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
}),
|
||||
);
|
||||
// Check for Azure Blob Storage provider configuration and register it
|
||||
if (config.has('catalog.providers.azureBlob')) {
|
||||
catalog.addEntityProvider(
|
||||
AzureBlobStorageEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Check for Azure DevOps provider configuration and register it
|
||||
if (config.has('catalog.providers.azureDevOps')) {
|
||||
catalog.addEntityProvider(
|
||||
AzureDevOpsEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
@@ -14,4 +14,4 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { catalogModuleAzureDevOpsEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider';
|
||||
export { catalogModuleAzureEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider';
|
||||
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
SchedulerService,
|
||||
SchedulerServiceTaskRunner,
|
||||
SchedulerServiceTaskInvocationDefinition,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
|
||||
import { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
|
||||
class PersistingTaskRunner implements SchedulerServiceTaskRunner {
|
||||
private tasks: SchedulerServiceTaskInvocationDefinition[] = [];
|
||||
|
||||
getTasks() {
|
||||
return this.tasks;
|
||||
}
|
||||
|
||||
run(task: SchedulerServiceTaskInvocationDefinition): Promise<void> {
|
||||
this.tasks.push(task);
|
||||
return Promise.resolve(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const blobs = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key4.yaml'];
|
||||
const createBlobList = (blobsArray: string[]) => {
|
||||
return blobsArray.map(blob => ({
|
||||
name: blob,
|
||||
}));
|
||||
};
|
||||
// Mocking Azure Storage Blob Library
|
||||
jest.mock('@azure/storage-blob', () => {
|
||||
return {
|
||||
BlobServiceClient: jest.fn().mockImplementation(() => ({
|
||||
url: 'https://myaccount.blob.core.windows.net/',
|
||||
getContainerClient: jest.fn().mockImplementation(() => ({
|
||||
listBlobsFlat: jest.fn(async function* () {
|
||||
yield* createBlobList(blobs);
|
||||
}),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
const logger = mockServices.logger.mock();
|
||||
|
||||
describe('AzureBlobStorageEntityProvider', () => {
|
||||
const containerName = 'container-1';
|
||||
|
||||
const expectMutation = async (
|
||||
providerId: string,
|
||||
providerConfig: object,
|
||||
expectedBaseUrl: string,
|
||||
names: Record<string, string>,
|
||||
integrationConfig?: object,
|
||||
scheduleInConfig?: boolean,
|
||||
) => {
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
azureBlobStorage: integrationConfig ? [integrationConfig] : [],
|
||||
},
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: {
|
||||
[providerId]: providerConfig,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const schedulingConfig: Record<string, any> = {};
|
||||
const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/')
|
||||
? expectedBaseUrl
|
||||
: `${expectedBaseUrl}/`;
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
|
||||
if (scheduleInConfig) {
|
||||
schedulingConfig.scheduler = {
|
||||
createScheduledTaskRunner: (_: any) => schedule,
|
||||
} as unknown as SchedulerService;
|
||||
} else {
|
||||
schedulingConfig.schedule = schedule;
|
||||
}
|
||||
|
||||
const provider = AzureBlobStorageEntityProvider.fromConfig(config, {
|
||||
...schedulingConfig,
|
||||
logger,
|
||||
})[0];
|
||||
|
||||
expect(provider.getProviderName()).toEqual(
|
||||
`azureBlobStorage-provider:${providerId}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await provider.connect(entityProviderConnection);
|
||||
} catch (error) {
|
||||
console.error('Error during provider connection:', error);
|
||||
}
|
||||
|
||||
const taskDef = schedule.getTasks()[0];
|
||||
expect(taskDef.id).toEqual(
|
||||
`azureBlobStorage-provider:${providerId}:refresh`,
|
||||
);
|
||||
|
||||
await (taskDef.fn as () => Promise<void>)();
|
||||
|
||||
const expectedEntities = blobs.map(blob => {
|
||||
const url = encodeURI(`${normalizedExpectedBaseUrl}${blob}`);
|
||||
return {
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Location',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location': `url:${url}`,
|
||||
'backstage.io/managed-by-origin-location': `url:${url}`,
|
||||
},
|
||||
name: expect.stringMatching(/generated-[a-f0-9]{40}/),
|
||||
},
|
||||
spec: {
|
||||
presence: 'required',
|
||||
target: `${url}`,
|
||||
type: 'url',
|
||||
},
|
||||
},
|
||||
locationKey: `azureBlobStorage-provider:${providerId}`,
|
||||
};
|
||||
});
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'full',
|
||||
entities: expectedEntities,
|
||||
});
|
||||
};
|
||||
|
||||
it('apply full update on scheduled execution', async () => {
|
||||
return expectMutation(
|
||||
'staticContainer',
|
||||
{
|
||||
containerName,
|
||||
},
|
||||
'https://myaccount.blob.core.windows.net/container-1/',
|
||||
{
|
||||
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
|
||||
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
|
||||
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
|
||||
'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
|
||||
},
|
||||
{
|
||||
accountName: 'myaccount',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('apply full update no prefix', async () => {
|
||||
return expectMutation(
|
||||
'staticContainerNoPrefix',
|
||||
{
|
||||
containerName,
|
||||
schedule: {
|
||||
frequency: { minutes: 30 },
|
||||
timeout: { minutes: 3 },
|
||||
},
|
||||
},
|
||||
'https://myaccount.blob.core.windows.net/container-1/',
|
||||
{
|
||||
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
|
||||
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
|
||||
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
|
||||
'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
|
||||
},
|
||||
{
|
||||
host: 'blob.core.windows.net',
|
||||
accountName: 'myaccount',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('fail without schedule and scheduler', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: {
|
||||
test: {
|
||||
containerName: 'container-1',
|
||||
prefix: 'sub/dir/',
|
||||
accountName: 'myaccount',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
AzureBlobStorageEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
}),
|
||||
).toThrow('Either schedule or scheduler must be provided');
|
||||
});
|
||||
|
||||
it('fail with scheduler but no schedule config', () => {
|
||||
const scheduler = {
|
||||
createScheduledTaskRunner: (_: any) => jest.fn(),
|
||||
} as unknown as SchedulerService;
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: {
|
||||
test: {
|
||||
containerName: 'container-1',
|
||||
prefix: 'sub/dir/',
|
||||
accountName: 'myaccount',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
AzureBlobStorageEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
scheduler,
|
||||
}),
|
||||
).toThrow(
|
||||
'No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:test.',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,228 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
AnonymousCredential,
|
||||
BlobServiceClient,
|
||||
ContainerClient,
|
||||
StorageSharedKeyCredential,
|
||||
} from '@azure/storage-blob';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
LoggerService,
|
||||
SchedulerService,
|
||||
SchedulerServiceTaskRunner,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import {
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
locationSpecToLocationEntity,
|
||||
} from '@backstage/plugin-catalog-node';
|
||||
import { LocationSpec } from '@backstage/plugin-catalog-common';
|
||||
import * as uuid from 'uuid';
|
||||
import { readAzureBlobStorageConfigs } from './config';
|
||||
import {
|
||||
AzureBlobStorageIntergation,
|
||||
DefaultAzureCredentialsManager,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import { TokenCredential } from '@azure/identity';
|
||||
import { AzureBlobStorageConfig } from './types';
|
||||
|
||||
export class AzureBlobStorageEntityProvider implements EntityProvider {
|
||||
private readonly logger: LoggerService;
|
||||
private connection?: EntityProviderConnection;
|
||||
private blobServiceClient?: BlobServiceClient;
|
||||
private readonly scheduleFn: () => Promise<void>;
|
||||
|
||||
static fromConfig(
|
||||
configRoot: Config,
|
||||
options: {
|
||||
logger: LoggerService;
|
||||
schedule?: SchedulerServiceTaskRunner;
|
||||
scheduler?: SchedulerService;
|
||||
},
|
||||
): AzureBlobStorageEntityProvider[] {
|
||||
const providerConfigs = readAzureBlobStorageConfigs(configRoot);
|
||||
|
||||
const scmIntegration = ScmIntegrations.fromConfig(configRoot);
|
||||
const credentialsProvider =
|
||||
DefaultAzureCredentialsManager.fromIntegrations(scmIntegration);
|
||||
if (!options.schedule && !options.scheduler) {
|
||||
throw new Error('Either schedule or scheduler must be provided.');
|
||||
}
|
||||
|
||||
return providerConfigs.map(providerConfig => {
|
||||
const integration = scmIntegration.azureBlobStorage.list()[0];
|
||||
if (!integration) {
|
||||
throw new Error(
|
||||
`There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!options.schedule && !providerConfig.schedule) {
|
||||
throw new Error(
|
||||
`No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:${providerConfig.id}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const taskRunner =
|
||||
options.schedule ??
|
||||
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
|
||||
|
||||
return new AzureBlobStorageEntityProvider(
|
||||
providerConfig,
|
||||
integration,
|
||||
credentialsProvider,
|
||||
options.logger,
|
||||
taskRunner,
|
||||
);
|
||||
});
|
||||
}
|
||||
constructor(
|
||||
private readonly config: AzureBlobStorageConfig,
|
||||
private readonly integration: AzureBlobStorageIntergation,
|
||||
private readonly credentialsProvider: DefaultAzureCredentialsManager,
|
||||
logger: LoggerService,
|
||||
schedule: SchedulerServiceTaskRunner,
|
||||
) {
|
||||
this.logger = logger.child({ target: this.getProviderName() });
|
||||
this.scheduleFn = this.createScheduleFn(schedule);
|
||||
}
|
||||
|
||||
private createScheduleFn(
|
||||
taskRunner: SchedulerServiceTaskRunner,
|
||||
): () => Promise<void> {
|
||||
return async () => {
|
||||
const taskId = `${this.getProviderName()}:refresh`;
|
||||
return taskRunner.run({
|
||||
id: taskId,
|
||||
fn: async () => {
|
||||
const logger = this.logger.child({
|
||||
class: AzureBlobStorageEntityProvider.prototype.constructor.name,
|
||||
taskId,
|
||||
taskInstanceId: uuid.v4(),
|
||||
});
|
||||
|
||||
try {
|
||||
await this.refresh(logger);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`${this.getProviderName()} refresh failed, ${error}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
getProviderName(): string {
|
||||
return `azureBlobStorage-provider:${this.config.id}`;
|
||||
}
|
||||
|
||||
async connect(connection: EntityProviderConnection): Promise<void> {
|
||||
this.connection = connection;
|
||||
let credential:
|
||||
| TokenCredential
|
||||
| StorageSharedKeyCredential
|
||||
| AnonymousCredential;
|
||||
if (this.integration.config.accountKey) {
|
||||
credential = new StorageSharedKeyCredential(
|
||||
this.integration.config.accountName as string,
|
||||
this.integration.config.accountKey as string,
|
||||
); // StorageSharedKeyCredential is only allowed in node.js runtime not in browser
|
||||
} else {
|
||||
credential = await this.credentialsProvider.getCredentials(
|
||||
this.integration.config.accountName as string,
|
||||
);
|
||||
}
|
||||
let blobServiceClientUrl: string;
|
||||
|
||||
if (this.integration.config.endpoint) {
|
||||
if (this.integration.config.sasToken) {
|
||||
blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`;
|
||||
} else {
|
||||
blobServiceClientUrl = `${this.integration.config.endpoint}`;
|
||||
}
|
||||
} else {
|
||||
blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`;
|
||||
}
|
||||
|
||||
this.blobServiceClient = new BlobServiceClient(
|
||||
blobServiceClientUrl,
|
||||
credential,
|
||||
);
|
||||
await this.scheduleFn();
|
||||
}
|
||||
|
||||
async refresh(logger: LoggerService) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
logger.info('Discovering Azure Blob Storage blobs');
|
||||
|
||||
const keys = await this.listAllBlobKeys();
|
||||
logger.info(`Discovered ${keys.length} Azure Blob Storage blobs`);
|
||||
|
||||
const locations = keys.map(key => this.createLocationSpec(key));
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
entities: locations.map(location => {
|
||||
return {
|
||||
locationKey: this.getProviderName(),
|
||||
entity: locationSpecToLocationEntity({ location }),
|
||||
};
|
||||
}),
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`Committed ${locations.length} Locations for Azure Blob Storage blobs`,
|
||||
);
|
||||
}
|
||||
|
||||
private async listAllBlobKeys(): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
const containerClient = this.blobServiceClient?.getContainerClient(
|
||||
this.config.containerName,
|
||||
);
|
||||
|
||||
for await (const blob of (
|
||||
containerClient as ContainerClient
|
||||
).listBlobsFlat()) {
|
||||
if (blob.name) {
|
||||
keys.push(blob.name);
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
private createLocationSpec(key: string): LocationSpec {
|
||||
return {
|
||||
type: 'url',
|
||||
target: this.createObjectUrl(key),
|
||||
presence: 'required',
|
||||
};
|
||||
}
|
||||
|
||||
private createObjectUrl(key: string): string {
|
||||
const endpoint = this.blobServiceClient?.url;
|
||||
return `${endpoint}${this.config.containerName}/${key}`;
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { readAzureDevOpsConfigs } from './config';
|
||||
import { readAzureBlobStorageConfigs, readAzureDevOpsConfigs } from './config';
|
||||
|
||||
describe('readAzureDevOpsConfigs', () => {
|
||||
it('reads all provider configs and set default values', () => {
|
||||
@@ -106,3 +106,69 @@ describe('readAzureDevOpsConfigs', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('readAzureBlobStorageConfigs', () => {
|
||||
it('reads single and multiple Azure Blob Storage provider configs', () => {
|
||||
const provider1 = {
|
||||
containerName: 'container-1',
|
||||
schedule: {
|
||||
frequency: 'PT30M',
|
||||
timeout: {
|
||||
minutes: 3,
|
||||
},
|
||||
},
|
||||
};
|
||||
const provider2 = {
|
||||
containerName: 'container-2',
|
||||
};
|
||||
|
||||
const configSingle = {
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: provider2,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const configMulti = {
|
||||
catalog: {
|
||||
providers: {
|
||||
azureBlob: {
|
||||
provider1,
|
||||
provider2,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Single provider case
|
||||
const actualSingle = readAzureBlobStorageConfigs(
|
||||
new ConfigReader(configSingle),
|
||||
);
|
||||
expect(actualSingle).toHaveLength(1);
|
||||
expect(actualSingle[0]).toEqual({
|
||||
id: 'default',
|
||||
containerName: 'container-2',
|
||||
schedule: undefined, // no schedule provided in this case
|
||||
});
|
||||
|
||||
// Multiple providers case
|
||||
const actualMulti = readAzureBlobStorageConfigs(
|
||||
new ConfigReader(configMulti),
|
||||
);
|
||||
expect(actualMulti).toHaveLength(2);
|
||||
expect(actualMulti[0]).toEqual({
|
||||
id: 'provider1',
|
||||
containerName: 'container-1',
|
||||
schedule: {
|
||||
...provider1.schedule,
|
||||
frequency: { minutes: 30 },
|
||||
},
|
||||
});
|
||||
expect(actualMulti[1]).toEqual({
|
||||
id: 'provider2',
|
||||
containerName: 'container-2',
|
||||
schedule: undefined, // no schedule provided
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,7 +16,9 @@
|
||||
|
||||
import { readSchedulerServiceTaskScheduleDefinitionFromConfig } from '@backstage/backend-plugin-api';
|
||||
import { Config } from '@backstage/config';
|
||||
import { AzureDevOpsConfig } from './types';
|
||||
import { AzureDevOpsConfig, AzureBlobStorageConfig } from './types';
|
||||
|
||||
const DEFAULT_PROVIDER_ID = 'default';
|
||||
|
||||
export function readAzureDevOpsConfigs(config: Config): AzureDevOpsConfig[] {
|
||||
const configs: AzureDevOpsConfig[] = [];
|
||||
@@ -61,3 +63,50 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig {
|
||||
schedule,
|
||||
};
|
||||
}
|
||||
|
||||
export function readAzureBlobStorageConfigs(
|
||||
config: Config,
|
||||
): AzureBlobStorageConfig[] {
|
||||
const configs: AzureBlobStorageConfig[] = [];
|
||||
|
||||
const providerConfigs = config.getOptionalConfig(
|
||||
'catalog.providers.azureBlob',
|
||||
);
|
||||
|
||||
if (!providerConfigs) {
|
||||
return configs;
|
||||
}
|
||||
|
||||
if (providerConfigs.has('containerName')) {
|
||||
// simple/single config variant
|
||||
configs.push(
|
||||
readAzureBlobStorageConfig(DEFAULT_PROVIDER_ID, providerConfigs),
|
||||
);
|
||||
|
||||
return configs;
|
||||
}
|
||||
|
||||
for (const id of providerConfigs.keys()) {
|
||||
configs.push(readAzureBlobStorageConfig(id, providerConfigs.getConfig(id)));
|
||||
}
|
||||
|
||||
return configs;
|
||||
}
|
||||
|
||||
function readAzureBlobStorageConfig(
|
||||
id: string,
|
||||
config: Config,
|
||||
): AzureBlobStorageConfig {
|
||||
const containerName = config.getString('containerName');
|
||||
const schedule = config.has('schedule')
|
||||
? readSchedulerServiceTaskScheduleDefinitionFromConfig(
|
||||
config.getConfig('schedule'),
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
id,
|
||||
containerName,
|
||||
schedule,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -15,3 +15,4 @@
|
||||
*/
|
||||
|
||||
export { AzureDevOpsEntityProvider } from './AzureDevOpsEntityProvider';
|
||||
export { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider';
|
||||
|
||||
@@ -26,3 +26,9 @@ export type AzureDevOpsConfig = {
|
||||
path: string;
|
||||
schedule?: SchedulerServiceTaskScheduleDefinition;
|
||||
};
|
||||
|
||||
export type AzureBlobStorageConfig = {
|
||||
id: string;
|
||||
containerName: string;
|
||||
schedule?: SchedulerServiceTaskScheduleDefinition;
|
||||
};
|
||||
|
||||
@@ -3605,6 +3605,8 @@ __metadata:
|
||||
"@aws-sdk/credential-providers": ^3.350.0
|
||||
"@aws-sdk/types": ^3.347.0
|
||||
"@aws-sdk/util-stream-node": ^3.350.0
|
||||
"@azure/identity": ^4.0.0
|
||||
"@azure/storage-blob": ^12.5.0
|
||||
"@backstage/backend-app-api": "workspace:^"
|
||||
"@backstage/backend-common": ^0.25.0
|
||||
"@backstage/backend-dev-utils": "workspace:^"
|
||||
@@ -4649,6 +4651,7 @@ __metadata:
|
||||
resolution: "@backstage/integration@workspace:packages/integration"
|
||||
dependencies:
|
||||
"@azure/identity": ^4.0.0
|
||||
"@azure/storage-blob": ^12.5.0
|
||||
"@backstage/cli": "workspace:^"
|
||||
"@backstage/config": "workspace:^"
|
||||
"@backstage/config-loader": "workspace:^"
|
||||
@@ -5396,6 +5399,8 @@ __metadata:
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure"
|
||||
dependencies:
|
||||
"@azure/identity": ^4.0.0
|
||||
"@azure/storage-blob": ^12.5.0
|
||||
"@backstage/backend-plugin-api": "workspace:^"
|
||||
"@backstage/backend-test-utils": "workspace:^"
|
||||
"@backstage/cli": "workspace:^"
|
||||
|
||||
Reference in New Issue
Block a user