feat: azure blob storage entity provider for catalog implemented

Signed-off-by: NIKUNJ LALITKUMAR HUDKA <nk856850@dal.ca>
This commit is contained in:
NIKUNJ LALITKUMAR HUDKA
2024-10-18 18:27:58 -03:00
parent 232a9600f0
commit 277092a876
33 changed files with 1912 additions and 15 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': major
---
Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider.
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-defaults': minor
---
Implemented `AzureBlobStorageUrlReader` to read from the url of committed location from the entity provider
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-azure': minor
---
Added the Azure Blob Storage as catalog entity provider to import all the desired entities from storage account provided in app-config.yaml
+18
View File
@@ -221,6 +221,17 @@ integrations:
# apiBaseUrl: server.bitbucket.com
# username: ${BITBUCKET_SERVER_USERNAME}
# appPassword: ${BITBUCKET_SERVER_APP_PASSWORD}
azureBlobStorage:
- accountName: ${ACCOUNT_NAME} # required
endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken
sasToken: ${SAS_TOKEN}
aadCredential:
clientId: ${CLIENT_ID}
tenantId: ${TENANT_ID}
clientSecret: ${CLIENT_SECRET}
accountKey: ${ACCOUNT_KEY}
azure:
- host: dev.azure.com
token: ${AZURE_TOKEN}
@@ -245,6 +256,13 @@ catalog:
- Domain
- Location
providers:
azureBlob:
containerName: ${CONTAINER_NAME}
schedule: # same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
backstageOpenapi:
plugins:
- catalog
+2
View File
@@ -119,6 +119,8 @@
"@aws-sdk/client-s3": "^3.350.0",
"@aws-sdk/credential-providers": "^3.350.0",
"@aws-sdk/types": "^3.347.0",
"@azure/identity": "^4.0.0",
"@azure/storage-blob": "^12.5.0",
"@backstage/backend-app-api": "workspace:^",
"@backstage/backend-common": "^0.25.0",
"@backstage/backend-dev-utils": "workspace:^",
@@ -0,0 +1,256 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
BlobServiceClient,
ContainerClient,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { ReaderFactory, ReadTreeResponseFactory } from './types';
import { ForwardedError, NotModifiedError } from '@backstage/errors';
import { Readable } from 'stream';
import { relative } from 'path/posix';
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
import {
AzureBlobStorageIntergation,
AzureCredentialsManager,
DefaultAzureCredentialsManager,
ScmIntegrations,
} from '@backstage/integration';
import {
UrlReaderService,
UrlReaderServiceReadTreeOptions,
UrlReaderServiceReadTreeResponse,
UrlReaderServiceReadUrlOptions,
UrlReaderServiceReadUrlResponse,
UrlReaderServiceSearchResponse,
} from '@backstage/backend-plugin-api';
export function parseUrl(
url: string,
config: AzureBlobStorageIntergation,
): { path: string; container: string } {
const parsedUrl = new URL(url);
const pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
if (pathSegments.length < 2) {
throw new Error(`Invalid Azure Blob Storage URL format: ${url}`);
}
// First segment is the container name, rest is the blob path
const container = pathSegments[0];
const path = pathSegments.slice(1).join('/');
return { path, container };
}
export class AzureBlobStorageUrlReader implements UrlReaderService {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const integrations = ScmIntegrations.fromConfig(config);
const credsManager =
DefaultAzureCredentialsManager.fromIntegrations(integrations);
return integrations.azureBlobStorage.list().map(integrationConfig => {
const reader = new AzureBlobStorageUrlReader(
credsManager,
integrationConfig,
{
treeResponseFactory,
},
);
const predicate = (url: URL) =>
url.host.endsWith(integrationConfig.config.host);
return { reader, predicate };
});
};
// private readonly blobServiceClient: BlobServiceClient;
constructor(
private readonly credsManager: AzureCredentialsManager,
private readonly integration: AzureBlobStorageIntergation,
private readonly deps: {
treeResponseFactory: ReadTreeResponseFactory;
},
) {}
private async createContainerClient(
containerName: string,
): Promise<ContainerClient> {
const accountName = this.integration.config.accountName; // Use the account name from the integration config
const accountKey = this.integration.config.accountKey; // Get the account key if it exists
if (accountKey && accountName) {
const creds = new StorageSharedKeyCredential(accountName, accountKey);
const blobServiceClient = new BlobServiceClient(
`https://${accountName}.${this.integration.config.host}`,
creds,
);
return blobServiceClient.getContainerClient(containerName);
}
// Use the credentials manager to get the correct credentials
const credential = await this.credsManager.getCredentials(
accountName as string,
);
let blobServiceClientUrl: string;
if (this.integration.config.endpoint) {
if (this.integration.config.sasToken) {
blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`;
} else {
blobServiceClientUrl = `${this.integration.config.endpoint}`;
}
} else {
blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`;
}
const blobServiceClient = new BlobServiceClient(
blobServiceClientUrl,
credential,
);
return blobServiceClient.getContainerClient(containerName);
}
async read(url: string): Promise<Buffer> {
const response = await this.readUrl(url);
return response.buffer();
}
async readUrl(
url: string,
options?: UrlReaderServiceReadUrlOptions,
): Promise<UrlReaderServiceReadUrlResponse> {
const { etag, lastModifiedAfter } = options ?? {};
try {
const { path, container } = parseUrl(url, this.integration);
const containerClient = await this.createContainerClient(container);
const blobClient = containerClient.getBlobClient(path);
const abortController = new AbortController();
const getBlobOptions = {
conditions: {
...(etag && { ifNoneMatch: etag }),
...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }),
},
};
options?.signal?.addEventListener('abort', () => abortController.abort());
const downloadBlockBlobResponse = await blobClient.download(
0,
undefined,
getBlobOptions,
);
const data = await this.retrieveAzureBlobData(
downloadBlockBlobResponse.readableStreamBody as Readable,
);
return ReadUrlResponseFactory.fromReadable(data, {
etag: downloadBlockBlobResponse.etag,
lastModifiedAt: downloadBlockBlobResponse.lastModified,
});
} catch (e) {
if (e.$metadata && e.$metadata.httpStatusCode === 304) {
throw new NotModifiedError();
}
throw new ForwardedError(
'Could not retrieve file from Azure Blob Storage',
e,
);
}
}
async readTree(
url: string,
options?: UrlReaderServiceReadTreeOptions,
): Promise<UrlReaderServiceReadTreeResponse> {
try {
const { path, container } = parseUrl(url, this.integration);
const containerClient = await this.createContainerClient(container);
const blobs = containerClient.listBlobsFlat({ prefix: path });
const responses = [];
const abortController = new AbortController();
for await (const blob of blobs) {
const blobClient = containerClient.getBlobClient(blob.name);
options?.signal?.addEventListener('abort', () =>
abortController.abort(),
);
const downloadBlockBlobResponse = await blobClient.download();
const data = await this.retrieveAzureBlobData(
downloadBlockBlobResponse.readableStreamBody as Readable,
);
responses.push({
data: Readable.from(data),
path: relative(path, blob.name),
lastModifiedAt: blob.properties.lastModified,
});
}
return this.deps.treeResponseFactory.fromReadableArray(responses);
} catch (e) {
throw new ForwardedError(
'Could not retrieve file tree from Azure Blob Storage',
e,
);
}
}
async search(): Promise<UrlReaderServiceSearchResponse> {
throw new Error('AzureBlobStorageUrlReader does not implement search');
}
toString() {
const accountName = this.integration.config.accountName;
const accountKey = this.integration.config.accountKey;
return `azureBlobStorage{accountName=${accountName},authed=${Boolean(
accountKey,
)}}`;
}
private parseUrl(url: string): { path: string } {
const parsedUrl = new URL(url);
const path = parsedUrl.pathname.substring(
parsedUrl.pathname.lastIndexOf('/') + 1,
);
return { path };
}
private async retrieveAzureBlobData(stream: Readable): Promise<Readable> {
return new Promise((resolve, reject) => {
try {
const chunks: any[] = [];
stream.on('data', chunk => chunks.push(chunk));
stream.on('error', (e: Error) =>
reject(new ForwardedError('Unable to read stream', e)),
);
stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks))));
} catch (e) {
throw new ForwardedError('Unable to parse the response data', e);
}
});
}
}
@@ -35,6 +35,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader';
import { GiteaUrlReader } from './GiteaUrlReader';
import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader';
import { HarnessUrlReader } from './HarnessUrlReader';
import { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader';
/**
* Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}.
@@ -99,6 +100,7 @@ export class UrlReaders {
GoogleGcsUrlReader.factory,
HarnessUrlReader.factory,
AwsS3UrlReader.factory,
AzureBlobStorageUrlReader.factory,
AwsCodeCommitUrlReader.factory,
FetchUrlReader.factory,
]),
@@ -24,6 +24,7 @@ export { GitlabUrlReader } from './GitlabUrlReader';
export { GiteaUrlReader } from './GiteaUrlReader';
export { HarnessUrlReader } from './HarnessUrlReader';
export { AwsS3UrlReader } from './AwsS3UrlReader';
export { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader';
export { FetchUrlReader } from './FetchUrlReader';
export { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
export type {
+1
View File
@@ -38,6 +38,7 @@
},
"dependencies": {
"@azure/identity": "^4.0.0",
"@azure/storage-blob": "^12.5.0",
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@octokit/auth-app": "^4.0.0",
@@ -39,6 +39,10 @@ import { GiteaIntegration, GiteaIntegrationConfig } from './gitea';
import { AwsCodeCommitIntegration } from './awsCodeCommit/AwsCodeCommitIntegration';
import { AwsCodeCommitIntegrationConfig } from './awsCodeCommit';
import { HarnessIntegration, HarnessIntegrationConfig } from './harness';
import {
AzureBlobStorageIntegrationConfig,
AzureBlobStorageIntergation,
} from './azureBlobStorage';
describe('ScmIntegrations', () => {
const awsS3 = new AwsS3Integration({
@@ -53,6 +57,10 @@ describe('ScmIntegrations', () => {
host: 'azure.local',
} as AzureIntegrationConfig);
const azureBlob = new AzureBlobStorageIntergation({
host: 'azureblobstorage.local',
} as AzureBlobStorageIntegrationConfig);
const bitbucket = new BitbucketIntegration({
host: 'bitbucket.local',
} as BitbucketIntegrationConfig);
@@ -89,6 +97,7 @@ describe('ScmIntegrations', () => {
awsS3: basicIntegrations([awsS3], item => item.config.host),
awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host),
azure: basicIntegrations([azure], item => item.config.host),
azureBlobStorage: basicIntegrations([azureBlob], item => item.config.host),
bitbucket: basicIntegrations([bitbucket], item => item.config.host),
bitbucketCloud: basicIntegrations([bitbucketCloud], item => item.title),
bitbucketServer: basicIntegrations(
@@ -108,6 +117,9 @@ describe('ScmIntegrations', () => {
awsCodeCommit,
);
expect(i.azure.byUrl('https://azure.local')).toBe(azure);
expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe(
azureBlob,
);
expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket);
expect(i.bitbucketCloud.byUrl('https://bitbucket.org')).toBe(
bitbucketCloud,
@@ -128,6 +140,7 @@ describe('ScmIntegrations', () => {
awsS3,
awsCodeCommit,
azure,
azureBlob,
bitbucket,
bitbucketCloud,
bitbucketServer,
@@ -144,6 +157,9 @@ describe('ScmIntegrations', () => {
expect(i.byUrl('https://awss3.local')).toBe(awsS3);
expect(i.byUrl('https://awscodecommit.local')).toBe(awsCodeCommit);
expect(i.byUrl('https://azure.local')).toBe(azure);
expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe(
azureBlob,
);
expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket);
expect(i.byUrl('https://bitbucket.org')).toBe(bitbucketCloud);
expect(i.byUrl('https://bitbucket-server.local')).toBe(bitbucketServer);
@@ -156,6 +172,7 @@ describe('ScmIntegrations', () => {
expect(i.byHost('awss3.local')).toBe(awsS3);
expect(i.byHost('awscodecommit.local')).toBe(awsCodeCommit);
expect(i.byHost('azure.local')).toBe(azure);
expect(i.byHost('azureblobstorage.local')).toBe(azureBlob);
expect(i.byHost('bitbucket.local')).toBe(bitbucket);
expect(i.byHost('bitbucket.org')).toBe(bitbucketCloud);
expect(i.byHost('bitbucket-server.local')).toBe(bitbucketServer);
@@ -29,6 +29,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types';
import { ScmIntegrationRegistry } from './registry';
import { GiteaIntegration } from './gitea';
import { HarnessIntegration } from './harness/HarnessIntegration';
import { AzureBlobStorageIntergation } from './azureBlobStorage';
/**
* The set of supported integrations.
@@ -38,6 +39,7 @@ import { HarnessIntegration } from './harness/HarnessIntegration';
export interface IntegrationsByType {
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
awsCodeCommit: ScmIntegrationsGroup<AwsCodeCommitIntegration>;
azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
azure: ScmIntegrationsGroup<AzureIntegration>;
/**
* @deprecated in favor of `bitbucketCloud` and `bitbucketServer`
@@ -64,6 +66,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
return new ScmIntegrations({
awsS3: AwsS3Integration.factory({ config }),
awsCodeCommit: AwsCodeCommitIntegration.factory({ config }),
azureBlobStorage: AzureBlobStorageIntergation.factory({ config }),
azure: AzureIntegration.factory({ config }),
bitbucket: BitbucketIntegration.factory({ config }),
bitbucketCloud: BitbucketCloudIntegration.factory({ config }),
@@ -88,6 +91,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
return this.byType.awsCodeCommit;
}
get azureBlobStorage(): ScmIntegrationsGroup<AzureBlobStorageIntergation> {
return this.byType.azureBlobStorage;
}
get azure(): ScmIntegrationsGroup<AzureIntegration> {
return this.byType.azure;
}
@@ -0,0 +1,78 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration';
describe('AzureBlobStorageIntegration', () => {
it('has a working factory', () => {
const integrations = AzureBlobStorageIntergation.factory({
config: new ConfigReader({
integrations: {
azureBlobStorage: [
{
endpoint: 'https://myaccount.blob.core.windows.net',
accountName: 'myaccount',
accountKey: 'someAccountKey',
},
],
},
}),
});
expect(integrations.list().length).toBe(2); // including default
expect(integrations.list()[0].config.host).toBe(
'myaccount.blob.core.windows.net',
);
expect(integrations.list()[1].config.host).toBe('blob.core.windows.net'); // default integration
});
it('returns the basics', () => {
const integration = new AzureBlobStorageIntergation({
host: 'myaccount.blob.core.windows.net',
} as any);
expect(integration.type).toBe('azureBlobStorage');
expect(integration.title).toBe('myaccount.blob.core.windows.net');
});
describe('resolveUrl', () => {
it('works for valid URLs', () => {
const integration = new AzureBlobStorageIntergation({
host: 'blob.core.windows.net',
} as any);
expect(
integration.resolveUrl({
url: 'https://myaccount.blob.core.windows.net/container/file.yaml',
base: 'https://myaccount.blob.core.windows.net/container/file.yaml',
}),
).toBe('https://myaccount.blob.core.windows.net/container/file.yaml');
});
});
it('resolve edit URL', () => {
const integration = new AzureBlobStorageIntergation({
host: 'myaccount.blob.core.windows.net',
} as any);
// TODO: The Azure Blob Storage integration doesn't support resolving an edit URL,
// instead we keep the input URL.
expect(
integration.resolveEditUrl(
'https://myaccount.blob.core.windows.net/container/file.yaml',
),
).toBe('https://myaccount.blob.core.windows.net/container/file.yaml');
});
});
@@ -0,0 +1,66 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
import {
AzureBlobStorageIntegrationConfig,
readAzureBlobStorageIntegrationConfigs,
} from './config';
export class AzureBlobStorageIntergation implements ScmIntegration {
static factory: ScmIntegrationsFactory<AzureBlobStorageIntergation> = ({
config,
}) => {
const configs = readAzureBlobStorageIntegrationConfigs(
config.getOptionalConfigArray('integrations.azureBlobStorage') ?? [],
);
return basicIntegrations(
configs.map(c => new AzureBlobStorageIntergation(c)),
i => i.config.host,
);
};
get type(): string {
return 'azureBlobStorage';
}
get title(): string {
return this.integrationConfig.host;
}
get config(): AzureBlobStorageIntegrationConfig {
return this.integrationConfig;
}
constructor(
private readonly integrationConfig: AzureBlobStorageIntegrationConfig,
) {}
resolveUrl(options: {
url: string;
base: string;
lineNumber?: number | undefined;
}): string {
const resolved = defaultScmResolveUrl(options);
return resolved;
}
resolveEditUrl(url: string): string {
// TODO: Implement edit URL for azureBlobStorage
return url;
}
}
@@ -0,0 +1,151 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
AccessToken,
ClientSecretCredential,
DefaultAzureCredential,
TokenCredential,
} from '@azure/identity';
import { AzureBlobStorageIntegrationConfig } from './config';
import { ScmIntegrationRegistry } from '../registry';
import { ConfigReader } from '@backstage/config';
import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider';
import { ScmIntegrations } from '../ScmIntegrations';
import { DateTime } from 'luxon';
const MockedClientSecretCredential = ClientSecretCredential as jest.MockedClass<
typeof ClientSecretCredential
>;
jest.mock('@azure/identity');
describe('DefaultAzureCredentialsManager', () => {
let mockIntegration: ScmIntegrationRegistry;
const buildProvider = (azureIntegrations: any[]) =>
DefaultAzureCredentialsManager.fromIntegrations(
ScmIntegrations.fromConfig(
new ConfigReader({
integrations: {
azureBlobStorage: azureIntegrations,
},
}),
),
);
beforeEach(() => {
mockIntegration = {
azureBlobStorage: {
list: jest.fn().mockReturnValue([
{
config: {
accountName: 'testaccount',
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
},
},
]),
},
} as unknown as ScmIntegrationRegistry;
MockedClientSecretCredential.prototype.getToken.mockImplementation(() =>
Promise.resolve({
expiresOnTimestamp: DateTime.local().plus({ days: 1 }).toSeconds(),
token: 'fake-client-secret-token',
} as AccessToken),
);
});
it('should create an instance from ScmIntegrationRegistry', () => {
const manager =
DefaultAzureCredentialsManager.fromIntegrations(mockIntegration);
expect(manager).toBeInstanceOf(DefaultAzureCredentialsManager);
});
it('should return cached credentials if available', async () => {
const manager = buildProvider([
{
accountName: 'testaccount',
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
},
]);
const mockCredential = new MockedClientSecretCredential(
'someTenantId',
'someClientId',
'someClientSecret',
);
const credential = await manager.getCredentials('testaccount');
const scopes = ['https://storage.azure.com/.default'];
const expectedToken = await mockCredential.getToken(scopes);
const receivedToken = await credential.getToken(scopes);
expect(receivedToken?.token).toEqual(expectedToken.token);
});
it('should use Azure AD credentials if aadCredential is provided', async () => {
const manager = buildProvider([
{
accountName: 'testaccount',
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
},
]);
const credential = await manager.getCredentials('testaccount');
expect(credential).toBeInstanceOf(ClientSecretCredential);
});
it('should use DefaultAzureCredential if no aadCredential is provided', async () => {
const manager = buildProvider([
{
accountName: 'testaccount',
},
]);
const credential = await manager.getCredentials('testaccount');
expect(credential).toBeInstanceOf(DefaultAzureCredential);
});
it('should cache credentials after first retrieval', async () => {
const manager = buildProvider([
{
accountName: 'testaccount',
},
]);
const credential = await manager.getCredentials('testaccount');
const cachedCredential = await manager.getCredentials('testaccount');
expect(cachedCredential).toBe(credential);
});
});
@@ -0,0 +1,121 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
DefaultAzureCredential,
ClientSecretCredential,
TokenCredential,
} from '@azure/identity';
import { AzureBlobStorageIntegrationConfig } from './config';
import { AzureCredentialsManager } from './types';
import { ScmIntegrationRegistry } from '../registry';
/**
* Default Azure Credentials Manager to dynamically select and manage Azure credentials.
* It supports Service Principal, Managed Identity, SAS Token, Connection String, Account Key, and Anonymous access.
*/
export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
private config: AzureBlobStorageIntegrationConfig;
private cachedCredentials: Map<string, TokenCredential>;
constructor(config: AzureBlobStorageIntegrationConfig) {
this.config = config;
this.cachedCredentials = new Map<string, TokenCredential>();
}
/**
* Creates an instance of DefaultAzureCredentialsManager from a Backstage Config.
*/
static fromIntegrations(
integration: ScmIntegrationRegistry,
): DefaultAzureCredentialsManager {
const azureConfig = integration.azureBlobStorage.list().length
? integration.azureBlobStorage.list()[0].config
: { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found
return new DefaultAzureCredentialsManager(azureConfig);
}
/**
* Determines the appropriate credential method and returns credentials for BlobServiceClient.
* Supports:
* - Service Principal
* - Managed Identity
* - SAS Token
* - Connection String
* - Account Key
* - Anonymous access
*/
async getCredentials(accountName: string): Promise<TokenCredential> {
// Check if the credentials are already cached
if (this.cachedCredentials.has(accountName)) {
return this.cachedCredentials.get(accountName)!;
}
let credential: TokenCredential;
// Check for SAS Token
// if (this.config.sasToken) {
// // console.log('Using SAS Token for Azure Blob Storage authentication');
// // SAS Token does not return a credential but can be used directly in BlobServiceClient
// // Here we can simply return undefined or keep a placeholder if needed
// return this.config.sasToken; // Or return a string for the URL using the SAS token
// }
// // Check for Connection String
// else if (this.config.connectionString) {
// // console.log(
// // 'Using Connection String for Azure Blob Storage authentication',
// // );
// // return undefined; // Connection string will also not return a specific credential object
// }
// Check for Account Key
// if (this.config.accountKey) {
// // console.log('Using Account Key for Azure Blob Storage authentication');
// credential = new StorageSharedKeyCredential(
// accountName,
// this.config.accountKey,
// );
// }
// Check for AAD credentials
if (
this.config.aadCredential &&
this.config.aadCredential.clientId &&
this.config.aadCredential.clientSecret &&
this.config.aadCredential.tenantId
) {
credential = new ClientSecretCredential(
this.config.aadCredential.tenantId,
this.config.aadCredential.clientId,
this.config.aadCredential.clientSecret,
);
}
// Check for Anonymous access
// else if (this.config.anonymousAccess) {
// console.log('Using Anonymous Credential for Azure Blob Storage access');
// credential = new AnonymousCredential();
// }
// Fallback to Managed Identity
else {
credential = new DefaultAzureCredential();
}
// Cache the credentials for future use
this.cachedCredentials.set(accountName, credential);
return credential;
}
}
@@ -0,0 +1,264 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config, ConfigReader } from '@backstage/config';
import {
AzureBlobStorageIntegrationConfig,
readAzureBlobStorageIntegrationConfig,
readAzureBlobStorageIntegrationConfigs,
} from './config';
describe('readAzureBlobStorageIntegrationConfig', () => {
function buildConfig(
data: Partial<AzureBlobStorageIntegrationConfig>,
): Config {
return new ConfigReader(data);
}
it('reads valid configuration with accountKey', () => {
const output = readAzureBlobStorageIntegrationConfig(
buildConfig({
accountName: 'mystorageaccount',
accountKey: 'someAccountKey',
}),
);
expect(output).toEqual({
host: 'blob.core.windows.net',
endpoint: undefined,
accountName: 'mystorageaccount',
accountKey: 'someAccountKey',
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
});
});
it('reads valid configuration with sasToken', () => {
const output = readAzureBlobStorageIntegrationConfig(
buildConfig({
endpoint: 'https://blob.core.windows.net',
accountName: 'mystorageaccount',
sasToken: 'someSASToken',
}),
);
expect(output).toEqual({
host: 'blob.core.windows.net',
endpoint: 'https://blob.core.windows.net',
accountName: 'mystorageaccount',
accountKey: undefined,
sasToken: 'someSASToken',
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
});
});
it('reads valid configuration with Azure AD credentials', () => {
const output = readAzureBlobStorageIntegrationConfig(
buildConfig({
accountName: 'mystorageaccount',
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
}),
);
expect(output).toEqual({
host: 'blob.core.windows.net',
endpoint: undefined,
accountName: 'mystorageaccount',
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
});
});
it('reads valid configuration with a custom endpoint', () => {
const output = readAzureBlobStorageIntegrationConfig(
buildConfig({
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'customaccount',
}),
);
expect(output).toEqual({
host: 'custom.blob.core.windows.net',
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'customaccount',
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
});
});
it('throws error for invalid endpoint URL', () => {
const config = buildConfig({
endpoint: 'invalid-url',
accountName: 'invalidaccount',
});
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
`invalid azureBlobStorage integration config, endpoint 'invalid-url' is not a valid URL`,
);
});
it('throws error if endpoint has a path', () => {
const config = buildConfig({
endpoint: 'https://blob.core.windows.net/path',
accountName: 'accountwithpath',
});
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
`invalid azureBlobStorage integration config, endpoints cannot contain path, got 'https://blob.core.windows.net/path'`,
);
});
it('throws error if both accountKey and sasToken are provided', () => {
const config = buildConfig({
accountName: 'mystorageaccount',
accountKey: 'someAccountKey',
sasToken: 'someSASToken',
});
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
`Invalid Azure Blob Storage config for mystorageaccount: Both account key and SAS token cannot be used simultaneously.`,
);
});
it('throws error if both aadCredential and accountKey/sasToken are provided', () => {
const config = buildConfig({
accountName: 'mystorageaccount',
accountKey: 'someAccountKey',
aadCredential: {
clientId: 'someClientId',
tenantId: 'someTenantId',
clientSecret: 'someClientSecret',
},
});
expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow(
`Invalid Azure Blob Storage config for mystorageaccount: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`,
);
});
});
describe('readAzureBlobStorageIntegrationConfigs', () => {
function buildConfigs(
data: Partial<AzureBlobStorageIntegrationConfig>[],
): Config[] {
return data.map(item => new ConfigReader(item));
}
it('reads all provided configurations', () => {
const output = readAzureBlobStorageIntegrationConfigs(
buildConfigs([
{
host: 'blob.core.windows.net',
accountName: 'account1',
accountKey: 'someAccountKey',
},
{
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'account2',
},
]),
);
expect(output).toEqual([
{
host: 'blob.core.windows.net',
endpoint: undefined,
accountName: 'account1',
accountKey: 'someAccountKey',
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
},
{
host: 'custom.blob.core.windows.net',
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'account2',
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
},
]);
});
it('adds default integration for blob.core.windows.net when missing', () => {
const output = readAzureBlobStorageIntegrationConfigs(
buildConfigs([
{
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'account2',
},
]),
);
expect(output).toEqual([
{
host: 'custom.blob.core.windows.net',
endpoint: 'https://custom.blob.core.windows.net',
accountName: 'account2',
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
},
{
host: 'blob.core.windows.net',
endpoint: undefined,
accountName: undefined,
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
},
]);
});
it('does not add default integration if blob.core.windows.net already exists', () => {
const output = readAzureBlobStorageIntegrationConfigs(
buildConfigs([
{ host: 'blob.core.windows.net', accountName: 'account1' },
]),
);
expect(output).toEqual([
{
host: 'blob.core.windows.net',
endpoint: undefined,
accountName: 'account1',
accountKey: undefined,
sasToken: undefined,
connectionString: undefined,
endpointSuffix: undefined,
aadCredential: undefined,
},
]);
});
});
@@ -0,0 +1,170 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
const AZURE_HOST = 'blob.core.windows.net';
/**
* The configuration parameters for a single Azure Blob Storage account.
*
* @public
*/
export type AzureBlobStorageIntegrationConfig = {
/**
* The name of the Azure Storage Account, e.g., "mystorageaccount".
*/
accountName?: string;
/**
* The primary or secondary key for the Azure Storage Account.
* Only required if connectionString or SAS token are not specified.
*/
accountKey?: string;
/**
* A Shared Access Signature (SAS) token for limited access to resources.
*/
sasToken?: string;
/**
* A full connection string for the Azure Storage Account.
* This includes the account name, key, and endpoint details.
*/
connectionString?: string;
/**
* Optional endpoint suffix for custom domains or sovereign clouds.
* e.g., "core.windows.net" for public Azure or "core.usgovcloudapi.net" for US Government cloud.
*/
endpointSuffix?: string;
/**
* The host of the target that this matches on, e.g., "blob.core.windows.net".
*/
host: string;
endpoint?: string;
/**
* Optional credential to use for Azure Active Directory authentication.
*/
aadCredential?: {
/**
* The client ID of the Azure AD application.
*/
clientId: string;
/**
* The tenant ID for Azure AD.
*/
tenantId: string;
/**
* The client secret for the Azure AD application.
*/
clientSecret: string;
};
};
/**
* Reads a single Azure Blob Storage integration config.
*
* @param config - The config object of a single integration.
* @public
*/
export function readAzureBlobStorageIntegrationConfig(
config: Config,
): AzureBlobStorageIntegrationConfig {
const endpoint = config.getOptionalString('endpoint');
const accountName = config.getString('accountName');
const accountKey = config.getOptionalString('accountKey')?.trim();
const sasToken = config.getOptionalString('sasToken')?.trim();
const connectionString = config.getOptionalString('connectionString')?.trim();
const endpointSuffix = config.getOptionalString('endpointSuffix')?.trim();
let host;
let pathname;
if (endpoint) {
try {
const url = new URL(endpoint);
host = url.host;
pathname = url.pathname;
} catch {
throw new Error(
`invalid azureBlobStorage integration config, endpoint '${endpoint}' is not a valid URL`,
);
}
if (pathname !== '/') {
throw new Error(
`invalid azureBlobStorage integration config, endpoints cannot contain path, got '${endpoint}'`,
);
}
} else {
host = AZURE_HOST;
}
const aadCredential = config.has('aadCredential')
? {
clientId: config.getString('aadCredential.clientId'),
tenantId: config.getString('aadCredential.tenantId'),
clientSecret: config.getString('aadCredential.clientSecret')?.trim(),
}
: undefined;
if (accountKey && sasToken) {
throw new Error(
`Invalid Azure Blob Storage config for ${accountName}: Both account key and SAS token cannot be used simultaneously.`,
);
}
if (aadCredential && (accountKey || sasToken)) {
throw new Error(
`Invalid Azure Blob Storage config for ${accountName}: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`,
);
}
return {
host,
endpoint,
accountName,
accountKey,
sasToken,
connectionString,
endpointSuffix,
aadCredential,
};
}
/**
* Reads a set of Azure Blob Storage integration configs.
*
* @param configs - All of the integration config objects.
* @public
*/
export function readAzureBlobStorageIntegrationConfigs(
configs: Config[],
): AzureBlobStorageIntegrationConfig[] {
// First read all the explicit integrations
const result = configs.map(readAzureBlobStorageIntegrationConfig);
// If no explicit blob.core.windows.net integration was added, put one in the list as
// a convenience
if (!result.some(c => c.host === AZURE_HOST)) {
result.push({
host: AZURE_HOST,
});
}
return result;
}
@@ -0,0 +1,23 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration';
export {
readAzureBlobStorageIntegrationConfig,
readAzureBlobStorageIntegrationConfigs,
} from './config';
export type { AzureBlobStorageIntegrationConfig } from './config';
export { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider';
export type { AzureCredentialsManager } from './types';
@@ -0,0 +1,29 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TokenCredential } from '@azure/identity';
import {
StorageSharedKeyCredential,
AnonymousCredential,
} from '@azure/storage-blob';
export interface AzureCredentialsManager {
getCredentials(
accountName: string,
): Promise<
TokenCredential | StorageSharedKeyCredential | AnonymousCredential
>;
}
+1
View File
@@ -22,6 +22,7 @@
export * from './awsS3';
export * from './awsCodeCommit';
export * from './azureBlobStorage';
export * from './azure';
export * from './bitbucket';
export * from './bitbucketCloud';
+2
View File
@@ -26,6 +26,7 @@ import { GithubIntegration } from './github/GithubIntegration';
import { GitLabIntegration } from './gitlab/GitLabIntegration';
import { GiteaIntegration } from './gitea/GiteaIntegration';
import { HarnessIntegration } from './harness/HarnessIntegration';
import { AzureBlobStorageIntergation } from './azureBlobStorage';
/**
* Holds all registered SCM integrations, of all types.
@@ -36,6 +37,7 @@ export interface ScmIntegrationRegistry
extends ScmIntegrationsGroup<ScmIntegration> {
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
awsCodeCommit: ScmIntegrationsGroup<AwsCodeCommitIntegration>;
azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
azure: ScmIntegrationsGroup<AzureIntegration>;
/**
* @deprecated in favor of `bitbucketCloud` and `bitbucketServer`
@@ -51,6 +51,8 @@
"test": "backstage-cli package test"
},
"dependencies": {
"@azure/identity": "^4.0.0",
"@azure/storage-blob": "^12.5.0",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/integration": "workspace:^",
@@ -23,3 +23,4 @@
export { default } from './module';
export { AzureDevOpsDiscoveryProcessor } from './processors';
export { AzureDevOpsEntityProvider } from './providers';
export { AzureBlobStorageEntityProvider } from './providers';
@@ -17,8 +17,11 @@
import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api';
import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
import { catalogModuleAzureDevOpsEntityProvider } from './catalogModuleAzureDevOpsEntityProvider';
import { AzureDevOpsEntityProvider } from '../providers';
import { catalogModuleAzureEntityProvider } from './catalogModuleAzureDevOpsEntityProvider';
import {
AzureBlobStorageEntityProvider,
AzureDevOpsEntityProvider,
} from '../providers';
describe('catalogModuleAzureDevOpsEntityProvider', () => {
it('should register provider at the catalog extension point', async () => {
@@ -58,7 +61,7 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => {
await startTestBackend({
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
features: [
catalogModuleAzureDevOpsEntityProvider,
catalogModuleAzureEntityProvider,
mockServices.rootConfig.factory({ data: config }),
mockServices.logger.factory(),
scheduler.factory,
@@ -73,4 +76,54 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => {
);
expect(runner).not.toHaveBeenCalled();
});
it('should register azure blob storage provider at the catalog extension point', async () => {
let addedProviders: Array<AzureBlobStorageEntityProvider> | undefined;
let usedSchedule: SchedulerServiceTaskScheduleDefinition | undefined;
const extensionPoint = {
addEntityProvider: (providers: any) => {
addedProviders = providers;
},
};
const runner = jest.fn();
const scheduler = mockServices.scheduler.mock({
createScheduledTaskRunner(schedule) {
usedSchedule = schedule;
return { run: runner };
},
});
const config = {
catalog: {
providers: {
azureBlob: {
containerName: 'test',
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
},
},
},
};
await startTestBackend({
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
features: [
catalogModuleAzureEntityProvider,
mockServices.rootConfig.factory({ data: config }),
mockServices.logger.factory(),
scheduler.factory,
],
});
expect(usedSchedule?.frequency).toEqual({ months: 1 });
expect(usedSchedule?.timeout).toEqual({ minutes: 3 });
expect(addedProviders?.length).toEqual(1);
expect(addedProviders?.pop()?.getProviderName()).toEqual(
'azureBlobStorage-provider:default',
);
expect(runner).not.toHaveBeenCalled();
});
});
@@ -19,16 +19,19 @@ import {
createBackendModule,
} from '@backstage/backend-plugin-api';
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
import { AzureDevOpsEntityProvider } from '../providers';
import {
AzureBlobStorageEntityProvider,
AzureDevOpsEntityProvider,
} from '../providers';
/**
* Registers the AzureDevOpsEntityProvider with the catalog processing extension point.
*
* @public
*/
export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({
export const catalogModuleAzureEntityProvider = createBackendModule({
pluginId: 'catalog',
moduleId: 'azure-dev-ops-entity-provider',
moduleId: 'azure-providers',
register(env) {
env.registerInit({
deps: {
@@ -38,12 +41,25 @@ export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({
scheduler: coreServices.scheduler,
},
async init({ config, catalog, logger, scheduler }) {
catalog.addEntityProvider(
AzureDevOpsEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
);
// Check for Azure Blob Storage provider configuration and register it
if (config.has('catalog.providers.azureBlob')) {
catalog.addEntityProvider(
AzureBlobStorageEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
);
}
// Check for Azure DevOps provider configuration and register it
if (config.has('catalog.providers.azureDevOps')) {
catalog.addEntityProvider(
AzureDevOpsEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
);
}
},
});
},
@@ -14,4 +14,4 @@
* limitations under the License.
*/
export { catalogModuleAzureDevOpsEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider';
export { catalogModuleAzureEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider';
@@ -0,0 +1,246 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
SchedulerService,
SchedulerServiceTaskRunner,
SchedulerServiceTaskInvocationDefinition,
} from '@backstage/backend-plugin-api';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider';
import { mockServices } from '@backstage/backend-test-utils';
class PersistingTaskRunner implements SchedulerServiceTaskRunner {
private tasks: SchedulerServiceTaskInvocationDefinition[] = [];
getTasks() {
return this.tasks;
}
run(task: SchedulerServiceTaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
const blobs = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key4.yaml'];
const createBlobList = (blobsArray: string[]) => {
return blobsArray.map(blob => ({
name: blob,
}));
};
// Mocking Azure Storage Blob Library
jest.mock('@azure/storage-blob', () => {
return {
BlobServiceClient: jest.fn().mockImplementation(() => ({
url: 'https://myaccount.blob.core.windows.net/',
getContainerClient: jest.fn().mockImplementation(() => ({
listBlobsFlat: jest.fn(async function* () {
yield* createBlobList(blobs);
}),
})),
})),
};
});
const logger = mockServices.logger.mock();
describe('AzureBlobStorageEntityProvider', () => {
const containerName = 'container-1';
const expectMutation = async (
providerId: string,
providerConfig: object,
expectedBaseUrl: string,
names: Record<string, string>,
integrationConfig?: object,
scheduleInConfig?: boolean,
) => {
const config = new ConfigReader({
integrations: {
azureBlobStorage: integrationConfig ? [integrationConfig] : [],
},
catalog: {
providers: {
azureBlob: {
[providerId]: providerConfig,
},
},
},
});
const schedulingConfig: Record<string, any> = {};
const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/')
? expectedBaseUrl
: `${expectedBaseUrl}/`;
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
if (scheduleInConfig) {
schedulingConfig.scheduler = {
createScheduledTaskRunner: (_: any) => schedule,
} as unknown as SchedulerService;
} else {
schedulingConfig.schedule = schedule;
}
const provider = AzureBlobStorageEntityProvider.fromConfig(config, {
...schedulingConfig,
logger,
})[0];
expect(provider.getProviderName()).toEqual(
`azureBlobStorage-provider:${providerId}`,
);
try {
await provider.connect(entityProviderConnection);
} catch (error) {
console.error('Error during provider connection:', error);
}
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual(
`azureBlobStorage-provider:${providerId}:refresh`,
);
await (taskDef.fn as () => Promise<void>)();
const expectedEntities = blobs.map(blob => {
const url = encodeURI(`${normalizedExpectedBaseUrl}${blob}`);
return {
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: expect.stringMatching(/generated-[a-f0-9]{40}/),
},
spec: {
presence: 'required',
target: `${url}`,
type: 'url',
},
},
locationKey: `azureBlobStorage-provider:${providerId}`,
};
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
};
it('apply full update on scheduled execution', async () => {
return expectMutation(
'staticContainer',
{
containerName,
},
'https://myaccount.blob.core.windows.net/container-1/',
{
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
},
{
accountName: 'myaccount',
},
);
});
it('apply full update no prefix', async () => {
return expectMutation(
'staticContainerNoPrefix',
{
containerName,
schedule: {
frequency: { minutes: 30 },
timeout: { minutes: 3 },
},
},
'https://myaccount.blob.core.windows.net/container-1/',
{
'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb',
'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11',
'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646',
'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9',
},
{
host: 'blob.core.windows.net',
accountName: 'myaccount',
},
);
});
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
catalog: {
providers: {
azureBlob: {
test: {
containerName: 'container-1',
prefix: 'sub/dir/',
accountName: 'myaccount',
},
},
},
},
});
expect(() =>
AzureBlobStorageEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as SchedulerService;
const config = new ConfigReader({
catalog: {
providers: {
azureBlob: {
test: {
containerName: 'container-1',
prefix: 'sub/dir/',
accountName: 'myaccount',
},
},
},
},
});
expect(() =>
AzureBlobStorageEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:test.',
);
});
});
@@ -0,0 +1,228 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
AnonymousCredential,
BlobServiceClient,
ContainerClient,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { Config } from '@backstage/config';
import {
LoggerService,
SchedulerService,
SchedulerServiceTaskRunner,
} from '@backstage/backend-plugin-api';
import {
EntityProvider,
EntityProviderConnection,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-node';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import * as uuid from 'uuid';
import { readAzureBlobStorageConfigs } from './config';
import {
AzureBlobStorageIntergation,
DefaultAzureCredentialsManager,
ScmIntegrations,
} from '@backstage/integration';
import { TokenCredential } from '@azure/identity';
import { AzureBlobStorageConfig } from './types';
export class AzureBlobStorageEntityProvider implements EntityProvider {
private readonly logger: LoggerService;
private connection?: EntityProviderConnection;
private blobServiceClient?: BlobServiceClient;
private readonly scheduleFn: () => Promise<void>;
static fromConfig(
configRoot: Config,
options: {
logger: LoggerService;
schedule?: SchedulerServiceTaskRunner;
scheduler?: SchedulerService;
},
): AzureBlobStorageEntityProvider[] {
const providerConfigs = readAzureBlobStorageConfigs(configRoot);
const scmIntegration = ScmIntegrations.fromConfig(configRoot);
const credentialsProvider =
DefaultAzureCredentialsManager.fromIntegrations(scmIntegration);
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
return providerConfigs.map(providerConfig => {
const integration = scmIntegration.azureBlobStorage.list()[0];
if (!integration) {
throw new Error(
`There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`,
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
return new AzureBlobStorageEntityProvider(
providerConfig,
integration,
credentialsProvider,
options.logger,
taskRunner,
);
});
}
constructor(
private readonly config: AzureBlobStorageConfig,
private readonly integration: AzureBlobStorageIntergation,
private readonly credentialsProvider: DefaultAzureCredentialsManager,
logger: LoggerService,
schedule: SchedulerServiceTaskRunner,
) {
this.logger = logger.child({ target: this.getProviderName() });
this.scheduleFn = this.createScheduleFn(schedule);
}
private createScheduleFn(
taskRunner: SchedulerServiceTaskRunner,
): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return taskRunner.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
class: AzureBlobStorageEntityProvider.prototype.constructor.name,
taskId,
taskInstanceId: uuid.v4(),
});
try {
await this.refresh(logger);
} catch (error) {
logger.error(
`${this.getProviderName()} refresh failed, ${error}`,
error,
);
}
},
});
};
}
getProviderName(): string {
return `azureBlobStorage-provider:${this.config.id}`;
}
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
let credential:
| TokenCredential
| StorageSharedKeyCredential
| AnonymousCredential;
if (this.integration.config.accountKey) {
credential = new StorageSharedKeyCredential(
this.integration.config.accountName as string,
this.integration.config.accountKey as string,
); // StorageSharedKeyCredential is only allowed in node.js runtime not in browser
} else {
credential = await this.credentialsProvider.getCredentials(
this.integration.config.accountName as string,
);
}
let blobServiceClientUrl: string;
if (this.integration.config.endpoint) {
if (this.integration.config.sasToken) {
blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`;
} else {
blobServiceClientUrl = `${this.integration.config.endpoint}`;
}
} else {
blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`;
}
this.blobServiceClient = new BlobServiceClient(
blobServiceClientUrl,
credential,
);
await this.scheduleFn();
}
async refresh(logger: LoggerService) {
if (!this.connection) {
throw new Error('Not initialized');
}
logger.info('Discovering Azure Blob Storage blobs');
const keys = await this.listAllBlobKeys();
logger.info(`Discovered ${keys.length} Azure Blob Storage blobs`);
const locations = keys.map(key => this.createLocationSpec(key));
await this.connection.applyMutation({
type: 'full',
entities: locations.map(location => {
return {
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
};
}),
});
logger.info(
`Committed ${locations.length} Locations for Azure Blob Storage blobs`,
);
}
private async listAllBlobKeys(): Promise<string[]> {
const keys: string[] = [];
const containerClient = this.blobServiceClient?.getContainerClient(
this.config.containerName,
);
for await (const blob of (
containerClient as ContainerClient
).listBlobsFlat()) {
if (blob.name) {
keys.push(blob.name);
}
}
return keys;
}
private createLocationSpec(key: string): LocationSpec {
return {
type: 'url',
target: this.createObjectUrl(key),
presence: 'required',
};
}
private createObjectUrl(key: string): string {
const endpoint = this.blobServiceClient?.url;
return `${endpoint}${this.config.containerName}/${key}`;
}
}
@@ -15,7 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { readAzureDevOpsConfigs } from './config';
import { readAzureBlobStorageConfigs, readAzureDevOpsConfigs } from './config';
describe('readAzureDevOpsConfigs', () => {
it('reads all provider configs and set default values', () => {
@@ -106,3 +106,69 @@ describe('readAzureDevOpsConfigs', () => {
});
});
});
describe('readAzureBlobStorageConfigs', () => {
it('reads single and multiple Azure Blob Storage provider configs', () => {
const provider1 = {
containerName: 'container-1',
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
};
const provider2 = {
containerName: 'container-2',
};
const configSingle = {
catalog: {
providers: {
azureBlob: provider2,
},
},
};
const configMulti = {
catalog: {
providers: {
azureBlob: {
provider1,
provider2,
},
},
},
};
// Single provider case
const actualSingle = readAzureBlobStorageConfigs(
new ConfigReader(configSingle),
);
expect(actualSingle).toHaveLength(1);
expect(actualSingle[0]).toEqual({
id: 'default',
containerName: 'container-2',
schedule: undefined, // no schedule provided in this case
});
// Multiple providers case
const actualMulti = readAzureBlobStorageConfigs(
new ConfigReader(configMulti),
);
expect(actualMulti).toHaveLength(2);
expect(actualMulti[0]).toEqual({
id: 'provider1',
containerName: 'container-1',
schedule: {
...provider1.schedule,
frequency: { minutes: 30 },
},
});
expect(actualMulti[1]).toEqual({
id: 'provider2',
containerName: 'container-2',
schedule: undefined, // no schedule provided
});
});
});
@@ -16,7 +16,9 @@
import { readSchedulerServiceTaskScheduleDefinitionFromConfig } from '@backstage/backend-plugin-api';
import { Config } from '@backstage/config';
import { AzureDevOpsConfig } from './types';
import { AzureDevOpsConfig, AzureBlobStorageConfig } from './types';
const DEFAULT_PROVIDER_ID = 'default';
export function readAzureDevOpsConfigs(config: Config): AzureDevOpsConfig[] {
const configs: AzureDevOpsConfig[] = [];
@@ -61,3 +63,50 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig {
schedule,
};
}
export function readAzureBlobStorageConfigs(
config: Config,
): AzureBlobStorageConfig[] {
const configs: AzureBlobStorageConfig[] = [];
const providerConfigs = config.getOptionalConfig(
'catalog.providers.azureBlob',
);
if (!providerConfigs) {
return configs;
}
if (providerConfigs.has('containerName')) {
// simple/single config variant
configs.push(
readAzureBlobStorageConfig(DEFAULT_PROVIDER_ID, providerConfigs),
);
return configs;
}
for (const id of providerConfigs.keys()) {
configs.push(readAzureBlobStorageConfig(id, providerConfigs.getConfig(id)));
}
return configs;
}
function readAzureBlobStorageConfig(
id: string,
config: Config,
): AzureBlobStorageConfig {
const containerName = config.getString('containerName');
const schedule = config.has('schedule')
? readSchedulerServiceTaskScheduleDefinitionFromConfig(
config.getConfig('schedule'),
)
: undefined;
return {
id,
containerName,
schedule,
};
}
@@ -15,3 +15,4 @@
*/
export { AzureDevOpsEntityProvider } from './AzureDevOpsEntityProvider';
export { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider';
@@ -26,3 +26,9 @@ export type AzureDevOpsConfig = {
path: string;
schedule?: SchedulerServiceTaskScheduleDefinition;
};
export type AzureBlobStorageConfig = {
id: string;
containerName: string;
schedule?: SchedulerServiceTaskScheduleDefinition;
};
+5
View File
@@ -3605,6 +3605,8 @@ __metadata:
"@aws-sdk/credential-providers": ^3.350.0
"@aws-sdk/types": ^3.347.0
"@aws-sdk/util-stream-node": ^3.350.0
"@azure/identity": ^4.0.0
"@azure/storage-blob": ^12.5.0
"@backstage/backend-app-api": "workspace:^"
"@backstage/backend-common": ^0.25.0
"@backstage/backend-dev-utils": "workspace:^"
@@ -4649,6 +4651,7 @@ __metadata:
resolution: "@backstage/integration@workspace:packages/integration"
dependencies:
"@azure/identity": ^4.0.0
"@azure/storage-blob": ^12.5.0
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
"@backstage/config-loader": "workspace:^"
@@ -5396,6 +5399,8 @@ __metadata:
version: 0.0.0-use.local
resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure"
dependencies:
"@azure/identity": ^4.0.0
"@azure/storage-blob": ^12.5.0
"@backstage/backend-plugin-api": "workspace:^"
"@backstage/backend-test-utils": "workspace:^"
"@backstage/cli": "workspace:^"