Merge pull request #6882 from splunk/seant-splunk/awsS3_readTree_processor

Add readTree() to AwsS3UrlReader, ReadableArrayResponse type, and AwsS3ReadTreeProcessor

plugins/catalog-backend/api-report.md

@@ -158,6 +158,20 @@ export type AwsOrganizationProviderConfig = {
   roleArn?: string;
 };
 
+// Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3DiscoveryProcessor implements CatalogProcessor {
+  constructor(reader: UrlReader);
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean>;
+}
+
 // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/catalog-backend/package.json

@@ -73,6 +73,7 @@
     "@types/supertest": "^2.0.8",
     "@types/uuid": "^8.0.0",
     "@types/yup": "^0.29.8",
+    "aws-sdk-mock": "^5.2.1",
     "msw": "^0.29.0",
     "sqlite3": "^5.0.1",
     "supertest": "^6.1.3",

plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger, UrlReaders } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
+import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
+import { defaultEntityDataParser } from './util/parse';
+import AWSMock from 'aws-sdk-mock';
+import aws from 'aws-sdk';
+import path from 'path';
+
+AWSMock.setSDKInstance(aws);
+const object: aws.S3.Types.Object = {
+  Key: 'awsS3-mock-object.txt',
+};
+const objectList: aws.S3.ObjectList = [object];
+const output: aws.S3.Types.ListObjectsV2Output = {
+  Contents: objectList,
+};
+AWSMock.mock('S3', 'listObjectsV2', output);
+AWSMock.mock(
+  'S3',
+  'getObject',
+  Buffer.from(
+    require('fs').readFileSync(
+      path.resolve(
+        'src',
+        'ingestion',
+        'processors',
+        '__fixtures__',
+        'fileReaderProcessor',
+        'awsS3',
+        'awsS3-mock-object.txt',
+      ),
+    ),
+  ),
+);
+
+const logger = getVoidLogger();
+const reader = UrlReaders.default({
+  logger,
+  config: new ConfigReader({
+    backend: { reading: { allow: [{ host: 'localhost' }] } },
+  }),
+});
+
+describe('readLocation', () => {
+  const processor = new AwsS3DiscoveryProcessor(reader);
+  const spec = {
+    type: 's3-discovery',
+    target: 'https://testbucket.s3.us-east-2.amazonaws.com',
+  };
+
+  it('should load from url', async () => {
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
+    )) as CatalogProcessorEntityResult;
+    expect(generated.type).toBe('entity');
+    expect(generated.location).toEqual({
+      target: 'awsS3-mock-object.txt',
+      type: 's3-discovery',
+    });
+    expect(generated.entity).toEqual({ site_name: 'Test' });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import limiterFactory from 'p-limit';
+import * as result from './results';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+} from './types';
+
+export class AwsS3DiscoveryProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean> {
+    if (location.type !== 's3-discovery') {
+      return false;
+    }
+
+    try {
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          emit(parseResult);
+        }
+      }
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
+
+      if (error.name === 'NotFoundError') {
+        if (!optional) {
+          emit(result.notFoundError(location, message));
+        }
+      } else {
+        emit(result.generalError(location, message));
+      }
+    }
+    return true;
+  }
+
+  private async doRead(
+    location: string,
+  ): Promise<{ data: Buffer; url: string }[]> {
+    const limiter = limiterFactory(5);
+    const response = await this.reader.readTree(location);
+    const responseFiles = await response.files();
+    const output = responseFiles.map(async file => ({
+      url: file.path,
+      data: await limiter(file.content),
+    }));
+    return Promise.all(output);
+  }
+}

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt

@@ -0,0 +1 @@
+site_name: Test

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -20,6 +20,7 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
 export type { AwsOrganizationProviderConfig } from './awsOrganization/config';
+export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';