paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

upgrade aws-sdk to v3 for search-backend-module-elasticsearch

Browse Source 
This also enables opensearch serverless support

Signed-off-by: Andrew Ochsner <andrew.ochsner@cognizant.com>
This commit is contained in:
Andrew Ochsner
2023-05-02 13:45:16 -05:00
parent 1ebb0a3944
commit 3d72bdb41c
7 changed files with 256 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/fluffy-crabs-reply.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-search-backend-module-elasticsearch': minor
---
Upgrade to aws-sdk v3 and include Opensearch Serverless support
plugins/search-backend-module-elasticsearch/api-report.md
+6 -2
View File
@@ -8,8 +8,8 @@
import { ApiResponse } from '@opensearch-project/opensearch';
import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch';
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
import { BulkHelper } from '@opensearch-project/opensearch/lib/Helpers';
import { BulkStats } from '@opensearch-project/opensearch/lib/Helpers';
import { BulkHelper } from '@elastic/elasticsearch/lib/Helpers';
import { BulkStats } from '@elastic/elasticsearch/lib/Helpers';
import { Config } from '@backstage/config';
import type { ConnectionOptions } from 'tls';
import { IndexableDocument } from '@backstage/plugin-search-common';
@@ -424,6 +424,10 @@ export interface OpenSearchElasticSearchClientOptions
nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
// (undocumented)
provider?: 'aws' | 'opensearch';
// (undocumented)
region?: string;
// (undocumented)
service?: 'es' | 'aoss';
}
// @public (undocumented)
plugins/search-backend-module-elasticsearch/package.json
+3 -3
View File
@@ -36,15 +36,15 @@
"clean": "backstage-cli package clean"
},
"dependencies": {
"@aws-sdk/credential-provider-node": "^3.321.1",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/plugin-search-backend-node": "workspace:^",
"@backstage/plugin-search-common": "workspace:^",
"@elastic/elasticsearch": "^7.13.0",
"@opensearch-project/opensearch": "^2.0.0",
"aws-os-connection": "^0.2.0",
"aws-sdk": "^2.948.0",
"@opensearch-project/opensearch": "^2.2.1",
"aws4": "^1.12.0",
"elastic-builder": "^2.16.0",
"lodash": "^4.17.21",
"uuid": "^8.3.2",
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
+2
View File
@@ -51,6 +51,8 @@ export type ElasticSearchClientOptions =
export interface OpenSearchElasticSearchClientOptions
extends BaseElasticSearchClientOptions {
provider?: 'aws' | 'opensearch';
region?: string;
service?: 'es' | 'aoss';
auth?: OpenSearchAuth;
connection?: OpenSearchConnectionConstructor;
node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts
+1
View File
@@ -229,6 +229,7 @@ describe('ElasticSearchClientWrapper', () => {
osOptions = {
provider: 'aws',
node: 'https://my-es-cluster.eu-west-1.es.amazonaws.com',
region: 'eu-west-1',
// todo(backstage/techdocs-core): Remove the following ts-ignore when
// @short.io/opensearch-mock is updated to work w/opensearch >= 2.0.0
// @ts-ignore
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
+17 -7
View File
@@ -21,9 +21,11 @@ import {
SearchEngine,
SearchQuery,
} from '@backstage/plugin-search-common';
import { awsGetCredentials, createAWSConnection } from 'aws-os-connection';
import { isEmpty, isNumber, isNaN as nan } from 'lodash';
import { defaultProvider } from '@aws-sdk/credential-provider-node'; // V3 SDK.
import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws';
import { RequestSigner } from 'aws4';
import { Config } from '@backstage/config';
import { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
@@ -462,15 +464,14 @@ export async function createElasticSearchClientOptions(
};
}
if (config.getOptionalString('provider') === 'aws') {
const awsCredentials = await awsGetCredentials();
const AWSConnection = createAWSConnection(awsCredentials);
const requestSigner = new RequestSigner(config.getString('node'));
const service =
config.getOptionalString('service') ?? requestSigner.service;
if (service !== 'es' && service !== 'aoss')
throw new Error(`Unrecognized serivce type: ${service}`);
return {
provider: 'aws',
// todo(backstage/techdocs-core): Remove the following ts-ignore when
// aws-os-connection is updated to work with opensearch >= 2.0.0
// @ts-ignore
node: config.getString('node'),
...AWSConnection,
...(sslConfig
? {
ssl: {
@@ -479,6 +480,15 @@ export async function createElasticSearchClientOptions(
},
}
: {}),
...AwsSigv4Signer({
region: config.getOptionalString('region') ?? requestSigner.region, // for backwards compatibility
service: service,
getCredentials: () => {
// Any other method to acquire a new Credentials object can be used.
const credentialsProvider = defaultProvider();
return credentialsProvider();
},
}),
};
}
if (config.getOptionalString('provider') === 'opensearch') {
yarn.lock
+222 -20
View File
@@ -741,6 +741,46 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/client-sso-oidc@npm:3.321.1":
version: 3.321.1
resolution: "@aws-sdk/client-sso-oidc@npm:3.321.1"
dependencies:
"@aws-crypto/sha256-browser": 3.0.0
"@aws-crypto/sha256-js": 3.0.0
"@aws-sdk/config-resolver": 3.310.0
"@aws-sdk/fetch-http-handler": 3.310.0
"@aws-sdk/hash-node": 3.310.0
"@aws-sdk/invalid-dependency": 3.310.0
"@aws-sdk/middleware-content-length": 3.310.0
"@aws-sdk/middleware-endpoint": 3.310.0
"@aws-sdk/middleware-host-header": 3.310.0
"@aws-sdk/middleware-logger": 3.310.0
"@aws-sdk/middleware-recursion-detection": 3.310.0
"@aws-sdk/middleware-retry": 3.310.0
"@aws-sdk/middleware-serde": 3.310.0
"@aws-sdk/middleware-stack": 3.310.0
"@aws-sdk/middleware-user-agent": 3.319.0
"@aws-sdk/node-config-provider": 3.310.0
"@aws-sdk/node-http-handler": 3.321.1
"@aws-sdk/protocol-http": 3.310.0
"@aws-sdk/smithy-client": 3.316.0
"@aws-sdk/types": 3.310.0
"@aws-sdk/url-parser": 3.310.0
"@aws-sdk/util-base64": 3.310.0
"@aws-sdk/util-body-length-browser": 3.310.0
"@aws-sdk/util-body-length-node": 3.310.0
"@aws-sdk/util-defaults-mode-browser": 3.316.0
"@aws-sdk/util-defaults-mode-node": 3.316.0
"@aws-sdk/util-endpoints": 3.319.0
"@aws-sdk/util-retry": 3.310.0
"@aws-sdk/util-user-agent-browser": 3.310.0
"@aws-sdk/util-user-agent-node": 3.310.0
"@aws-sdk/util-utf8": 3.310.0
tslib: ^2.5.0
checksum: 25653c4d788b1cf95fa1246e82c6be4eba588bc5ff150c2b6b7495b73b08ad89efa75a9d52312079326feea56ebfd8f715150bd4a1bcf59dd2e0a282397ff09f
languageName: node
linkType: hard
"@aws-sdk/client-sso@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/client-sso@npm:3.310.0"
@@ -781,6 +821,46 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/client-sso@npm:3.321.1":
version: 3.321.1
resolution: "@aws-sdk/client-sso@npm:3.321.1"
dependencies:
"@aws-crypto/sha256-browser": 3.0.0
"@aws-crypto/sha256-js": 3.0.0
"@aws-sdk/config-resolver": 3.310.0
"@aws-sdk/fetch-http-handler": 3.310.0
"@aws-sdk/hash-node": 3.310.0
"@aws-sdk/invalid-dependency": 3.310.0
"@aws-sdk/middleware-content-length": 3.310.0
"@aws-sdk/middleware-endpoint": 3.310.0
"@aws-sdk/middleware-host-header": 3.310.0
"@aws-sdk/middleware-logger": 3.310.0
"@aws-sdk/middleware-recursion-detection": 3.310.0
"@aws-sdk/middleware-retry": 3.310.0
"@aws-sdk/middleware-serde": 3.310.0
"@aws-sdk/middleware-stack": 3.310.0
"@aws-sdk/middleware-user-agent": 3.319.0
"@aws-sdk/node-config-provider": 3.310.0
"@aws-sdk/node-http-handler": 3.321.1
"@aws-sdk/protocol-http": 3.310.0
"@aws-sdk/smithy-client": 3.316.0
"@aws-sdk/types": 3.310.0
"@aws-sdk/url-parser": 3.310.0
"@aws-sdk/util-base64": 3.310.0
"@aws-sdk/util-body-length-browser": 3.310.0
"@aws-sdk/util-body-length-node": 3.310.0
"@aws-sdk/util-defaults-mode-browser": 3.316.0
"@aws-sdk/util-defaults-mode-node": 3.316.0
"@aws-sdk/util-endpoints": 3.319.0
"@aws-sdk/util-retry": 3.310.0
"@aws-sdk/util-user-agent-browser": 3.310.0
"@aws-sdk/util-user-agent-node": 3.310.0
"@aws-sdk/util-utf8": 3.310.0
tslib: ^2.5.0
checksum: be867824663577684520e1abefb3d7989affabd8a89cbabd3bf0eb4d60bb7d47488b8e8fce4ac29b36c479fa97d27f2de6169d5ef98e748c7ac700fdedb41580
languageName: node
linkType: hard
"@aws-sdk/client-sts@npm:3.312.0, @aws-sdk/client-sts@npm:^3.208.0":
version: 3.312.0
resolution: "@aws-sdk/client-sts@npm:3.312.0"
@@ -890,7 +970,24 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/credential-provider-node@npm:3.310.0, @aws-sdk/credential-provider-node@npm:^3.208.0":
"@aws-sdk/credential-provider-ini@npm:3.321.1":
version: 3.321.1
resolution: "@aws-sdk/credential-provider-ini@npm:3.321.1"
dependencies:
"@aws-sdk/credential-provider-env": 3.310.0
"@aws-sdk/credential-provider-imds": 3.310.0
"@aws-sdk/credential-provider-process": 3.310.0
"@aws-sdk/credential-provider-sso": 3.321.1
"@aws-sdk/credential-provider-web-identity": 3.310.0
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/shared-ini-file-loader": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: ee3d615535efca5cfdd4749f23b02782a5f685289a114799dae46c84e03ba1ebf24ee3b846daaf3b042df43943bd132a83559cc76c4f261cdaadd1150f02a08f
languageName: node
linkType: hard
"@aws-sdk/credential-provider-node@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/credential-provider-node@npm:3.310.0"
dependencies:
@@ -908,6 +1005,24 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/credential-provider-node@npm:^3.208.0, @aws-sdk/credential-provider-node@npm:^3.321.1":
version: 3.321.1
resolution: "@aws-sdk/credential-provider-node@npm:3.321.1"
dependencies:
"@aws-sdk/credential-provider-env": 3.310.0
"@aws-sdk/credential-provider-imds": 3.310.0
"@aws-sdk/credential-provider-ini": 3.321.1
"@aws-sdk/credential-provider-process": 3.310.0
"@aws-sdk/credential-provider-sso": 3.321.1
"@aws-sdk/credential-provider-web-identity": 3.310.0
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/shared-ini-file-loader": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: 899af9f4636b60be7a5dc022bf03d00bb5df8ff1b39214604d2157322766fc368c69457435bfb21a56fc5997a4b13feca32ec8bb53a57562ee2994a9cee4a968
languageName: node
linkType: hard
"@aws-sdk/credential-provider-process@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/credential-provider-process@npm:3.310.0"
@@ -934,6 +1049,20 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/credential-provider-sso@npm:3.321.1":
version: 3.321.1
resolution: "@aws-sdk/credential-provider-sso@npm:3.321.1"
dependencies:
"@aws-sdk/client-sso": 3.321.1
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/shared-ini-file-loader": 3.310.0
"@aws-sdk/token-providers": 3.321.1
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: a7c588394dbe29a2c9ade15dd5d151131963d1da4d14272432f397aa73aa551e5eb7bb5fb7829a628f136927ee689bfad579b7adc18bbc02d7f47e37d745105e
languageName: node
linkType: hard
"@aws-sdk/credential-provider-web-identity@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/credential-provider-web-identity@npm:3.310.0"
@@ -1326,6 +1455,18 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/middleware-user-agent@npm:3.319.0":
version: 3.319.0
resolution: "@aws-sdk/middleware-user-agent@npm:3.319.0"
dependencies:
"@aws-sdk/protocol-http": 3.310.0
"@aws-sdk/types": 3.310.0
"@aws-sdk/util-endpoints": 3.319.0
tslib: ^2.5.0
checksum: a8bcb55eb774a96bc7018fe59ac5c3728743e538f7202815887daeebf3186b102835ac5b3bfd7addea1c3215f04ad8017d19d3f00cfe5635cf0eb4645b379c40
languageName: node
linkType: hard
"@aws-sdk/node-config-provider@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/node-config-provider@npm:3.310.0"
@@ -1338,7 +1479,7 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/node-http-handler@npm:3.310.0, @aws-sdk/node-http-handler@npm:^3.208.0":
"@aws-sdk/node-http-handler@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/node-http-handler@npm:3.310.0"
dependencies:
@@ -1351,6 +1492,19 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/node-http-handler@npm:3.321.1, @aws-sdk/node-http-handler@npm:^3.208.0":
version: 3.321.1
resolution: "@aws-sdk/node-http-handler@npm:3.321.1"
dependencies:
"@aws-sdk/abort-controller": 3.310.0
"@aws-sdk/protocol-http": 3.310.0
"@aws-sdk/querystring-builder": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: b8546b1b20572e1ca02d3792319967712b40c925e4e06f1fb098d70ce55e32a15e514d655e8809650349567e1f46056dbbc3fb9f55df1257c8619127ca01f8fe
languageName: node
linkType: hard
"@aws-sdk/property-provider@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/property-provider@npm:3.310.0"
@@ -1452,6 +1606,17 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/smithy-client@npm:3.316.0":
version: 3.316.0
resolution: "@aws-sdk/smithy-client@npm:3.316.0"
dependencies:
"@aws-sdk/middleware-stack": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: 2a42969efdf6f3e2383aae3f400f9d21623864ffefb4b83da275866621cfa216315e13e4629c5683a280ee61e3bd9908e90b1f4623fbb51fab212fa492a68226
languageName: node
linkType: hard
"@aws-sdk/token-providers@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/token-providers@npm:3.310.0"
@@ -1465,6 +1630,19 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/token-providers@npm:3.321.1":
version: 3.321.1
resolution: "@aws-sdk/token-providers@npm:3.321.1"
dependencies:
"@aws-sdk/client-sso-oidc": 3.321.1
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/shared-ini-file-loader": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: 2a4af584a5bdaa131ee85d62f67cef210d096f558064923c109f8773b402ef2c4830bba3486422b55f33e31371c794dbaecb07cf3d889250704bd815491569df
languageName: node
linkType: hard
"@aws-sdk/types@npm:3.310.0, @aws-sdk/types@npm:^3.208.0, @aws-sdk/types@npm:^3.222.0":
version: 3.310.0
resolution: "@aws-sdk/types@npm:3.310.0"
@@ -1553,6 +1731,18 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/util-defaults-mode-browser@npm:3.316.0":
version: 3.316.0
resolution: "@aws-sdk/util-defaults-mode-browser@npm:3.316.0"
dependencies:
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/types": 3.310.0
bowser: ^2.11.0
tslib: ^2.5.0
checksum: abd989642a7f04dd87fef911623820e2f2e80932a663b157a716eb87907fbea99c28cbf706c5284b019db71641b464c8ae1448dc22ebfa9e4b3e6d627f800522
languageName: node
linkType: hard
"@aws-sdk/util-defaults-mode-node@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/util-defaults-mode-node@npm:3.310.0"
@@ -1567,6 +1757,20 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/util-defaults-mode-node@npm:3.316.0":
version: 3.316.0
resolution: "@aws-sdk/util-defaults-mode-node@npm:3.316.0"
dependencies:
"@aws-sdk/config-resolver": 3.310.0
"@aws-sdk/credential-provider-imds": 3.310.0
"@aws-sdk/node-config-provider": 3.310.0
"@aws-sdk/property-provider": 3.310.0
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: 28408b0ab9717750774d642d30ddf2b547093837bb69706bd2d0d9dc11f1d331c98c0f44412fc1b361b9a17b36f9bb970225b45ed0ec42ed17d2af242d864106
languageName: node
linkType: hard
"@aws-sdk/util-endpoints@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/util-endpoints@npm:3.310.0"
@@ -1577,6 +1781,16 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/util-endpoints@npm:3.319.0":
version: 3.319.0
resolution: "@aws-sdk/util-endpoints@npm:3.319.0"
dependencies:
"@aws-sdk/types": 3.310.0
tslib: ^2.5.0
checksum: 7b3e67494bab98f7e979fed0e172ea2179f29184fffea0954e0173f89128367996c09ee9bb2a86b500f8d44ad024b4efc550610b2daa943e223b2050c7a32b56
languageName: node
linkType: hard
"@aws-sdk/util-hex-encoding@npm:3.310.0":
version: 3.310.0
resolution: "@aws-sdk/util-hex-encoding@npm:3.310.0"
@@ -8402,6 +8616,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "@backstage/plugin-search-backend-module-elasticsearch@workspace:plugins/search-backend-module-elasticsearch"
dependencies:
"@aws-sdk/credential-provider-node": ^3.321.1
"@backstage/backend-common": "workspace:^"
"@backstage/backend-plugin-api": "workspace:^"
"@backstage/cli": "workspace:^"
@@ -8410,10 +8625,9 @@ __metadata:
"@backstage/plugin-search-common": "workspace:^"
"@elastic/elasticsearch": ^7.13.0
"@elastic/elasticsearch-mock": ^1.0.0
"@opensearch-project/opensearch": ^2.0.0
"@opensearch-project/opensearch": ^2.2.1
"@short.io/opensearch-mock": ^0.3.1
aws-os-connection: ^0.2.0
aws-sdk: ^2.948.0
aws4: ^1.12.0
elastic-builder: ^2.16.0
lodash: ^4.17.21
uuid: ^8.3.2
@@ -13447,7 +13661,7 @@ __metadata:
languageName: node
linkType: hard
"@opensearch-project/opensearch@npm:^2.0.0":
"@opensearch-project/opensearch@npm:^2.2.1":
version: 2.2.1
resolution: "@opensearch-project/opensearch@npm:2.2.1"
dependencies:
@@ -18377,18 +18591,6 @@ __metadata:
languageName: node
linkType: hard
"aws-os-connection@npm:^0.2.0":
version: 0.2.0
resolution: "aws-os-connection@npm:0.2.0"
dependencies:
aws4: ^1.11.0
peerDependencies:
"@opensearch-project/opensearch": 2
aws-sdk: ">=2"
checksum: 1708ffd9bdc5f7bd6b8447f97859936a8657f4df567f61f12dacd040e6f1a25020bbea8027bb28f19fe27d171a0d3a5090c2d84d6f752bb04e387640413fec7c
languageName: node
linkType: hard
"aws-sdk-client-mock-jest@npm:^2.0.0":
version: 2.1.1
resolution: "aws-sdk-client-mock-jest@npm:2.1.1"
@@ -18423,7 +18625,7 @@ __metadata:
languageName: node
linkType: hard
"aws-sdk@npm:^2.1231.0, aws-sdk@npm:^2.840.0, aws-sdk@npm:^2.948.0":
"aws-sdk@npm:^2.1231.0, aws-sdk@npm:^2.840.0":
version: 2.1279.0
resolution: "aws-sdk@npm:2.1279.0"
dependencies:
@@ -18448,7 +18650,7 @@ __metadata:
languageName: node
linkType: hard
"aws4@npm:^1.11.0, aws4@npm:^1.8.0":
"aws4@npm:^1.11.0, aws4@npm:^1.12.0, aws4@npm:^1.8.0":
version: 1.12.0
resolution: "aws4@npm:1.12.0"
checksum: 68f79708ac7c335992730bf638286a3ee0a645cf12575d557860100767c500c08b30e24726b9f03265d74116417f628af78509e1333575e9f8d52a80edfe8cbc