From c05a6982eabffaddc1915faa720bffe65ac1777e Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Tue, 4 Feb 2025 08:06:37 +0200 Subject: [PATCH] feat: support postgres store for rate limiting Signed-off-by: Hellgren Heikki --- .changeset/famous-terms-rescue.md | 4 +- app-config.yaml | 2 +- packages/backend-defaults/config.d.ts | 16 +++ packages/backend-defaults/package.json | 1 + .../http/RateLimitStoreFactory.test.ts | 21 ++- .../http/RateLimitStoreFactory.ts | 15 ++ yarn.lock | 130 +++++++++++++++++- 7 files changed, 184 insertions(+), 5 deletions(-) diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md index 9f38986934..68fd1f3591 100644 --- a/.changeset/famous-terms-rescue.md +++ b/.changeset/famous-terms-rescue.md @@ -4,12 +4,12 @@ Added new rate limit middleware to allow rate limiting requests to the backend -If you are using the `configure` callback of the root HTTP router service and do NOT call `applyDefaults()` inside it, please see [the relevant changes](https://github.com/backstage/backstage/pull/26725/files#diff-86ad1b6a694dd250823aee39d410428dd837c9d9a04ca8c33bd1081fbe3f22af) that were made, to see if you want to apply them as well to your custom configuration. +If you are using the `configure` callback of the root HTTP router service and do NOT call `applyDefaults()` inside it, please see [the relevant changes](https://github.com/backstage/backstage/pull/28708/files#diff-86ad1b6a694dd250823aee39d410428dd837c9d9a04ca8c33bd1081fbe3f22af) that were made, to see if you want to apply them as well to your custom configuration. Rate limiting can be turned on by adding the following configuration to `app-config.yaml`: ```yaml backend: rateLimit: - window: 6000ms + window: 6s incomingRequestLimit: 100 ``` diff --git a/app-config.yaml b/app-config.yaml index df80c7add0..89926357c2 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -38,7 +38,7 @@ backend: # Used for testing rate limiting locally # rateLimit: - # windowMs: 60000 + # windowMs: 1m # incomingRequestLimit: 1 # ipAllowList: [] diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 6cd1aa183a..67183896bb 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -801,6 +801,22 @@ export interface Config { client: 'redis'; connection: string; } + | { + client: 'postgres'; + connection: + | string + | { + /** + * @visibility secret + */ + password?: string; + /** + * Other connection settings + * @see https://node-postgres.com/apis/client + */ + [key: string]: unknown; + }; + } | { client: 'memory'; }; diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index b1f9513f2c..bca883c44e 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -130,6 +130,7 @@ "test": "backstage-cli package test" }, "dependencies": { + "@acpr/rate-limit-postgresql": "^1.4.1", "@aws-sdk/abort-controller": "^3.347.0", "@aws-sdk/client-codecommit": "^3.350.0", "@aws-sdk/client-s3": "^3.350.0", diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts index c1f690474e..6fe4a2d327 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -15,6 +15,8 @@ */ import { mockServices } from '@backstage/backend-test-utils'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; +import { RedisStore } from 'rate-limit-redis'; +import { PostgresStore } from '@acpr/rate-limit-postgresql'; jest.mock('@keyv/redis', () => { const Actual = jest.requireActual('@keyv/redis'); @@ -62,6 +64,23 @@ describe('CacheRateLimitStoreFactory', () => { }, }); const store = RateLimitStoreFactory.create(config); - expect(store).not.toBeUndefined(); + expect(store).toBeInstanceOf(RedisStore); + }); + + it('should return postgres store if configured explicitly', async () => { + const config = mockServices.rootConfig({ + data: { + backend: { + rateLimit: { + store: { + client: 'postgres', + connection: 'postgres://localhost:5432', + }, + }, + }, + }, + }); + const store = RateLimitStoreFactory.create(config); + expect(store).toBeInstanceOf(PostgresStore); }); }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts index 1275095aca..3905655587 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -16,6 +16,8 @@ import { Config } from '@backstage/config'; import type { Store } from 'express-rate-limit'; import { RedisStore } from 'rate-limit-redis'; +import { parsePgConnectionString } from '../../database/connectors/postgres.ts'; +import { PostgresStore } from '@acpr/rate-limit-postgresql'; /** * Creates a store for `express-rate-limit` based on the configuration. @@ -32,6 +34,9 @@ export class RateLimitStoreFactory { switch (client) { case 'redis': return this.redis(store); + case 'postgres': + return this.postgres(store); + case 'memory': default: return undefined; } @@ -48,4 +53,14 @@ export class RateLimitStoreFactory { }, }); } + + private static postgres(storeConfig: Config): Store { + const connection = storeConfig.get('connection') as any; + const isConnectionString = + typeof connection === 'string' || connection instanceof String; + const connectionOptions = isConnectionString + ? parsePgConnectionString(connection as string) + : connection; + return new PostgresStore(connectionOptions, 'rl'); + } } diff --git a/yarn.lock b/yarn.lock index aebb875a10..66f819db26 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,6 +12,20 @@ __metadata: languageName: node linkType: hard +"@acpr/rate-limit-postgresql@npm:^1.4.1": + version: 1.4.1 + resolution: "@acpr/rate-limit-postgresql@npm:1.4.1" + dependencies: + "@types/pg-pool": "npm:2.0.3" + pg: "npm:8.11.3" + pg-pool: "npm:3.6.1" + postgres-migrations: "npm:5.3.0" + peerDependencies: + express-rate-limit: ">=6.0.0" + checksum: 10/9295f86890ea10f0be24a211f100cfe9dde40df20d8328be36a66736e36ee7043dc6fcae785e39bea19de3f43ad344f3e0fa3f9d40bc8d89d38bf6ce457bcc28 + languageName: node + linkType: hard + "@adobe/css-tools@npm:^4.4.0": version: 4.4.0 resolution: "@adobe/css-tools@npm:4.4.0" @@ -3580,6 +3594,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/backend-defaults@workspace:packages/backend-defaults" dependencies: + "@acpr/rate-limit-postgresql": "npm:^1.4.1" "@aws-sdk/abort-controller": "npm:^3.347.0" "@aws-sdk/client-codecommit": "npm:^3.350.0" "@aws-sdk/client-s3": "npm:^3.350.0" @@ -20987,6 +21002,15 @@ __metadata: languageName: node linkType: hard +"@types/pg-pool@npm:2.0.3": + version: 2.0.3 + resolution: "@types/pg-pool@npm:2.0.3" + dependencies: + "@types/pg": "npm:*" + checksum: 10/9ea0bcdbdd09c9de6f774e59465189e552ee094901724278082c41ba6287e7fddffb9ba4b4107c242bba4e8f8a1f0016e6a1eb0c6ca306d43c08b5ddd7f34549 + languageName: node + linkType: hard + "@types/pg-pool@npm:2.0.6": version: 2.0.6 resolution: "@types/pg-pool@npm:2.0.6" @@ -25149,6 +25173,13 @@ __metadata: languageName: node linkType: hard +"buffer-writer@npm:2.0.0": + version: 2.0.0 + resolution: "buffer-writer@npm:2.0.0" + checksum: 10/fdca8e28c55704de7af2f41c8f875293de69ad22005d5041d54aa916d125cead00afa969bc09e4702ae6b66e098409958c06bebfc97fcf8fa4ea5afcae088cd9 + languageName: node + linkType: hard + "buffer-xor@npm:^1.0.3": version: 1.0.3 resolution: "buffer-xor@npm:1.0.3" @@ -39963,6 +39994,13 @@ __metadata: languageName: node linkType: hard +"packet-reader@npm:1.0.0": + version: 1.0.0 + resolution: "packet-reader@npm:1.0.0" + checksum: 10/8504cc8c32672380867e933516a029b1d4dd784c139213c85c9042ffc1162de48ec914f8c71260a9311518694cf5d0be11c67357f4b536129d2ea42aa7257ec0 + languageName: node + linkType: hard + "pacote@npm:^12.0.0, pacote@npm:^12.0.2": version: 12.0.3 resolution: "pacote@npm:12.0.3" @@ -40492,7 +40530,7 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.7.0": +"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.6.2, pg-connection-string@npm:^2.7.0": version: 2.7.0 resolution: "pg-connection-string@npm:2.7.0" checksum: 10/68015a8874b7ca5dad456445e4114af3d2602bac2fdb8069315ecad0ff9660ec93259b9af7186606529ac4f6f72a06831e6f20897a689b16cc7fda7ca0e247fd @@ -40520,6 +40558,24 @@ __metadata: languageName: node linkType: hard +"pg-pool@npm:3.6.1": + version: 3.6.1 + resolution: "pg-pool@npm:3.6.1" + peerDependencies: + pg: ">=8.0" + checksum: 10/5d1b02b959e6c849004d8f3d2222c48d3b3b67b7b1eb5f2e5819ed9412129ea6b0f0376bc74ddf197973c99575d325cbb3f64a8017ab520535c011329b12fffb + languageName: node + linkType: hard + +"pg-pool@npm:^3.6.1, pg-pool@npm:^3.7.0": + version: 3.7.0 + resolution: "pg-pool@npm:3.7.0" + peerDependencies: + pg: ">=8.0" + checksum: 10/a07a4f9e26eec9d7ac3597dc7b3469c62983edff9a321dbb7acbe1bbc7f5e9b2d33438e277d4cf8145071f3d63c7ebdc287a539fd69dfb8cdddb15b33eefe1a2 + languageName: node + linkType: hard + "pg-pool@npm:^3.8.0": version: 3.8.0 resolution: "pg-pool@npm:3.8.0" @@ -40536,6 +40592,13 @@ __metadata: languageName: node linkType: hard +"pg-protocol@npm:^1.6.0, pg-protocol@npm:^1.7.0": + version: 1.7.0 + resolution: "pg-protocol@npm:1.7.0" + checksum: 10/ffffdf74426c9357b57050f1c191e84447c0e8b2a701b3ab302ac7dd0eb27b862d92e5e3b2d38876a1051de83547eb9165d6a58b3a8e90bb050dae97f9993d54 + languageName: node + linkType: hard + "pg-types@npm:^2.1.0, pg-types@npm:^2.2.0": version: 2.2.0 resolution: "pg-types@npm:2.2.0" @@ -40564,6 +40627,30 @@ __metadata: languageName: node linkType: hard +"pg@npm:8.11.3": + version: 8.11.3 + resolution: "pg@npm:8.11.3" + dependencies: + buffer-writer: "npm:2.0.0" + packet-reader: "npm:1.0.0" + pg-cloudflare: "npm:^1.1.1" + pg-connection-string: "npm:^2.6.2" + pg-pool: "npm:^3.6.1" + pg-protocol: "npm:^1.6.0" + pg-types: "npm:^2.1.0" + pgpass: "npm:1.x" + peerDependencies: + pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true + peerDependenciesMeta: + pg-native: + optional: true + checksum: 10/f15f29c8e17723ee1da72abdf400cbed2c04602c58c93687f3f0068e71df2a6fb62b9a3543e13da21b10a0494f4c5b4cfc8d6cd8396617b76c4cbfd6ddab17e7 + languageName: node + linkType: hard + "pg@npm:^8.11.3, pg@npm:^8.9.0": version: 8.14.1 resolution: "pg@npm:8.14.1" @@ -40586,6 +40673,28 @@ __metadata: languageName: node linkType: hard +"pg@npm:^8.6.0": + version: 8.13.1 + resolution: "pg@npm:8.13.1" + dependencies: + pg-cloudflare: "npm:^1.1.1" + pg-connection-string: "npm:^2.7.0" + pg-pool: "npm:^3.7.0" + pg-protocol: "npm:^1.7.0" + pg-types: "npm:^2.1.0" + pgpass: "npm:1.x" + peerDependencies: + pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true + peerDependenciesMeta: + pg-native: + optional: true + checksum: 10/542aa49fcb37657cf5f779b4a31fe6eb336e683445ecca38e267eeb0ca85d873ffe51f04794f9f9e184187e9f74bf7895e932a0fa9507132ac0dfc76c7c73451 + languageName: node + linkType: hard + "pgpass@npm:1.x": version: 1.0.2 resolution: "pgpass@npm:1.0.2" @@ -41353,6 +41462,18 @@ __metadata: languageName: node linkType: hard +"postgres-migrations@npm:5.3.0": + version: 5.3.0 + resolution: "postgres-migrations@npm:5.3.0" + dependencies: + pg: "npm:^8.6.0" + sql-template-strings: "npm:^2.2.2" + bin: + pg-validate-migrations: dist/bin/validate.js + checksum: 10/520d95f01144f88689d5c0a7575743c4f99536935deb1ffff7b3765883a688c4f001d98e8b493ca9b342cd2609593970c3d2198b41fade648f102008e3607226 + languageName: node + linkType: hard + "postgres-range@npm:^1.1.1": version: 1.1.3 resolution: "postgres-range@npm:1.1.3" @@ -45172,6 +45293,13 @@ __metadata: languageName: node linkType: hard +"sql-template-strings@npm:^2.2.2": + version: 2.2.2 + resolution: "sql-template-strings@npm:2.2.2" + checksum: 10/594378a44acbaf3db8a4067137c0c315d0656fcc1b6b8fa76c760d032c1970bf6ede2b31690a3bdc6482d86cbff8b202bb14f6528aa1d9d6bf19d48b03ba2744 + languageName: node + linkType: hard + "sqlstring@npm:^2.3.2": version: 2.3.2 resolution: "sqlstring@npm:2.3.2"