From c6bc67daf65887ccc9e68f51c96b006b4e9a6cff Mon Sep 17 00:00:00 2001 From: Jacob Bulbul Date: Mon, 28 Apr 2025 19:22:49 +0100 Subject: [PATCH] add Valkey cache support alongside Redis and relevant tests Signed-off-by: Jacob Bulbul --- .changeset/true-breads-rhyme.md | 6 ++ .../config/vocabularies/Backstage/accept.txt | 1 + packages/backend-defaults/config.d.ts | 71 ++++++++++++++++ packages/backend-defaults/package.json | 1 + .../entrypoints/cache/CacheManager.test.ts | 14 ++++ .../src/entrypoints/cache/CacheManager.ts | 45 +++++++++- packages/backend-test-utils/package.json | 1 + packages/backend-test-utils/report.api.md | 2 +- .../src/cache/TestCaches.ts | 16 ++++ .../backend-test-utils/src/cache/types.ts | 9 +- .../src/cache/valkey.test.ts | 34 ++++++++ .../backend-test-utils/src/cache/valkey.ts | 82 +++++++++++++++++++ yarn.lock | 35 ++++++++ 13 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 .changeset/true-breads-rhyme.md create mode 100644 packages/backend-test-utils/src/cache/valkey.test.ts create mode 100644 packages/backend-test-utils/src/cache/valkey.ts diff --git a/.changeset/true-breads-rhyme.md b/.changeset/true-breads-rhyme.md new file mode 100644 index 0000000000..1a81edc71b --- /dev/null +++ b/.changeset/true-breads-rhyme.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-test-utils': minor +'@backstage/backend-defaults': minor +--- + +Added Valkey support alongside Redis in backend-defaults cache clients, using the new Keyv Valkey package. Also extended backend-test-utils to support Valkey in tests. diff --git a/.github/vale/config/vocabularies/Backstage/accept.txt b/.github/vale/config/vocabularies/Backstage/accept.txt index f5defc22f2..c845a6060b 100644 --- a/.github/vale/config/vocabularies/Backstage/accept.txt +++ b/.github/vale/config/vocabularies/Backstage/accept.txt @@ -508,6 +508,7 @@ utils Valentina validator validators +Valkey varchar vite VMware diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index b221edfcb5..14732bbccd 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -660,6 +660,77 @@ export interface Config { }; }; } + | { + store: 'valkey'; + /** + * A valkey connection string in the form `redis://user:pass@host:port`. + * @visibility secret + */ + connection: string; + /** An optional default TTL (in milliseconds, if given as a number). */ + defaultTtl?: number | HumanDuration | string; + valkey?: { + /** + * An optional Valkey client configuration. These options are passed to the `@keyv/valkey` client. + */ + client?: { + /** + * Namespace for the current instance. + */ + namespace?: string; + /** + * Separator to use between namespace and key. + */ + keyPrefixSeparator?: string; + /** + * Number of keys to delete in a single batch. + */ + clearBatchSize?: number; + /** + * Enable Unlink instead of using Del for clearing keys. This is more performant but may not be supported by all Redis versions. + */ + useUnlink?: boolean; + /** + * Whether to allow clearing all keys when no namespace is set. + * If set to true and no namespace is set, iterate() will return all keys. + * Defaults to `false`. + */ + noNamespaceAffectsAll?: boolean; + }; + /** + * An optional Valkey cluster (redis cluster under the hood) configuration. + */ + cluster?: { + /** + * Cluster configuration options to be passed to the `@keyv/valkey` client (and node-redis under the hood) + * https://github.com/redis/node-redis/blob/master/docs/clustering.md + * + * @visibility secret + */ + rootNodes: Array; + /** + * Cluster node default configuration options to be passed to the `@keyv/redis` client (and node-redis under the hood) + * https://github.com/redis/node-redis/blob/master/docs/clustering.md + * + * @visibility secret + */ + defaults?: Partial; + /** + * When `true`, `.connect()` will only discover the cluster topology, without actually connecting to all the nodes. + * Useful for short-term or PubSub-only connections. + */ + minimizeConnections?: boolean; + /** + * When `true`, distribute load by executing readonly commands (such as `GET`, `GEOSEARCH`, etc.) across all cluster nodes. When `false`, only use master nodes. + */ + useReplicas?: boolean; + /** + * The maximum number of times a command will be redirected due to `MOVED` or `ASK` errors. + */ + maxCommandRedirections?: number; + }; + }; + } | { store: 'memcache'; /** diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 4b81d5d1e8..98538600b2 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -144,6 +144,7 @@ "@google-cloud/storage": "^7.0.0", "@keyv/memcache": "^2.0.1", "@keyv/redis": "^4.0.1", + "@keyv/valkey": "^1.0.1", "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^19.0.3", "@opentelemetry/api": "^1.9.0", diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts index 3a7886098f..4e9475b66e 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts @@ -16,6 +16,7 @@ import { mockServices, TestCaches } from '@backstage/backend-test-utils'; import KeyvRedis, { createCluster } from '@keyv/redis'; +import KeyvValkey from '@keyv/valkey'; import KeyvMemcache from '@keyv/memcache'; import { CacheManager } from './CacheManager'; @@ -33,6 +34,16 @@ jest.mock('@keyv/redis', () => { createCluster: jest.fn(), }; }); +jest.mock('@keyv/valkey', () => { + const Actual = jest.requireActual('@keyv/valkey'); + const DefaultConstructor = Actual.default; + return { + ...Actual, + __esModule: true, + default: jest.fn((...args: any[]) => new DefaultConstructor(...args)), + createCluster: jest.fn(), + }; +}); jest.mock('@keyv/memcache', () => { const Actual = jest.requireActual('@keyv/memcache'); const DefaultConstructor = Actual.default; @@ -70,6 +81,9 @@ describe('CacheManager integration', () => { } else if (store === 'memcache') { // eslint-disable-next-line jest/no-conditional-expect expect(KeyvMemcache).toHaveBeenCalledTimes(3); + } else if (store === 'valkey') { + // eslint-disable-next-line jest/no-conditional-expect + expect(KeyvValkey).toHaveBeenCalledTimes(3); } }, ); diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts index ab8b03eb3b..b1a662619d 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts @@ -47,6 +47,7 @@ export class CacheManager { */ private readonly storeFactories = { redis: this.createRedisStoreFactory(), + valkey: this.createValkeyStoreFactory(), memcache: this.createMemcacheStoreFactory(), memory: this.createMemoryStoreFactory(), }; @@ -80,7 +81,7 @@ export class CacheManager { if (config.has('backend.cache.useRedisSets')) { logger?.warn( - "The 'backend.cache.useRedisSets' configuration key is deprecated and no longer has any effect. The underlying '@keyv/redis' library no longer supports redis sets.", + "The 'backend.cache.useRedisSets' configuration key is deprecated and no longer has any effect. The underlying '@keyv/redis' and '@keyv/redis' libraries no longer support redis sets.", ); } @@ -111,7 +112,7 @@ export class CacheManager { /** * Parse store-specific options from configuration. * - * @param store - The cache store type ('redis', 'memcache', or 'memory') + * @param store - The cache store type ('redis', 'valkey', 'memcache', or 'memory') * @param config - The configuration service * @param logger - Optional logger for warnings * @returns The parsed store options @@ -123,7 +124,10 @@ export class CacheManager { ): CacheStoreOptions | undefined { const storeConfigPath = `backend.cache.${store}`; - if (store === 'redis' && config.has(storeConfigPath)) { + if ( + (store === 'redis' || store === 'valkey') && + config.has(storeConfigPath) + ) { return CacheManager.parseRedisOptions(storeConfigPath, config, logger); } @@ -255,6 +259,41 @@ export class CacheManager { }; } + private createValkeyStoreFactory(): StoreFactory { + const KeyvValkey = require('@keyv/valkey').default; + const { createCluster } = require('@keyv/valkey'); + const stores: Record = {}; + + return (pluginId, defaultTtl) => { + if (!stores[pluginId]) { + const valkeyOptions = this.storeOptions?.client || { + keyPrefixSeparator: ':', + }; + if (this.storeOptions?.cluster) { + // Create a Valkey cluster (Redis cluster under the hood) + const cluster = createCluster(this.storeOptions?.cluster); + stores[pluginId] = new KeyvValkey(cluster, valkeyOptions); + } else { + // Create a regular Valkey connection + stores[pluginId] = new KeyvValkey(this.connection, valkeyOptions); + } + + // Always provide an error handler to avoid stopping the process + stores[pluginId].on('error', (err: Error) => { + this.logger?.error('Failed to create valkey cache client', err); + this.errorHandler?.(err); + }); + } + return new Keyv({ + namespace: pluginId, + ttl: defaultTtl, + store: stores[pluginId], + emitErrors: false, + useKeyPrefix: false, + }); + }; + } + private createMemcacheStoreFactory(): StoreFactory { const KeyvMemcache = require('@keyv/memcache').default; const stores: Record = {}; diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index 4a0450cf40..faea90db30 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -55,6 +55,7 @@ "@backstage/types": "workspace:^", "@keyv/memcache": "^2.0.1", "@keyv/redis": "^4.0.1", + "@keyv/valkey": "^1.0.1", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", "@types/keyv": "^4.2.0", diff --git a/packages/backend-test-utils/report.api.md b/packages/backend-test-utils/report.api.md index 947862fae7..7edcf6e1b7 100644 --- a/packages/backend-test-utils/report.api.md +++ b/packages/backend-test-utils/report.api.md @@ -470,7 +470,7 @@ export interface TestBackendOptions { } // @public -export type TestCacheId = 'MEMORY' | 'REDIS_7' | 'MEMCACHED_1'; +export type TestCacheId = 'MEMORY' | 'REDIS_7' | 'VALKEY_8' | 'MEMCACHED_1'; // @public export class TestCaches { diff --git a/packages/backend-test-utils/src/cache/TestCaches.ts b/packages/backend-test-utils/src/cache/TestCaches.ts index e9ba0e5c4f..d4553697bf 100644 --- a/packages/backend-test-utils/src/cache/TestCaches.ts +++ b/packages/backend-test-utils/src/cache/TestCaches.ts @@ -19,6 +19,7 @@ import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; import { connectToExternalMemcache, startMemcachedContainer } from './memcache'; import { connectToExternalRedis, startRedisContainer } from './redis'; import { Instance, TestCacheId, TestCacheProperties, allCaches } from './types'; +import { connectToExternalValkey, startValkeyContainer } from './valkey'; /** * Encapsulates the creation of ephemeral test cache instances for use inside @@ -156,6 +157,8 @@ export class TestCaches { return this.initMemcached(properties); case 'redis': return this.initRedis(properties); + case 'valkey': + return this.initValkey(properties); case 'memory': return { store: 'memory', @@ -196,6 +199,19 @@ export class TestCaches { return await startRedisContainer(properties.dockerImageName!); } + private async initValkey(properties: TestCacheProperties): Promise { + // Use the connection string if provided + const envVarName = properties.connectionStringEnvironmentVariableName; + if (envVarName) { + const connectionString = process.env[envVarName]; + if (connectionString) { + return connectToExternalValkey(connectionString); + } + } + + return await startValkeyContainer(properties.dockerImageName!); + } + private async shutdown() { const instances = [...this.instanceById.values()]; this.instanceById.clear(); diff --git a/packages/backend-test-utils/src/cache/types.ts b/packages/backend-test-utils/src/cache/types.ts index 1aea522042..4781ed42d2 100644 --- a/packages/backend-test-utils/src/cache/types.ts +++ b/packages/backend-test-utils/src/cache/types.ts @@ -22,7 +22,7 @@ import { getDockerImageForName } from '../util/getDockerImageForName'; * * @public */ -export type TestCacheId = 'MEMORY' | 'REDIS_7' | 'MEMCACHED_1'; +export type TestCacheId = 'MEMORY' | 'REDIS_7' | 'VALKEY_8' | 'MEMCACHED_1'; export type TestCacheProperties = { name: string; @@ -58,4 +58,11 @@ export const allCaches: Record = name: 'In-memory', store: 'memory', }, + VALKEY_8: { + name: 'Valkey 8.x', + store: 'valkey', + dockerImageName: getDockerImageForName('valkey/valkey:8'), + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_CACHE_VALKEY8_CONNECTION_STRING', + }, }); diff --git a/packages/backend-test-utils/src/cache/valkey.test.ts b/packages/backend-test-utils/src/cache/valkey.test.ts new file mode 100644 index 0000000000..d706458659 --- /dev/null +++ b/packages/backend-test-utils/src/cache/valkey.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +import { v4 as uuid } from 'uuid'; +import { startValkeyContainer } from './valkey'; + +const itIfDocker = isDockerDisabledForTests() ? it.skip : it; + +jest.setTimeout(60_000); + +describe('startValkeyContainer', () => { + itIfDocker('successfully launches the container', async () => { + const { stop, keyv } = await startValkeyContainer('valkey/valkey:8'); + const value = uuid(); + await keyv.set('test', value); + // eslint-disable-next-line jest/no-standalone-expect + await expect(keyv.get('test')).resolves.toBe(value); + await stop(); + }); +}); diff --git a/packages/backend-test-utils/src/cache/valkey.ts b/packages/backend-test-utils/src/cache/valkey.ts new file mode 100644 index 0000000000..05bb161c94 --- /dev/null +++ b/packages/backend-test-utils/src/cache/valkey.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Keyv from 'keyv'; +import KeyvValkey from '@keyv/valkey'; +import { v4 as uuid } from 'uuid'; +import { Instance } from './types'; + +async function attemptValkeyConnection(connection: string): Promise { + const startTime = Date.now(); + + for (;;) { + try { + const store = new KeyvValkey(connection); + const keyv = new Keyv({ store }); + const value = uuid(); + await keyv.set('test', value); + if ((await keyv.get('test')) === value) { + return keyv; + } + } catch (e) { + if (Date.now() - startTime > 30_000) { + throw new Error( + `Timed out waiting for valkey to be ready for connections, ${e}`, + ); + } + } + + await new Promise(resolve => setTimeout(resolve, 100)); + } +} + +export async function connectToExternalValkey( + connection: string, +): Promise { + const keyv = await attemptValkeyConnection(connection); + return { + store: 'valkey', + connection, + keyv, + stop: async () => await keyv.disconnect(), + }; +} + +export async function startValkeyContainer(image: string): Promise { + // Lazy-load to avoid side-effect of importing testcontainers + const { GenericContainer } = + require('testcontainers') as typeof import('testcontainers'); + + const container = await new GenericContainer(image) + .withExposedPorts(6379) + .start(); + + const host = container.getHost(); + const port = container.getMappedPort(6379); + const connection = `redis://${host}:${port}`; + + const keyv = await attemptValkeyConnection(connection); + + return { + store: 'valkey', + connection, + keyv, + stop: async () => { + await keyv.disconnect(); + await container.stop({ timeout: 10_000 }); + }, + }; +} diff --git a/yarn.lock b/yarn.lock index a6aa5e382c..c49ff64a58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3576,6 +3576,7 @@ __metadata: "@google-cloud/storage": "npm:^7.0.0" "@keyv/memcache": "npm:^2.0.1" "@keyv/redis": "npm:^4.0.1" + "@keyv/valkey": "npm:^1.0.1" "@manypkg/get-packages": "npm:^1.1.3" "@octokit/rest": "npm:^19.0.3" "@opentelemetry/api": "npm:^1.9.0" @@ -3772,6 +3773,7 @@ __metadata: "@backstage/types": "workspace:^" "@keyv/memcache": "npm:^2.0.1" "@keyv/redis": "npm:^4.0.1" + "@keyv/valkey": "npm:^1.0.1" "@types/express": "npm:^4.17.6" "@types/express-serve-static-core": "npm:^4.17.5" "@types/jest": "npm:*" @@ -10909,6 +10911,13 @@ __metadata: languageName: node linkType: hard +"@iovalkey/commands@npm:^0.1.0": + version: 0.1.0 + resolution: "@iovalkey/commands@npm:0.1.0" + checksum: 10/9226ad4b26b8b3bf8446f4aa95bc0ae45bef0d15af7f087a3484e7f4f50f3f8741ba03f4355ebc3b2982d47a2960cb7f39bb83f33256c258fe1ae34bccbc71e1 + languageName: node + linkType: hard + "@isaacs/cliui@npm:^8.0.2": version: 8.0.2 resolution: "@isaacs/cliui@npm:8.0.2" @@ -11419,6 +11428,15 @@ __metadata: languageName: node linkType: hard +"@keyv/valkey@npm:^1.0.1": + version: 1.0.3 + resolution: "@keyv/valkey@npm:1.0.3" + dependencies: + iovalkey: "npm:^0.3.1" + checksum: 10/ff6ba62e4d19c426e45a1437fe215ed2baddc58e811d97507dd75ead0058c3105d679c8c7c4241ddf732abe56320357c2e568c014620e50d7c0eaef1f2528b88 + languageName: node + linkType: hard + "@kubernetes-models/apimachinery@npm:^2.0.0, @kubernetes-models/apimachinery@npm:^2.0.2": version: 2.0.2 resolution: "@kubernetes-models/apimachinery@npm:2.0.2" @@ -32745,6 +32763,23 @@ __metadata: languageName: node linkType: hard +"iovalkey@npm:^0.3.1": + version: 0.3.1 + resolution: "iovalkey@npm:0.3.1" + dependencies: + "@iovalkey/commands": "npm:^0.1.0" + cluster-key-slot: "npm:^1.1.0" + debug: "npm:^4.3.4" + denque: "npm:^2.1.0" + lodash.defaults: "npm:^4.2.0" + lodash.isarguments: "npm:^3.1.0" + redis-errors: "npm:^1.2.0" + redis-parser: "npm:^3.0.0" + standard-as-callback: "npm:^2.1.0" + checksum: 10/afe5e0218810d902263dca2b22dd4501fb74698111f1850804d0948bd6a97793a7f5006757f9b6e8c8131bac6bd532d07ad971e7776bed7f6dc1f6e471706c53 + languageName: node + linkType: hard + "ip-address@npm:^9.0.5": version: 9.0.5 resolution: "ip-address@npm:9.0.5"