Adding MySQL support to the catalog-backend plugin

Signed-off-by: tonedef <kobylk@gmail.com>

.changeset/nice-pants-boil.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Adds MySQL support for the catalog-backend

packages/backend-common/src/database/util.ts

@@ -29,6 +29,8 @@ export function isDatabaseConflictError(e: unknown) {
     typeof message === 'string' &&
     (/SQLITE_CONSTRAINT(?:_UNIQUE)?: UNIQUE/.test(message) ||
       /UNIQUE constraint failed:/.test(message) ||
-      /unique constraint/.test(message))
+      /unique constraint/.test(message) ||
+      /Duplicate entry/.test(message) // MySQL uniqueness error msg
+    )
   );
 }

plugins/catalog-backend/migrations/20200511113813_init.js

@@ -59,7 +59,7 @@ exports.up = async function up(knex) {
             'An opaque string that changes for each update operation to any part of the entity, including metadata.',
           );
         table
-          .string('generation')
+          .integer('generation')
           .notNullable()
           .unsigned()
           .comment(

plugins/catalog-backend/migrations/20200702153613_entities.js

@@ -31,6 +31,7 @@ exports.up = async function up(knex) {
   }
   await knex.schema.alterTable('entities', table => {
     table.dropUnique([], 'entities_unique_name');
+    table.dropForeign(['location_id']);
   });
   // Setup temporary tables
   await knex.schema.renameTable('entities_search', 'tmp_entities_search');
@@ -56,7 +57,7 @@ exports.up = async function up(knex) {
           'An opaque string that changes for each update operation to any part of the entity, including metadata.',
         );
       table
-        .string('generation')
+        .integer('generation')
         .notNullable()
         .unsigned()
         .comment(

plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js

@@ -24,8 +24,8 @@ exports.up = async function up(knex) {
     .where({ namespace: null })
     .update({ namespace: 'default' });
   await knex('entities_search').update({
-    key: knex.raw('LOWER(key)'),
-    value: knex.raw('LOWER(value)'),
+    key: knex.raw('LOWER(??)', ['key']),
+    value: knex.raw('LOWER(??)', ['value']),
   });
 };
 

plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js

@@ -21,19 +21,20 @@
  */
 exports.up = async function up(knex) {
   await knex.schema.alterTable('entities', table => {
-    table.text('full_name').nullable();
+    table.string('full_name').nullable();
   });
 
   await knex('entities').update({
     full_name: knex.raw(
-      "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)",
+      "LOWER(??) || ':' || LOWER(COALESCE(??, 'default')) || '/' || LOWER(??)",
+      ['kind', 'namespace', 'name'],
     ),
   });
 
   // SQLite does not support alter column
   if (!knex.client.config.client.includes('sqlite3')) {
     await knex.schema.alterTable('entities', table => {
-      table.text('full_name').notNullable().alter({ alterNullable: true });
+      table.string('full_name').notNullable().alter();
     });
   }
 

plugins/catalog-backend/migrations/20201006130744_entity_data_column.js

@@ -31,7 +31,8 @@ exports.up = async function up(knex) {
     // apiVersion and kind should not contain any JSON unsafe chars, and both
     // metadata and spec are already valid serialized JSON
     data: knex.raw(
-      `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`,
+      `'{"apiVersion":"' || ?? || '","kind":"' || ?? || '","metadata":' || ?? || COALESCE(',"spec":' || ??, '') || '}'`,
+      ['api_version', 'kind', 'metadata', 'spec'],
     ),
   });
 

plugins/catalog-backend/migrations/20201007201501_index_entity_search.js

@@ -21,8 +21,17 @@
  */
 exports.up = async function up(knex) {
   await knex.schema.alterTable('entities_search', table => {
-    table.index(['key'], 'entities_search_key');
-    table.index(['value'], 'entities_search_value');
+    if (knex.client.config.client.includes('mysql')) {
+      table.index(['key'], 'entities_search_key', {
+        indexType: 'FULLTEXT',
+      });
+      table.index(['value'], 'entities_search_value', {
+        indexType: 'FULLTEXT',
+      });
+    } else {
+      table.index(['key'], 'entities_search_key');
+      table.index(['value'], 'entities_search_value');
+    }
   });
 };
 

plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js

@@ -24,7 +24,7 @@ exports.up = async function up(knex) {
     // sqlite doesn't support dropPrimary so we recreate it properly instead
     await knex.schema.dropTable('entities_relations');
     await knex.schema.createTable('entities_relations', table => {
-      table.comment('All relations between entities in the catalog');
+      table.comment('All relations between entities');
       table
         .uuid('originating_entity_id')
         .references('id')
@@ -61,7 +61,7 @@ exports.down = async function down(knex) {
   if (knex.client.config.client.includes('sqlite3')) {
     await knex.schema.dropTable('entities_relations');
     await knex.schema.createTable('entities_relations', table => {
-      table.comment('All relations between entities in the catalog');
+      table.comment('All relations between entities');
       table
         .uuid('originating_entity_id')
         .references('id')

plugins/catalog-backend/migrations/20210302150147_refresh_state.js

@@ -21,38 +21,28 @@
  */
 exports.up = async function up(knex) {
   await knex.schema.createTable('refresh_state', table => {
-    table.comment(
-      'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.',
-    );
+    table.comment('Location refresh states');
     table
-      .text('entity_id')
+      .string('entity_id')
       .primary()
       .notNullable()
-      .comment(
-        'Primary ID, which will also be used as the uid of the resulting entity',
-      );
+      .comment('Primary ID, also used as the uid of the entity');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .notNullable()
-      .comment('A reference to the entity that the refresh state is tied to');
+      .comment('A reference to the entity for this refresh state');
     table
       .text('unprocessed_entity')
       .notNullable()
-      .comment(
-        'The unprocessed entity (in its source form, before being run through all of the processors) as JSON',
-      );
+      .comment('The unprocessed entity (in original form) as JSON');
     table
       .text('processed_entity')
       .nullable()
-      .comment(
-        'The processed entity (after running through all processors, but before being stitched together with state and relations) as JSON',
-      );
+      .comment('The processed entity (not yet stitched) as JSON');
     table
       .text('cache')
       .nullable()
-      .comment(
-        'Cache information tied to the refreshing of this entity, such as etag information or actual response caching',
-      );
+      .comment('Cache information tied to refreshes of this entity');
     table
       .text('errors')
       .notNullable()
@@ -64,41 +54,28 @@ exports.up = async function up(knex) {
     table
       .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar
       .notNullable()
-      .comment('The last timestamp of which this entity was discovered');
-    table.unique(['entity_ref'], {
-      indexName: 'refresh_state_entity_ref_uniq',
-    });
+      .comment('The last timestamp that this entity was discovered');
+    table.unique(['entity_ref'], 'refresh_state_entity_ref_uniq');
     table.index('entity_id', 'refresh_state_entity_id_idx');
     table.index('entity_ref', 'refresh_state_entity_ref_idx');
     table.index('next_update_at', 'refresh_state_next_update_at_idx');
   });
 
   await knex.schema.createTable('final_entities', table => {
-    table.comment(
-      'This table contains the final entity result after processing and stitching',
-    );
+    table.comment('Final entities after processing and stitching');
     table
-      .text('entity_id')
+      .string('entity_id')
       .primary()
       .notNullable()
       .references('entity_id')
       .inTable('refresh_state')
       .onDelete('CASCADE')
-      .comment(
-        'Entity ID which corresponds to the ID in the refresh_state table',
-      );
+      .comment('Entity ID -> refresh_state table');
+    table.text('hash').notNullable().comment('Stable hash of the entity data');
     table
-      .text('hash')
+      .string('stitch_ticket')
       .notNullable()
-      .comment(
-        'Stable hash of the entity data, to be used for caching and avoiding redundant work',
-      );
-    table
-      .text('stitch_ticket')
-      .notNullable()
-      .comment(
-        'A random value representing a unique stitch attempt ticket, that gets updated each time that a stitching attempt is made on the entity',
-      );
+      .comment('Random value representing a unique stitch attempt ticket');
     table
       .text('final_entity')
       .nullable()
@@ -107,29 +84,23 @@ exports.up = async function up(knex) {
   });
 
   await knex.schema.createTable('refresh_state_references', table => {
-    table.comment(
-      'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.',
-    );
+    table.comment('Edges between refresh state rows');
     table
       .increments('id')
       .comment('Primary key to distinguish unique lines from each other');
     table
-      .text('source_key')
+      .string('source_key')
       .nullable()
-      .comment(
-        'When the reference source is not an entity, this is an opaque identifier for that source.',
-      );
+      .comment('Opaque identifier for non-entity sources');
     table
-      .text('source_entity_ref')
+      .string('source_entity_ref')
       .nullable()
       .references('entity_ref')
       .inTable('refresh_state')
       .onDelete('CASCADE')
-      .comment(
-        'When the reference source is an entity, this is the EntityRef of the source entity.',
-      );
+      .comment('EntityRef of entity sources');
     table
-      .text('target_entity_ref')
+      .string('target_entity_ref')
       .notNullable()
       .references('entity_ref')
       .inTable('refresh_state')
@@ -147,36 +118,34 @@ exports.up = async function up(knex) {
   });
 
   await knex.schema.createTable('relations', table => {
-    table.comment('All relations between entities in the catalog');
+    table.comment('All relations between entities');
     table
-      .text('originating_entity_id')
+      .string('originating_entity_id')
       .references('entity_id')
       .inTable('refresh_state')
       .onDelete('CASCADE')
       .notNullable()
       .comment('The entity that provided the relation');
     table
-      .text('source_entity_ref')
+      .string('source_entity_ref')
       .notNullable()
-      .comment('The entity reference of the source entity of the relation');
+      .comment('Entity reference of the source entity of the relation');
     table
-      .text('type')
+      .string('type')
       .notNullable()
       .comment('The type of the relation between the entities');
     table
-      .text('target_entity_ref')
+      .string('target_entity_ref')
       .notNullable()
-      .comment('The entity reference of the target entity of the relation');
+      .comment('Entity reference of the target entity of the relation');
     table.index('source_entity_ref', 'relations_source_entity_ref_idx');
     table.index('originating_entity_id', 'relations_source_entity_id_idx');
   });
 
   await knex.schema.createTable('search', table => {
-    table.comment(
-      'Flattened key-values from the entities, used for quick filtering',
-    );
+    table.comment('Flattened key-values from the entities, for filtering');
     table
-      .text('entity_id')
+      .string('entity_id')
       .references('entity_id')
       .inTable('refresh_state')
       .onDelete('CASCADE')

plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js

@@ -24,9 +24,7 @@ exports.up = async function up(knex) {
     table
       .text('location_key')
       .nullable()
-      .comment(
-        'An opaque key that uniquely identifies the location of an entity in order to support conflict resolution',
-      );
+      .comment('Opaque conflict resolution key');
   });
 };
 

plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js

@@ -24,7 +24,7 @@ exports.up = async function up(knex) {
     table
       .text('unprocessed_hash')
       .nullable()
-      .comment('A hash of the unprocessed contents, used to detect changes');
+      .comment('A hash of the unprocessed contents');
   });
 };
 

plugins/catalog-backend/migrations/20220616202842_refresh_keys.js

@@ -23,14 +23,14 @@ exports.up = async function up(knex) {
       'This table contains relations between entities and keys to trigger refreshes with',
     );
     table
-      .text('entity_id')
+      .string('entity_id')
       .notNullable()
       .references('entity_id')
       .inTable('refresh_state')
       .onDelete('CASCADE')
       .comment('A reference to the entity that the refresh key is tied to');
     table
-      .text('key')
+      .string('key')
       .notNullable()
       .comment(
         'A reference to a key which should be used to trigger a refresh on this entity',

plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts

@@ -36,7 +36,7 @@ import { generateStableHash } from './util';
 describe('Default Processing Database', () => {
   const defaultLogger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   async function createDatabase(
@@ -1447,10 +1447,12 @@ describe('Default Processing Database', () => {
         const result1 = await db.transaction(async tx =>
           db.listParents(tx, { entityRef: 'component:default/foobar' }),
         );
-        expect(result1.entityRefs).toEqual([
-          'location:default/root-1',
-          'location:default/root-2',
-        ]);
+        expect(result1.entityRefs).toEqual(
+          expect.arrayContaining([
+            'location:default/root-1',
+            'location:default/root-2',
+          ]),
+        );
 
         const result2 = await db.transaction(async tx =>
           db.listParents(tx, { entityRef: 'location:default/root-1' }),

plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts

@@ -82,6 +82,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
       refreshKeys,
       locationKey,
     } = options;
+    const configClient = tx.client.config.client;
     const refreshResult = await tx<DbRefreshStateRow>('refresh_state')
       .update({
         processed_entity: JSON.stringify(processedEntity),
@@ -114,10 +115,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     // Delete old relations
     // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.
     let previousRelationRows: DbRelationsRow[];
-    if (
-      tx.client.config.client.includes('sqlite3') ||
-      tx.client.config.client.includes('mysql')
-    ) {
+    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {
       previousRelationRows = await tx<DbRelationsRow>('relations')
         .select('*')
         .where({ originating_entity_id: id });
@@ -663,11 +661,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         last_discovery_at: tx.fn.now(),
       });
 
-      // TODO(Rugvip): only tested towards Postgres and SQLite
+      // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite.
       // We have to do this because the only way to detect if there was a conflict with
       // SQLite is to catch the error, while Postgres needs to ignore the conflict to not
       // break the ongoing transaction.
-      if (!tx.client.config.client.includes('sqlite3')) {
+      if (tx.client.config.client.includes('pg')) {
         query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime
       }
 
@@ -675,10 +673,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
       const result: { rowCount?: number; length?: number } = await query;
       return result.rowCount === 1 || result.length === 1;
     } catch (error) {
-      // SQLite reached this rather than the rowCount check above
+      // SQLite, or MySQL reached this rather than the rowCount check above
       if (
         isError(error) &&
-        error.message.includes('UNIQUE constraint failed')
+        (error.message.includes('UNIQUE constraint failed') ||
+          error.message.includes('Duplicate entry')) // MySQL failure
       ) {
         return false;
       }

plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts

@@ -20,7 +20,7 @@ import { DefaultLocationStore } from './DefaultLocationStore';
 
 describe('DefaultLocationStore', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   async function createLocationStore(databaseId: TestDatabaseId) {

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts

@@ -30,7 +30,7 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
 
 describe('DefaultEntitiesCatalog', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
   const stitch = jest.fn();
   const stitcher: Stitcher = { stitch } as any;
@@ -239,7 +239,10 @@ describe('DefaultEntitiesCatalog', () => {
           expect.arrayContaining([
             {
               entity: expect.objectContaining({ metadata: { name: 'root' } }),
-              parentEntityRefs: ['k:default/parent1', 'k:default/parent2'],
+              parentEntityRefs: expect.arrayContaining([
+                'k:default/parent1',
+                'k:default/parent2',
+              ]),
             },
             {
               entity: expect.objectContaining({

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -238,6 +238,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
   }
 
   async removeEntityByUid(uid: string): Promise<void> {
+    const dbConfig = this.database.client.config;
+
     // Clear the hashed state of the immediate parents of the deleted entity.
     // This makes sure that when they get reprocessed, their output is written
     // down again. The reason for wanting to do this, is that if the user
@@ -246,21 +248,49 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     // means it'll never try to write down the children again (it assumes that
     // they already exist). This means that without the code below, the database
     // never "heals" from accidental deletes.
-    await this.database<DbRefreshStateRow>('refresh_state')
-      .update({
-        result_hash: 'child-was-deleted',
-        next_update_at: this.database.fn.now(),
-      })
-      .whereIn('entity_ref', function parents(builder) {
-        return builder
-          .from<DbRefreshStateRow>('refresh_state')
-          .innerJoin<DbRefreshStateReferencesRow>('refresh_state_references', {
-            'refresh_state_references.target_entity_ref':
-              'refresh_state.entity_ref',
-          })
-          .where('refresh_state.entity_id', '=', uid)
-          .select('refresh_state_references.source_entity_ref');
-      });
+    if (dbConfig.client.includes('mysql')) {
+      // MySQL doesn't support the syntax we need to do this in a single query,
+      // http://dev.mysql.com/doc/refman/5.6/en/update.html
+      const results = await this.database<DbRefreshStateRow>('refresh_state')
+        .select('entity_id')
+        .whereIn('entity_ref', function parents(builder) {
+          return builder
+            .from<DbRefreshStateRow>('refresh_state')
+            .innerJoin<DbRefreshStateReferencesRow>(
+              'refresh_state_references',
+              {
+                'refresh_state_references.target_entity_ref':
+                  'refresh_state.entity_ref',
+              },
+            )
+            .where('refresh_state.entity_id', '=', uid)
+            .select('refresh_state_references.source_entity_ref');
+        });
+      await this.database<DbRefreshStateRow>('refresh_state')
+        .update({ result_hash: 'child-was-deleted' })
+        .whereIn(
+          'entity_id',
+          results.map(key => key.entity_id),
+        );
+    } else {
+      await this.database<DbRefreshStateRow>('refresh_state')
+        .update({
+          result_hash: 'child-was-deleted',
+        })
+        .whereIn('entity_ref', function parents(builder) {
+          return builder
+            .from<DbRefreshStateRow>('refresh_state')
+            .innerJoin<DbRefreshStateReferencesRow>(
+              'refresh_state_references',
+              {
+                'refresh_state_references.target_entity_ref':
+                  'refresh_state.entity_ref',
+              },
+            )
+            .where('refresh_state.entity_id', '=', uid)
+            .select('refresh_state_references.source_entity_ref');
+        });
+    }
 
     // Stitch the entities that the deleted one had relations to. If we do not
     // do this, the entities in the other end of the relations will still look
@@ -285,7 +315,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
           .select({ ref: 'relations.source_entity_ref' }),
       );
 
-    // Perform the actual deletion
     await this.database<DbRefreshStateRow>('refresh_state')
       .where('entity_id', uid)
       .delete();

plugins/catalog-backend/src/service/DefaultRefreshService.test.ts

@@ -36,7 +36,7 @@ import { DefaultRefreshService } from './DefaultRefreshService';
 describe('Refresh integration', () => {
   const defaultLogger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   async function createDatabase(

plugins/catalog-backend/src/stitching/Stitcher.test.ts

@@ -29,7 +29,7 @@ import { Stitcher } from './Stitcher';
 
 describe('Stitcher', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
   const logger = getVoidLogger();