Merge pull request #18921 from PeteLevineA/continued-mysql-support

patch: Add Continued MySQL Support

plugins/app-backend/migrations/20211229105307_init.js

@@ -20,11 +20,20 @@
  * @param {import('knex').Knex} knex
  */
 exports.up = async function up(knex) {
-  return knex.schema.createTable('static_assets_cache', table => {
+  const isMySQL = knex.client.config.client.includes('mysql');
+  await knex.schema.createTable('static_assets_cache', table => {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table.text('path').primary().notNullable().comment('The path of the file');
+    if (!isMySQL) {
+      table
+        .text('path')
+        .primary()
+        .notNullable()
+        .comment('The path of the file');
+    } else {
+      table.text('path').notNullable().comment('The path of the file');
+    }
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())
@@ -35,6 +44,12 @@ exports.up = async function up(knex) {
     table.binary('content').notNullable().comment('The asset content');
     table.index('last_modified_at', 'static_asset_cache_last_modified_at_idx');
   });
+  // specifically for mysql specify a unique index up to 254 characters(mysql limit)
+  if (isMySQL) {
+    await knex.schema.raw(
+      'create unique index static_assets_cache_path_idx on static_assets_cache(path(254));',
+    );
+  }
 };
 
 /**

plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts

@@ -37,7 +37,7 @@ jest.setTimeout(60_000);
 
 describe('StaticAssetsStore', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   it.each(databases.eachSupportedId())(
@@ -153,14 +153,18 @@ describe('StaticAssetsStore', () => {
           content: async () => Buffer.alloc(0),
         },
       ]);
-
+      // interval check for postgresql
+      let hourPast = `now() + interval '-3600 seconds'`;
+      if (knex.client.config.client.includes('mysql')) {
+        hourPast = `date_sub(now(), interval 3600 second)`;
+      } else if (knex.client.config.client.includes('sqlite3')) {
+        hourPast = `datetime('now', '-3600 seconds')`;
+      }
       // Rewrite modified time of "old" to be 1h in the past
       const updated = await knex('static_assets_cache')
         .where({ path: 'old' })
         .update({
-          last_modified_at: knex.client.config.client.includes('sqlite3')
-            ? knex.raw(`datetime('now', '-3600 seconds')`)
-            : knex.raw(`now() + interval '-3600 seconds'`),
+          last_modified_at: knex.raw(hourPast),
         });
       expect(updated).toBe(1);
 

plugins/app-backend/src/lib/assets/StaticAssetsStore.ts

@@ -138,14 +138,20 @@ export class StaticAssetsStore implements StaticAssetProvider {
    */
   async trimAssets(options: { maxAgeSeconds: number }) {
     const { maxAgeSeconds } = options;
+    let lastModifiedInterval = this.#db.raw(
+      `now() + interval '${-maxAgeSeconds} seconds'`,
+    );
+    if (this.#db.client.config.client.includes('mysql')) {
+      lastModifiedInterval = this.#db.raw(
+        `date_sub(now(), interval ${maxAgeSeconds} second)`,
+      );
+    } else if (this.#db.client.config.client.includes('sqlite3')) {
+      lastModifiedInterval = this.#db.raw(`datetime('now', ?)`, [
+        `-${maxAgeSeconds} seconds`,
+      ]);
+    }
     await this.#db<StaticAssetRow>('static_assets_cache')
-      .where(
-        'last_modified_at',
-        '<=',
-        this.#db.client.config.client.includes('sqlite3')
-          ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`])
-          : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`),
-      )
+      .where('last_modified_at', '<=', lastModifiedInterval)
       .delete();
   }
 }

plugins/bazaar-backend/migrations/20211014144054_init.js

@@ -23,7 +23,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('metadata', table => {
     table.comment('The table of Bazaar metadata');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .notNullable()
       .unique()
       .comment('The ref of the entity');
@@ -49,7 +49,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('members', table => {
     table.comment('The table of Bazaar members');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .notNullable()
       .references('metadata.entity_ref')
       .onDelete('CASCADE')

plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js

@@ -89,9 +89,11 @@ exports.up = async function up(knex) {
     await knex.schema.alterTable('members', table => {
       table
         .integer('item_id')
+        .unsigned()
         .references('metadata.id')
         .onDelete('CASCADE')
         .comment('Id of the associated item');
+      table.dropForeign('entity_ref');
       table.dropColumn('entity_ref');
     });
   }

plugins/bazaar-backend/src/service/DatabaseHandler.test.ts

@@ -36,7 +36,7 @@ jest.setTimeout(60_000);
 
 describe('DatabaseHandler', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   function createDatabaseManager(

plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts

@@ -24,7 +24,7 @@ import { WrapperProviders } from './WrapperProviders';
 describe('WrapperProviders', () => {
   const applyDatabaseMigrations = jest.fn();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
   const config = new ConfigReader({});
   const logger = getVoidLogger();

plugins/catalog-backend/migrations/20210302150147_refresh_state.js

@@ -37,7 +37,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('The unprocessed entity (in original form) as JSON');
     table
-      .text('processed_entity')
+      .text('processed_entity', 'longtext')
       .nullable()
       .comment('The processed entity (not yet stitched) as JSON');
     table
@@ -83,7 +83,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('Random value representing a unique stitch attempt ticket');
     table
-      .text('final_entity')
+      .text('final_entity', 'longtext')
       .nullable()
       .comment('The JSON encoded final entity');
     table.index('entity_id', 'final_entities_entity_id_idx');

plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts

@@ -138,6 +138,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         type,
       }),
     );
+
     await tx.batchInsert(
       'relations',
       this.deduplicateRelations(relationRows),

plugins/code-coverage-backend/migrations/20210302_init.js

@@ -34,7 +34,7 @@ exports.up = async function up(knex) {
       .comment('An insert counter to ensure ordering');
     table.uuid('id').notNullable().comment('The ID of the code coverage');
     table
-      .text('entity')
+      .string('entity')
       .notNullable()
       .comment('The entity ref that this code coverage applies to');
     table

plugins/linguist-backend/migrations/20221115_init.js

@@ -32,7 +32,7 @@ exports.up = async function up(knex) {
       .comment('An insert counter to ensure ordering');
     table.uuid('id').notNullable().comment('The ID of the Linguist result');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .unique()
       .notNullable()
       .comment('The entity ref that this Linguist result applies to');

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -261,18 +261,21 @@ export class DatabaseTaskStore implements TaskStore {
     tasks: { taskId: string }[];
   }> {
     const { timeoutS } = options;
-
+    let heartbeatInterval = this.db.raw(`? - interval '${timeoutS} seconds'`, [
+      this.db.fn.now(),
+    ]);
+    if (this.db.client.config.client.includes('mysql')) {
+      heartbeatInterval = this.db.raw(
+        `date_sub(now(), interval ${timeoutS} second)`,
+      );
+    } else if (this.db.client.config.client.includes('sqlite3')) {
+      heartbeatInterval = this.db.raw(`datetime('now', ?)`, [
+        `-${timeoutS} seconds`,
+      ]);
+    }
     const rawRows = await this.db<RawDbTaskRow>('tasks')
       .where('status', 'processing')
-      .andWhere(
-        'last_heartbeat_at',
-        '<=',
-        this.db.client.config.client.includes('sqlite3')
-          ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`])
-          : this.db.raw(`? - interval '${timeoutS} seconds'`, [
-              this.db.fn.now(),
-            ]),
-      );
+      .andWhere('last_heartbeat_at', '<=', heartbeatInterval);
     const tasks = rawRows.map(row => ({
       taskId: row.id,
     }));

plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js

@@ -25,7 +25,7 @@ exports.up = async function up(knex) {
       'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.',
     );
     table
-      .text('id')
+      .string('id')
       .notNullable()
       .comment('Identifier of the fact retriever plugin/package');
     table

plugins/tech-insights-backend/migrations/202109061212_facts.js

@@ -25,7 +25,7 @@ exports.up = async function up(knex) {
       'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.',
     );
     table
-      .text('id')
+      .string('id')
       .notNullable()
       .comment('Unique identifier of the fact retriever plugin/package');
     table
@@ -40,7 +40,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('The timestamp when this entry was created');
     table
-      .text('entity')
+      .string('entity')
       .notNullable()
       .comment('Identifier of the entity these facts relate to');
     table