From e9d3c48ff41cb834de0d15cb47179f320d6aa285 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 16:31:13 +0100 Subject: [PATCH 1/5] Fixing images to use a locked SHA, this means that we shouldn't have issues with CI breaking due to using images which are receiving updates (we've experienced two CI breakages due to MariaDB and MS-SQL updates). --- .github/workflows/budibase_ci.yml | 21 ++++++++++++------- globalSetup.ts | 2 +- packages/server/datasource-sha.env | 5 +++++ .../server/src/api/routes/tests/table.spec.ts | 8 +++---- .../src/integrations/tests/utils/images.ts | 17 +++++++++++++++ .../src/integrations/tests/utils/index.ts | 8 ++++++- .../src/integrations/tests/utils/mariadb.ts | 3 ++- .../src/integrations/tests/utils/mongodb.ts | 3 ++- .../src/integrations/tests/utils/mssql.ts | 5 ++--- .../src/integrations/tests/utils/mysql.ts | 3 ++- .../src/integrations/tests/utils/postgres.ts | 3 ++- 11 files changed, 57 insertions(+), 21 deletions(-) create mode 100644 packages/server/datasource-sha.env create mode 100644 packages/server/src/integrations/tests/utils/images.ts diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1bc1915a71..5489771441 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -108,7 +108,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sqs & + docker pull budibase/couchdb:v3.3.3 & docker pull redis & wait $(jobs -p) @@ -162,17 +162,22 @@ jobs: node-version: 20.x cache: yarn + - name: Load dotenv + run: | + npm install -g dotenv-cli & + dotenv -e packages/server/datasource-sha.env -- env > $GITHUB_ENV + - name: Pull testcontainers images run: | - docker pull mcr.microsoft.com/mssql/server:2022-CU13-ubuntu-22.04 & - docker pull mysql:8.3 & - docker pull postgres:16.1-bullseye & - docker pull mongo:7.0-jammy & - docker pull mariadb:lts & - docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sqs & + docker pull mcr.microsoft.com/mssql/server@${{ env.MSSQL_SHA }} & + docker pull mysql@${{ env.MYSQL_SHA }} & + docker pull postgres@${{ env.POSTGRES_SHA }} & + docker pull mongo@${{ env.MONGODB_SHA }} & + docker pull mariadb@${{ env.MARIADB_SHA }} & docker pull minio/minio & docker pull redis & + docker pull testcontainers/ryuk:0.5.1 & + docker pull budibase/couchdb:v3.3.3 & wait $(jobs -p) diff --git a/globalSetup.ts b/globalSetup.ts index dd1454b6e1..aa1cb00fe1 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -46,7 +46,7 @@ export default async function setup() { await killContainers(containers) try { - const couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs") + const couchdb = new GenericContainer("budibase/couchdb:v3.3.3") .withExposedPorts(5984, 4984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env new file mode 100644 index 0000000000..9b935ed8eb --- /dev/null +++ b/packages/server/datasource-sha.env @@ -0,0 +1,5 @@ +MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe +POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e +MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d +MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 20c83549d2..54464227fa 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,11 +28,11 @@ const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ describe.each([ - ["internal", undefined], + // ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/tables (%s)", (_, dsProvider) => { const isInternal: boolean = !dsProvider let datasource: Datasource | undefined diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts new file mode 100644 index 0000000000..4141b9b059 --- /dev/null +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -0,0 +1,17 @@ +import dotenv from "dotenv" +import { join } from "path" + +const path = join(__dirname, "..", "..", "..", "..", "datasource-sha.env") +dotenv.config({ + path, +}) + +export function getImageSHAs() { + return { + mssql: `mcr.microsoft.com/mssql/server@${process.env.MSSQL_SHA}`, + mysql: `mysql@${process.env.MYSQL_SHA}`, + postgres: `postgres@${process.env.POSTGRES_SHA}`, + mongodb: `mongo@${process.env.MONGODB_SHA}`, + mariadb: `mariadb@${process.env.MARIADB_SHA}`, + } +} diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index b888f1adc1..5945d47088 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -1,3 +1,4 @@ +import "./images" import { Datasource, SourceName } from "@budibase/types" import * as postgres from "./postgres" import * as mongodb from "./mongodb" @@ -67,7 +68,12 @@ export async function knexClient(ds: Datasource) { export async function startContainer(container: GenericContainer) { const imageName = (container as any).imageName.string as string - const key = imageName.replaceAll("/", "-").replaceAll(":", "-") + let key: string + if (imageName.includes("@sha256")) { + key = imageName.split("@")[0] + } else { + key = imageName.replaceAll("/", "-").replaceAll(":", "-") + } container = container .withReuse() diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index 3a90b554ee..004b79e60f 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -4,6 +4,7 @@ import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait- import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import { knexClient } from "./mysql" +import { getImageSHAs } from "./images" let ports: Promise @@ -27,7 +28,7 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer("mariadb:lts") + new GenericContainer(getImageSHAs().mariadb) .withExposedPorts(3306) .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MariaDBWaitStrategy()) diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts index 0bdbb2808c..08cbf81b17 100644 --- a/packages/server/src/integrations/tests/utils/mongodb.ts +++ b/packages/server/src/integrations/tests/utils/mongodb.ts @@ -2,13 +2,14 @@ import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { startContainer } from "." +import { getImageSHAs } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer("mongo:7.0-jammy") + new GenericContainer(getImageSHAs().mongodb) .withExposedPorts(27017) .withEnvironment({ MONGO_INITDB_ROOT_USERNAME: "mongo", diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index ed94477814..e3a49f6cec 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -3,15 +3,14 @@ import { GenericContainer, Wait } from "testcontainers" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" +import { getImageSHAs } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer( - "mcr.microsoft.com/mssql/server:2022-CU13-ubuntu-22.04" - ) + new GenericContainer(getImageSHAs().mssql) .withExposedPorts(1433) .withEnvironment({ ACCEPT_EULA: "Y", diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index c35be0689e..f26a751d1e 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -4,6 +4,7 @@ import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait- import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" +import { getImageSHAs } from "./images" let ports: Promise @@ -30,7 +31,7 @@ class MySQLWaitStrategy extends AbstractWaitStrategy { export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer("mysql:8.3") + new GenericContainer(getImageSHAs().mysql) .withExposedPorts(3306) .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000)) diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 74f5722737..d7639ce7c7 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -3,13 +3,14 @@ import { GenericContainer, Wait } from "testcontainers" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" +import { getImageSHAs } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer("postgres:16.1-bullseye") + new GenericContainer(getImageSHAs().postgres) .withExposedPorts(5432) .withEnvironment({ POSTGRES_PASSWORD: "password" }) .withWaitStrategy( From 95bf8b78f54704754ff6b7b8563aed1c007b4449 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 16:38:00 +0100 Subject: [PATCH 2/5] Updating how dotenv is loaded. --- .github/workflows/budibase_ci.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 5489771441..b92431f227 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -163,17 +163,16 @@ jobs: cache: yarn - name: Load dotenv - run: | - npm install -g dotenv-cli & - dotenv -e packages/server/datasource-sha.env -- env > $GITHUB_ENV + id: dotenv + uses: ./.github/actions/dotenv-action - name: Pull testcontainers images run: | - docker pull mcr.microsoft.com/mssql/server@${{ env.MSSQL_SHA }} & - docker pull mysql@${{ env.MYSQL_SHA }} & - docker pull postgres@${{ env.POSTGRES_SHA }} & - docker pull mongo@${{ env.MONGODB_SHA }} & - docker pull mariadb@${{ env.MARIADB_SHA }} & + docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} & + docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} & + docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} & + docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} & + docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} & docker pull minio/minio & docker pull redis & docker pull testcontainers/ryuk:0.5.1 & From 2e2a61366f43ef3fb79c772adece6be7011b8545 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 16:42:31 +0100 Subject: [PATCH 3/5] Another attempt to fix. --- .github/workflows/budibase_ci.yml | 4 +++- packages/server/src/api/routes/tests/table.spec.ts | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index b92431f227..2e22e76ac6 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -164,7 +164,9 @@ jobs: - name: Load dotenv id: dotenv - uses: ./.github/actions/dotenv-action + uses: falti/dotenv-action@v1.1.3 + with: + path: ./packages/server/datasource-sha.env - name: Pull testcontainers images run: | diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 54464227fa..20c83549d2 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,11 +28,11 @@ const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ describe.each([ - // ["internal", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/tables (%s)", (_, dsProvider) => { const isInternal: boolean = !dsProvider let datasource: Datasource | undefined From b68697002a0ef990b8fad8b767409659b7aedba4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 16:48:35 +0100 Subject: [PATCH 4/5] Anoter fix. --- packages/server/src/integrations/tests/utils/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 5945d47088..8e317d9099 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -68,12 +68,11 @@ export async function knexClient(ds: Datasource) { export async function startContainer(container: GenericContainer) { const imageName = (container as any).imageName.string as string - let key: string + let key: string = imageName if (imageName.includes("@sha256")) { key = imageName.split("@")[0] - } else { - key = imageName.replaceAll("/", "-").replaceAll(":", "-") } + key = key.replaceAll("/", "-").replaceAll(":", "-") container = container .withReuse() From ea7dffa3d7b67f7a224d6733ba77d39bb9f827b9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 17:33:39 +0100 Subject: [PATCH 5/5] PR comment. --- .../server/src/integrations/tests/utils/images.ts | 14 +++++--------- .../server/src/integrations/tests/utils/mariadb.ts | 4 ++-- .../server/src/integrations/tests/utils/mongodb.ts | 4 ++-- .../server/src/integrations/tests/utils/mssql.ts | 4 ++-- .../server/src/integrations/tests/utils/mysql.ts | 4 ++-- .../src/integrations/tests/utils/postgres.ts | 4 ++-- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index 4141b9b059..8c2bb9b220 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -6,12 +6,8 @@ dotenv.config({ path, }) -export function getImageSHAs() { - return { - mssql: `mcr.microsoft.com/mssql/server@${process.env.MSSQL_SHA}`, - mysql: `mysql@${process.env.MYSQL_SHA}`, - postgres: `postgres@${process.env.POSTGRES_SHA}`, - mongodb: `mongo@${process.env.MONGODB_SHA}`, - mariadb: `mariadb@${process.env.MARIADB_SHA}`, - } -} +export const MSSQL_IMAGE = `mcr.microsoft.com/mssql/server@${process.env.MSSQL_SHA}` +export const MYSQL_IMAGE = `mysql@${process.env.MYSQL_SHA}` +export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}` +export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` +export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index 004b79e60f..529ac0b76b 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -4,7 +4,7 @@ import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait- import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import { knexClient } from "./mysql" -import { getImageSHAs } from "./images" +import { MARIADB_IMAGE } from "./images" let ports: Promise @@ -28,7 +28,7 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer(getImageSHAs().mariadb) + new GenericContainer(MARIADB_IMAGE) .withExposedPorts(3306) .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MariaDBWaitStrategy()) diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts index 08cbf81b17..a62d895042 100644 --- a/packages/server/src/integrations/tests/utils/mongodb.ts +++ b/packages/server/src/integrations/tests/utils/mongodb.ts @@ -2,14 +2,14 @@ import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { startContainer } from "." -import { getImageSHAs } from "./images" +import { MONGODB_IMAGE } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer(getImageSHAs().mongodb) + new GenericContainer(MONGODB_IMAGE) .withExposedPorts(27017) .withEnvironment({ MONGO_INITDB_ROOT_USERNAME: "mongo", diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index e3a49f6cec..709ebb9439 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -3,14 +3,14 @@ import { GenericContainer, Wait } from "testcontainers" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" -import { getImageSHAs } from "./images" +import { MSSQL_IMAGE } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer(getImageSHAs().mssql) + new GenericContainer(MSSQL_IMAGE) .withExposedPorts(1433) .withEnvironment({ ACCEPT_EULA: "Y", diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index f26a751d1e..68e591837b 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -4,7 +4,7 @@ import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait- import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" -import { getImageSHAs } from "./images" +import { MYSQL_IMAGE } from "./images" let ports: Promise @@ -31,7 +31,7 @@ class MySQLWaitStrategy extends AbstractWaitStrategy { export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer(getImageSHAs().mysql) + new GenericContainer(MYSQL_IMAGE) .withExposedPorts(3306) .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000)) diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index d7639ce7c7..bf8d76e39d 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -3,14 +3,14 @@ import { GenericContainer, Wait } from "testcontainers" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" -import { getImageSHAs } from "./images" +import { POSTGRES_IMAGE } from "./images" let ports: Promise export async function getDatasource(): Promise { if (!ports) { ports = startContainer( - new GenericContainer(getImageSHAs().postgres) + new GenericContainer(POSTGRES_IMAGE) .withExposedPorts(5432) .withEnvironment({ POSTGRES_PASSWORD: "password" }) .withWaitStrategy(