diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index c07f9b2c28..e0263546ff 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -11,7 +11,6 @@ on: branches: - master - develop - - release workflow_dispatch: env: @@ -20,9 +19,53 @@ env: PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - run: yarn + - run: yarn lint + build: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - run: yarn + - run: yarn bootstrap + - run: yarn build + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - run: yarn + - run: yarn bootstrap + - run: yarn test + - uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml + name: codecov-umbrella + verbose: true + + integration-test: + runs-on: ubuntu-latest services: couchdb: image: ibmcom/couchdb3 @@ -31,39 +74,18 @@ jobs: COUCHDB_USER: budibase ports: - 4567:5984 - - strategy: - matrix: - node-version: [14.x] - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH - - - run: yarn - - run: yarn bootstrap - - run: yarn lint - - run: yarn build - - run: yarn test - env: - CI: true - name: Budibase CI - - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml - name: codecov-umbrella - verbose: true - - - name: QA Core Integration Tests - run: | - cd qa-core - yarn - yarn api:test:ci \ No newline at end of file + - uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - run: yarn + - run: yarn bootstrap + - run: yarn build + - run: | + cd qa-core + yarn + yarn api:test:ci diff --git a/.husky/pre-commit b/.husky/pre-commit index 3b614330e0..6700f51282 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,2 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" - -yarn run lint diff --git a/lerna.json b/lerna.json index 2d1b05887b..0a3d923d6c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.3.17-alpha.7", + "version": "2.3.18-alpha.6", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 0483fb073a..1e69797e71 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -9,15 +9,9 @@ const baseConfig: Config.InitialProjectOptions = { transform: { "^.+\\.ts?$": "@swc/jest", }, -} - -if (!process.env.CI) { - // use sources when not in CI - baseConfig.moduleNameMapper = { + moduleNameMapper: { "@budibase/types": "/../types/src", - } -} else { - console.log("Running tests with compiled dependency sources") + }, } const config: Config.InitialOptions = { diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index f991cb07bf..479a54bd94 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.3.17-alpha.7", + "version": "2.3.18-alpha.6", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -18,13 +18,13 @@ "build:pro": "../../scripts/pro/build.sh", "postbuild": "yarn run build:pro", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", - "test": "jest --coverage", + "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" }, "dependencies": { "@budibase/nano": "10.1.1", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.3.17-alpha.7", + "@budibase/types": "2.3.18-alpha.6", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/scripts/test.sh b/packages/backend-core/scripts/test.sh new file mode 100644 index 0000000000..4bf1900984 --- /dev/null +++ b/packages/backend-core/scripts/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ -n $CI ]] +then + # --runInBand performs better in ci where resources are limited + echo "jest --coverage --runInBand" + jest --coverage --runInBand +else + # --maxWorkers performs better in development + echo "jest --coverage" + jest --coverage +fi \ No newline at end of file diff --git a/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts index eb8ffc9b71..d0689a1f0a 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts @@ -19,7 +19,7 @@ describe("google", () => { const callbackUrl = generator.url() it("should create successfully create a google strategy", async () => { - await google.strategyFactory(googleConfig, callbackUrl) + await google.strategyFactory(googleConfig, callbackUrl, mockSaveUserFn) const expectedOptions = { clientID: googleConfig.clientID, diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 79c7eb55ea..2e3524775f 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -4,7 +4,7 @@ import { StaticDatabases, getAllApps, getGlobalDBName, - doWithDB, + getDB, } from "../db" import environment from "../environment" import * as platform from "../platform" @@ -86,66 +86,65 @@ export const runMigration = async ( count++ const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - await doWithDB(dbName, async (db: any) => { - try { - const doc = await getMigrationsDoc(db) + const db = getDB(dbName) + try { + const doc = await getMigrationsDoc(db) - // the migration has already been run - if (doc[migrationName]) { - // check for force - if ( - options.force && - options.force[migrationType] && - options.force[migrationType].includes(migrationName) - ) { - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` - ) - } else { - // no force, exit - return - } - } - - // check if the migration is not a no-op - if (!options.noOp) { + // the migration has already been run + if (doc[migrationName]) { + // check for force + if ( + options.force && + options.force[migrationType] && + options.force[migrationType].includes(migrationName) + ) { log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` - ) - - if (migration.preventRetry) { - // eagerly set the completion date - // so that we never run this migration twice even upon failure - doc[migrationName] = Date.now() - const response = await db.put(doc) - doc._rev = response.rev - } - - // run the migration - if (migrationType === MigrationType.APP) { - await context.doInAppContext(db.name, async () => { - await migration.fn(db) - }) - } else { - await migration.fn(db) - } - - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` ) + } else { + // no force, exit + return } - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, - err - ) - throw err } - }) + + // check if the migration is not a no-op + if (!options.noOp) { + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` + ) + + if (migration.preventRetry) { + // eagerly set the completion date + // so that we never run this migration twice even upon failure + doc[migrationName] = Date.now() + const response = await db.put(doc) + doc._rev = response.rev + } + + // run the migration + if (migrationType === MigrationType.APP) { + await context.doInAppContext(db.name, async () => { + await migration.fn(db) + }) + } else { + await migration.fn(db) + } + + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` + ) + } + + // mark as complete + doc[migrationName] = Date.now() + await db.put(doc) + } catch (err) { + console.error( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, + err + ) + throw err + } } } @@ -185,7 +184,10 @@ export const runMigrations = async ( // for all migrations for (const migration of migrations) { // run the migration - await context.doInTenant(tenantId, () => runMigration(migration, options)) + await context.doInTenant( + tenantId, + async () => await runMigration(migration, options) + ) } } console.log("Migrations complete") diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap b/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap similarity index 100% rename from packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap rename to packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js deleted file mode 100644 index c1915510c3..0000000000 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ /dev/null @@ -1,57 +0,0 @@ -require("../../../tests") -const { runMigrations, getMigrationsDoc } = require("../index") -const { getGlobalDBName, getDB } = require("../../db") - -const { structures, testEnv } = require("../../../tests") -testEnv.multiTenant() - -let db - -describe("migrations", () => { - - const migrationFunction = jest.fn() - - const MIGRATIONS = [{ - type: "global", - name: "test", - fn: migrationFunction - }] - - let tenantId - - beforeEach(() => { - tenantId = structures.tenant.id() - db = getDB(getGlobalDBName(tenantId)) - }) - - afterEach(async () => { - jest.clearAllMocks() - await db.destroy() - }) - - const migrate = () => { - return runMigrations(MIGRATIONS, { tenantIds: [tenantId]}) - } - - it("should run a new migration", async () => { - await migrate() - expect(migrationFunction).toHaveBeenCalled() - const doc = await getMigrationsDoc(db) - expect(doc.test).toBeDefined() - }) - - it("should match snapshot", async () => { - await migrate() - const doc = await getMigrationsDoc(db) - expect(doc).toMatchSnapshot() - }) - - it("should skip a previously run migration", async () => { - await migrate() - const previousMigrationTime = await getMigrationsDoc(db).test - await migrate() - const currentMigrationTime = await getMigrationsDoc(db).test - expect(migrationFunction).toHaveBeenCalledTimes(1) - expect(currentMigrationTime).toBe(previousMigrationTime) - }) -}) \ No newline at end of file diff --git a/packages/backend-core/src/migrations/tests/migrations.spec.ts b/packages/backend-core/src/migrations/tests/migrations.spec.ts new file mode 100644 index 0000000000..c74ab816c1 --- /dev/null +++ b/packages/backend-core/src/migrations/tests/migrations.spec.ts @@ -0,0 +1,64 @@ +import { testEnv, DBTestConfiguration } from "../../../tests" +import * as migrations from "../index" +import * as context from "../../context" +import { MigrationType } from "@budibase/types" + +testEnv.multiTenant() + +describe("migrations", () => { + const config = new DBTestConfiguration() + + const migrationFunction = jest.fn() + + const MIGRATIONS = [ + { + type: MigrationType.GLOBAL, + name: "test" as any, + fn: migrationFunction, + }, + ] + + beforeEach(() => { + config.newTenant() + }) + + afterEach(async () => { + jest.clearAllMocks() + }) + + const migrate = () => { + return migrations.runMigrations(MIGRATIONS, { + tenantIds: [config.tenantId], + }) + } + + it("should run a new migration", async () => { + await config.doInTenant(async () => { + await migrate() + expect(migrationFunction).toHaveBeenCalled() + const db = context.getGlobalDB() + const doc = await migrations.getMigrationsDoc(db) + expect(doc.test).toBeDefined() + }) + }) + + it("should match snapshot", async () => { + await config.doInTenant(async () => { + await migrate() + const doc = await migrations.getMigrationsDoc(context.getGlobalDB()) + expect(doc).toMatchSnapshot() + }) + }) + + it("should skip a previously run migration", async () => { + await config.doInTenant(async () => { + const db = context.getGlobalDB() + await migrate() + const previousDoc = await migrations.getMigrationsDoc(db) + await migrate() + const currentDoc = await migrations.getMigrationsDoc(db) + expect(migrationFunction).toHaveBeenCalledTimes(1) + expect(currentDoc.test).toBe(previousDoc.test) + }) + }) +}) diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index 00329ffb84..485268edad 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -20,13 +20,17 @@ async function init() { ).init() } -process.on("exit", async () => { +export async function shutdown() { if (userClient) await userClient.finish() if (sessionClient) await sessionClient.finish() if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() if (lockClient) await lockClient.finish() +} + +process.on("exit", async () => { + await shutdown() }) export async function getUserClient() { diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 2669cd816a..951369496a 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -91,6 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) { } // attach handlers client.on("end", (err: Error) => { + if (env.isTest()) { + // don't try to re-connect in test env + // allow the process to exit + return + } connectionError(selectDb, timeout, err) }) client.on("error", (err: Error) => { diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index 2021da2b56..136d7f5d33 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -85,17 +85,20 @@ export const doWithLock = async (opts: LockOptions, task: any) => { const result = await task() return result } catch (e: any) { - console.log("lock error") + console.warn("lock error") // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { // don't throw for try-once locks, they will always error // due to retry count (0) exceeded + console.warn(e) return } else { + console.error(e) throw e } } else { + console.error(e) throw e } } finally { diff --git a/packages/backend-core/tests/utilities/DBTestConfiguration.ts b/packages/backend-core/tests/utilities/DBTestConfiguration.ts index cad62e2979..e5e57a99a3 100644 --- a/packages/backend-core/tests/utilities/DBTestConfiguration.ts +++ b/packages/backend-core/tests/utilities/DBTestConfiguration.ts @@ -12,6 +12,10 @@ class DBTestConfiguration { this.tenantId = structures.tenant.id() } + newTenant() { + this.tenantId = structures.tenant.id() + } + // TENANCY doInTenant(task: any) { diff --git a/packages/bbui/package.json b/packages/bbui/package.json index e925115c81..6cfeb44a7b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.3.17-alpha.7", + "version": "2.3.18-alpha.6", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.3.17-alpha.7", + "@budibase/string-templates": "2.3.18-alpha.6", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 71d498f06e..77b09fdbf3 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.3.17-alpha.7", + "version": "2.3.18-alpha.6", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.3.17-alpha.7", - "@budibase/client": "2.3.17-alpha.7", - "@budibase/frontend-core": "2.3.17-alpha.7", - "@budibase/string-templates": "2.3.17-alpha.7", + "@budibase/bbui": "2.3.18-alpha.6", + "@budibase/client": "2.3.18-alpha.6", + "@budibase/frontend-core": "2.3.18-alpha.6", + "@budibase/string-templates": "2.3.18-alpha.6", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index bf07cddf23..f56d7a78d4 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -254,8 +254,8 @@ {:else if filter.type === "datetime"} {:else} diff --git a/packages/builder/src/helpers/userInitials.js b/packages/builder/src/helpers/userInitials.js new file mode 100644 index 0000000000..c87d38c494 --- /dev/null +++ b/packages/builder/src/helpers/userInitials.js @@ -0,0 +1,13 @@ +const getUserInitials = user => { + if (user.firstName && user.lastName) { + return user.firstName[0] + user.lastName[0] + } else if (user.firstName) { + return user.firstName[0] + } else if (user.email) { + return user.email[0] + } + + return "U" +} + +export default getUserInitials diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/ActionsRenderer.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/ActionsRenderer.svelte index 7dc302186a..4787112760 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/ActionsRenderer.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/ActionsRenderer.svelte @@ -3,31 +3,26 @@ ActionMenu, MenuItem, Icon, - Input, Heading, Body, Modal, } from "@budibase/bbui" import ConfirmDialog from "components/common/ConfirmDialog.svelte" import CreateRestoreModal from "./CreateRestoreModal.svelte" - import { createEventDispatcher, onMount } from "svelte" + import { createEventDispatcher } from "svelte" export let row let deleteDialog let restoreDialog - let updateDialog - let name let restoreBackupModal const dispatch = createEventDispatcher() - const onClickRestore = name => { + const onClickRestore = () => { dispatch("buttonclick", { type: "backupRestore", - name, backupId: row._id, - restoreBackupName: name, }) } @@ -38,21 +33,9 @@ }) } - const onClickUpdate = () => { - dispatch("buttonclick", { - type: "backupUpdate", - backupId: row._id, - name, - }) - } - async function downloadExport() { window.open(`/api/apps/${row.appId}/backups/${row._id}/file`, "_blank") } - - onMount(() => { - name = row.name - })
@@ -66,12 +49,11 @@ Delete Download {/if} - Rename
- onClickRestore(name)} /> + - Are you sure you wish to delete the backup - {row.name}? - This action cannot be undone. + Are you sure you wish to delete this backup? This action cannot be undone. - {row.name || "Backup"} + Backup {new Date(row.timestamp).toLocaleString()} - - - - diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/NameRenderer.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/NameRenderer.svelte deleted file mode 100644 index 93eda410fe..0000000000 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/NameRenderer.svelte +++ /dev/null @@ -1,8 +0,0 @@ - - -{truncatedValue} diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/TimeAgoRenderer.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/TimeAgoRenderer.svelte new file mode 100644 index 0000000000..fd67b4010b --- /dev/null +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/TimeAgoRenderer.svelte @@ -0,0 +1,10 @@ + + +{dayjs(value).fromNow()} diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/UserRenderer.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/UserRenderer.svelte index abab314d05..a9aabae857 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/UserRenderer.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/UserRenderer.svelte @@ -1,17 +1,14 @@ -
- {#if value != null} -
{username}
- {/if} +
+