diff --git a/packages/client/package.json b/packages/client/package.json index 39ddb4bd49..227c7b25d4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -37,7 +37,6 @@ "downloadjs": "1.4.7", "html5-qrcode": "^2.2.1", "leaflet": "^1.7.1", - "regexparam": "^1.3.0", "sanitize-html": "^2.7.0", "screenfull": "^6.0.1", "shortid": "^2.2.15", diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js index 8488b702b6..d4c8faa4d2 100644 --- a/packages/client/src/api/api.js +++ b/packages/client/src/api/api.js @@ -77,4 +77,10 @@ export const API = createAPIClient({ // Log all errors to console console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`) }, + onMigrationDetected: _appId => { + if (!window.MIGRATING_APP) { + // We will force a reload, that will display the updating screen until the migration is running + window.location.reload() + } + }, }) diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte new file mode 100644 index 0000000000..74e5500715 --- /dev/null +++ b/packages/client/src/components/UpdatingApp.svelte @@ -0,0 +1,23 @@ + + +
+ +
+ + diff --git a/packages/client/src/index.js b/packages/client/src/index.js index a3cb4206c3..f6ed23b2a9 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -1,4 +1,5 @@ import ClientApp from "./components/ClientApp.svelte" +import UpdatingApp from "./components/UpdatingApp.svelte" import { builderStore, appStore, @@ -52,6 +53,13 @@ const loadBudibase = async () => { window["##BUDIBASE_APP_EMBEDDED##"] === "true" ) + if (window.MIGRATING_APP) { + new UpdatingApp({ + target: window.document.body, + }) + return + } + // Fetch environment info if (!get(environmentStore)?.loaded) { await environmentStore.actions.fetchEnvironment() diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index 33d277dd64..5535e0772e 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -7,7 +7,7 @@ "../shared-core", "../string-templates" ], - "ext": "js,ts,json", + "ext": "js,ts,json,svelte", "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"], "exec": "yarn build && node ./dist/index.js" } diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 0a7430aa94..8c177f1704 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -14,10 +14,7 @@ import { DatasourcePlus, FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, - IntegrationBase, - Schema, SourceName, - Table, UpdateDatasourceResponse, UserCtx, VerifyDatasourceRequest, @@ -28,65 +25,6 @@ import { builderSocket } from "../../websockets" import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" import { isEqual } from "lodash" -async function getConnector( - datasource: Datasource -): Promise { - const Connector = await getIntegration(datasource.source) - // can't enrich if it doesn't have an ID yet - if (datasource._id) { - datasource = await sdk.datasources.enrich(datasource) - } - // Connect to the DB and build the schema - return new Connector(datasource.config) -} - -async function getAndMergeDatasource(datasource: Datasource) { - let existingDatasource: undefined | Datasource - if (datasource._id) { - existingDatasource = await sdk.datasources.get(datasource._id) - } - let enrichedDatasource = datasource - if (existingDatasource) { - enrichedDatasource = sdk.datasources.mergeConfigs( - datasource, - existingDatasource - ) - } - return await sdk.datasources.enrich(enrichedDatasource) -} - -async function buildSchemaHelper(datasource: Datasource): Promise { - const connector = (await getConnector(datasource)) as DatasourcePlus - return await connector.buildSchema( - datasource._id!, - datasource.entities! as Record - ) -} - -async function buildFilteredSchema( - datasource: Datasource, - filter?: string[] -): Promise { - let schema = await buildSchemaHelper(datasource) - if (!filter) { - return schema - } - - let filteredSchema: Schema = { tables: {}, errors: {} } - for (let key in schema.tables) { - if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { - filteredSchema.tables[key] = schema.tables[key] - } - } - - for (let key in schema.errors) { - if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { - filteredSchema.errors[key] = schema.errors[key] - } - } - return filteredSchema -} - export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() } @@ -95,8 +33,10 @@ export async function verify( ctx: UserCtx ) { const { datasource } = ctx.request.body - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = await getConnector(enrichedDatasource) + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( + datasource + ) + const connector = await sdk.datasources.getConnector(enrichedDatasource) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } @@ -112,8 +52,12 @@ export async function information( ctx: UserCtx ) { const { datasource } = ctx.request.body - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( + datasource + ) + const connector = (await sdk.datasources.getConnector( + enrichedDatasource + )) as DatasourcePlus if (!connector.getTableNames) { ctx.throw(400, "Table name fetching not supported by datasource") } @@ -128,7 +72,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) { const tablesFilter = ctx.request.body.tablesFilter const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter) + const { tables, errors } = await sdk.datasources.buildFilteredSchema( + datasource, + tablesFilter + ) datasource.entities = tables setDefaultDisplayColumns(datasource) @@ -280,7 +227,10 @@ export async function save( let errors: Record = {} if (fetchSchema) { - const schema = await buildFilteredSchema(datasource, tablesFilter) + const schema = await sdk.datasources.buildFilteredSchema( + datasource, + tablesFilter + ) datasource.entities = schema.tables setDefaultDisplayColumns(datasource) errors = schema.errors @@ -384,8 +334,10 @@ export async function query(ctx: UserCtx) { export async function getExternalSchema(ctx: UserCtx) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = await getConnector(enrichedDatasource) + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( + datasource + ) + const connector = await sdk.datasources.getConnector(enrichedDatasource) if (!connector.getExternalSchema) { ctx.throw(400, "Datasource does not support exporting external schema") diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 2963546e7f..5f383e837d 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -25,8 +25,12 @@ import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types" +import { + getAppMigrationVersion, + getLatestMigrationId, +} from "../../../appMigrations" -const send = require("koa-send") +import send from "koa-send" export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` @@ -125,7 +129,26 @@ export const deleteObjects = async function (ctx: Ctx) { ) } +const requiresMigration = async (ctx: Ctx) => { + const appId = context.getAppId() + if (!appId) { + ctx.throw("AppId could not be found") + } + + const latestMigration = getLatestMigrationId() + if (!latestMigration) { + return false + } + + const latestMigrationApplied = await getAppMigrationVersion(appId) + + const requiresMigrations = latestMigrationApplied !== latestMigration + return requiresMigrations +} + export const serveApp = async function (ctx: Ctx) { + const needMigrations = await requiresMigration(ctx) + const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -145,8 +168,8 @@ export const serveApp = async function (ctx: Ctx) { let appId = context.getAppId() if (!env.isJest()) { - const App = require("./templates/BudibaseApp.svelte").default const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins) + const App = require("./templates/BudibaseApp.svelte").default const { head, html, css } = App.render({ metaImage: branding?.metaImageUrl || @@ -167,6 +190,7 @@ export const serveApp = async function (ctx: Ctx) { config?.logoUrl !== "" ? objectStore.getGlobalFileUrl("settings", "logoUrl") : "", + appMigrating: needMigrations, }) const appHbs = loadHandlebarsFile(appHbsPath) ctx.body = await processString(appHbs, { @@ -273,7 +297,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) { const { bucket, key } = ctx.request.body || {} if (!bucket || !key) { ctx.throw(400, "bucket and key values are required") - return } try { const s3 = new AWS.S3({ diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte index 32edb6dc7b..7819368fc0 100644 --- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte +++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte @@ -8,6 +8,7 @@ export let clientLibPath export let usedPlugins + export let appMigrating @@ -110,6 +111,11 @@ + {#if appMigrating} + + {/if} diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index de3bf0e59e..2d61d05a01 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -362,8 +362,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { }) } - let finalizedTables = finaliseExternalTables(tables, entities) - let errors = checkExternalTables(finalizedTables) + const finalizedTables = finaliseExternalTables(tables, entities) + const errors = checkExternalTables(finalizedTables) return { tables: finalizedTables, errors } } catch (err) { // @ts-ignore diff --git a/packages/server/src/sdk/app/datasources/index.ts b/packages/server/src/sdk/app/datasources/index.ts index 1ce6b0e689..8f06e989d3 100644 --- a/packages/server/src/sdk/app/datasources/index.ts +++ b/packages/server/src/sdk/app/datasources/index.ts @@ -1,5 +1,7 @@ import * as datasources from "./datasources" +import * as plus from "./plus" export default { ...datasources, + ...plus, } diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts new file mode 100644 index 0000000000..117d19a6a7 --- /dev/null +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -0,0 +1,62 @@ +import { + Datasource, + DatasourcePlus, + IntegrationBase, + Schema, +} from "@budibase/types" +import * as datasources from "./datasources" +import { getIntegration } from "../../../integrations" + +export async function buildFilteredSchema( + datasource: Datasource, + filter?: string[] +): Promise { + const schema = await buildSchemaHelper(datasource) + if (!filter) { + return schema + } + + let filteredSchema: Schema = { tables: {}, errors: {} } + for (let key in schema.tables) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.tables[key] = schema.tables[key] + } + } + + for (let key in schema.errors) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.errors[key] = schema.errors[key] + } + } + return filteredSchema +} + +async function buildSchemaHelper(datasource: Datasource): Promise { + const connector = (await getConnector(datasource)) as DatasourcePlus + const externalSchema = await connector.buildSchema( + datasource._id!, + datasource.entities! + ) + return externalSchema +} + +export async function getConnector( + datasource: Datasource +): Promise { + const Connector = await getIntegration(datasource.source) + // can't enrich if it doesn't have an ID yet + if (datasource._id) { + datasource = await datasources.enrich(datasource) + } + // Connect to the DB and build the schema + return new Connector(datasource.config) +} + +export async function getAndMergeDatasource(datasource: Datasource) { + if (datasource._id) { + const existingDatasource = await datasources.get(datasource._id) + + datasource = datasources.mergeConfigs(datasource, existingDatasource) + } + return await datasources.enrich(datasource) +} diff --git a/yarn.lock b/yarn.lock index 150585ce1a..85dd6ab251 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18312,11 +18312,6 @@ regexparam@2.0.1: resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa" integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw== -regexparam@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f" - integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g== - regexpu-core@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"