diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index b26d0386fc..79c97b69af 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -32,7 +32,9 @@ jobs: uses: cypress-io/github-action@v2 with: install: false - command: yarn test:e2e:ci + command: yarn test:e2e:ci:record + env: + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} # TODO: upload recordings to s3 # - name: Configure AWS Credentials diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index c80cfa2ecc..3bbe718d73 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -34,6 +34,7 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: secretKeyRef: @@ -44,6 +45,7 @@ spec: secretKeyRef: name: {{ template "couchdb.fullname" . }} key: adminPassword + {{ end }} - name: ENABLE_ANALYTICS value: {{ .Values.globals.enableAnalytics | quote }} - name: INTERNAL_API_KEY @@ -112,6 +114,8 @@ spec: value: {{ .Values.globals.google.secret | quote }} - name: AUTOMATION_MAX_ITERATIONS value: {{ .Values.globals.automationMaxIterations | quote }} + - name: EXCLUDE_QUOTAS_TENANTS + value: {{ .Values.globals.excludeQuotasTenants | quote }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index c2180aca2b..15ff05e214 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -29,6 +29,7 @@ spec: - env: - name: CLUSTER_PORT value: {{ .Values.services.worker.port | quote }} + {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: secretKeyRef: @@ -39,6 +40,7 @@ spec: secretKeyRef: name: {{ template "couchdb.fullname" . }} key: adminPassword + {{ end }} - name: COUCH_DB_URL {{ if .Values.services.couchdb.url }} value: {{ .Values.services.couchdb.url }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 116931a147..52ead6d076 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -215,7 +215,7 @@ couchdb: ## The CouchDB image image: repository: couchdb - tag: 3.1.0 + tag: 3.2.1 pullPolicy: IfNotPresent ## Experimental integration with Lucene-powered fulltext search diff --git a/lerna.json b/lerna.json index aad3485095..319ea561a1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 727104d830..fb6d9da990 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "test:e2e": "lerna run cy:test --stream", "test:e2e:ci": "lerna run cy:ci --stream", + "test:e2e:ci:record": "lerna run cy:ci:record --stream", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 96d12dbd13..3a06793939 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index 26e0d486f7..c4fc4a87c8 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -6,6 +6,10 @@ function isTest() { ) } +function isDev() { + return process.env.NODE_ENV !== "production" +} + module.exports = { JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", @@ -32,6 +36,7 @@ module.exports = { TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, USE_COUCH: process.env.USE_COUCH || true, isTest, + isDev, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 07df005e8c..9e973a5d32 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.130-alpha.3", + "@budibase/string-templates": "^1.0.143-alpha.1", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Table/DateTimeRenderer.svelte b/packages/bbui/src/Table/DateTimeRenderer.svelte index 5d856968e7..f4b0821069 100644 --- a/packages/bbui/src/Table/DateTimeRenderer.svelte +++ b/packages/bbui/src/Table/DateTimeRenderer.svelte @@ -2,17 +2,22 @@ import dayjs from "dayjs" export let value + export let schema // adding the 0- will turn a string like 00:00:00 into a valid ISO // date, but will make actual ISO dates invalid $: time = new Date(`0-${value}`) - $: isTime = !isNaN(time) + $: isTimeOnly = !isNaN(time) || schema?.timeOnly + $: isDateOnly = schema?.dateOnly + $: format = isTimeOnly + ? "HH:mm:ss" + : isDateOnly + ? "MMMM D YYYY" + : "MMMM D YYYY, HH:mm"
- {dayjs(isTime ? time : value).format( - isTime ? "HH:mm:ss" : "MMMM D YYYY, HH:mm" - )} + {dayjs(isTimeOnly ? time : value).format(format)}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte new file mode 100644 index 0000000000..c30e8fc2ee --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte @@ -0,0 +1,145 @@ + + + + + btn_google_dark_normal_ios + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 1441d3834b..95a5f54e32 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -49,6 +49,10 @@ filters = [...filters, duplicate] } + const getSchema = filter => { + return schemaFields.find(field => field.name === filter.field) + } + const onFieldChange = (expression, field) => { // Update the field type expression.type = enrichedSchemaFields.find(x => x.name === field)?.type @@ -150,7 +154,12 @@ bind:value={filter.value} /> {:else if filter.type === "datetime"} - + {:else} {/if} diff --git a/packages/builder/src/components/integration/QueryViewer.svelte b/packages/builder/src/components/integration/QueryViewer.svelte index 8f6f9eeb53..49b9b38d86 100644 --- a/packages/builder/src/components/integration/QueryViewer.svelte +++ b/packages/builder/src/components/integration/QueryViewer.svelte @@ -44,6 +44,20 @@ $: readQuery = query.queryVerb === "read" || query.readable $: queryInvalid = !query.name || (readQuery && data.length === 0) + //Cast field in query preview response to number if specified by schema + $: { + for (let i = 0; i < data.length; i++) { + let row = data[i] + for (let fieldName of Object.keys(fields)) { + if (fields[fieldName] === "number" && !isNaN(Number(row[fieldName]))) { + row[fieldName] = Number(row[fieldName]) + } else { + row[fieldName] = row[fieldName]?.toString() + } + } + } + } + // seed the transformer if (query && !query.transformer) { query.transformer = "return data" diff --git a/packages/builder/src/pages/builder/portal/manage/email/index.svelte b/packages/builder/src/pages/builder/portal/manage/email/index.svelte index 4ef59d2daa..56242f0fe4 100644 --- a/packages/builder/src/pages/builder/portal/manage/email/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/email/index.svelte @@ -13,7 +13,7 @@ Table, Checkbox, } from "@budibase/bbui" - import { email } from "stores/portal" + import { email, admin } from "stores/portal" import { API } from "api" import { cloneDeep } from "lodash/fp" import analytics, { Events } from "analytics" @@ -58,6 +58,7 @@ const savedConfig = await API.saveConfig(smtp) smtpConfig._rev = savedConfig._rev smtpConfig._id = savedConfig._id + await admin.getChecklist() notifications.success(`Settings saved`) analytics.captureEvent(Events.SMTP.SAVED) } catch (error) { diff --git a/packages/builder/src/stores/portal/admin.js b/packages/builder/src/stores/portal/admin.js index 088d396291..69a2c82349 100644 --- a/packages/builder/src/stores/portal/admin.js +++ b/packages/builder/src/stores/portal/admin.js @@ -24,14 +24,8 @@ export function createAdminStore() { const admin = writable(DEFAULT_CONFIG) async function init() { - const tenantId = get(auth).tenantId - const checklist = await API.getChecklist(tenantId) - const totalSteps = Object.keys(checklist).length - const completedSteps = Object.values(checklist).filter( - x => x?.checked - ).length + await getChecklist() await getEnvironment() - // enable system status checks in the cloud if (get(admin).cloud) { await getSystemStatus() @@ -40,8 +34,6 @@ export function createAdminStore() { admin.update(store => { store.loaded = true - store.checklist = checklist - store.onboardingProgress = (completedSteps / totalSteps) * 100 return store }) } @@ -81,6 +73,20 @@ export function createAdminStore() { }) } + async function getChecklist() { + const tenantId = get(auth).tenantId + const checklist = await API.getChecklist(tenantId) + const totalSteps = Object.keys(checklist).length + const completedSteps = Object.values(checklist).filter( + x => x?.checked + ).length + admin.update(store => { + store.checklist = checklist + store.onboardingProgress = (completedSteps / totalSteps) * 100 + return store + }) + } + function unload() { admin.update(store => { store.loaded = false @@ -93,6 +99,7 @@ export function createAdminStore() { init, checkImportComplete, unload, + getChecklist, } } diff --git a/packages/cli/package.json b/packages/cli/package.json index ebdd7677fa..081304fed1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index c72fdbe660..aa3da23beb 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.130-alpha.3", - "@budibase/frontend-core": "^1.0.130-alpha.3", - "@budibase/string-templates": "^1.0.130-alpha.3", + "@budibase/bbui": "^1.0.143-alpha.1", + "@budibase/frontend-core": "^1.0.143-alpha.1", + "@budibase/string-templates": "^1.0.143-alpha.1", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index 54ad1f3a80..b8815279dd 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -88,6 +88,10 @@ const schema = schemaFields.find(x => x.name === field) return schema?.constraints?.inclusion || [] } + + const getSchema = filter => { + return schemaFields.find(field => field.name === filter.field) + }
@@ -134,7 +138,12 @@ bind:value={filter.value} /> {:else if filter.type === "datetime"} - + {:else} {/if} diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte index 3ebfc5084f..b267f6caff 100644 --- a/packages/client/src/components/app/forms/Field.svelte +++ b/packages/client/src/components/app/forms/Field.svelte @@ -44,7 +44,6 @@ fieldApi = value?.fieldApi fieldSchema = value?.fieldSchema }) - onDestroy(() => unsubscribe?.()) // Determine label class from position $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}` @@ -52,6 +51,11 @@ const updateLabel = e => { builderStore.actions.updateProp("label", e.target.textContent) } + + onDestroy(() => { + fieldApi?.deregister() + unsubscribe?.() + }) diff --git a/packages/client/src/components/app/forms/FormStep.svelte b/packages/client/src/components/app/forms/FormStep.svelte index 4441f515ee..22972c5c48 100644 --- a/packages/client/src/components/app/forms/FormStep.svelte +++ b/packages/client/src/components/app/forms/FormStep.svelte @@ -22,7 +22,7 @@ if ( formContext && $builderStore.inBuilder && - $componentStore.selectedComponentPath?.includes($component.id) + $componentStore?.selectedComponentPath?.includes($component.id) ) { formContext.formApi.setStep(step) } diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte index 99dcbf4d5e..752bc9a2eb 100644 --- a/packages/client/src/components/app/forms/InnerForm.svelte +++ b/packages/client/src/components/app/forms/InnerForm.svelte @@ -329,6 +329,17 @@ } } + // We don't want to actually remove the field state when deregistering, just + // remove any errors and validation + const deregister = () => { + const fieldInfo = getField(field) + fieldInfo.update(state => { + state.fieldState.validator = null + state.fieldState.error = null + return state + }) + } + // Updates the disabled state of a certain field const setDisabled = fieldDisabled => { const fieldInfo = getField(field) @@ -348,6 +359,7 @@ reset, updateValidation, setDisabled, + deregister, validate: () => { // Validate the field by force setting the same value again const { fieldState } = get(getField(field)) diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index f054e93ca1..ce3448bd79 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.130-alpha.3", + "@budibase/bbui": "^1.0.143-alpha.1", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index bad7f1a97e..feb124ea52 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -68,10 +68,10 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.130-alpha.3", - "@budibase/client": "^1.0.130-alpha.3", - "@budibase/pro": "1.0.130-alpha.3", - "@budibase/string-templates": "^1.0.130-alpha.3", + "@budibase/backend-core": "^1.0.143-alpha.1", + "@budibase/client": "^1.0.143-alpha.1", + "@budibase/pro": "1.0.143-alpha.1", + "@budibase/string-templates": "^1.0.143-alpha.1", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index 0d858741dd..a3bcae0cee 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -86,3 +86,15 @@ exports.substituteLoopStep = (hbsString, substitute) => { return hbsString } + +exports.stringSplit = value => { + if (value == null) { + return [] + } + if (value.split("\n").length > 1) { + value = value.split("\n") + } else { + value = value.split(",") + } + return value +} diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 1b37b5df9a..a7b9906481 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -242,12 +242,10 @@ module MSSQLModule { if (typeof name !== "string") { continue } - const type: string = convertSqlType(def.DATA_TYPE) - schema[name] = { autocolumn: !!autoColumns.find((col: string) => col === name), name: name, - type, + ...convertSqlType(def.DATA_TYPE), } } tables[tableName] = { diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 065a1b2333..4fe996a019 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -15,6 +15,7 @@ import { } from "./utils" import { DatasourcePlus } from "./base/datasourcePlus" import dayjs from "dayjs" +import { FieldTypes } from "../constants" const { NUMBER_REGEX } = require("../utilities") module MySQLModule { @@ -101,7 +102,7 @@ module MySQLModule { } // if not a number, see if it is a date - important to do in this order as any // integer will be considered a valid date - else if (dayjs(binding).isValid()) { + else if (/^\d/.test(binding) && dayjs(binding).isValid()) { bindings[i] = dayjs(binding).toDate() } } @@ -151,20 +152,24 @@ module MySQLModule { async internalQuery( query: SqlQuery, - connect: boolean = true + opts: { connect?: boolean; disableCoercion?: boolean } = { + connect: true, + disableCoercion: false, + } ): Promise { try { - if (connect) { + if (opts?.connect) { await this.connect() } + const baseBindings = query.bindings || [] + const bindings = opts?.disableCoercion + ? baseBindings + : bindingTypeCoerce(baseBindings) // Node MySQL is callback based, so we must wrap our call in a promise - const response = await this.client.query( - query.sql, - bindingTypeCoerce(query.bindings || []) - ) + const response = await this.client.query(query.sql, bindings) return response[0] } finally { - if (connect) { + if (opts?.connect) { await this.disconnect() } } @@ -179,7 +184,7 @@ module MySQLModule { // get the tables first const tablesResp = await this.internalQuery( { sql: "SHOW TABLES;" }, - false + { connect: false } ) const tableNames = tablesResp.map( (obj: any) => @@ -191,7 +196,7 @@ module MySQLModule { const schema: TableSchema = {} const descResp = await this.internalQuery( { sql: `DESCRIBE \`${tableName}\`;` }, - false + { connect: false } ) for (let column of descResp) { const columnName = column.Field @@ -211,8 +216,8 @@ module MySQLModule { schema[columnName] = { name: columnName, autocolumn: isAuto, - type: convertSqlType(column.Type), constraints, + ...convertSqlType(column.Type), } } if (!tables[tableName]) { @@ -254,7 +259,8 @@ module MySQLModule { async query(json: QueryJson) { await this.connect() try { - const queryFn = (query: any) => this.internalQuery(query, false) + const queryFn = (query: any) => + this.internalQuery(query, { connect: false, disableCoercion: true }) return await this.queryWithReturning(json, queryFn) } finally { await this.disconnect() diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 7cb7ba88cf..94e51694c1 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -279,9 +279,9 @@ module OracleModule { ) } - private internalConvertType(column: OracleColumn): string { + private internalConvertType(column: OracleColumn): { type: string } { if (this.isBooleanType(column)) { - return FieldTypes.BOOLEAN + return { type: FieldTypes.BOOLEAN } } return convertSqlType(column.type) @@ -328,7 +328,7 @@ module OracleModule { fieldSchema = { autocolumn: OracleIntegration.isAutoColumn(oracleColumn), name: columnName, - type: this.internalConvertType(oracleColumn), + ...this.internalConvertType(oracleColumn), } table.schema[columnName] = fieldSchema } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 1dc6fd9d2d..01257f3aa0 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -227,7 +227,6 @@ module PostgresModule { } } - const type: string = convertSqlType(column.data_type) const identity = !!( column.identity_generation || column.identity_start || @@ -242,7 +241,7 @@ module PostgresModule { tables[tableName].schema[columnName] = { autocolumn: isAuto, name: columnName, - type, + ...convertSqlType(column.data_type), } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 326b213bc7..7e4efad84f 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -35,6 +35,9 @@ const SQL_DATE_TYPE_MAP = { date: FieldTypes.DATETIME, } +const SQL_DATE_ONLY_TYPES = ["date"] +const SQL_TIME_ONLY_TYPES = ["time"] + const SQL_STRING_TYPE_MAP = { varchar: FieldTypes.STRING, char: FieldTypes.STRING, @@ -85,9 +88,9 @@ export function breakExternalTableId(tableId: string | undefined) { return {} } const parts = tableId.split(DOUBLE_SEPARATOR) - let tableName = parts.pop() + let datasourceId = parts.shift() // if they need joined - let datasourceId = parts.join(DOUBLE_SEPARATOR) + let tableName = parts.join(DOUBLE_SEPARATOR) return { datasourceId, tableName } } @@ -137,12 +140,20 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } export function convertSqlType(type: string) { + let foundType = FieldTypes.STRING + const lcType = type.toLowerCase() for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { - if (type.toLowerCase().includes(external)) { - return internal + if (lcType.includes(external)) { + foundType = internal + break } } - return FieldTypes.STRING + const schema: any = { type: foundType } + if (foundType === FieldTypes.DATETIME) { + schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType) + schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType) + } + return schema } export function getSqlQuery(query: SqlQuery | string): SqlQuery { diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index db462f5a8d..98c45e4af3 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -100,10 +100,10 @@ class Orchestrator { let automation = this._automation const app = await this.getApp() let stopped = false - let loopStep + let loopStep = null let stepCount = 0 - let loopStepNumber + let loopStepNumber = null let loopSteps = [] for (let step of automation.definition.steps) { stepCount++ @@ -117,15 +117,17 @@ class Orchestrator { if (loopStep) { input = await processObject(loopStep.inputs, this._context) } - let iterations = loopStep ? input.binding.length : 1 + let iterations = loopStep + ? Array.isArray(input.binding) + ? input.binding.length + : automationUtils.stringSplit(input.binding).length + : 1 let iterationCount = 0 for (let index = 0; index < iterations; index++) { let originalStepInput = cloneDeep(step.inputs) // Handle if the user has set a max iteration count or if it reaches the max limit set by us if (loopStep) { - // lets first of all handle the input - // if the input is array then use it, if it is a string then split it on every new line let newInput = await processObject( loopStep.inputs, cloneDeep(this._context) @@ -134,9 +136,6 @@ class Orchestrator { newInput, loopStep.schema.inputs ) - this._context.steps[loopStepNumber] = { - currentItem: newInput.binding[index], - } let tempOutput = { items: loopSteps, iterations: iterationCount } if ( @@ -154,6 +153,20 @@ class Orchestrator { break } + let item + if ( + typeof loopStep.inputs.binding === "string" && + loopStep.inputs.option === "String" + ) { + item = automationUtils.stringSplit(newInput.binding) + } else { + item = loopStep.inputs.binding + } + + this._context.steps[loopStepNumber] = { + currentItem: item[index], + } + // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it // Pretty hacky because we need to account for the row object for (let [key, value] of Object.entries(originalStepInput)) { @@ -178,7 +191,6 @@ class Orchestrator { } } } - if ( index === parseInt(env.AUTOMATION_MAX_ITERATIONS) || index === loopStep.inputs.iterations @@ -192,10 +204,25 @@ class Orchestrator { break } + let isFailure = false if ( - this._context.steps[loopStepNumber]?.currentItem === - loopStep.inputs.failure + typeof this._context.steps[loopStepNumber]?.currentItem === "object" ) { + isFailure = Object.keys( + this._context.steps[loopStepNumber].currentItem + ).some(value => { + return ( + this._context.steps[loopStepNumber].currentItem[value] === + loopStep.inputs.failure + ) + }) + } else { + isFailure = + this._context.steps[loopStepNumber]?.currentItem === + loopStep.inputs.failure + } + + if (isFailure) { this.updateContextAndOutput(loopStepNumber, step, tempOutput, { status: AutomationErrors.FAILURE_CONDITION, success: false, @@ -286,18 +313,16 @@ class Orchestrator { module.exports = (input, callback) => { const appId = input.data.event.appId - doInAppContext(appId, () => { + doInAppContext(appId, async () => { const automationOrchestrator = new Orchestrator( input.data.automation, input.data.event ) - automationOrchestrator - .execute() - .then(response => { - callback(null, response) - }) - .catch(err => { - callback(err) - }) + try { + const response = await automationOrchestrator.execute() + callback(null, response) + } catch (err) { + callback(err) + } }) } diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 71994a7244..ec9d9a6fa6 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -191,14 +191,13 @@ class QueryRunner { } module.exports = (input, callback) => { - doInAppContext(input.appId, () => { + doInAppContext(input.appId, async () => { const Runner = new QueryRunner(input) - Runner.execute() - .then(response => { - callback(null, response) - }) - .catch(err => { - callback(err) - }) + try { + const response = await Runner.execute() + callback(null, response) + } catch (err) { + callback(err) + } }) } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 8a02afc5b3..e2a76c49a1 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -2,7 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") const uuid = require("uuid/v4") -const { doWithDB } = require("@budibase/backend-core/db") +const { + doWithDB, + dangerousGetDB, + closeDB, +} = require("@budibase/backend-core/db") const { ObjectStoreBuckets } = require("../../constants") const { upload, @@ -151,14 +155,18 @@ exports.streamBackup = async appId => { * @return {*} either a readable stream or a string */ exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - return doWithDB(dbName, async db => { - // Stream the dump if required - if (stream) { - const memStream = new MemoryStream() - db.dump(memStream, { filter }) - return memStream - } + // streaming a DB dump is a bit more complicated, can't close DB + if (stream) { + const db = dangerousGetDB(dbName) + const memStream = new MemoryStream() + memStream.on("end", async () => { + await closeDB(db) + }) + db.dump(memStream, { filter }) + return memStream + } + return doWithDB(dbName, async db => { // Write the dump to file if required if (exportName) { const path = join(budibaseTempDir(), exportName) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 5f506a5b34..d787cf79da 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.130-alpha.3": - version "1.0.130-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.130-alpha.3.tgz#1e51fb22fc9e6a6db1b588f6111bcf372047797c" - integrity sha512-R9RCmZPoJR+vFRiW5XYBWzImMdhsIRPGI7MJHmigb1/TMB1bKaPDFvjnU5rDQ5Qv6NZ1E81FGmMWvDYOzouV5w== +"@budibase/backend-core@1.0.143-alpha.1": + version "1.0.143-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.143-alpha.1.tgz#9915d17a2c46f4fd30fad1d33165e25eb3c4e0fc" + integrity sha512-Mo/OMvpbH+SgDx6t0Mg2AbD6hzU4ZOYCL0J6/AAr+W5xZtLXlIa/yznwB93UZPggkHSxVVxkXYKTvGVZgUfbHg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.130-alpha.3": - version "1.0.130-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.130-alpha.3.tgz#3311901846b761aa8e57980f8bd1baf36c9d9eb9" - integrity sha512-3QydQ0PGOrC8ho6cvaBFf/xm50+ygFYyy2P0fNj860+khid66lrrr5AGSozqtFCPGCz4vWn0J1irjDBhYSUHlw== +"@budibase/pro@1.0.143-alpha.1": + version "1.0.143-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.143-alpha.1.tgz#8d4de72bbebd68f935928d7eb495d35c74537793" + integrity sha512-YbTcEsKLUc0X272LLlq3Xa5Wp0MkPp/srIL3vaBfVRvCyj2f7mNWj5f1dWPe+aNbdILLdsUcAS51Bqg9r4ekdQ== dependencies: - "@budibase/backend-core" "1.0.130-alpha.3" + "@budibase/backend-core" "1.0.143-alpha.1" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": @@ -3432,11 +3432,6 @@ async-limiter@~1.0.0: resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== -async@0.9.x: - version "0.9.2" - resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d" - integrity sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0= - async@^2.6.3: version "2.6.3" resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" @@ -3814,6 +3809,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" @@ -4086,7 +4088,7 @@ chalk@^3.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.0.0, chalk@^4.1.0: +chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -5055,11 +5057,11 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= ejs@^3.1.6: - version "3.1.6" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.6.tgz#5bfd0a0689743bb5268b3550cceeebbc1702822a" - integrity sha512-9lt9Zse4hPucPkoP7FHDF0LQAlGyF9JVpnClFLFH3aSSbxmyoqINRpp/9wePWJTUl4KOQwRL72Iw3InHPDkoGw== + version "3.1.7" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.7.tgz#c544d9c7f715783dd92f0bddcf73a59e6962d006" + integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== dependencies: - jake "^10.6.1" + jake "^10.8.5" electron-to-chromium@^1.4.17: version "1.4.71" @@ -5910,11 +5912,11 @@ file-uri-to-path@1.0.0: integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== filelist@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b" - integrity sha512-z7O0IS8Plc39rTCq6i6iHxk43duYOn8uFJiWSewIq0Bww1RNybVHSCjahmcC87ZqAm4OTvFzlzeGu3XAzG1ctQ== + version "1.0.3" + resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.3.tgz#448607750376484932f67ef1b9ff07386b036c83" + integrity sha512-LwjCsruLWQULGYKy7TX0OPtrL9kLpojOFKc5VCTxdFTV7w5zbsgqVKfnkKG7Qgjtq50gKfO56hJv88OfcGb70Q== dependencies: - minimatch "^3.0.4" + minimatch "^5.0.1" filename-reserved-regex@^2.0.0: version "2.0.0" @@ -7462,13 +7464,13 @@ isurl@^1.0.0-alpha5: has-to-string-tag-x "^1.2.0" is-object "^1.0.1" -jake@^10.6.1: - version "10.8.2" - resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.2.tgz#ebc9de8558160a66d82d0eadc6a2e58fbc500a7b" - integrity sha512-eLpKyrfG3mzvGE2Du8VoPbeSkRry093+tyNjdYaBbJS9v17knImYGNXQCUV0gLxQtF82m3E8iRb/wdSQZLoq7A== +jake@^10.8.5: + version "10.8.5" + resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" + integrity sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw== dependencies: - async "0.9.x" - chalk "^2.4.2" + async "^3.2.3" + chalk "^4.0.2" filelist "^1.0.1" minimatch "^3.0.4" @@ -9451,6 +9453,13 @@ min-document@^2.19.0: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b" + integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index dc38fed2ad..6e436a63d5 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 6e464c27c4..f4feceac4b 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -70,7 +70,7 @@ function createTemplate(string, opts) { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The structure input, as fully updated as possible. */ module.exports.processObject = async (object, context, opts) => { @@ -101,7 +101,7 @@ module.exports.processObject = async (object, context, opts) => { * then nothing will occur. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The enriched string, all templates should have been replaced if they can be. */ module.exports.processString = async (string, context, opts) => { @@ -115,7 +115,7 @@ module.exports.processString = async (string, context, opts) => { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {object|array} The structure input, as fully updated as possible. */ module.exports.processObjectSync = (object, context, opts) => { @@ -136,7 +136,7 @@ module.exports.processObjectSync = (object, context, opts) => { * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {string} The enriched string, all templates should have been replaced if they can be. */ module.exports.processStringSync = (string, context, opts) => { @@ -194,7 +194,7 @@ module.exports.makePropSafe = property => { /** * Checks whether or not a template string contains totally valid syntax (simply tries running it) * @param string The string to test for valid syntax - this may contain no templates and will be considered valid. - * @param opts optional - specify some options for processing. + * @param [opts] optional - specify some options for processing. * @returns {boolean} Whether or not the input string is valid. */ module.exports.isValid = (string, opts) => { @@ -205,6 +205,7 @@ module.exports.isValid = (string, opts) => { "array", "cannot read property", "undefined", + "json at position 0", ] // this is a portion of a specific string always output by handlebars in the case of a syntax error const invalidCases = [`expecting '`] diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.js index 0d39660d59..17e6876bba 100644 --- a/packages/string-templates/test/helpers.spec.js +++ b/packages/string-templates/test/helpers.spec.js @@ -360,6 +360,13 @@ describe("Test the literal helper", () => { }) }) +describe("Test that JSONpase helper", () => { + it("should state that the JSONparse helper is valid", async () => { + const output = isValid(`{{ JSONparse input }}`) + expect(output).toBe(true) + }) +}) + describe("Cover a few complex use cases", () => { it("should allow use of three different collection helpers", async () => { const output = await processString( diff --git a/packages/worker/package.json b/packages/worker/package.json index e5b8ab97a1..13d9a75cbc 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.130-alpha.3", + "version": "1.0.143-alpha.1", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -31,9 +31,9 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.130-alpha.3", - "@budibase/pro": "1.0.130-alpha.3", - "@budibase/string-templates": "^1.0.130-alpha.3", + "@budibase/backend-core": "^1.0.143-alpha.1", + "@budibase/pro": "1.0.143-alpha.1", + "@budibase/string-templates": "^1.0.143-alpha.1", "@koa/router": "^8.0.0", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index fc9b6ac529..50e1173eb1 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.130-alpha.3": - version "1.0.130-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.130-alpha.3.tgz#1e51fb22fc9e6a6db1b588f6111bcf372047797c" - integrity sha512-R9RCmZPoJR+vFRiW5XYBWzImMdhsIRPGI7MJHmigb1/TMB1bKaPDFvjnU5rDQ5Qv6NZ1E81FGmMWvDYOzouV5w== +"@budibase/backend-core@1.0.143-alpha.1": + version "1.0.143-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.143-alpha.1.tgz#9915d17a2c46f4fd30fad1d33165e25eb3c4e0fc" + integrity sha512-Mo/OMvpbH+SgDx6t0Mg2AbD6hzU4ZOYCL0J6/AAr+W5xZtLXlIa/yznwB93UZPggkHSxVVxkXYKTvGVZgUfbHg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.130-alpha.3": - version "1.0.130-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.130-alpha.3.tgz#3311901846b761aa8e57980f8bd1baf36c9d9eb9" - integrity sha512-3QydQ0PGOrC8ho6cvaBFf/xm50+ygFYyy2P0fNj860+khid66lrrr5AGSozqtFCPGCz4vWn0J1irjDBhYSUHlw== +"@budibase/pro@1.0.143-alpha.1": + version "1.0.143-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.143-alpha.1.tgz#8d4de72bbebd68f935928d7eb495d35c74537793" + integrity sha512-YbTcEsKLUc0X272LLlq3Xa5Wp0MkPp/srIL3vaBfVRvCyj2f7mNWj5f1dWPe+aNbdILLdsUcAS51Bqg9r4ekdQ== dependencies: - "@budibase/backend-core" "1.0.130-alpha.3" + "@budibase/backend-core" "1.0.143-alpha.1" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":