MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into cypress-testing

This commit is contained in:
Mitch-Budibase 2022-05-11 16:50:23 +01:00
parent 2e53a80219 cb9655c360
commit 47a6b2d5ed
45 changed files with 407 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/smoke_test.yaml vendored
View File

@ -31,6 +31,7 @@ jobs:
continue-on-error: true
uses: cypress-io/github-action@v2
with:
record: true
install: false
command: yarn test:e2e:ci:record
env:
@ -48,7 +49,7 @@ jobs:
uses: tsickert/discord-webhook@v4.0.0
with:
webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }}
content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}"
content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.outputs.dashboardUrl }}"
embed-title: ${{ steps.cypress.outcome }}
embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }}

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -114,8 +114,8 @@ spec:
value: {{ .Values.globals.google.secret | quote }}
- name: AUTOMATION_MAX_ITERATIONS
value: {{ .Values.globals.automationMaxIterations | quote }}
- name: EXCLUDE_QUOTAS_TENANTS
value: {{ .Values.globals.excludeQuotasTenants | quote }}
- name: TENANT_FEATURE_FLAGS
value: {{ .Values.globals.tenantFeatureFlags | quote }}
image: budibase/apps:{{ .Values.globals.appVersion }}
imagePullPolicy: Always

10
hosting/nginx.prod.conf.hbs
View File

@ -30,7 +30,7 @@ http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
map $http_upgrade $connection_upgrade {
default "upgrade";
}
@ -42,13 +42,13 @@ http {
client_max_body_size 1000m;
ignore_invalid_headers off;
proxy_buffering off;
set $csp_default "default-src 'self'";
set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com";
set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
@ -58,7 +58,7 @@ http {
error_page 502 503 504 /error.html;
location = /error.html {
root /usr/share/nginx/html;
root /usr/share/nginx/html;
internal;
}
@ -154,4 +154,4 @@ http {
gzip_comp_level 6;
gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
}
}
}

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"npmClient": "yarn",
"packages": [
"packages/*"

2
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

32
packages/backend-core/src/db/conversions.js
View File

@ -23,24 +23,30 @@ exports.isDevApp = app => {
}
/**
* Convert a development app ID to a deployed app ID.
* Generates a development app ID from a real app ID.
* @returns {string} the dev app ID which can be used for dev database.
*/
exports.getProdAppID = appId => {
// if dev, convert it
if (appId.startsWith(APP_DEV_PREFIX)) {
const id = appId.split(APP_DEV_PREFIX)[1]
return `${APP_PREFIX}${id}`
exports.getDevelopmentAppID = appId => {
if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
return appId
// split to take off the app_ element, then join it together incase any other app_ exist
const split = appId.split(APP_PREFIX)
split.shift()
const rest = split.join(APP_PREFIX)
return `${APP_DEV_PREFIX}${rest}`
}
/**
* Convert a deployed app ID to a development app ID.
* Convert a development app ID to a deployed app ID.
*/
exports.getDevelopmentAppID = appId => {
if (!appId.startsWith(APP_DEV_PREFIX)) {
const id = appId.split(APP_PREFIX)[1]
return `${APP_DEV_PREFIX}${id}`
exports.getProdAppID = appId => {
if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
return appId
// split to take off the app_dev element, then join it together incase any other app_ exist
const split = appId.split(APP_DEV_PREFIX)
split.shift()
const rest = split.join(APP_DEV_PREFIX)
return `${APP_PREFIX}${rest}`
}

61
packages/backend-core/src/db/tests/utils.spec.js Normal file
View File

@ -0,0 +1,61 @@
const {
generateAppID,
getDevelopmentAppID,
getProdAppID,
isDevAppID,
isProdAppID,
} = require("../utils")
function getID() {
const appId = generateAppID()
const split = appId.split("_")
const uuid = split[split.length - 1]
const devAppId = `app_dev_${uuid}`
return { appId, devAppId, split, uuid }
}
describe("app ID manipulation", () => {
it("should be able to generate a new app ID", () => {
expect(generateAppID().startsWith("app_")).toEqual(true)
})
it("should be able to convert a production app ID to development", () => {
const { appId, uuid } = getID()
expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development app ID to development", () => {
const { devAppId, uuid } = getID()
expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development ID to a production", () => {
const { devAppId, uuid } = getID()
expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
})
it("should be able to convert a production ID to production", () => {
const { appId, uuid } = getID()
expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
})
it("should be able to confirm dev app ID is development", () => {
const { devAppId } = getID()
expect(isDevAppID(devAppId)).toEqual(true)
})
it("should be able to confirm prod app ID is not development", () => {
const { appId } = getID()
expect(isDevAppID(appId)).toEqual(false)
})
it("should be able to confirm prod app ID is prod", () => {
const { appId } = getID()
expect(isProdAppID(appId)).toEqual(true)
})
it("should be able to confirm dev app ID is not prod", () => {
const { devAppId } = getID()
expect(isProdAppID(devAppId)).toEqual(false)
})
})

12
packages/backend-core/src/db/utils.js
View File

@ -43,6 +43,18 @@ exports.isDevAppID = isDevAppID
exports.getDevelopmentAppID = getDevelopmentAppID
exports.getProdAppID = getProdAppID
/**
* Generates a new app ID.
* @returns {string} The new app ID which the app doc can be stored under.
*/
exports.generateAppID = (tenantId = null) => {
let id = APP_PREFIX
if (tenantId) {
id += `${tenantId}${SEPARATOR}`
}
return `${id}${newid()}`
}
/**
* If creating DB allDocs/query params with only a single top level ID this can be used, this
* is usually the case as most of our docs are top level e.g. tables, automations, users and so on.

6
packages/backend-core/src/environment.js
View File

@ -34,6 +34,12 @@ module.exports = {
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
PLATFORM_URL: process.env.PLATFORM_URL,
TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates",
GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global",
GLOBAL_CLOUD_BUCKET_NAME:
process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
USE_COUCH: process.env.USE_COUCH || true,
isTest,
isDev,

1
packages/backend-core/src/featureFlags/index.js
View File

@ -49,4 +49,5 @@ exports.getTenantFeatureFlags = tenantId => {
exports.FeatureFlag = {
LICENSING: "LICENSING",
GOOGLE_SHEETS: "GOOGLE_SHEETS",
}

11
packages/backend-core/src/objectStore/utils.js
View File

@ -1,12 +1,13 @@
const { join } = require("path")
const { tmpdir } = require("os")
const env = require("../environment")
exports.ObjectStoreBuckets = {
BACKUPS: "backups",
APPS: "prod-budi-app-assets",
TEMPLATES: "templates",
GLOBAL: "global",
GLOBAL_CLOUD: "prod-budi-tenant-uploads",
BACKUPS: env.BACKUPS_BUCKET_NAME,
APPS: env.APPS_BUCKET_NAME,
TEMPLATES: env.TEMPLATES_BUCKET_NAME,
GLOBAL: env.GLOBAL_BUCKET_NAME,
GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
}
exports.budibaseTempDir = function () {

4
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
"@budibase/string-templates": "^1.0.143-alpha.2",
"@budibase/string-templates": "^1.0.151-alpha.2",
"@spectrum-css/actionbutton": "^1.0.1",
"@spectrum-css/actiongroup": "^1.0.1",
"@spectrum-css/avatar": "^3.0.2",

2
packages/builder/cypress/integration/createScreen.js
View File

@ -9,7 +9,7 @@ filterTests(["smoke", "all"], () => {
})
it("Should successfully create a screen", () => {
cy.createScreen("/test")
cy.createScreen("test")
cy.get(".nav-items-container").within(() => {
cy.contains("/test").should("exist")
})

3
packages/builder/cypress/integration/revertApp.spec.js
View File

@ -26,6 +26,8 @@ filterTests(['smoke', 'all'], () => {
})
it("should revert a published app", () => {
cy.navigateToFrontend()
// Add initial component - Paragraph
cy.addComponent("Elements", "Paragraph")
// Publish app
@ -37,6 +39,7 @@ filterTests(['smoke', 'all'], () => {
cy.get(".spectrum-ButtonGroup").within(() => {
cy.get(".spectrum-Button").contains("Done").click({ force: true })
})
// Add second component - Button
cy.addComponent("Elements", "Button")
// Click Revert

1
packages/builder/cypress/support/commands.js
View File

@ -142,7 +142,6 @@ Cypress.Commands.add("createTestApp", () => {
const appName = "Cypress Tests"
cy.deleteApp(appName)
cy.createApp(appName, "This app is used for Cypress testing.")
cy.createScreen("home")
})
Cypress.Commands.add("createTestTableWithData", () => {

10
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -67,10 +67,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.143-alpha.2",
"@budibase/client": "^1.0.143-alpha.2",
"@budibase/frontend-core": "^1.0.143-alpha.2",
"@budibase/string-templates": "^1.0.143-alpha.2",
"@budibase/bbui": "^1.0.151-alpha.2",
"@budibase/client": "^1.0.151-alpha.2",
"@budibase/frontend-core": "^1.0.151-alpha.2",
"@budibase/string-templates": "^1.0.151-alpha.2",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

2
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -53,6 +53,7 @@
: { schema: {} }
$: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
$: schemaFields = Object.values(schema || {})
$: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
const onChange = Utils.sequential(async (e, key) => {
try {
@ -330,6 +331,7 @@
on:change={e => onChange(e, key)}
{bindings}
updateOnChange={false}
placeholder={value.customType === "queryLimit" ? queryLimit : ""}
/>
</div>
{/if}

4
packages/builder/src/components/start/AppRow.svelte
View File

@ -19,6 +19,7 @@
export let unpublishApp
export let releaseLock
export let editIcon
export let copyAppId
</script>
<div class="title">
@ -102,6 +103,9 @@
<MenuItem on:click={() => unpublishApp(app)} icon="GlobeRemove">
Unpublish
</MenuItem>
<MenuItem on:click={() => copyAppId(app)} icon="Copy">
Copy App ID
</MenuItem>
{/if}
{#if !app.deployed}
<MenuItem on:click={() => updateApp(app)} icon="Edit">Edit</MenuItem>

20
packages/builder/src/components/start/CreateAppModal.svelte
View File

@ -10,6 +10,8 @@
import { createValidationStore } from "helpers/validation/yup"
import * as appValidation from "helpers/validation/yup/app"
import TemplateCard from "components/common/TemplateCard.svelte"
import createFromScratchScreen from "builderStore/store/screenTemplates/createFromScratchScreen"
import { Roles } from "constants/backend"
export let template
@ -81,7 +83,7 @@
}
data.append("useTemplate", template != null)
if (template) {
data.append("templateName", template.name)
data.append("templateName", template.name) //or here?
data.append("templateKey", template.key)
data.append("templateFile", $values.file)
}
@ -104,6 +106,22 @@
// Create user
await API.updateOwnMetadata({ roleId: $values.roleId })
await auth.setInitInfo({})
// Create a default home screen if no template was selected
if (template == null) {
let defaultScreenTemplate = createFromScratchScreen.create()
defaultScreenTemplate.routing.route = "/home"
defaultScreenTemplate.routing.roldId = Roles.BASIC
try {
await store.actions.screens.save(defaultScreenTemplate)
} catch (err) {
console.error("Could not create a default application screen", err)
notifications.warning(
"Encountered an issue creating the default screen."
)
}
}
$goto(`/builder/app/${createdApp.instance._id}`)
} catch (error) {
creating = false

7
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -12,6 +12,7 @@
Body,
Search,
Divider,
Helpers,
} from "@budibase/bbui"
import TemplateDisplay from "components/common/TemplateDisplay.svelte"
import Spinner from "components/common/Spinner.svelte"
@ -261,6 +262,11 @@
}
}
const copyAppId = async app => {
await Helpers.copyToClipboard(app.prodId)
notifications.success("App ID copied to clipboard.")
}
function createAppFromTemplateUrl(templateKey) {
// validate the template key just to make sure
const templateParts = templateKey.split("/")
@ -394,6 +400,7 @@
<div class="appTable">
{#each filteredApps as app (app.appId)}
<AppRow
{copyAppId}
{releaseLock}
{editIcon}
{app}

2
packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte
View File

@ -47,7 +47,7 @@
</Body>
<Input
type="email"
label="Username"
label="Email"
bind:value={$email}
error={$touched && $error}
/>

2
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

8
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.143-alpha.2",
"@budibase/frontend-core": "^1.0.143-alpha.2",
"@budibase/string-templates": "^1.0.143-alpha.2",
"@budibase/bbui": "^1.0.151-alpha.2",
"@budibase/frontend-core": "^1.0.151-alpha.2",
"@budibase/string-templates": "^1.0.151-alpha.2",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

2
packages/client/src/components/app/forms/FormStep.svelte
View File

@ -22,7 +22,7 @@
if (
formContext &&
$builderStore.inBuilder &&
$componentStore?.selectedComponentPath?.includes($component.id)
$componentStore.selectedComponentPath?.includes($component.id)
) {
formContext.formApi.setStep(step)
}

2
packages/client/src/components/preview/SettingsBar.svelte
View File

@ -155,7 +155,7 @@
icon="Duplicate"
on:click={() => {
builderStore.actions.duplicateComponent(
$builderStore.selectedComponent._id
$builderStore.selectedComponentId
)
}}
title="Duplicate component"

2
packages/client/src/sdk.js
View File

@ -7,6 +7,7 @@ import {
builderStore,
uploadStore,
rowSelectionStore,
componentStore,
} from "stores"
import { styleable } from "utils/styleable"
import { linkable } from "utils/linkable"
@ -24,6 +25,7 @@ export default {
screenStore,
builderStore,
uploadStore,
componentStore,
styleable,
linkable,
getAction,

4
packages/frontend-core/package.json
View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "^1.0.143-alpha.2",
"@budibase/bbui": "^1.0.151-alpha.2",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

36
packages/server/__mocks__/@google-cloud/firestore.ts Normal file
View File

@ -0,0 +1,36 @@
module FirebaseMock {
const firebase: any = {}
firebase.Firestore = function () {
this.get = jest.fn(() => [
{
data: jest.fn(() => ({ result: "test" })),
},
])
this.update = jest.fn()
this.set = jest.fn()
this.delete = jest.fn()
this.doc = jest.fn(() => ({
update: this.update,
set: this.set,
delete: this.delete,
get: jest.fn(() => ({
data: jest.fn(() => ({ result: "test" })),
})),
id: "test_id",
}))
this.where = jest.fn(() => ({
get: this.get,
}))
this.collection = jest.fn(() => ({
doc: this.doc,
where: this.where,
}))
}
module.exports = firebase
}

10
packages/server/__mocks__/pg.ts
View File

@ -14,21 +14,13 @@ module PgMock {
function Client() {}
Client.prototype.query = query
Client.prototype.end = jest.fn()
Client.prototype.connect = jest.fn()
Client.prototype.release = jest.fn()
function Pool() {}
const on = jest.fn()
Pool.prototype.query = query
Pool.prototype.connect = jest.fn(() => {
// @ts-ignore
return new Client()
})
Pool.prototype.on = on
pg.Client = Client
pg.Pool = Pool
pg.queryMock = query
pg.on = on

10
packages/server/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -68,10 +68,10 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.143-alpha.2",
"@budibase/client": "^1.0.143-alpha.2",
"@budibase/pro": "1.0.143-alpha.2",
"@budibase/string-templates": "^1.0.143-alpha.2",
"@budibase/backend-core": "^1.0.151-alpha.2",
"@budibase/client": "^1.0.151-alpha.2",
"@budibase/pro": "1.0.151-alpha.2",
"@budibase/string-templates": "^1.0.151-alpha.2",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",

11
packages/server/src/api/controllers/integration.js
View File

@ -1,21 +1,16 @@
const { cloneDeep } = require("lodash")
const { definitions } = require("../../integrations")
const { getTenantId } = require("@budibase/backend-core/tenancy")
const { SourceNames } = require("../../definitions/datasource")
const googlesheets = require("../../integrations/googlesheets")
const env = require("../../environment")
const { featureFlags } = require("@budibase/backend-core")
exports.fetch = async function (ctx) {
ctx.status = 200
const defs = cloneDeep(definitions)
// for google sheets integration google verification
if (env.EXCLUDE_QUOTAS_TENANTS) {
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.split(",")
const tenantId = getTenantId()
if (excludedTenants.includes(tenantId)) {
defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
}
if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
}
ctx.body = defs

5
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -175,9 +175,10 @@ module External {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
for (let fieldName of Object.keys(table.schema)) {
const value = row[`${table.name}.${fieldName}`] || row[fieldName]
const pathValue = row[`${table.name}.${fieldName}`]
const value = pathValue != null ? pathValue : row[fieldName]
// all responses include "select col as table.col" so that overlaps are handled
if (value) {
if (value != null) {
thisRow[fieldName] = value
}
}

2
packages/server/src/api/controllers/view/exporters.js
View File

@ -6,7 +6,7 @@ exports.csv = function (headers, rows) {
.map(header => {
let val = row[header]
val =
typeof val === "object"
typeof val === "object" && !(val instanceof Date)
? `"${JSON.stringify(val).replace(/"/g, "'")}"`
: `"${val}"`
return val.trim()

1
packages/server/src/automations/steps/queryRows.js
View File

@ -50,6 +50,7 @@ exports.definition = {
limit: {
type: "number",
title: "Limit",
customType: "queryLimit",
},
},
required: ["tableId"],

26
packages/server/src/db/utils.js
View File

@ -9,6 +9,8 @@ const {
StaticDatabases,
isDevAppID,
isProdAppID,
getDevelopmentAppID,
generateAppID,
} = require("@budibase/backend-core/db")
const UNICODE_MAX = "\ufff0"
@ -80,6 +82,8 @@ exports.UNICODE_MAX = UNICODE_MAX
exports.SearchIndexes = SearchIndexes
exports.AppStatus = AppStatus
exports.BudibaseInternalDB = BudibaseInternalDB
exports.generateAppID = generateAppID
exports.generateDevAppID = getDevelopmentAppID
exports.generateRoleID = generateRoleID
exports.getRoleParams = getRoleParams
@ -243,28 +247,6 @@ exports.getLinkParams = (otherProps = {}) => {
return getDocParams(DocumentTypes.LINK, null, otherProps)
}
/**
* Generates a new app ID.
* @returns {string} The new app ID which the app doc can be stored under.
*/
exports.generateAppID = (tenantId = null) => {
let id = `${DocumentTypes.APP}${SEPARATOR}`
if (tenantId) {
id += `${tenantId}${SEPARATOR}`
}
return `${id}${newid()}`
}
/**
* Generates a development app ID from a real app ID.
* @returns {string} the dev app ID which can be used for dev database.
*/
exports.generateDevAppID = appId => {
const prefix = `${DocumentTypes.APP}${SEPARATOR}`
const rest = appId.split(prefix)[1]
return `${DocumentTypes.APP_DEV}${SEPARATOR}${rest}`
}
/**
* Generates a new layout ID.
* @returns {string} The new layout ID which the layout doc can be stored under.

20
packages/server/src/integrations/firebase.ts
View File

@ -92,13 +92,13 @@ module Firebase {
class FirebaseIntegration implements IntegrationBase {
private config: FirebaseConfig
private db: Firestore
private client: Firestore
constructor(config: FirebaseConfig) {
this.config = config
if (config.serviceAccount) {
const serviceAccount = JSON.parse(config.serviceAccount)
this.db = new Firestore({
this.client = new Firestore({
projectId: serviceAccount.project_id,
credentials: {
client_email: serviceAccount.client_email,
@ -106,7 +106,7 @@ module Firebase {
},
})
} else {
this.db = new Firestore({
this.client = new Firestore({
projectId: config.projectId,
credentials: {
client_email: config.email,
@ -118,7 +118,7 @@ module Firebase {
async create(query: { json: object; extra: { [key: string]: string } }) {
try {
const documentReference = this.db
const documentReference = this.client
.collection(query.extra.collection)
.doc()
await documentReference.set({ ...query.json, id: documentReference.id })
@ -133,7 +133,7 @@ module Firebase {
async read(query: { json: object; extra: { [key: string]: string } }) {
try {
let snapshot
const collectionRef = this.db.collection(query.extra.collection)
const collectionRef = this.client.collection(query.extra.collection)
if (
query.extra.filterField &&
query.extra.filter &&
@ -164,19 +164,19 @@ module Firebase {
extra: { [key: string]: string }
}) {
try {
await this.db
await this.client
.collection(query.extra.collection)
.doc(query.json.id)
.update(query.json)
return (
await this.db
await this.client
.collection(query.extra.collection)
.doc(query.json.id)
.get()
).data()
} catch (err) {
console.error("Error writing to firebase", err)
console.error("Error writing to Firestore", err)
throw err
}
}
@ -186,13 +186,13 @@ module Firebase {
extra: { [key: string]: string }
}) {
try {
await this.db
await this.client
.collection(query.extra.collection)
.doc(query.json.id)
.delete()
return true
} catch (err) {
console.error("Error writing to mongodb", err)
console.error("Error deleting from Firestore", err)
throw err
}
}

1
packages/server/src/integrations/index.ts
View File

@ -46,6 +46,7 @@ const INTEGRATIONS = {
[SourceNames.FIREBASE]: firebase.integration,
[SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
[SourceNames.REDIS]: redis.integration,
[SourceNames.FIREBASE]: firebase.integration,
}
// optionally add oracle integration if the oracle binary can be installed

96
packages/server/src/integrations/postgres.ts
View File

@ -16,7 +16,7 @@ import {
import { DatasourcePlus } from "./base/datasourcePlus"
module PostgresModule {
const { Pool } = require("pg")
const { Client } = require("pg")
const Sql = require("./base/sql")
const { escapeDangerousCharacters } = require("../utilities")
@ -104,7 +104,6 @@ module PostgresModule {
}
class PostgresIntegration extends Sql implements DatasourcePlus {
static pool: any
private readonly client: any
private readonly config: PostgresConfig
private index: number = 1
@ -136,11 +135,7 @@ module PostgresModule {
}
: undefined,
}
if (!this.pool) {
this.pool = new Pool(newConfig)
}
this.client = this.pool
this.client = new Client(newConfig)
this.setSchema()
}
@ -152,7 +147,7 @@ module PostgresModule {
return parts.join(" || ")
}
async internalQuery(query: SqlQuery) {
async internalQuery(query: SqlQuery, close: boolean = true) {
const client = this.client
this.index = 1
// need to handle a specific issue with json data types in postgres,
@ -169,18 +164,20 @@ module PostgresModule {
try {
return await client.query(query.sql, query.bindings || [])
} catch (err) {
await this.client.end()
// @ts-ignore
throw new Error(err)
} finally {
if (close) await this.client.end()
}
}
setSchema() {
async setSchema() {
await this.client.connect()
if (!this.config.schema) {
this.config.schema = "public"
}
this.client.on("connect", (client: any) => {
client.query(`SET search_path TO ${this.config.schema}`)
})
this.client.query(`SET search_path TO ${this.config.schema}`)
this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
}
@ -210,44 +207,52 @@ module PostgresModule {
tableKeys = {}
}
const columnsResponse = await this.client.query(this.COLUMNS_SQL)
const tables: { [key: string]: Table } = {}
try {
const columnsResponse = await this.client.query(this.COLUMNS_SQL)
for (let column of columnsResponse.rows) {
const tableName: string = column.table_name
const columnName: string = column.column_name
const tables: { [key: string]: Table } = {}
// table key doesn't exist yet
if (!tables[tableName] || !tables[tableName].schema) {
tables[tableName] = {
_id: buildExternalTableId(datasourceId, tableName),
primary: tableKeys[tableName] || [],
name: tableName,
schema: {},
for (let column of columnsResponse.rows) {
const tableName: string = column.table_name
const columnName: string = column.column_name
// table key doesn't exist yet
if (!tables[tableName] || !tables[tableName].schema) {
tables[tableName] = {
_id: buildExternalTableId(datasourceId, tableName),
primary: tableKeys[tableName] || [],
name: tableName,
schema: {},
}
}
const identity = !!(
column.identity_generation ||
column.identity_start ||
column.identity_increment
)
const hasDefault =
typeof column.column_default === "string" &&
column.column_default.startsWith("nextval")
const isGenerated =
column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = {
autocolumn: isAuto,
name: columnName,
...convertSqlType(column.data_type),
}
}
const identity = !!(
column.identity_generation ||
column.identity_start ||
column.identity_increment
)
const hasDefault =
typeof column.column_default === "string" &&
column.column_default.startsWith("nextval")
const isGenerated =
column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = {
autocolumn: isAuto,
name: columnName,
...convertSqlType(column.data_type),
}
const final = finaliseExternalTables(tables, entities)
this.tables = final.tables
this.schemaErrors = final.errors
} catch (err) {
// @ts-ignore
throw new Error(err)
} finally {
await this.client.end()
}
const final = finaliseExternalTables(tables, entities)
this.tables = final.tables
this.schemaErrors = final.errors
}
async create(query: SqlQuery | string) {
@ -276,8 +281,9 @@ module PostgresModule {
if (Array.isArray(input)) {
const responses = []
for (let query of input) {
responses.push(await this.internalQuery(query))
responses.push(await this.internalQuery(query, false))
}
await this.client.end()
return responses
} else {
const response = await this.internalQuery(input)

92
packages/server/src/integrations/tests/firebase.spec.js Normal file
View File

@ -0,0 +1,92 @@
const firebase = require("@google-cloud/firestore")
const FirebaseIntegration = require("../firebase")
jest.mock("@google-cloud/firestore")
class TestConfiguration {
constructor(config = {}) {
this.integration = new FirebaseIntegration.integration(config)
}
}
describe("Firebase Integration", () => {
let config
let tableName = "Users"
beforeEach(() => {
config = new TestConfiguration({
serviceAccount: "{}"
})
})
it("calls the create method with the correct params", async () => {
await config.integration.create({
table: tableName,
json: {
Name: "Test Name"
},
extra: {
collection: "test"
}
})
expect(config.integration.client.collection).toHaveBeenCalledWith("test")
expect(config.integration.client.set).toHaveBeenCalledWith({
Name: "Test Name",
id: "test_id"
})
})
it("calls the read method with the correct params", async () => {
const response = await config.integration.read({
table: tableName,
json: {
Name: "Test"
},
extra: {
collection: "test",
filterField: "field",
filter: "==",
filterValue: "value",
}
})
expect(config.integration.client.collection).toHaveBeenCalledWith("test")
expect(config.integration.client.where).toHaveBeenCalledWith("field", "==", "value")
expect(response).toEqual([{ result: "test"}])
})
it("calls the update method with the correct params", async () => {
const response = await config.integration.update({
table: tableName,
json: {
id: "test",
Name: "Test"
},
extra: {
collection: "test"
}
})
expect(config.integration.client.collection).toHaveBeenCalledWith("test")
expect(config.integration.client.update).toHaveBeenCalledWith({
Name: "Test",
id: "test"
})
expect(response).toEqual({
result: "test"
})
})
it("calls the delete method with the correct params", async () => {
const response = await config.integration.delete({
table: tableName,
json: {
id: "test",
Name: "Test"
},
extra: {
collection: "test"
}
})
expect(config.integration.client.collection).toHaveBeenCalledWith("test")
expect(config.integration.client.doc).toHaveBeenCalledWith("test")
expect(config.integration.client.delete).toHaveBeenCalled()
})
})

4
packages/server/src/integrations/tests/postgres.spec.js
View File

@ -15,10 +15,6 @@ describe("Postgres Integration", () => {
config = new TestConfiguration()
})
it("calls the connection callback", async () => {
expect(pg.on).toHaveBeenCalledWith('connect', expect.anything())
})
it("calls the create method with the correct params", async () => {
const sql = "insert into users (name, age) values ('Joe', 123);"
await config.integration.create({

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.138":
version "1.0.138"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0"
integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg==
"@budibase/backend-core@1.0.147":
version "1.0.147"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18"
integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.138":
version "1.0.138"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544"
integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA==
"@budibase/pro@1.0.147":
version "1.0.147"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1"
integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw==
dependencies:
"@budibase/backend-core" "1.0.138"
"@budibase/backend-core" "1.0.147"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

8
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.143-alpha.2",
"version": "1.0.151-alpha.2",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -31,9 +31,9 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.143-alpha.2",
"@budibase/pro": "1.0.143-alpha.2",
"@budibase/string-templates": "^1.0.143-alpha.2",
"@budibase/backend-core": "^1.0.151-alpha.2",
"@budibase/pro": "1.0.151-alpha.2",
"@budibase/string-templates": "^1.0.151-alpha.2",
"@koa/router": "^8.0.0",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "^0.3.0",

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.138":
version "1.0.138"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0"
integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg==
"@budibase/backend-core@1.0.147":
version "1.0.147"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18"
integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.138":
version "1.0.138"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544"
integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA==
"@budibase/pro@1.0.147":
version "1.0.147"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1"
integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw==
dependencies:
"@budibase/backend-core" "1.0.138"
"@budibase/backend-core" "1.0.147"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":

6
pull_request_template.md
View File

@ -1,8 +1,12 @@
## Description
_Describe the problem or feature in addition to a link to the relevant github issues._
Addresses:
- `<Enter the Link to the issue(s) this PR addresses>`
- ...more if required
## Screenshots
_If a UI facing feature, some screenshots of the new functionality._
_If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._