MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into feature/table-fetching-frontend

This commit is contained in:
Michael Drury 2023-05-26 08:44:39 +01:00
parent d3eb262bed 631988a563
commit c08d67d299
67 changed files with 2737 additions and 1194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/budibase_ci.yml vendored
View File

@ -44,9 +44,9 @@ jobs:
node-version: 14.x
cache: "yarn"
- run: yarn
- run: yarn build
- run: yarn nx run-many -t=build --configuration=production
test:
test-libraries:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@ -59,8 +59,27 @@ jobs:
node-version: 14.x
cache: "yarn"
- run: yarn
- run: yarn build --scope=@budibase/types --scope=@budibase/shared-core --scope=@budibase/string-templates
- run: yarn test --ignore=@budibase/pro
- run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
- uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
name: codecov-umbrella
verbose: true
test-services:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- name: Use Node.js 14.x
uses: actions/setup-node@v3
with:
node-version: 14.x
cache: "yarn"
- run: yarn
- run: yarn test --scope=@budibase/worker --scope=@budibase/server
- uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

4
.github/workflows/release-develop.yml vendored
View File

@ -37,7 +37,7 @@ jobs:
with:
node-version: 14.x
- run: yarn
- run: yarn install --frozen-lockfile
- name: Update versions
run: |
version=$(cat lerna.json \
@ -51,7 +51,7 @@ jobs:
node scripts/syncLocalDependencies.js $version
echo "Syncing yarn workspace"
yarn
- run: yarn build
- run: yarn build --configuration=production
- run: yarn build:sdk
- name: Publish budibase packages to NPM

4
.github/workflows/release-master.yml vendored
View File

@ -42,7 +42,7 @@ jobs:
with:
node-version: 14.x
- run: yarn
- run: yarn install --frozen-lockfile
- name: Update versions
run: |
version=$(cat lerna.json \
@ -57,7 +57,7 @@ jobs:
echo "Syncing yarn workspace"
yarn
- run: yarn lint
- run: yarn build
- run: yarn build --configuration=production
- run: yarn build:sdk
- name: Publish budibase packages to NPM

2
.nvmrc
View File

@ -1 +1 @@
v14.20.1
v14.20.1

2
.python-version
View File

@ -1 +1 @@
3.10.0
3.10.0

4
docs/CONTRIBUTING.md
View File

@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running (
`yarn` to install project dependencies
`yarn bootstrap` will install all budibase modules and symlink them together using lerna.
`yarn build` will build all budibase packages.
#### 4. Running
@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README.
Note that only budibase maintainers will be able to access the pro repo.
The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
### Troubleshooting

77
hosting/docker-compose.build.yaml Normal file
View File

@ -0,0 +1,77 @@
version: "3"
# optional ports are specified throughout for more advanced use cases.
services:
app-service:
build: ../packages/server
container_name: build-bbapps
environment:
SELF_HOSTED: 1
COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
WORKER_URL: http://worker-service:4003
MINIO_URL: http://minio-service:9000
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
PORT: 4002
API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
JWT_SECRET: ${JWT_SECRET}
LOG_LEVEL: info
SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
ENABLE_ANALYTICS: "true"
REDIS_URL: redis-service:6379
REDIS_PASSWORD: ${REDIS_PASSWORD}
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
PLUGINS_DIR: ${PLUGINS_DIR}
depends_on:
- worker-service
- redis-service
# volumes:
# - /some/path/to/plugins:/plugins
worker-service:
build: ../packages/worker
container_name: build-bbworker
environment:
SELF_HOSTED: 1
PORT: 4003
CLUSTER_PORT: ${MAIN_PORT}
API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
JWT_SECRET: ${JWT_SECRET}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
MINIO_URL: http://minio-service:9000
APPS_URL: http://app-service:4002
COUCH_DB_USERNAME: ${COUCH_DB_USER}
COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
REDIS_URL: redis-service:6379
REDIS_PASSWORD: ${REDIS_PASSWORD}
depends_on:
- redis-service
- minio-service
proxy-service-docker:
ports:
- "${MAIN_PORT}:10000"
container_name: build-bbproxy
image: budibase/proxy
environment:
- PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
- PROXY_RATE_LIMIT_API_PER_SECOND=20
- APPS_UPSTREAM_URL=http://app-service:4002
- WORKER_UPSTREAM_URL=http://worker-service:4003
- MINIO_UPSTREAM_URL=http://minio-service:9000
- COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
- WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
- RESOLVER=127.0.0.11
depends_on:
- minio-service
- worker-service
- app-service
- couchdb-service

12
hosting/single/Dockerfile
View File

@ -1,22 +1,22 @@
FROM node:14-slim as build
FROM node:16-slim as build
# install node-gyp dependencies
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
# add pin script
WORKDIR /
ADD scripts/pinVersions.js scripts/cleanup.sh ./
ADD scripts/cleanup.sh ./
RUN chmod +x /cleanup.sh
# build server
WORKDIR /app
ADD packages/server .
RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
# build worker
WORKDIR /worker
ADD packages/worker .
RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
FROM budibase/couchdb
ARG TARGETARCH
@ -31,9 +31,7 @@ COPY --from=build /worker /worker
# install base dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \
apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
apt-get update
apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
WORKDIR /nodejs

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.6.19-alpha.4",
"version": "2.6.19-alpha.11",
"npmClient": "yarn",
"packages": [
"packages/backend-core",

10
nx.json
View File

@ -6,5 +6,15 @@
"cacheableOperations": ["build", "test"]
}
}
},
"targetDefaults": {
"dev:builder": {
"dependsOn": [
{
"projects": ["@budibase/string-templates"],
"target": "build"
}
]
}
}
}

31
package.json
View File

@ -2,17 +2,24 @@
"name": "root",
"private": true,
"devDependencies": {
"@esbuild-plugins/node-resolve": "^0.2.2",
"@esbuild-plugins/tsconfig-paths": "^0.1.2",
"@nx/esbuild": "16.2.1",
"@nx/js": "16.2.1",
"@rollup/plugin-json": "^4.0.2",
"@typescript-eslint/parser": "5.45.0",
"babel-eslint": "^10.0.3",
"esbuild": "^0.17.18",
"eslint": "^7.28.0",
"eslint-plugin-cypress": "^2.11.3",
"eslint-plugin-svelte3": "^3.2.0",
"husky": "^8.0.3",
"js-yaml": "^4.1.0",
"kill-port": "^1.6.1",
"lerna": "^6.6.1",
"lerna": "7.0.0-alpha.0",
"madge": "^6.0.0",
"minimist": "^1.2.8",
"nx": "^16.2.1",
"prettier": "^2.3.1",
"prettier-plugin-svelte": "^2.3.0",
"rimraf": "^3.0.2",
@ -23,9 +30,9 @@
},
"scripts": {
"preinstall": "node scripts/syncProPackage.js",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
"bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
"build": "lerna run --stream build",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
"bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
"build": "yarn nx run-many -t=build",
"build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
"backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
"backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
@ -41,10 +48,11 @@
"kill-builder": "kill-port 3000",
"kill-server": "kill-port 4001 4002",
"kill-all": "yarn run kill-builder && yarn run kill-server",
"dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream",
"dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
"dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
"dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream",
"dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
"dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
"dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ",
"test": "lerna run --stream test --stream",
"lint:eslint": "eslint packages && eslint qa-core",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@ -53,16 +61,16 @@
"lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
"lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
"build:specs": "lerna run --stream specs",
"build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
"build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
"build:docker:pre": "lerna run --stream build && lerna run --stream predocker",
"build:docker:proxy": "docker build hosting/proxy -t proxy-service",
"build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
"build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
"build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
"build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
"build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
"build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
"build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
"build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image",
"build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image",
"build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
"publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
"publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
@ -101,5 +109,6 @@
"packages/worker",
"packages/pro/packages/pro"
]
}
},
"dependencies": {}
}

14
packages/backend-core/package.json
View File

@ -88,5 +88,19 @@
"tsconfig-paths": "4.0.0",
"typescript": "4.7.3"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/types"
],
"target": "build"
}
]
}
}
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
}

14
packages/bbui/package.json
View File

@ -90,5 +90,19 @@
"resolutions": {
"loader-utils": "1.4.1"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/string-templates"
],
"target": "build"
}
]
}
}
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
}

4
packages/bbui/src/ActionButton/ActionButton.svelte
View File

@ -102,7 +102,9 @@
margin-left: 0;
transition: color ease-out 130ms;
}
.is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) {
.is-selected:not(.spectrum-ActionButton--emphasized):not(
.spectrum-ActionButton--quiet
) {
background: var(--spectrum-global-color-gray-300);
border-color: var(--spectrum-global-color-gray-500);
}

28
packages/builder/package.json
View File

@ -9,7 +9,7 @@
"dev:builder": "routify -c dev:vite",
"dev:vite": "vite --host 0.0.0.0",
"rollup": "rollup -c -w",
"test": "vitest"
"test": "vitest run"
},
"jest": {
"globals": {
@ -117,5 +117,31 @@
"vite": "^3.0.8",
"vitest": "^0.29.2"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/string-templates",
"@budibase/shared-core"
],
"target": "build"
}
]
},
"test": {
"dependsOn": [
{
"projects": [
"@budibase/shared-core",
"@budibase/string-templates"
],
"target": "build"
}
]
}
}
},
"gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
}

1
packages/builder/src/components/backend/DataTable/DataTable.svelte
View File

@ -32,6 +32,7 @@
<Grid
{API}
tableId={id}
tableType={$tables.selected?.type}
allowAddRows={!isUsersTable}
allowDeleteRows={!isUsersTable}
schemaOverrides={isUsersTable ? userSchemaOverrides : null}

3
packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
View File

@ -3,6 +3,7 @@
import ImportModal from "../modals/ImportModal.svelte"
export let tableId
export let tableType
export let disabled
let modal
@ -12,5 +13,5 @@
Import
</ActionButton>
<Modal bind:this={modal}>
<ImportModal {tableId} on:importrows />
<ImportModal {tableId} {tableType} on:importrows />
</Modal>

3
packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
View File

@ -4,11 +4,12 @@
export let disabled = false
const { rows, tableId } = getContext("grid")
const { rows, tableId, tableType } = getContext("grid")
</script>
<ImportButton
{disabled}
tableId={$tableId}
{tableType}
on:importrows={rows.actions.refreshData}
/>

12
packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
View File

@ -13,15 +13,18 @@
const dispatch = createEventDispatcher()
export let tableId
export let tableType
let rows = []
let allValid = false
let displayColumn = null
let identifierFields = []
async function importData() {
try {
await API.importTableData({
tableId,
rows,
identifierFields,
})
notifications.success("Rows successfully imported")
} catch (error) {
@ -45,6 +48,13 @@
</Body>
<Layout gap="XS" noPadding>
<Label grey extraSmall>CSV or JSON file to import</Label>
<TableDataImport {tableId} bind:rows bind:allValid bind:displayColumn />
<TableDataImport
{tableId}
{tableType}
bind:rows
bind:allValid
bind:displayColumn
bind:identifierFields
/>
</Layout>
</ModalContent>

21
packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
View File

@ -1,5 +1,5 @@
<script>
import { Select } from "@budibase/bbui"
import { Select, Toggle, Multiselect } from "@budibase/bbui"
import { FIELDS } from "constants/backend"
import { API } from "api"
import { parseFile } from "./utils"
@ -9,14 +9,17 @@
let fileType = null
let loading = false
let updateExistingRows = false
let validation = {}
let validateHash = ""
let schema = null
let invalidColumns = []
export let tableId = null
export let tableType
export let rows = []
export let allValid = false
export let identifierFields = []
const typeOptions = [
{
@ -159,6 +162,22 @@
</div>
{/each}
</div>
{#if tableType === "internal"}
<br />
<Toggle
bind:value={updateExistingRows}
on:change={() => (identifierFields = [])}
thin
text="Update existing rows"
/>
{#if updateExistingRows}
<Multiselect
label="Identifier field(s)"
options={Object.keys(validation)}
bind:value={identifierFields}
/>
{/if}
{/if}
{#if invalidColumns.length > 0}
<p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
The following columns are present in the data you wish to import, but do

11
packages/builder/src/components/common/CodeMirrorEditor.svelte
View File

@ -146,15 +146,18 @@
/* Override default active line highlight colour in dark theme */
div
:global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
.CodeMirror-activeline-background) {
:global(
.CodeMirror-focused.cm-s-tomorrow-night-eighties
.CodeMirror-activeline-background
) {
background: rgba(255, 255, 255, 0.075);
}
/* Remove active line styling when not focused */
div
:global(.CodeMirror:not(.CodeMirror-focused)
.CodeMirror-activeline-background) {
:global(
.CodeMirror:not(.CodeMirror-focused) .CodeMirror-activeline-background
) {
background: unset;
}

24
packages/builder/src/pages/builder/portal/settings/auth/scim.svelte
View File

@ -115,27 +115,6 @@
align-items: center;
}
input[type="file"] {
display: none;
}
.sso-link-icon {
padding-top: 4px;
margin-left: 3px;
}
.sso-link {
margin-top: 12px;
display: flex;
flex-direction: row;
align-items: center;
}
.enforce-sso-title {
margin-right: 10px;
}
.enforce-sso-heading-container {
display: flex;
flex-direction: row;
align-items: start;
}
.provider-title {
display: flex;
flex-direction: row;
@ -143,9 +122,6 @@
align-items: center;
gap: var(--spacing-m);
}
.provider-title span {
flex: 1 1 auto;
}
.inputContainer {
display: flex;
flex-direction: row;

14
packages/cli/package.json
View File

@ -63,5 +63,19 @@
"renamer": "^4.0.0",
"ts-node": "^10.9.1",
"typescript": "4.7.3"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/backend-core"
],
"target": "build"
}
]
}
}
}
}

15
packages/client/package.json
View File

@ -65,5 +65,20 @@
"resolutions": {
"loader-utils": "1.4.1"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/string-templates",
"@budibase/shared-core"
],
"target": "build"
}
]
}
}
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
}

5
packages/client/src/components/app/charts/ApexChart.svelte
View File

@ -54,8 +54,9 @@
color: white;
}
div
:global(.apexcharts-theme-dark
.apexcharts-tooltip-series-group.apexcharts-active) {
:global(
.apexcharts-theme-dark .apexcharts-tooltip-series-group.apexcharts-active
) {
padding-bottom: 0;
}
</style>

8
packages/client/src/components/app/forms/JSONField.svelte
View File

@ -72,9 +72,11 @@
:global(.spectrum-Form-itemField .spectrum-Textfield--multiline) {
min-height: calc(var(--height) - 24px);
}
:global(.spectrum-Form--labelsAbove
.spectrum-Form-itemField
.spectrum-Textfield--multiline) {
:global(
.spectrum-Form--labelsAbove
.spectrum-Form-itemField
.spectrum-Textfield--multiline
) {
min-height: calc(var(--height) - 24px);
}
</style>

6
packages/frontend-core/src/api/tables.js
View File

@ -62,13 +62,15 @@ export const buildTableEndpoints = API => ({
/**
* Imports data into an existing table
* @param tableId the table ID to import to
* @param data the data import object
* @param rows the data import object
* @param identifierFields column names to be used as keys for overwriting existing rows
*/
importTableData: async ({ tableId, rows }) => {
importTableData: async ({ tableId, rows, identifierFields }) => {
return await API.post({
url: `/api/tables/${tableId}/import`,
body: {
rows,
identifierFields,
},
})
},

2
packages/frontend-core/src/components/grid/layout/Grid.svelte
View File

@ -33,6 +33,7 @@
export let API = null
export let tableId = null
export let tableType = null
export let schemaOverrides = null
export let allowAddRows = true
export let allowAddColumns = true
@ -62,6 +63,7 @@
rand,
config,
tableId: tableIdStore,
tableType,
schemaOverrides: schemaOverridesStore,
}
context = { ...context, ...createEventManagers() }

2
packages/pro

@ -1 +1 @@
Subproject commit a590dc237a16983b8f39dc8e65005b7736d23467
Subproject commit aea8a4acb0bae6a1036520bf4c6d8cae428cc7d9

13
packages/server/.dockerignore
View File

@ -1,6 +1,7 @@
node_modules
npm-debug.log
Dockerfile
.dockerignore
.git
.gitignore
*
!/dist/
!/scripts/integrations/oracle/
!/package.json
!/docker_run.sh
!/builder/
!/client/

24
packages/server/Dockerfile
View File

@ -15,22 +15,28 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
# copy files and install dependencies
COPY . ./
# handle node-gyp
RUN apt-get update \
&& apt-get install -y --no-install-recommends g++ make python \
&& yarn \
&& yarn cache clean \
&& apt-get remove -y --purge --auto-remove g++ make python \
&& rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
&& apt-get install -y --no-install-recommends g++ make python
RUN yarn global add pm2
RUN yarn build
# Install client for oracle datasource
RUN apt-get install unzip libaio1
COPY scripts/integrations/oracle/ scripts/integrations/oracle/
RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
COPY package.json .
RUN yarn install --frozen-lockfile --production=true
# Remove unneeded data from file system to reduce image size
RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
&& rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
COPY dist/ dist/
COPY docker_run.sh .
COPY builder/ builder/
COPY client/ client/
EXPOSE 4001
# have to add node environment production after install
@ -38,4 +44,6 @@ EXPOSE 4001
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
ENV CLUSTER_MODE=${CLUSTER_MODE}
ENV TOP_LEVEL_PATH=/app
CMD ["./docker_run.sh"]

3
packages/server/jest.config.ts
View File

@ -1,6 +1,7 @@
import { Config } from "@jest/types"
import * as fs from "fs"
import { join } from "path"
const preset = require("ts-jest/jest-preset")
const baseConfig: Config.InitialProjectOptions = {
@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
coverageReporters: ["lcov", "json", "clover"],
}
process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
export default config

2
packages/server/nodemon.json
View File

@ -6,5 +6,5 @@
"src/**/*.spec.js",
"../backend-core/dist/**/*"
],
"exec": "ts-node src/index.ts"
"exec": "node ./scripts/build.js && node ./dist/index.js"
}

32
packages/server/package.json
View File

@ -10,22 +10,22 @@
},
"scripts": {
"prebuild": "rimraf dist/",
"build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
"build": "node ./scripts/build.js",
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
"postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
"test": "bash scripts/test.sh",
"test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
"test:watch": "jest --watch",
"predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
"build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
"predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
"build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
"build:docs": "node ./scripts/docs/generate.js open",
"run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.config.js",
"dev:stack:up": "node scripts/dev/manage.js up",
"dev:stack:down": "node scripts/dev/manage.js down",
"dev:stack:nuke": "node scripts/dev/manage.js nuke",
"dev:builder": "yarn run dev:stack:up && nodemon",
"dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
"dev:built": "yarn run dev:stack:up && yarn run run:docker",
"specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
"initialise": "node scripts/initialise.js",
@ -47,7 +47,7 @@
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "0.0.1",
"@budibase/client": "0.0.1",
"@budibase/pro": "0.0.1",
"@budibase/pro": "develop",
"@budibase/shared-core": "0.0.1",
"@budibase/string-templates": "0.0.1",
"@budibase/types": "0.0.1",
@ -99,6 +99,7 @@
"mysql2": "2.3.3",
"node-fetch": "2.6.7",
"open": "8.4.0",
"openai": "^3.2.1",
"pg": "8.10.0",
"posthog-node": "1.3.0",
"pouchdb": "7.3.0",
@ -118,8 +119,8 @@
"validate.js": "0.13.1",
"vm2": "3.9.17",
"worker-farm": "1.7.0",
"xml2js": "0.5.0",
"yargs": "13.2.4"
"yargs": "13.2.4",
"xml2js": "0.5.0"
},
"devDependencies": {
"@babel/core": "7.17.4",
@ -177,5 +178,20 @@
"optionalDependencies": {
"oracledb": "5.3.0"
},
"nx": {
"targets": {
"test": {
"dependsOn": [
{
"projects": [
"@budibase/string-templates",
"@budibase/shared-core"
],
"target": "build"
}
]
}
}
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
}

2
packages/server/pm2.config.js
View File

@ -1,7 +1,7 @@
module.exports = {
apps: [
{
script: "dist/index.js",
script: "./dist/index.js",
instances: "max",
exec_mode: "cluster",
},

48
packages/server/scripts/build.js Executable file
View File

@ -0,0 +1,48 @@
#!/usr/bin/node
const { join } = require("path")
const fs = require("fs")
const coreBuild = require("../../../scripts/build")
const dir = join(__dirname, "..")
const entryPath = join(dir, "src")
const outfilePath = join(dir, "dist")
/**
* The reasoning for this is that now our built version is simple
* dist/index.js - any kind of threaded approach in Node.js requires
* a runner file to work from - I played around with a lot of
* different methods, but we really want to be able to use forks.
*
* Rather than trying to rewrite so that forks run the whole system,
* I instead went down a path of building the individual threads so
* that we have runner files for each of them e.g. dist/automations.js
* and dist/query.js - these can be ran totally independently and then
* the parent process can pass down data for processing to them.
*
* The ignoring is simply to remove the files which really don't need
* to be built - they could be built and it wouldn't cause any issues,
* but this just means if any further threads are added in future
* they will naturally work (rather than including, which would mean
* adjustments to the build files).
*/
const ignoredFiles = ["definitions", "index", "utils"]
const threadNames = fs
.readdirSync(join(dir, "src", "threads"))
.filter(path => !ignoredFiles.find(file => path.includes(file)))
.map(path => path.replace(".ts", ""))
const files = [
{
entry: join(entryPath, "index.ts"),
out: join(outfilePath, "index.js"),
},
]
for (let name of threadNames) {
files.push({
entry: join(entryPath, "threads", `${name}.ts`),
out: join(outfilePath, `${name}.js`),
})
}
for (let file of files) {
coreBuild(file.entry, file.out)
}

6
packages/server/src/api/controllers/static/index.ts
View File

@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
? objectStore.getGlobalFileUrl("settings", "logoUrl")
: "",
})
const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
ctx.body = await processString(appHbs, {
head,
body: html,
@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
: "",
})
const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
ctx.body = await processString(appHbs, {
head,
body: html,
@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {
if (!env.isJest()) {
let appId = context.getAppId()
const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
ctx.body = await processString(previewHbs, {
clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
})

8
packages/server/src/api/controllers/table/internal.ts
View File

@ -186,11 +186,7 @@ export async function destroy(ctx: any) {
export async function bulkImport(ctx: any) {
const db = context.getAppDB()
const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows } = ctx.request.body
await handleDataImport(ctx.user, table, rows)
// Ensure auto id and other table updates are persisted
await db.put(table)
const { rows, identifierFields } = ctx.request.body
await handleDataImport(ctx.user, table, rows, identifierFields)
return table
}

33
packages/server/src/api/controllers/table/utils.ts
View File

@ -149,7 +149,12 @@ export function importToRows(
return finalData
}
export async function handleDataImport(user: any, table: any, rows: any) {
export async function handleDataImport(
user: any,
table: any,
rows: any,
identifierFields: Array<string> = []
) {
const schema: unknown = table.schema
if (!rows || !isRows(rows) || !isSchema(schema)) {
@ -161,6 +166,32 @@ export async function handleDataImport(user: any, table: any, rows: any) {
let finalData: any = importToRows(data, table, user)
//Set IDs of finalData to match existing row if an update is expected
if (identifierFields.length > 0) {
const allDocs = await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
allDocs.rows
.map(existingRow => existingRow.doc)
.forEach((doc: any) => {
finalData.forEach((finalItem: any) => {
let match = true
for (const field of identifierFields) {
if (finalItem[field] !== doc[field]) {
match = false
break
}
}
if (match) {
finalItem._id = doc._id
finalItem._rev = doc._rev
}
})
})
}
await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
tableId: table._id,
})

4
packages/server/src/api/routes/static.ts
View File

@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
import { permissions } from "@budibase/backend-core"
import env from "../../environment"
import { paramResource } from "../../middleware/resourceId"
import { devClientLibPath } from "../../utilities/fileSystem"
const { BUILDER, PermissionType, PermissionLevel } = permissions
const router: Router = new Router()
@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
}
// test serves from require
if (env.isTest()) {
ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
const path = devClientLibPath()
ctx.devPath = path.split(ctx.file)[0]
} else if (env.isDev()) {
// Serving the client library from your local dir in dev
ctx.devPath = budibaseTempDir()

5
packages/server/src/automations/actions.ts
View File

@ -71,10 +71,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
// ran at all
if (env.SELF_HOSTED) {
const bash = require("./steps/bash")
const openai = require("./steps/openai")
// @ts-ignore
ACTION_IMPLS["EXECUTE_BASH"] = bash.run
// @ts-ignore
BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
ACTION_IMPLS.OPENAI = openai.run
BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
}
export async function getActionDefinitions() {

105
packages/server/src/automations/steps/openai.ts Normal file
View File

@ -0,0 +1,105 @@
import { Configuration, OpenAIApi } from "openai"
import {
AutomationActionStepId,
AutomationStepSchema,
AutomationStepInput,
AutomationStepType,
AutomationIOType,
} from "@budibase/types"
import * as automationUtils from "../automationUtils"
import environment from "../../environment"
enum Model {
GPT_35_TURBO = "gpt-3.5-turbo",
// will only work with api keys that have access to the GPT4 API
GPT_4 = "gpt-4",
}
export const definition: AutomationStepSchema = {
name: "OpenAI",
tagline: "Send prompts to ChatGPT",
icon: "Algorithm",
description: "Interact with the OpenAI ChatGPT API.",
type: AutomationStepType.ACTION,
internal: true,
stepId: AutomationActionStepId.OPENAI,
inputs: {
prompt: "",
},
schema: {
inputs: {
properties: {
prompt: {
type: AutomationIOType.STRING,
title: "Prompt",
},
model: {
type: AutomationIOType.STRING,
title: "Model",
enum: Object.values(Model),
},
},
required: ["prompt", "model"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
response: {
type: AutomationIOType.STRING,
description: "What was output",
},
},
required: ["success", "response"],
},
},
}
export async function run({ inputs, context }: AutomationStepInput) {
if (!environment.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
if (inputs.prompt == null) {
return {
success: false,
response: "Budibase OpenAI Automation Failed: No prompt supplied",
}
}
try {
const configuration = new Configuration({
apiKey: environment.OPENAI_API_KEY,
})
const openai = new OpenAIApi(configuration)
const completion = await openai.createChatCompletion({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
const response = completion?.data?.choices[0]?.message?.content
return {
response,
success: true,
}
} catch (err) {
return {
success: false,
response: automationUtils.getError(err),
}
}
}

86
packages/server/src/automations/tests/openai.spec.ts Normal file
View File

@ -0,0 +1,86 @@
const setup = require("./utilities")
import environment from "../../environment"
import openai from "openai"
jest.mock(
"openai",
jest.fn(() => ({
Configuration: jest.fn(),
OpenAIApi: jest.fn(() => ({
createChatCompletion: jest.fn(() => ({
data: {
choices: [
{
message: {
content: "This is a test",
},
},
],
},
})),
})),
}))
)
const OPENAI_PROMPT = "What is the meaning of life?"
describe("test the openai action", () => {
let config = setup.getConfig()
beforeAll(async () => {
await config.init()
})
beforeEach(() => {
environment.OPENAI_API_KEY = "abc123"
})
afterAll(setup.afterAll)
it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => {
delete environment.OPENAI_API_KEY
let res = await setup.runStep("OPENAI", {
prompt: OPENAI_PROMPT,
})
expect(res.response).toEqual(
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable."
)
expect(res.success).toBeFalsy()
})
it("should be able to receive a response from ChatGPT given a prompt", async () => {
const res = await setup.runStep("OPENAI", {
prompt: OPENAI_PROMPT,
})
expect(res.response).toEqual("This is a test")
expect(res.success).toBeTruthy()
})
it("should present the correct error message when a prompt is not provided", async () => {
const res = await setup.runStep("OPENAI", {
prompt: null,
})
expect(res.response).toEqual(
"Budibase OpenAI Automation Failed: No prompt supplied"
)
expect(res.success).toBeFalsy()
})
it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => {
openai.OpenAIApi.mockImplementation(() => ({
createChatCompletion: jest.fn(() => {
throw new Error("An error occurred while calling createChatCompletion")
}),
}))
const res = await setup.runStep("OPENAI", {
prompt: OPENAI_PROMPT,
})
expect(res.response).toEqual(
"Error: An error occurred while calling createChatCompletion"
)
expect(res.success).toBeFalsy()
})
})

33
packages/server/src/constants/index.ts
View File

@ -1,4 +1,5 @@
import { objectStore, roles, constants } from "@budibase/backend-core"
import { FieldType as FieldTypes } from "@budibase/types"
export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"
export enum FilterTypes {
@ -24,14 +25,14 @@ export const NoEmptyFilterStrings = [
]
export const CanSwitchTypes = [
[exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
[FieldTypes.JSON, FieldTypes.ARRAY],
[
exports.FieldTypes.STRING,
exports.FieldTypes.OPTIONS,
exports.FieldTypes.LONGFORM,
exports.FieldTypes.BARCODEQR,
FieldTypes.STRING,
FieldTypes.OPTIONS,
FieldTypes.LONGFORM,
FieldTypes.BARCODEQR,
],
[exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
[FieldTypes.BOOLEAN, FieldTypes.NUMBER],
]
export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
@ -77,9 +78,9 @@ export const USERS_TABLE_SCHEMA = {
// TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
schema: {
email: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
constraints: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
email: true,
length: {
maximum: "",
@ -92,27 +93,27 @@ export const USERS_TABLE_SCHEMA = {
firstName: {
name: "firstName",
fieldName: "firstName",
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
constraints: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
presence: false,
},
},
lastName: {
name: "lastName",
fieldName: "lastName",
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
constraints: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
presence: false,
},
},
roleId: {
fieldName: "roleId",
name: "roleId",
type: exports.FieldTypes.OPTIONS,
type: FieldTypes.OPTIONS,
constraints: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
presence: false,
inclusion: Object.values(roles.BUILTIN_ROLE_IDS),
},
@ -120,9 +121,9 @@ export const USERS_TABLE_SCHEMA = {
status: {
fieldName: "status",
name: "status",
type: exports.FieldTypes.OPTIONS,
type: FieldTypes.OPTIONS,
constraints: {
type: exports.FieldTypes.STRING,
type: FieldTypes.STRING,
presence: false,
inclusion: Object.values(constants.UserStatus),
},

4
packages/server/src/db/inMemoryView.ts
View File

@ -25,7 +25,9 @@ export async function runView(
}))
)
let fn = (doc: Document, emit: any) => emit(doc._id)
eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)"))
;(0, eval)(
"fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")
)
const queryFns: any = {
meta: view.meta,
map: fn,

2
packages/server/src/environment.ts
View File

@ -71,6 +71,7 @@ const environment = {
BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
// flags
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,
@ -96,6 +97,7 @@ const environment = {
isInThread: () => {
return process.env.FORKED_PROCESS
},
TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
}
// threading can cause memory issues with node-ts in development

1
packages/server/src/integrations/base/sqlTable.ts
View File

@ -206,4 +206,3 @@ class SqlTableQueryBuilder {
}
export default SqlTableQueryBuilder
module.exports = SqlTableQueryBuilder

2
packages/server/src/utilities/fileSystem/app.ts
View File

@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
const filename = "manifest.json"
if (env.isDev() || env.isTest()) {
const path = join(TOP_LEVEL_PATH, "../client", filename)
const path = join(TOP_LEVEL_PATH, "packages/client", filename)
// always load from new so that updates are refreshed
delete require.cache[require.resolve(path)]
return require(path)

11
packages/server/src/utilities/fileSystem/clientLibrary.ts
View File

@ -1,4 +1,4 @@
import { join } from "path"
import path, { join } from "path"
import { ObjectStoreBuckets } from "../../constants"
import fs from "fs"
import { objectStore } from "@budibase/backend-core"
@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
import env from "../../environment"
import { TOP_LEVEL_PATH } from "./filesystem"
export function devClientLibPath() {
return require.resolve("@budibase/client")
}
/**
* Client library paths in the object store:
* Previously, the entire client library package was downloaded from NPM
@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
let manifest, client
if (env.isDev()) {
const clientPath = devClientLibPath()
// Load the symlinked version in dev which is always the newest
manifest = require.resolve("@budibase/client/manifest.json")
client = require.resolve("@budibase/client")
manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
client = clientPath
} else {
// Load the bundled version in prod
manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")

4
packages/server/src/utilities/fileSystem/filesystem.ts
View File

@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
import { join } from "path"
import env from "../../environment"
import tar from "tar"
import environment from "../../environment"
const uuid = require("uuid/v4")
export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
export const TOP_LEVEL_PATH =
environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")
/**
* Upon first startup of instance there may not be everything we need in tmp directory, set it up.

10
packages/server/tsconfig.build.json
View File

@ -10,7 +10,15 @@
"incremental": true,
"types": ["node", "jest"],
"outDir": "dist/src",
"skipLibCheck": true
"skipLibCheck": true,
"baseUrl": ".",
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"],
"@budibase/shared-core": ["../shared-core/src"],
"@budibase/pro": ["../pro/packages/pro/src"]
}
},
"include": ["src/**/*"],
"exclude": [

15
packages/server/tsconfig.json
View File

@ -5,25 +5,12 @@
"declaration": true,
"sourceMap": true,
"baseUrl": ".",
"outDir": "dist",
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"],
"@budibase/shared-core": ["../shared-core/src"],
"@budibase/pro": ["../pro/packages/pro/src"]
}
"outDir": "dist"
},
"ts-node": {
"require": ["tsconfig-paths/register"],
"swc": true
},
"references": [
{ "path": "../types" },
{ "path": "../backend-core" },
{ "path": "../shared-core" },
{ "path": "../../../budibase-pro/packages/pro" }
],
"include": ["src/**/*", "specs"],
"exclude": ["node_modules", "dist"]
}

14
packages/shared-core/package.json
View File

@ -26,5 +26,19 @@
"concurrently": "^7.6.0",
"rimraf": "3.0.2",
"typescript": "4.7.3"
},
"nx": {
"targets": {
"build": {
"dependsOn": [
{
"projects": [
"@budibase/types"
],
"target": "build"
}
]
}
}
}
}

4
packages/shared-core/src/filters.ts
View File

@ -328,8 +328,8 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
return (
docValue == null ||
docValue === "" ||
docValue < testValue.low ||
docValue > testValue.high
+docValue < testValue.low ||
+docValue > testValue.high
)
}
)

3
packages/string-templates/tsconfig.json
View File

@ -1,12 +1,11 @@
{
"include": ["src/**/*"],
"compilerOptions": {
"allowJs": true,
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "dist",
"esModuleInterop": true,
"types": ["node"]
"types": ["node", "jest"]
}
}

1
packages/types/src/documents/app/automation.ts
View File

@ -57,6 +57,7 @@ export enum AutomationActionStepId {
FILTER = "FILTER",
QUERY_ROWS = "QUERY_ROWS",
LOOP = "LOOP",
OPENAI = "OPENAI",
// these used to be lowercase step IDs, maintain for backwards compat
discord = "discord",
slack = "slack",

10
packages/worker/.dockerignore
View File

@ -1,7 +1,3 @@
node_modules
npm-debug.log
Dockerfile
.dockerignore
.git
.gitignore
*
!/dist/
!/docker_run.sh

15
packages/worker/Dockerfile
View File

@ -7,13 +7,20 @@ LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-ho
WORKDIR /app
# copy files and install dependencies
COPY . ./
# handle node-gyp
RUN apk add --no-cache --virtual .gyp python3 make g++ \
&& yarn && apk del .gyp
RUN apk add --no-cache --virtual .gyp python3 make g++
RUN yarn global add pm2
COPY dist/package.json .
RUN yarn install --frozen-lockfile --production=true
# Remove unneeded data from file system to reduce image size
RUN apk del .gyp \
&& yarn cache clean
COPY dist/ dist/
COPY docker_run.sh .
EXPOSE 4001
# have to add node environment production after install

2
packages/worker/nodemon.json
View File

@ -6,5 +6,5 @@
"src/**/*.spec.js",
"../backend-core/dist/**/*"
],
"exec": "ts-node src/index.ts"
"exec": "yarn build && node dist/index.js"
}

10
packages/worker/package.json
View File

@ -13,15 +13,15 @@
],
"scripts": {
"prebuild": "rimraf dist/",
"build": "tsc -p tsconfig.build.json",
"postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
"build": "cd ../.. && nx build @budibase/worker",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"run:docker": "node dist/index.js",
"debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.config.js",
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"predocker": "yarn build --configuration=production",
"build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init",
"dev:builder": "npm run dev:stack:init && nodemon",
"dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",
"dev:built": "yarn run dev:stack:init && yarn run run:docker",
"test": "bash scripts/test.sh",
"test:watch": "jest --watch",
@ -38,7 +38,7 @@
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "0.0.1",
"@budibase/pro": "0.0.1",
"@budibase/pro": "develop",
"@budibase/string-templates": "0.0.1",
"@budibase/types": "0.0.1",
"@koa/router": "8.0.8",

48
packages/worker/project.json Normal file
View File

@ -0,0 +1,48 @@
{
"name": "@budibase/worker",
"$schema": "../../node_modules/nx/schemas/project-schema.json",
"targets": {
"build": {
"executor": "@nx/esbuild:esbuild",
"outputs": ["{options.outputPath}"],
"options": {
"main": "packages/worker/src/index.ts",
"outputPath": "packages/worker/dist",
"outputFileName": "index.js",
"tsConfig": "packages/worker/tsconfig.build.json",
"assets": [
{
"glob": "**/*.hbs",
"input": "packages/worker/src/constants/templates",
"output": "."
}
],
"external": ["graphql/*", "deasync", "mock-aws-s3", "nock"],
"format": ["cjs"],
"esbuildOptions": {
"outExtension": {
".js": ".js"
},
"sourcemap": true
},
"minify": true,
"generatePackageJson": true,
"skipTypeCheck": true
},
"configurations": {
"production": {
"skipTypeCheck": false,
"esbuildOptions": {
"sourcemap": false
}
}
},
"dependsOn": [
{
"projects": ["@budibase/types", "@budibase/string-templates"],
"target": "build"
}
]
}
}
}

17
packages/worker/tsconfig.build.json
View File

@ -8,13 +8,18 @@
"esModuleInterop": true,
"resolveJsonModule": true,
"incremental": true,
"types": [ "node", "jest" ],
"types": ["node", "jest"],
"outDir": "dist",
"skipLibCheck": true
"skipLibCheck": true,
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"],
"@budibase/shared-core": ["../shared-core/src"],
"@budibase/pro": ["../pro/packages/pro/src"]
}
},
"include": [
"src/**/*"
],
"include": ["src/**/*"],
"exclude": [
"node_modules",
"dist",
@ -22,4 +27,4 @@
"**/*.spec.js",
"**/*.spec.ts"
]
}
}

9
packages/worker/tsconfig.json
View File

@ -4,19 +4,12 @@
"composite": true,
"declaration": true,
"sourceMap": true,
"baseUrl": ".",
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"],
"@budibase/pro": ["../pro/packages/pro/src"]
}
"baseUrl": "."
},
"ts-node": {
"require": ["tsconfig-paths/register"],
"swc": true
},
"references": [{ "path": "../types" }, { "path": "../backend-core" }],
"include": ["src/**/*"],
"exclude": ["dist"]
}

4
qa-core/tsconfig.json
View File

@ -22,10 +22,6 @@
"ts-node": {
"require": ["tsconfig-paths/register"]
},
"references": [
{ "path": "../packages/types" },
{ "path": "../packages/backend-core" }
],
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

1
scripts/bootstrap.sh
View File

@ -3,5 +3,4 @@ if [ -d "packages/pro/packages" ]; then
yarn
lerna bootstrap
yarn setup
fi

73
scripts/build.js Executable file
View File

@ -0,0 +1,73 @@
#!/usr/bin/node
const start = Date.now()
const glob = require("glob")
const fs = require("fs")
const path = require("path")
const { build } = require("esbuild")
const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
const {
default: TsconfigPathsPlugin,
} = require("@esbuild-plugins/tsconfig-paths")
var argv = require("minimist")(process.argv.slice(2))
function runBuild(entry, outfile) {
const isDev = process.env.NODE_ENV !== "production"
const tsconfig = argv["p"] || `tsconfig.build.json`
const sharedConfig = {
entryPoints: [entry],
bundle: true,
minify: !isDev,
sourcemap: isDev,
tsconfig,
plugins: [
TsconfigPathsPlugin({ tsconfig }),
NodeResolve({
extensions: [".ts", ".js"],
onResolved: resolved => {
if (resolved.includes("node_modules")) {
return {
external: true,
}
}
return resolved
},
}),
],
target: "node14",
preserveSymlinks: true,
loader: {
".svelte": "copy",
},
}
build({
...sharedConfig,
platform: "node",
outfile,
}).then(() => {
glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
for (const file of files) {
fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
}
console.log(
"\x1b[32m%s\x1b[0m",
`Build successfully in ${(Date.now() - start) / 1000} seconds`
)
})
})
}
if (require.main === module) {
const entry = argv["e"] || "./src/index.ts"
const outfile = `dist/${entry.split("/").pop().replace(".ts", ".js")}`
runBuild(entry, outfile)
} else {
module.exports = runBuild
}

2905
yarn.lock
View File

File diff suppressed because it is too large Load Diff