Merge branch 'develop' of github.com:Budibase/budibase into feature/table-fetching-frontend

.github/workflows/release-develop.yml

@@ -48,19 +48,8 @@ jobs:
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
-        run: |
-          version=$(cat lerna.json \
-            | grep version \
-            | head -1 \
-            | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \
-            | sed 's/[",]//g')
-          echo "Setting version $version"
-          yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
-          echo "Updating dependencies"
-          node scripts/syncLocalDependencies.js $version
-          echo "Syncing yarn workspace"
-          yarn
-      - run: yarn build --configuration=production
+        run: ./scripts/updateVersions.sh
+      - run: yarn build
       - run: yarn build:sdk
 
       - name: Publish budibase packages to NPM

.github/workflows/release-master.yml

@@ -41,20 +41,9 @@ jobs:
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
-        run: |
-          version=$(cat lerna.json \
-            | grep version \
-            | head -1 \
-            | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \
-            | sed 's/[",]//g')
-          echo "Setting version $version"
-          yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
-          echo "Updating dependencies"
-          node scripts/syncLocalDependencies.js $version
-          echo "Syncing yarn workspace"
-          yarn
+        run: ./scripts/updateVersions.sh
       - run: yarn lint
-      - run: yarn build --configuration=production
+      - run: yarn build
       - run: yarn build:sdk
 
       - name: Publish budibase packages to NPM

.github/workflows/release-selfhost.yml

@@ -17,6 +17,8 @@ jobs:
 
       - uses: actions/checkout@v2
         with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
 
       - name: Fail if tag is not in master

.github/workflows/release-singleimage.yml

@@ -5,7 +5,7 @@ on:
 
 env:
   CI: true
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   REGISTRY_URL: registry.hub.docker.com
 jobs:
   build:
@@ -24,6 +24,8 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
         with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
 
       - name: Fail if tag is not in master
@@ -32,7 +34,7 @@ jobs:
             echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
             exit 1
           fi
-          
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
@@ -44,10 +46,12 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - name: Run Yarn
         run: yarn
-      - name: Run Yarn Bootstrap
-        run: yarn bootstrap
+      - name: Update versions
+        run: ./scripts/updateVersions.sh
       - name: Runt Yarn Lint
         run: yarn lint
+      - name: Update versions
+        run: ./scripts/updateVersions.sh
       - name: Run Yarn Build
         run: yarn build:docker:pre
       - name: Login to Docker Hub

hosting/single/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-slim as build
+FROM node:14-slim as build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
@@ -11,12 +11,16 @@ RUN chmod +x /cleanup.sh
 # build server
 WORKDIR /app
 ADD packages/server .
-RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
+COPY yarn.lock .
+RUN yarn install --production=true
+RUN /cleanup.sh
 
 # build worker
 WORKDIR /worker
 ADD packages/worker .
-RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
+COPY yarn.lock .
+RUN yarn install --production=true
+RUN /cleanup.sh
 
 FROM budibase/couchdb
 ARG TARGETARCH

hosting/single/runner.sh

@@ -17,6 +17,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
 [[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002
 [[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001
+[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
 #  export CUSTOM_DOMAIN=budi001.custom.com
 
 # Azure App Service customisations

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.7.1-alpha.1",
+  "version": "2.7.6",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",
@@ -31,4 +31,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

package.json

@@ -52,7 +52,7 @@
     "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
-    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@@ -62,7 +62,7 @@
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
     "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:pre": "lerna run --stream build && lerna run --stream predocker",
+    "build:docker:pre": "yarn build && lerna run --stream predocker",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
     "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",

packages/server/Dockerfile

@@ -14,7 +14,7 @@ ENV SERVICE=app-service
 ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
-
+ENV TOP_LEVEL_PATH=/
 
 # handle node-gyp
 RUN apt-get update \
@@ -27,7 +27,8 @@ COPY scripts/integrations/oracle/ scripts/integrations/oracle/
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
 
 COPY package.json .
-RUN yarn install --frozen-lockfile --production=true
+COPY dist/yarn.lock .
+RUN yarn install --production=true
 # Remove unneeded data from file system to reduce image size
 RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp

packages/server/package.json

@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build && cp ../../yarn.lock ./dist/",
     "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",

packages/server/src/api/controllers/static/index.ts

@@ -1,6 +1,6 @@
 require("svelte/register")
 
-import { resolve, join } from "../../../utilities/centralPath"
+import { join } from "../../../utilities/centralPath"
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
@@ -49,7 +49,7 @@ export const toggleBetaUiFeature = async function (ctx: any) {
     return
   }
 
-  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+  let builderPath = join(TOP_LEVEL_PATH, "new_design_ui")
 
   // // download it from S3
   if (!fs.existsSync(builderPath)) {
@@ -67,7 +67,7 @@ export const toggleBetaUiFeature = async function (ctx: any) {
 }
 
 export const serveBuilder = async function (ctx: any) {
-  const builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 

packages/server/src/api/controllers/table/external.ts

@@ -208,6 +208,7 @@ export async function save(ctx: UserCtx) {
   let tableToSave: TableRequest = {
     type: "table",
     _id: buildExternalTableId(datasourceId, inputs.name),
+    sourceId: datasourceId,
     ...inputs,
   }
 

packages/server/src/environment.ts

@@ -98,7 +98,8 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
-  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
+  TOP_LEVEL_PATH:
+    process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
 }
 
 // threading can cause memory issues with node-ts in development

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -1,14 +1,14 @@
 import { PathLike } from "fs"
 import fs from "fs"
 import { budibaseTempDir } from "../budibaseDir"
-import { join } from "path"
+import { resolve, join } from "path"
 import env from "../../environment"
 import tar from "tar"
 import environment from "../../environment"
 const uuid = require("uuid/v4")
 
 export const TOP_LEVEL_PATH =
-  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")
+  environment.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

packages/worker/Dockerfile

@@ -13,7 +13,8 @@ RUN yarn global add pm2
 
 
 COPY package.json .
-RUN yarn install --frozen-lockfile --production=true
+COPY dist/yarn.lock .
+RUN yarn install --production=true
 # Remove unneeded data from file system to reduce image size
 RUN apk del .gyp \
     && yarn cache clean

packages/worker/package.json

@@ -19,7 +19,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "predocker": "yarn build --configuration=production",
+    "predocker": "yarn build && cp ../../yarn.lock ./dist/",
     "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",

scripts/resetVersions.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+echo "Resetting package versions"
+yarn lerna exec "yarn version --no-git-tag-version --new-version=0.0.0"
+echo "Updating dependencies"
+node scripts/syncLocalDependencies.js "0.0.0"
+git checkout package.json
+echo "Package versions reset!"

scripts/updateVersions.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+version=$(cat lerna.json \
+  | grep version \
+  | head -1 \
+  | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \
+  | sed 's/[",]//g')
+echo "Setting version $version"
+yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
+echo "Updating dependencies"
+node scripts/syncLocalDependencies.js $version
+echo "Syncing yarn workspace"
+yarn