poc wip

2022-02-04 15:56:48 +00:00 · 2022-02-04 15:56:48 +00:00 · 684a915a30
parent c8da7b9cf0
commit 684a915a30
19 changed files with 3015 additions and 5985 deletions
--- a/.github/workflows/pro-release-develop.yml
+++ b/.github/workflows/pro-release-develop.yml
@ -0,0 +1,47 @@
+name: Budibase Release Staging (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  push:
+    branches:
+      - pro-develop
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
+
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
+      - run: yarn test
+
+      - name: Build/release Docker images (Pro)
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:develop:pro
+          yarn release:docker:pro develop
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
--- a/.github/workflows/pro-release-selfhost.yml
+++ b/.github/workflows/pro-release-selfhost.yml
@ -0,0 +1,41 @@
+name: Budibase Release Selfhost (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          node-version: 14.x
+          fetch_depth: 0
+
+      - name: Tag and release Docker images (Pro) (Self Host)
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+          # Get latest release version
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+          release_tag=v$release_version
+
+          # Pull apps and worker images
+          docker pull budibase/apps-pro:$release_tag
+          docker pull budibase/worker-pro:$release_tag
+
+          # Tag apps and worker images
+          docker tag budibase/apps-pro:$release_tag budibase/apps-pro:$SELFHOST_TAG
+          docker tag budibase/worker-pro:$release_tag budibase/worker-pro:$SELFHOST_TAG
+
+          # Push images
+          docker push budibase/apps-pro:$SELFHOST_TAG
+          docker push budibase/worker-pro:$SELFHOST_TAG
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          SELFHOST_TAG: latest
--- a/.github/workflows/pro-release.yml
+++ b/.github/workflows/pro-release.yml
@ -0,0 +1,53 @@
+name: Budibase Release (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  push:
+    branches:
+      - pro-master
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
+
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
+      - run: yarn test
+
+      - name: "Get Previous tag"
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+
+      - name: Build/release Docker images
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:pro
+          yarn release:docker:pro ${BUDIBASE_RELEASE_VERSION}
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -1,38 +1,38 @@
 name: Budibase Release Staging

-on: 
- push:
+on:
+  push:
    branches:
      - develop
    paths:
-      - '.aws/**'
-      - '.github/**'
-      - 'charts/**'
-      - 'packages/**'
-      - 'scripts/**'
-      - 'package.json'
-      - 'yarn.lock'
-      - 'package.json'
-      - 'yarn.lock'
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"

 env:
  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-      
+
 jobs:
  release:
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-node@v1
        with:
          node-version: 14.x
-      - run: yarn 
-      - run: yarn bootstrap 
-      - run: yarn lint 
-      - run: yarn build 
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
      - run: yarn test

      - name: Configure AWS Credentials
@ -45,7 +45,7 @@ jobs:
      - name: Publish budibase packages to NPM
        env:
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: | 
+        run: |
          # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
          git config user.name "Budibase Staging Release Bot"
          git config user.email "<>"
@ -53,10 +53,10 @@ jobs:
          yarn release:develop

      - name: Build/release Docker images
-        run: | 
+        run: |
          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build
          yarn build:docker:develop
+          yarn release:docker develop
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -1,19 +1,19 @@
 name: Budibase Release

 on:
- push:
+  push:
    branches:
      - master
    paths:
-      - '.aws/**'
-      - '.github/**'
-      - 'charts/**'
-      - 'packages/**'
-      - 'scripts/**'
-      - 'package.json'
-      - 'yarn.lock'
-      - 'package.json'
-      - 'yarn.lock'
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"

 env:
  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
@ -53,15 +53,15 @@ jobs:
          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
          yarn release

-      - name: 'Get Previous tag'
+      - name: "Get Previous tag"
        id: previoustag
        uses: "WyriHaximus/github-action-get-previous-tag@v1"

      - name: Build/release Docker images
-        run: | 
+        run: |
          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build
          yarn build:docker
+          yarn release:docker ${BUDIBASE_RELEASE_VERSION}
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@ -31,6 +31,34 @@ services:
      - worker-service
      - redis-service

+  app-service-pro:
+    restart: always
+    image: app-service-pro
+    container_name: bbapps-pro
+    ports:
+      - "${APP_PORT}:4002"
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    volumes:
+      - ./logs:/logs
+    depends_on:
+      - worker-service
+      - redis-service
+
  worker-service:
    restart: always
    image: budibase.docker.scarf.sh/budibase/worker
--- a/hosting/scripts/linux/docker-release/budibase-pro.sh
+++ b/hosting/scripts/linux/docker-release/budibase-pro.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+tag=$1
+
+if [[ ! "$tag" ]]; then
+	echo "No tag present. You must pass a tag to this script"
+	exit 1
+fi
+
+echo "Tagging images with tag: $tag"
+
+docker tag app-service-pro budibase/apps-pro:$tag
+docker tag worker-service-pro budibase/worker-pro:$tag
+
+docker push --all-tags budibase/apps-pro
+docker push --all-tags budibase/worker-pro
--- a/hosting/scripts/linux/docker-release/budibase.sh
+++ b/hosting/scripts/linux/docker-release/budibase.sh
--- a/hosting/scripts/pro/download.js
+++ b/hosting/scripts/pro/download.js
@ -0,0 +1,52 @@
+const fetch = require('node-fetch')
+const fs = require('fs')
+const util = require('util')
+const streamPipeline = util.promisify(require('stream').pipeline)
+
+async function download (url, opts, path) {
+  const response = await fetch(url, opts)
+  if (!response.ok) throw new Error(`unexpected response ${response.statusText}`)
+  await streamPipeline(response.body, fs.createWriteStream(path))
+}
+
+const install = async () => {
+  // get github token
+  const token = process.env.GITHUB_TOKEN
+  if (!token) {
+    throw new Error("Missing token")
+  }
+  
+  // get version from package.json
+  let packageJson = JSON.parse(fs.readFileSync('package.json'))
+  let version = packageJson.dependencies['@budibase/pro']
+  if (version.startsWith('^')) {
+    version = version.substring(1)
+  }
+  const ghVersion = `v${version}`
+  
+  // get the package url
+  // TODO: Add semver support
+  // TODO: Potentially can use yarn to download directly from github if pro becomes a non monorepo
+  const resp = await fetch(`https://api.github.com/repos/budibase/budibase-pro/releases`, { 
+    headers: { 
+      Authorization: `token ${token}`,
+      Accept: "application/vnd.github.v3.raw"  
+    } 
+  })
+
+  if (!resp.ok) {
+    throw new Error("Failed to get release info. Aborting")
+  }
+  const releases = await resp.json()
+  const release = releases.filter(r => r.name === ghVersion)[0]
+  const url = release.assets.filter(a => a.name === "pro.zip")[0].url
+  
+  console.log("Downloading pro package")
+  // download the package
+  await download(url, { headers: { Authorization: `token ${token}`, Accept: 'application/octet-stream' }}, './pro.zip')
+}
+
+install().catch(e => {
+  console.error(e)
+  process.exit(1)
+})
--- a/hosting/scripts/pro/install.sh
+++ b/hosting/scripts/pro/install.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ ! -z "${PRO}" ]]; then
+  echo "Installing pro package"
+  rm -rf node_modules/@budibase/pro
+  unzip pro.zip -d node_modules/@budibase
+  rm pro.zip
+fi
--- a/package.json
+++ b/package.json
@ -19,9 +19,9 @@
  },
  "scripts": {
    "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "install:pro": "lerna run install:pro --",
    "bootstrap": "lerna link && lerna bootstrap",
    "build": "lerna run build",
-    "publishdev": "lerna run publishdev",
    "publishnpm": "yarn build && lerna publish --force-publish",
    "release": "lerna publish patch --yes --force-publish",
    "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop",
@ -30,10 +30,12 @@
    "nuke:packages": "yarn run restore",
    "nuke:docker": "lerna run --parallel dev:stack:nuke",
    "clean": "lerna clean",
-    "kill-port": "kill-port 4001",
-    "dev": "yarn run kill-port && lerna link && lerna run --parallel dev:builder --concurrency 1",
-    "dev:noserver": "lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
+    "kill-builder": "kill-port 3000",
+    "kill-server": "kill-port 4001 4002",
+    "kill-all": "yarn run kill-builder && yarn run kill-server",
+    "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
+    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
    "test": "lerna run test",
    "lint:eslint": "eslint packages",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
--- a/packages/builder/src/components/common/bindings/BindingPanel.svelte
+++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte
@ -146,7 +146,7 @@
                  <div class="helper__description">
                    {@html helper.description}
                  </div>
-                  <pre class="helper__example">{helper.example || ''}</pre>
+                  <pre class="helper__example">{helper.example || ""}</pre>
                </div>
              </li>
            {/each}
--- a/packages/client/stats.html
+++ b/packages/client/stats.html
--- a/packages/server/.gitignore
+++ b/packages/server/.gitignore
@ -7,4 +7,6 @@ public/
 db/dev.db/
 dist
 coverage/
-watchtower-hook.json
+watchtower-hook.json
+pro.zip
+scripts/pro/*
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@ -16,10 +16,19 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION
 # copy files and install dependencies
 COPY . ./
 RUN yarn
+
+RUN apt-get install unzip
+
+# add the pro package if PRO is set
+ARG PRO
+ENV PRO=$PRO
+RUN yarn install:pro
+
+# build
 RUN yarn build

 # Install client for oracle datasource
-RUN apt-get install unzip libaio1
+RUN apt-get install libaio1
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 

 EXPOSE 4001
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -13,8 +13,12 @@
    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
    "test": "jest --coverage --maxWorkers=2",
    "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../hosting/scripts/pro/install.sh scripts/pro",
    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "download:pro": "node ../../hosting/scripts/pro/download.js",
+    "predocker:pro": "yarn run predocker && yarn download:pro",
+    "build:docker:pro": "yarn run predocker:pro && docker build --build-arg PRO=1 . -t app-service-pro --label version=$BUDIBASE_RELEASE_VERSION",
+    "install:pro": "./scripts/pro/install.sh",
    "build:docs": "node ./scripts/docs/generate.js open",
    "run:docker": "node dist/index.js",
    "dev:stack:up": "node scripts/dev/manage.js up",
@ -73,6 +77,7 @@
    "@budibase/backend-core": "^1.0.65",
    "@budibase/client": "^1.0.65",
    "@budibase/string-templates": "^1.0.65",
+    "@budibase/pro": "0.0.2-alpha.27",
    "@bull-board/api": "^3.7.0",
    "@bull-board/koa": "^3.7.0",
    "@elastic/elasticsearch": "7.10.0",
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@ -2,6 +2,26 @@
 import { ExtendableContext } from "koa"

 import * as env from "./environment"
+
+// check we can import pro types no matter the installation method (budibase-pro or budibase)
+import * as pro from "@budibase/pro"
+import { poc, License } from "@budibase/pro"
+
+// check we can use pro types no matter the installation
+const license: License = { id: "123" }
+console.log(license)
+
+// log the difference between how pro is imported
+// in budibase these will be undefined
+// in budibase-pro these will be defined
+console.log(poc)
+console.log(pro)
+
+if (env.PRO) {
+  // check only pro logic runs in pro
+  console.log("PRO")
+}
+
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@ -50,6 +50,7 @@ module.exports = {
  JEST_WORKER_ID: process.env.JEST_WORKER_ID,
  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  PRO: process.env.PRO,
  // minor
  SALT_ROUNDS: process.env.SALT_ROUNDS,
  LOGGER: process.env.LOGGER,
--- a/yarn.lock
+++ b/yarn.lock