Merge branch 'develop' of github.com:Budibase/budibase into rest-pagination

.github/stale.yml

@@ -1,7 +1,9 @@
-# Number of days of inactivity before an issue becomes stale
+# Configuration for probot-stale - https://github.com/probot/stale
+# Number of days of inactivity before an Issue or Pull Request becomes stale
 daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-daysUntilClose: 7
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: false
 # Issues with these labels will never be considered stale
 exemptLabels:
   - pinned

.github/workflows/release-selfhost.yml

@@ -9,11 +9,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
         with:
-          fetch_depth: 0
+          node-version: 14.x
 
       - name: Tag and release Docker images (Self Host)
-        run: | 
+        run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
 
           # Get latest release version
@@ -28,7 +29,7 @@ jobs:
           # Tag apps and worker images
           docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG
           docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG
-          
+
           # Push images
           docker push budibase/apps:$SELFHOST_TAG
           docker push budibase/worker:$SELFHOST_TAG
@@ -36,13 +37,20 @@ jobs:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
           SELFHOST_TAG: latest
-      
+
-      - name: Setup Helm 
+      - name: Setup Helm
         uses: azure/setup-helm@v1
         id: helm-install
 
+      - name: Build CLI executables
+        run: |
+          pushd packages/cli
+          yarn
+          yarn build
+          popd
+
       # - name: Build and release helm chart
-      #   run: | 
+      #   run: |
       #     git config user.name "Budibase Helm Bot"
       #     git config user.email "<>"
       #     mv budibase-${{ env.RELEASE_VERSION }}.tgz docs
@@ -66,4 +74,8 @@ jobs:
         with:
           name: v${{ env.RELEASE_VERSION }}
           tag_name: v${{ env.RELEASE_VERSION }}
-          generate_release_notes: true
+          generate_release_notes: true
+          files: |
+            packages/cli/build/cli-win.exe
+            packages/cli/build/cli-linux
+            packages/cli/build/cli-macos

.gitignore

@@ -93,3 +93,5 @@ hosting/.generated-envoy.dev.yaml
 # Sublime text
 *.sublime-project
 *.sublime-workspace
+
+bin/

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.2",
+    "@budibase/bbui": "^1.0.27-alpha.5",
-    "@budibase/client": "^1.0.27-alpha.2",
+    "@budibase/client": "^1.0.27-alpha.5",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.27-alpha.2",
+    "@budibase/string-templates": "^1.0.27-alpha.5",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -13,7 +13,9 @@
   },
   "pkg": {
     "targets": [
-      "node14"
+      "node14-linux",
+      "node14-win",
+      "node14-macos"
     ],
     "outputPath": "build"
   },

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.2",
+    "@budibase/bbui": "^1.0.27-alpha.5",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.27-alpha.2",
+    "@budibase/string-templates": "^1.0.27-alpha.5",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/__mocks__/node-fetch.ts

@@ -1,5 +1,6 @@
 module FetchMock {
   const fetch = jest.requireActual("node-fetch")
+  let failCount = 0
 
   module.exports = async (url: any, opts: any) => {
     function json(body: any, status = 200) {
@@ -63,6 +64,17 @@ module FetchMock {
         opts,
         value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
       })
+    } else if (url.includes("failonce.com")) {
+      failCount++
+      if (failCount === 1) {
+        return json({ message: "error" }, 500)
+      } else {
+        return json({
+          fails: failCount - 1,
+          url,
+          opts,
+        })
+      }
     }
     return fetch(url, opts)
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.2",
+    "@budibase/auth": "^1.0.27-alpha.5",
-    "@budibase/client": "^1.0.27-alpha.2",
+    "@budibase/client": "^1.0.27-alpha.5",
-    "@budibase/string-templates": "^1.0.27-alpha.2",
+    "@budibase/string-templates": "^1.0.27-alpha.5",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/routes/tests/query.spec.js

@@ -11,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -239,33 +240,7 @@ describe("/queries", () => {
       })
     }
 
-    it("should work with static variables", async () => {
+    async function dynamicVariableDatasource() {
-      const datasource = await restDatasource({
-        staticVariables: {
-          variable: "google",
-          variable2: "1",
-        },
-      })
-      const res = await request
-        .post(`/api/queries/preview`)
-        .send({
-          datasourceId: datasource._id,
-          parameters: {},
-          fields: {
-            path: "www.{{ variable }}.com",
-            queryString: "test={{ variable2 }}",
-          },
-          queryVerb: "read",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      // these responses come from the mock
-      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
-    })
-
-    it("should work with dynamic variables", async () => {
       const datasource = await restDatasource()
       const basedOnQuery = await config.createQuery({
         ...basicQuery(datasource._id),
@@ -281,22 +256,62 @@ describe("/queries", () => {
           ]
         }
       })
-      const res = await request
+      return { datasource, query: basedOnQuery }
+    }
+
+    async function preview(datasource, fields) {
+      return await request
         .post(`/api/queries/preview`)
         .send({
           datasourceId: datasource._id,
           parameters: {},
-          fields: {
+          fields,
-            path: "www.google.com",
-            queryString: "test={{ variable3 }}",
-          },
           queryVerb: "read",
         })
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
       expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
       expect(res.body.rows[0].url).toContain("doctype html")
     })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
   })
 })

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.5",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^1.0.27-alpha.2",
+    "@budibase/auth": "^1.0.27-alpha.5",
-    "@budibase/string-templates": "^1.0.27-alpha.2",
+    "@budibase/string-templates": "^1.0.27-alpha.5",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",