Merge branch 'develop' of github.com:Budibase/budibase into lab-day/refactor-app-db

2022-02-01 14:33:02 +00:00 · 2022-02-01 14:33:02 +00:00 · 53aabb4d05
parent 9fa10ff02f beefc0158c
commit 53aabb4d05
30 changed files with 1941 additions and 2435 deletions
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/package.json
+++ b/package.json
@ -36,10 +36,10 @@
    "dev:server": "lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
    "test": "lerna run test",
    "lint:eslint": "eslint packages",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
    "lint": "yarn run lint:eslint && yarn run lint:prettier",
    "lint:fix:eslint": "eslint --fix packages",
-    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,svelte}\"",
+    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\"",
    "lint:fix:ts": "lerna run lint:fix",
    "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
    "test:e2e": "lerna run cy:test",
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/backend-core",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "description": "Budibase backend core libraries used in server and worker",
  "main": "src/index.js",
  "author": "Budibase",
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/bbui",
  "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "license": "MPL-2.0",
  "svelte": "src/index.js",
  "module": "dist/bbui.es.js",
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/builder",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "license": "GPL-3.0",
  "private": true,
  "scripts": {
@ -66,10 +66,10 @@
    }
  },
  "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.2",
+    "@budibase/bbui": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
    "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
    "@sentry/browser": "5.19.1",
    "@spectrum-css/page": "^3.0.1",
    "@spectrum-css/vars": "^3.0.1",
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/cli",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "description": "Budibase CLI, for developers, self hosting and migrations.",
  "main": "src/index.js",
  "bin": {
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/client",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "license": "MPL-2.0",
  "module": "dist/budibase-client.js",
  "main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
    "dev:builder": "rollup -cw"
  },
  "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.2",
+    "@budibase/bbui": "^1.0.49-alpha.5",
    "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
    "regexparam": "^1.3.0",
    "rollup-plugin-polyfill-node": "^0.8.0",
    "shortid": "^2.2.15",
--- a/packages/client/src/utils/fetch/QueryFetch.js
+++ b/packages/client/src/utils/fetch/QueryFetch.js
@ -16,7 +16,12 @@ export default class QueryFetch extends DataFetch {
    if (!datasource?._id) {
      return null
    }
-    return await fetchQueryDefinition(datasource._id)
+    const definition = await fetchQueryDefinition(datasource._id)
+    // After getting the definition of query, it loses "fields" attribute because of security reason from the server. However, this attribute needs to be inside of defintion for pagination.
+    if (!definition.fields) {
+      definition.fields = datasource.fields
+    }
+    return definition
  }

  async getData() {
--- a/packages/server/mocks/arangojs.ts
+++ b/packages/server/mocks/arangojs.ts
@ -9,7 +9,7 @@ module ArangoMock {
    this.close = jest.fn()
  }

-// @ts-ignore
+  // @ts-ignore
  arangodb.aql = (strings, ...args) => {
    let str = strings.join("{}")

--- a/packages/server/mocks/aws-sdk.ts
+++ b/packages/server/mocks/aws-sdk.ts
@ -1,7 +1,7 @@
 module AwsMock {
  const aws: any = {}

-  const response = (body: any) => () => ({promise: () => body})
+  const response = (body: any) => () => ({ promise: () => body })

  function DocumentClient() {
    // @ts-ignore
@ -39,9 +39,9 @@ module AwsMock {
    )
  }

-  aws.DynamoDB = {DocumentClient}
+  aws.DynamoDB = { DocumentClient }
  aws.S3 = S3
-  aws.config = {update: jest.fn()}
+  aws.config = { update: jest.fn() }

  module.exports = aws
 }
--- a/packages/server/mocks/mongodb.ts
+++ b/packages/server/mocks/mongodb.ts
@ -5,15 +5,14 @@ module MongoMock {
    this.connect = jest.fn()
    this.close = jest.fn()
    this.insertOne = jest.fn()
-    this.insertMany = jest.fn(() => ({toArray: () => []}))
-    this.find = jest.fn(() => ({toArray: () => []}))
+    this.insertMany = jest.fn(() => ({ toArray: () => [] }))
+    this.find = jest.fn(() => ({ toArray: () => [] }))
    this.findOne = jest.fn()
    this.count = jest.fn()
    this.deleteOne = jest.fn()
-    this.deleteMany = jest.fn(() => ({toArray: () => []}))
+    this.deleteMany = jest.fn(() => ({ toArray: () => [] }))
    this.updateOne = jest.fn()
-    this.updateMany = jest.fn(() => ({toArray: () => []}))
-
+    this.updateMany = jest.fn(() => ({ toArray: () => [] }))

    this.collection = jest.fn(() => ({
      insertOne: this.insertOne,
--- a/packages/server/mocks/mssql.ts
+++ b/packages/server/mocks/mssql.ts
@ -10,12 +10,12 @@ module MsSqlMock {
    ],
  }))

-// mssql.connect = jest.fn(() => ({ recordset: [] }))
+  // mssql.connect = jest.fn(() => ({ recordset: [] }))

  mssql.ConnectionPool = jest.fn(() => ({
    connect: jest.fn(() => ({
      request: jest.fn(() => ({
-        query: jest.fn(sql => ({ recordset: [ sql ] })),
+        query: jest.fn(sql => ({ recordset: [sql] })),
      })),
    })),
  }))
--- a/packages/server/mocks/node-fetch.ts
+++ b/packages/server/mocks/node-fetch.ts
@ -62,7 +62,8 @@ module FetchMock {
      return json({
        url,
        opts,
-        value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
+        value:
+          '<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="en-GB"></html>',
      })
    } else if (url.includes("failonce.com")) {
      failCount++
--- a/packages/server/mocks/pg.ts
+++ b/packages/server/mocks/pg.ts
@ -10,16 +10,14 @@ module PgMock {
    ],
  }))

-// constructor
-  function Client() {
-  }
+  // constructor
+  function Client() {}

  Client.prototype.query = query
  Client.prototype.connect = jest.fn()
  Client.prototype.release = jest.fn()

-  function Pool() {
-  }
+  function Pool() {}

  const on = jest.fn()
  Pool.prototype.query = query
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/server",
  "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "description": "Budibase Web Server",
  "main": "src/index.ts",
  "repository": {
@ -70,9 +70,9 @@
  "license": "GPL-3.0",
  "dependencies": {
    "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/backend-core": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
    "@bull-board/api": "^3.7.0",
    "@bull-board/koa": "^3.7.0",
    "@elastic/elasticsearch": "7.10.0",
--- a/packages/server/src/migrations/functions/appUrls.ts
+++ b/packages/server/src/migrations/functions/appUrls.ts
@ -9,7 +9,16 @@ import { getAppUrl } from "../../api/controllers/application"
 * Add the url to the app metadata if it doesn't exist
 */
 export const run = async (appDb: any) => {
-  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  let metadata
+  try {
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  } catch (e) {
+    // sometimes the metadata document doesn't exist
+    // exit early instead of failing the migration
+    console.error("Error retrieving app metadata. Skipping", e)
+    return
+  }
+
  if (!metadata.url) {
    const context = {
      request: {
@ -20,6 +29,6 @@ export const run = async (appDb: any) => {
    }
    metadata.url = getAppUrl(context)
    console.log(`Adding url to app: ${metadata.url}`)
+    await appDb.put(metadata)
  }
-  await appDb.put(metadata)
 }
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/string-templates",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "description": "Handlebars wrapper for Budibase templating.",
  "main": "src/index.cjs",
  "module": "dist/bundle.mjs",
--- a/packages/worker/.eslintrc
+++ b/packages/worker/.eslintrc
@ -3,6 +3,7 @@
    "emit": true,
    "key": true
  },
+  "parser": "@typescript-eslint/parser",
  "env": {
    "node": true
  },
--- a/packages/worker/.gitignore
+++ b/packages/worker/.gitignore
@ -1,3 +1,4 @@
 node_modules/
 .env
 watchtower-hook.json
+dist/
--- a/packages/worker/nodemon.json
+++ b/packages/worker/nodemon.json
@ -1,3 +1,6 @@
 {
-  "watch": ["src", "../backend-core"]
+  "watch": ["src", "../backend-core"],
+  "ext": "js,ts,json",
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
+  "exec": "ts-node src/index.ts"
 }
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -1,9 +1,9 @@
 {
  "name": "@budibase/worker",
  "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
  "description": "Budibase background service",
-  "main": "src/index.js",
+  "main": "src/index.ts",
  "repository": {
    "type": "git",
    "url": "https://github.com/Budibase/budibase.git"
@ -12,10 +12,15 @@
    "budibase"
  ],
  "scripts": {
-    "run:docker": "node src/index.js",
+    "build": "rimraf dist/ && tsc",
+    "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
+    "run:docker": "node dist/index.js",
    "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
    "dev:stack:init": "node ./scripts/dev/manage.js init",
    "dev:builder": "npm run dev:stack:init && nodemon",
+    "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write",
+    "lint": "eslint --fix src/",
+    "lint:fix": "yarn run format && yarn run lint",
    "test": "jest --runInBand",
    "env:multi:enable": "node scripts/multiTenancy.js enable",
    "env:multi:disable": "node scripts/multiTenancy.js disable",
@ -29,8 +34,8 @@
  "author": "Budibase",
  "license": "GPL-3.0",
  "dependencies": {
-    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/backend-core": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
    "@koa/router": "^8.0.0",
    "@sentry/node": "^6.0.0",
    "@techpass/passport-openidconnect": "^0.3.0",
@ -58,13 +63,26 @@
    "server-destroy": "^1.0.1"
  },
  "devDependencies": {
-    "jest": "^26.6.3",
+    "@types/jest": "^26.0.23",
+    "@types/koa": "^2.13.3",
+    "@types/koa-router": "^7.4.2",
+    "@types/node": "^15.12.4",
+    "@typescript-eslint/parser": "4.28.0",
+    "copyfiles": "^2.4.1",
+    "eslint": "^6.8.0",
+    "jest": "^27.0.5",
    "nodemon": "^2.0.7",
    "pouchdb-adapter-memory": "^7.2.2",
+    "prettier": "2.3.1",
+    "rimraf": "^3.0.2",
    "supertest": "^6.1.3",
+    "ts-jest": "^27.0.3",
+    "ts-node": "^10.0.0",
+    "typescript": "4.3.5",
    "update-dotenv": "^1.1.1"
  },
  "jest": {
+    "preset": "ts-jest",
    "testEnvironment": "node",
    "setupFiles": [
      "./scripts/jestSetup.js"
--- a/packages/worker/src/api/routes/tests/auth.spec.js
+++ b/packages/worker/src/api/routes/tests/auth.spec.js
@ -1,10 +1,9 @@
+jest.mock("nodemailer")
 const setup = require("./utilities")
+const sendMailMock = setup.emailMock()

 const TENANT_ID = "default"

-jest.mock("nodemailer")
-const sendMailMock = setup.emailMock()
-
 describe("/api/global/auth", () => {
  let request = setup.getRequest()
  let config = setup.getConfig()
--- a/packages/worker/src/api/routes/tests/configs.spec.js
+++ b/packages/worker/src/api/routes/tests/configs.spec.js
@ -1,11 +1,7 @@
-const setup = require("./utilities")
-
 // mock the email system
 jest.mock("nodemailer")
-const nodemailer = require("nodemailer")
-nodemailer.createTransport.mockReturnValue({
-  verify: jest.fn(),
-})
+const setup = require("./utilities")
+setup.emailMock()

 describe("/api/global/configs/checklist", () => {
  let request = setup.getRequest()
--- a/packages/worker/src/api/routes/tests/email.spec.js
+++ b/packages/worker/src/api/routes/tests/email.spec.js
@ -1,16 +1,10 @@
+jest.mock("nodemailer")
 const setup = require("./utilities")
+const sendMailMock = setup.emailMock()
+
 const { EmailTemplatePurpose } = require("../../../constants")
 const { TENANT_ID } = require("./utilities/structures")

-// mock the email system
-const sendMailMock = jest.fn()
-jest.mock("nodemailer")
-const nodemailer = require("nodemailer")
-nodemailer.createTransport.mockReturnValue({
-  sendMail: sendMailMock,
-  verify: jest.fn(),
-})
-
 describe("/api/global/email", () => {
  let request = setup.getRequest()
  let config = setup.getConfig()
--- a/packages/worker/src/api/routes/tests/users.spec.js
+++ b/packages/worker/src/api/routes/tests/users.spec.js
@ -1,7 +1,5 @@
-const setup = require("./utilities")
-const { TENANT_ID } = require("./utilities/structures")
-
 jest.mock("nodemailer")
+const setup = require("./utilities")
 const sendMailMock = setup.emailMock()

 describe("/api/global/users", () => {
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@ -1,4 +1,8 @@
 // need to load environment first
+import { Scope } from "@sentry/node"
+import { Event } from "@sentry/types/dist/event"
+import Application from "koa"
+
 const env = require("./environment")
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
@ -13,7 +17,7 @@ const api = require("./api")
 const redis = require("./utilities/redis")
 const Sentry = require("@sentry/node")

-const app = new Koa()
+const app: Application = new Koa()

 app.keys = ["secret", "key"]

@ -42,8 +46,8 @@ if (env.isProd()) {
  Sentry.init()

  app.on("error", (err, ctx) => {
-    Sentry.withScope(function (scope) {
-      scope.addEventProcessor(function (event) {
+    Sentry.withScope(function (scope: Scope) {
+      scope.addEventProcessor(function (event: Event) {
        return Sentry.Handlers.parseRequest(event, ctx.request)
      })
      Sentry.captureException(err)
--- a/packages/worker/tsconfig.json
+++ b/packages/worker/tsconfig.json
@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["es2019"],
+    "allowJs": true,
+    "outDir": "dist",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true
+  },
+  "include": [
+    "./src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "**/*.json",
+    "**/*.spec.ts",
+    "**/*.spec.js"
+  ]
+}
--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock