Merge branch 'master' of github.com:Budibase/budibase into fix/BUDI-8258-update-presence

charts/budibase/values.yaml

@@ -641,7 +641,7 @@ couchdb:
     # @ignore
     repository: budibase/couchdb
     # @ignore
-    tag: v3.2.1
+    tag: v3.3.3
     # @ignore
     pullPolicy: Always
 

packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap

@@ -1,91 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`/datasources fetch returns all the datasources from the server 1`] = `
-[
-  {
-    "config": {},
-    "entities": [
-      {
-        "_id": "ta_users",
-        "_rev": "1-73b7912e6cbdd3d696febc60f3715844",
-        "createdAt": "2020-01-01T00:00:00.000Z",
-        "name": "Users",
-        "primaryDisplay": "email",
-        "schema": {
-          "email": {
-            "constraints": {
-              "email": true,
-              "length": {
-                "maximum": "",
-              },
-              "presence": true,
-              "type": "string",
-            },
-            "name": "email",
-            "type": "string",
-          },
-          "firstName": {
-            "constraints": {
-              "presence": false,
-              "type": "string",
-            },
-            "name": "firstName",
-            "type": "string",
-          },
-          "lastName": {
-            "constraints": {
-              "presence": false,
-              "type": "string",
-            },
-            "name": "lastName",
-            "type": "string",
-          },
-          "roleId": {
-            "constraints": {
-              "inclusion": [
-                "ADMIN",
-                "POWER",
-                "BASIC",
-                "PUBLIC",
-              ],
-              "presence": false,
-              "type": "string",
-            },
-            "name": "roleId",
-            "type": "options",
-          },
-          "status": {
-            "constraints": {
-              "inclusion": [
-                "active",
-                "inactive",
-              ],
-              "presence": false,
-              "type": "string",
-            },
-            "name": "status",
-            "type": "options",
-          },
-        },
-        "sourceId": "bb_internal",
-        "sourceType": "internal",
-        "type": "table",
-        "updatedAt": "2020-01-01T00:00:00.000Z",
-        "views": {},
-      },
-    ],
-    "name": "Budibase DB",
-    "source": "BUDIBASE",
-    "type": "budibase",
-  },
-  {
-    "config": {},
-    "createdAt": "2020-01-01T00:00:00.000Z",
-    "isSQL": true,
-    "name": "Test",
-    "source": "POSTGRES",
-    "type": "datasource",
-    "updatedAt": "2020-01-01T00:00:00.000Z",
-  },
-]
-`;

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -4,14 +4,12 @@ import { getCachedVariable } from "../../../threads/utils"
 import { context, events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 
-import tk from "timekeeper"
+import { generator } from "@budibase/backend-core/tests"
-import { mocks } from "@budibase/backend-core/tests"
 import {
   Datasource,
   FieldSchema,
   BBReferenceFieldSubType,
   FieldType,
-  QueryPreview,
   RelationshipType,
   SourceName,
   Table,
@@ -21,36 +19,34 @@ import {
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import { tableForDatasource } from "../../../tests/utilities/structures"
 
-tk.freeze(mocks.date.MOCK_DATE)
-
-let { basicDatasource } = setup.structures
-
 describe("/datasources", () => {
-  let request = setup.getRequest()
+  const config = setup.getConfig()
-  let config = setup.getConfig()
+  let datasource: Datasource
-  let datasource: any
 
+  beforeAll(async () => {
+    await config.init()
+  })
   afterAll(setup.afterAll)
 
-  async function setupTest() {
+  beforeEach(async () => {
-    await config.init()
+    datasource = await config.api.datasource.create({
-    datasource = await config.createDatasource()
+      type: "datasource",
+      name: "Test",
+      source: SourceName.POSTGRES,
+      config: {},
+    })
     jest.clearAllMocks()
-  }
+  })
-
-  beforeAll(setupTest)
 
   describe("create", () => {
     it("should create a new datasource", async () => {
-      const res = await request
+      const ds = await config.api.datasource.create({
-        .post(`/api/datasources`)
+        type: "datasource",
-        .send(basicDatasource())
+        name: "Test",
-        .set(config.defaultHeaders())
+        source: SourceName.POSTGRES,
-        .expect("Content-Type", /json/)
+        config: {},
-        .expect(200)
+      })
-
+      expect(ds.name).toEqual("Test")
-      expect(res.body.datasource.name).toEqual("Test")
-      expect(res.body.errors).toEqual({})
       expect(events.datasource.created).toHaveBeenCalledTimes(1)
     })
 
@@ -72,88 +68,71 @@ describe("/datasources", () => {
     })
   })
 
-  describe("update", () => {
+  describe("dynamic variables", () => {
-    it("should update an existing datasource", async () => {
+    it("should invalidate changed or removed variables", async () => {
-      datasource.name = "Updated Test"
+      let datasource = await config.api.datasource.create({
-      const res = await request
+        type: "datasource",
-        .put(`/api/datasources/${datasource._id}`)
+        name: "Rest",
-        .send(datasource)
+        source: SourceName.REST,
-        .set(config.defaultHeaders())
+        config: {},
-        .expect("Content-Type", /json/)
+      })
-        .expect(200)
 
-      expect(res.body.datasource.name).toEqual("Updated Test")
+      const query = await config.api.query.save({
-      expect(res.body.errors).toBeUndefined()
+        datasourceId: datasource._id!,
-      expect(events.datasource.updated).toHaveBeenCalledTimes(1)
+        fields: {
-    })
+          path: "www.google.com",
+        },
+        parameters: [],
+        transformer: null,
+        queryVerb: "read",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      })
 
-    describe("dynamic variables", () => {
+      datasource = await config.api.datasource.update({
-      async function preview(
+        ...datasource,
-        datasource: any,
+        config: {
-        fields: { path: string; queryString: string }
+          dynamicVariables: [
-      ) {
+            {
-        const queryPreview: QueryPreview = {
+              queryId: query._id,
-          fields,
+              name: "variable3",
-          datasourceId: datasource._id,
+              value: "{{ data.0.[value] }}",
-          parameters: [],
+            },
-          transformer: null,
+          ],
-          queryVerb: "read",
+        },
-          name: datasource.name,
+      })
-          schema: {},
-          readable: true,
-        }
-        return config.api.query.preview(queryPreview)
-      }
 
-      it("should invalidate changed or removed variables", async () => {
+      // preview once to cache variables
-        const { datasource, query } = await config.dynamicVariableDatasource()
+      await config.api.query.preview({
-        // preview once to cache variables
+        fields: {
-        await preview(datasource, {
           path: "www.example.com",
           queryString: "test={{ variable3 }}",
-        })
+        },
-        // check variables in cache
+        datasourceId: datasource._id!,
-        let contents = await getCachedVariable(query._id!, "variable3")
+        parameters: [],
-        expect(contents.rows.length).toEqual(1)
+        transformer: null,
-
+        queryVerb: "read",
-        // update the datasource to remove the variables
+        name: datasource.name!,
-        datasource.config!.dynamicVariables = []
+        schema: {},
-        const res = await request
+        readable: true,
-          .put(`/api/datasources/${datasource._id}`)
-          .send(datasource)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect(res.body.errors).toBeUndefined()
-
-        // check variables no longer in cache
-        contents = await getCachedVariable(query._id!, "variable3")
-        expect(contents).toBe(null)
       })
+
+      // check variables in cache
+      let contents = await getCachedVariable(query._id!, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // update the datasource to remove the variables
+      datasource.config!.dynamicVariables = []
+      await config.api.datasource.update(datasource)
+
+      // check variables no longer in cache
+      contents = await getCachedVariable(query._id!, "variable3")
+      expect(contents).toBe(null)
     })
   })
 
-  describe("fetch", () => {
+  describe("permissions", () => {
-    beforeAll(setupTest)
-
-    it("returns all the datasources from the server", async () => {
-      const res = await request
-        .get(`/api/datasources`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      const datasources = res.body
-
-      // remove non-deterministic fields
-      for (let source of datasources) {
-        delete source._id
-        delete source._rev
-      }
-
-      expect(datasources).toMatchSnapshot()
-    })
-
     it("should apply authorization to endpoint", async () => {
       await checkBuilderEndpoint({
         config,
@@ -161,41 +140,8 @@ describe("/datasources", () => {
         url: `/api/datasources`,
       })
     })
-  })
 
-  describe("find", () => {
+    it("should apply authorization to delete endpoint", async () => {
-    it("should be able to find a datasource", async () => {
-      const res = await request
-        .get(`/api/datasources/${datasource._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-      expect(res.body._rev).toBeDefined()
-      expect(res.body._id).toEqual(datasource._id)
-    })
-  })
-
-  describe("destroy", () => {
-    beforeAll(setupTest)
-
-    it("deletes queries for the datasource after deletion and returns a success message", async () => {
-      await config.createQuery()
-
-      await request
-        .delete(`/api/datasources/${datasource._id}/${datasource._rev}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-
-      const res = await request
-        .get(`/api/datasources`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      expect(res.body.length).toEqual(1)
-      expect(events.datasource.deleted).toHaveBeenCalledTimes(1)
-    })
-
-    it("should apply authorization to endpoint", async () => {
       await checkBuilderEndpoint({
         config,
         method: "DELETE",
@@ -204,175 +150,227 @@ describe("/datasources", () => {
     })
   })
 
-  describe("check secret replacement", () => {
-    async function makeDatasource() {
-      datasource = basicDatasource()
-      datasource.datasource.config.password = "testing"
-      const res = await request
-        .post(`/api/datasources`)
-        .send(datasource)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      return res.body.datasource
-    }
-
-    it("should save a datasource with password", async () => {
-      const datasource = await makeDatasource()
-      expect(datasource.config.password).toBe("--secret-value--")
-    })
-
-    it("should not the password on update with the --secret-value--", async () => {
-      const datasource = await makeDatasource()
-      await request
-        .put(`/api/datasources/${datasource._id}`)
-        .send(datasource)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      await context.doInAppContext(config.getAppId(), async () => {
-        const dbDatasource: any = await sdk.datasources.get(datasource._id)
-        expect(dbDatasource.config.password).toBe("testing")
-      })
-    })
-  })
-
   describe.each([
     [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
     [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
     [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
     [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  ])("fetch schema (%s)", (_, dsProvider) => {
+  ])("%s", (_, dsProvider) => {
-    beforeAll(async () => {
+    beforeEach(async () => {
       datasource = await config.api.datasource.create(await dsProvider)
     })
 
-    it("fetching schema will not drop tables or columns", async () => {
+    describe("get", () => {
-      const datasourceId = datasource!._id!
+      it("should be able to get a datasource", async () => {
+        const ds = await config.api.datasource.get(datasource._id!)
+        expect(ds._id).toEqual(datasource._id)
+        expect(ds._rev).toBeDefined()
+      })
 
-      const simpleTable = await config.api.table.save(
+      it("should not return database password", async () => {
-        tableForDatasource(datasource, {
+        const ds = await config.api.datasource.get(datasource._id!)
-          name: "simple",
+        expect(ds.config!.password).toBe("--secret-value--")
-          schema: {
+      })
-            name: {
+    })
-              name: "name",
+
-              type: FieldType.STRING,
+    describe("list", () => {
+      it("returns all the datasources", async () => {
+        const datasources = await config.api.datasource.fetch()
+        expect(datasources).toContainEqual(expect.objectContaining(datasource))
+      })
+    })
+
+    describe("put", () => {
+      it("should update an existing datasource", async () => {
+        const newName = generator.guid()
+        datasource.name = newName
+        const updatedDs = await config.api.datasource.update(datasource)
+        expect(updatedDs.name).toEqual(newName)
+        expect(events.datasource.updated).toHaveBeenCalledTimes(1)
+      })
+
+      it("should not overwrite database password with --secret-value--", async () => {
+        const password = await context.doInAppContext(
+          config.getAppId(),
+          async () => {
+            const ds = await sdk.datasources.get(datasource._id!)
+            return ds.config!.password
+          }
+        )
+
+        expect(password).not.toBe("--secret-value--")
+
+        const ds = await config.api.datasource.get(datasource._id!)
+        expect(ds.config!.password).toBe("--secret-value--")
+
+        await config.api.datasource.update(
+          await config.api.datasource.get(datasource._id!)
+        )
+
+        const newPassword = await context.doInAppContext(
+          config.getAppId(),
+          async () => {
+            const ds = await sdk.datasources.get(datasource._id!)
+            return ds.config!.password
+          }
+        )
+
+        expect(newPassword).not.toBe("--secret-value--")
+        expect(newPassword).toBe(password)
+      })
+    })
+
+    describe("destroy", () => {
+      it("deletes queries for the datasource after deletion and returns a success message", async () => {
+        await config.api.query.save({
+          datasourceId: datasource._id!,
+          name: "Test Query",
+          parameters: [],
+          fields: {},
+          schema: {},
+          queryVerb: "read",
+          transformer: null,
+          readable: true,
+        })
+
+        await config.api.datasource.delete(datasource)
+        const datasources = await config.api.datasource.fetch()
+        expect(datasources).not.toContainEqual(
+          expect.objectContaining(datasource)
+        )
+        expect(events.datasource.deleted).toHaveBeenCalledTimes(1)
+      })
+    })
+
+    describe("schema", () => {
+      it("fetching schema will not drop tables or columns", async () => {
+        const datasourceId = datasource!._id!
+
+        const simpleTable = await config.api.table.save(
+          tableForDatasource(datasource, {
+            name: "simple",
+            schema: {
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+
+        const fullSchema: {
+          [type in SupportedSqlTypes]: FieldSchema & { type: type }
+        } = {
+          [FieldType.STRING]: {
+            name: "string",
+            type: FieldType.STRING,
+            constraints: {
+              presence: true,
             },
           },
-        })
+          [FieldType.LONGFORM]: {
-      )
+            name: "longform",
-
+            type: FieldType.LONGFORM,
-      const fullSchema: {
-        [type in SupportedSqlTypes]: FieldSchema & { type: type }
-      } = {
-        [FieldType.STRING]: {
-          name: "string",
-          type: FieldType.STRING,
-          constraints: {
-            presence: true,
           },
-        },
+          [FieldType.OPTIONS]: {
-        [FieldType.LONGFORM]: {
+            name: "options",
-          name: "longform",
+            type: FieldType.OPTIONS,
-          type: FieldType.LONGFORM,
+            constraints: {
-        },
+              presence: { allowEmpty: false },
-        [FieldType.OPTIONS]: {
-          name: "options",
-          type: FieldType.OPTIONS,
-          constraints: {
-            presence: { allowEmpty: false },
-          },
-        },
-        [FieldType.NUMBER]: {
-          name: "number",
-          type: FieldType.NUMBER,
-        },
-        [FieldType.BOOLEAN]: {
-          name: "boolean",
-          type: FieldType.BOOLEAN,
-        },
-        [FieldType.ARRAY]: {
-          name: "array",
-          type: FieldType.ARRAY,
-        },
-        [FieldType.DATETIME]: {
-          name: "datetime",
-          type: FieldType.DATETIME,
-          dateOnly: true,
-          timeOnly: false,
-        },
-        [FieldType.LINK]: {
-          name: "link",
-          type: FieldType.LINK,
-          tableId: simpleTable._id!,
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          fieldName: "link",
-        },
-        [FieldType.FORMULA]: {
-          name: "formula",
-          type: FieldType.FORMULA,
-          formula: "any formula",
-        },
-        [FieldType.BARCODEQR]: {
-          name: "barcodeqr",
-          type: FieldType.BARCODEQR,
-        },
-        [FieldType.BIGINT]: {
-          name: "bigint",
-          type: FieldType.BIGINT,
-        },
-        [FieldType.BB_REFERENCE]: {
-          name: "bb_reference",
-          type: FieldType.BB_REFERENCE,
-          subtype: BBReferenceFieldSubType.USER,
-        },
-        [FieldType.BB_REFERENCE_SINGLE]: {
-          name: "bb_reference_single",
-          type: FieldType.BB_REFERENCE_SINGLE,
-          subtype: BBReferenceFieldSubType.USER,
-        },
-      }
-
-      await config.api.table.save(
-        tableForDatasource(datasource, {
-          name: "full",
-          schema: fullSchema,
-        })
-      )
-
-      const persisted = await config.api.datasource.get(datasourceId)
-      await config.api.datasource.fetchSchema({ datasourceId })
-
-      const updated = await config.api.datasource.get(datasourceId)
-      const expected: Datasource = {
-        ...persisted,
-        entities:
-          persisted?.entities &&
-          Object.entries(persisted.entities).reduce<Record<string, Table>>(
-            (acc, [tableName, table]) => {
-              acc[tableName] = {
-                ...table,
-                primaryDisplay: expect.not.stringMatching(
-                  new RegExp(`^${table.primaryDisplay || ""}$`)
-                ),
-                schema: Object.entries(table.schema).reduce<TableSchema>(
-                  (acc, [fieldName, field]) => {
-                    acc[fieldName] = expect.objectContaining({
-                      ...field,
-                    })
-                    return acc
-                  },
-                  {}
-                ),
-              }
-              return acc
             },
-            {}
+          },
-          ),
+          [FieldType.NUMBER]: {
+            name: "number",
+            type: FieldType.NUMBER,
+          },
+          [FieldType.BOOLEAN]: {
+            name: "boolean",
+            type: FieldType.BOOLEAN,
+          },
+          [FieldType.ARRAY]: {
+            name: "array",
+            type: FieldType.ARRAY,
+          },
+          [FieldType.DATETIME]: {
+            name: "datetime",
+            type: FieldType.DATETIME,
+            dateOnly: true,
+            timeOnly: false,
+          },
+          [FieldType.LINK]: {
+            name: "link",
+            type: FieldType.LINK,
+            tableId: simpleTable._id!,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "link",
+          },
+          [FieldType.FORMULA]: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any formula",
+          },
+          [FieldType.BARCODEQR]: {
+            name: "barcodeqr",
+            type: FieldType.BARCODEQR,
+          },
+          [FieldType.BIGINT]: {
+            name: "bigint",
+            type: FieldType.BIGINT,
+          },
+          [FieldType.BB_REFERENCE]: {
+            name: "bb_reference",
+            type: FieldType.BB_REFERENCE,
+            subtype: BBReferenceFieldSubType.USER,
+          },
+          [FieldType.BB_REFERENCE_SINGLE]: {
+            name: "bb_reference_single",
+            type: FieldType.BB_REFERENCE_SINGLE,
+            subtype: BBReferenceFieldSubType.USER,
+          },
+        }
 
-        _rev: expect.any(String),
+        await config.api.table.save(
-      }
+          tableForDatasource(datasource, {
-      expect(updated).toEqual(expected)
+            name: "full",
+            schema: fullSchema,
+          })
+        )
+
+        const persisted = await config.api.datasource.get(datasourceId)
+        await config.api.datasource.fetchSchema({ datasourceId })
+
+        const updated = await config.api.datasource.get(datasourceId)
+        const expected: Datasource = {
+          ...persisted,
+          entities:
+            persisted?.entities &&
+            Object.entries(persisted.entities).reduce<Record<string, Table>>(
+              (acc, [tableName, table]) => {
+                acc[tableName] = {
+                  ...table,
+                  primaryDisplay: expect.not.stringMatching(
+                    new RegExp(`^${table.primaryDisplay || ""}$`)
+                  ),
+                  schema: Object.entries(table.schema).reduce<TableSchema>(
+                    (acc, [fieldName, field]) => {
+                      acc[fieldName] = expect.objectContaining({
+                        ...field,
+                      })
+                      return acc
+                    },
+                    {}
+                  ),
+                }
+                return acc
+              },
+              {}
+            ),
+
+          _rev: expect.any(String),
+          updatedAt: expect.any(String),
+        }
+        expect(updated).toEqual(expected)
+      })
     })
   })
 })

packages/server/src/integration-test/mysql.spec.ts

@@ -4,13 +4,7 @@ import {
   MakeRequestResponse,
 } from "../api/routes/public/tests/utils"
 import * as setup from "../api/routes/tests/utilities"
-import {
+import { Datasource, FieldType, Table, TableSourceType } from "@budibase/types"
-  Datasource,
-  FieldType,
-  Table,
-  TableRequest,
-  TableSourceType,
-} from "@budibase/types"
 import {
   DatabaseName,
   getDatasource,
@@ -231,57 +225,6 @@ describe("mysql integrations", () => {
     })
   })
 
-  describe("POST /api/tables/", () => {
-    it("will rename a column", async () => {
-      await makeRequest("post", "/api/tables/", primaryMySqlTable)
-
-      let renameColumnOnTable: TableRequest = {
-        ...primaryMySqlTable,
-        schema: {
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            autocolumn: true,
-            externalType: "unsigned integer",
-          },
-          name: {
-            name: "name",
-            type: FieldType.STRING,
-            externalType: "text",
-          },
-          description: {
-            name: "description",
-            type: FieldType.STRING,
-            externalType: "text",
-          },
-          age: {
-            name: "age",
-            type: FieldType.NUMBER,
-            externalType: "float(8,2)",
-          },
-        },
-      }
-
-      const response = await makeRequest(
-        "post",
-        "/api/tables/",
-        renameColumnOnTable
-      )
-
-      const ds = (
-        await makeRequest("post", `/api/datasources/${datasource._id}/schema`)
-      ).body.datasource
-
-      expect(response.status).toEqual(200)
-      expect(Object.keys(ds.entities![primaryMySqlTable.name].schema)).toEqual([
-        "id",
-        "name",
-        "description",
-        "age",
-      ])
-    })
-  })
-
   describe("POST /api/datasources/:datasourceId/schema", () => {
     let tableName: string
 

packages/server/src/tests/utilities/api/datasource.ts

@@ -61,6 +61,10 @@ export class DatasourceAPI extends TestAPI {
     })
   }
 
+  fetch = async (expectations?: Expectations) => {
+    return await this._get<Datasource[]>(`/api/datasources`, { expectations })
+  }
+
   query = async (
     query: Omit<QueryJson, "meta"> & Partial<Pick<QueryJson, "meta">>,
     expectations?: Expectations