Making really good progress removing the pg mocks. More to do, though.

packages/server/__mocks__/pg.ts

@@ -1,25 +0,0 @@
-const query = jest.fn(() => ({
-  rows: [
-    {
-      a: "string",
-      b: 1,
-    },
-  ],
-}))
-
-class Client {
-  query = query
-  end = jest.fn(cb => {
-    if (cb) cb()
-  })
-  connect = jest.fn()
-  release = jest.fn()
-}
-
-const on = jest.fn()
-
-module.exports = {
-  Client,
-  queryMock: query,
-  on,
-}

packages/server/jest.config.ts

@@ -60,6 +60,8 @@ const config: Config.InitialOptions = {
     "!src/db/views/staticViews.*",
     "!src/**/*.spec.{js,ts}",
     "!src/tests/**/*.{js,ts}",
+    // The use of coverage in the JS runner bundles breaks tests
+    "!src/jsRunner/bundles/**/*.{js,ts}",
   ],
   coverageReporters: ["lcov", "json", "clover"],
 }

packages/server/scripts/test.sh

@@ -10,5 +10,5 @@ else
   # --maxWorkers performs better in development
   export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
   echo "jest --coverage --maxWorkers=2 --forceExit $@"
-  jest --coverage --maxWorkers=2 --forceExit $@
+  jest --maxWorkers=2 --forceExit $@
 fi

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -1,9 +1,11 @@
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, QueryPreview, SourceName } from "@budibase/types"
 import * as setup from "../utilities"
 import { databaseTestProviders } from "../../../../integrations/tests/utils"
 import pg from "pg"
 import mysql from "mysql2/promise"
 import mssql from "mssql"
+import { Expectations } from "src/tests/utilities/api/base"
+import { events } from "@budibase/backend-core"
 
 jest.unmock("pg")
 
@@ -40,7 +42,10 @@ describe.each([
   const config = setup.getConfig()
   let datasource: Datasource
 
-  async function createQuery(query: Partial<Query>): Promise<Query> {
+  async function createQuery(
+    query: Partial<Query>,
+    expectations?: Expectations
+  ): Promise<Query> {
     const defaultQuery: Query = {
       datasourceId: datasource._id!,
       name: "New Query",
@@ -51,17 +56,16 @@ describe.each([
       transformer: "return data",
       readable: true,
     }
-    return await config.api.query.create({ ...defaultQuery, ...query })
+    return await config.api.query.save(
+      { ...defaultQuery, ...query },
+      expectations
+    )
   }
 
   async function rawQuery(sql: string): Promise<any> {
-    // We re-fetch the datasource here because the one returned by
+    switch (datasource.source) {
-    // config.api.datasource.create has the password field blanked out, and we
-    // need the password to connect to the database.
-    const ds = await dsProvider.datasource()
-    switch (ds.source) {
       case SourceName.POSTGRES: {
-        const client = new pg.Client(ds.config!)
+        const client = new pg.Client(datasource.config!)
         await client.connect()
         try {
           const { rows } = await client.query(sql)
@@ -71,7 +75,7 @@ describe.each([
         }
       }
       case SourceName.MYSQL: {
-        const con = await mysql.createConnection(ds.config!)
+        const con = await mysql.createConnection(datasource.config!)
         try {
           const [rows] = await con.query(sql)
           return rows
@@ -80,7 +84,9 @@ describe.each([
         }
       }
       case SourceName.SQL_SERVER: {
-        const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
+        const pool = new mssql.ConnectionPool(
+          datasource.config! as mssql.config
+        )
         const client = await pool.connect()
         try {
           const { recordset } = await client.query(sql)
@@ -94,17 +100,26 @@ describe.each([
 
   beforeAll(async () => {
     await config.init()
-    datasource = await config.api.datasource.create(
-      await dsProvider.datasource()
-    )
   })
 
   beforeEach(async () => {
+    const datasourceRequest = await dsProvider.datasource()
+    datasource = await config.api.datasource.create(datasourceRequest)
+
+    // The Datasource API does not return the password, but we need
+    // it later to connect to the underlying database, so we fill it
+    // back in here.
+    datasource.config!.password = datasourceRequest.config!.password
+
     await rawQuery(createTableSQL[datasource.source])
     await rawQuery(insertSQL)
+
+    jest.clearAllMocks()
   })
 
   afterEach(async () => {
+    const ds = await config.api.datasource.get(datasource._id!)
+    config.api.datasource.delete(ds)
     await rawQuery(dropTableSQL)
   })
 
@@ -113,78 +128,319 @@ describe.each([
     setup.afterAll()
   })
 
-  describe("create", () => {
+  describe("query admin", () => {
-    it("should be able to insert with bindings", async () => {
+    describe("create", () => {
-      const query = await createQuery({
+      it("should be able to create a query", async () => {
-        fields: {
+        const query = await createQuery({
-          sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+          name: "New Query",
-        },
+          fields: {
-        parameters: [
+            sql: "SELECT * FROM test_table",
-          {
-            name: "foo",
-            default: "bar",
           },
-        ],
+        })
-        queryVerb: "create",
+
+        expect(query).toMatchObject({
+          datasourceId: datasource._id!,
+          name: "New Query",
+          parameters: [],
+          fields: {
+            sql: "SELECT * FROM test_table",
+          },
+          schema: {},
+          queryVerb: "read",
+          transformer: "return data",
+          readable: true,
+          createdAt: expect.any(String),
+          updatedAt: expect.any(String),
+        })
+
+        expect(events.query.created).toHaveBeenCalledTimes(1)
+        expect(events.query.updated).not.toHaveBeenCalled()
       })
-
-      const result = await config.api.query.execute(query._id!, {
-        parameters: {
-          foo: "baz",
-        },
-      })
-
-      expect(result.data).toEqual([
-        {
-          created: true,
-        },
-      ])
-
-      const rows = await rawQuery("SELECT * FROM test_table WHERE name = 'baz'")
-      expect(rows).toHaveLength(1)
     })
 
-    it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
+    describe("update", () => {
-      "should coerce %s into a date",
+      it("should be able to update a query", async () => {
-      async datetimeStr => {
-        const date = new Date(datetimeStr)
         const query = await createQuery({
           fields: {
-            sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
+            sql: "SELECT * FROM test_table",
           },
-          parameters: [
+        })
+
+        jest.clearAllMocks()
+
+        const updatedQuery = await config.api.query.save({
+          ...query,
+          name: "Updated Query",
+          fields: {
+            sql: "SELECT * FROM test_table WHERE id = 1",
+          },
+        })
+
+        expect(updatedQuery).toMatchObject({
+          datasourceId: datasource._id!,
+          name: "Updated Query",
+          parameters: [],
+          fields: {
+            sql: "SELECT * FROM test_table WHERE id = 1",
+          },
+          schema: {},
+          queryVerb: "read",
+          transformer: "return data",
+          readable: true,
+        })
+
+        expect(events.query.created).not.toHaveBeenCalled()
+        expect(events.query.updated).toHaveBeenCalledTimes(1)
+      })
+    })
+
+    describe("delete", () => {
+      it("should be able to delete a query", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "SELECT * FROM test_table",
+          },
+        })
+
+        await config.api.query.delete(query)
+        await config.api.query.get(query._id!, { status: 404 })
+
+        const queries = await config.api.query.fetch()
+        expect(queries).not.toContainEqual(query)
+
+        expect(events.query.deleted).toHaveBeenCalledTimes(1)
+        expect(events.query.deleted).toHaveBeenCalledWith(datasource, query)
+      })
+    })
+
+    describe("read", () => {
+      it("should be able to list queries", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "SELECT * FROM test_table",
+          },
+        })
+
+        const queries = await config.api.query.fetch()
+        expect(queries).toContainEqual(query)
+      })
+
+      it("should strip sensitive fields for prod apps", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "SELECT * FROM test_table",
+          },
+        })
+
+        await config.publish()
+        const prodQuery = await config.api.query.getProd(query._id!)
+
+        expect(prodQuery._id).toEqual(query._id)
+        expect(prodQuery.fields).toBeUndefined()
+        expect(prodQuery.parameters).toBeUndefined()
+        expect(prodQuery.schema).toBeDefined()
+      })
+    })
+  })
+
+  describe("preview", () => {
+    it("should be able to preview a query", async () => {
+      const request: QueryPreview = {
+        datasourceId: datasource._id!,
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT * FROM test_table WHERE id = 1`,
+        },
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      }
+      const response = await config.api.query.previewQuery(request)
+      expect(response.schema).toEqual({
+        birthday: {
+          name: "birthday",
+          type: "string",
+        },
+        id: {
+          name: "id",
+          type: "number",
+        },
+        name: {
+          name: "name",
+          type: "string",
+        },
+      })
+      expect(response.rows).toEqual([
+        {
+          birthday: null,
+          id: 1,
+          name: "one",
+        },
+      ])
+      expect(events.query.previewed).toHaveBeenCalledTimes(1)
+
+      const dsWithoutConfig = { ...datasource }
+      delete dsWithoutConfig.config
+      expect(events.query.previewed).toHaveBeenCalledWith(
+        dsWithoutConfig,
+        request
+      )
+    })
+
+    it("should work with static variables", async () => {
+      await config.api.datasource.update({
+        ...datasource,
+        config: {
+          ...datasource.config,
+          staticVariables: {
+            foo: "bar",
+          },
+        },
+      })
+
+      const request: QueryPreview = {
+        datasourceId: datasource._id!,
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT '{{ foo }}' as foo`,
+        },
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      }
+
+      const response = await config.api.query.previewQuery(request)
+
+      expect(response.schema).toEqual({
+        foo: {
+          name: "foo",
+          type: "string",
+        },
+      })
+
+      expect(response.rows).toEqual([
+        {
+          foo: "bar",
+        },
+      ])
+    })
+
+    it("should work with dynamic variables", async () => {
+      const basedOnQuery = await createQuery({
+        fields: {
+          sql: "SELECT name FROM test_table WHERE id = 1",
+        },
+      })
+
+      await config.api.datasource.update({
+        ...datasource,
+        config: {
+          ...datasource.config,
+          dynamicVariables: [
             {
-              name: "birthday",
+              queryId: basedOnQuery._id!,
-              default: "",
+              name: "foo",
+              value: "{{ data[0].name }}",
             },
           ],
-          queryVerb: "create",
+        },
-        })
+      })
 
-        const result = await config.api.query.execute(query._id!, {
+      const preview = await config.api.query.previewQuery({
-          parameters: { birthday: datetimeStr },
+        datasourceId: datasource._id!,
-        })
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT '{{ foo }}' as foo`,
+        },
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      })
 
-        expect(result.data).toEqual([{ created: true }])
+      expect(preview.schema).toEqual({
+        foo: {
+          name: "foo",
+          type: "string",
+        },
+      })
 
-        const rows = await rawQuery(
+      expect(preview.rows).toEqual([
-          `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'`
+        {
-        )
+          foo: "one",
-        expect(rows).toHaveLength(1)
+        },
-      }
+      ])
-    )
+    })
 
-    it.each(["2021,02,05", "202205-1500"])(
+    it("should handle the dynamic base query being deleted", async () => {
-      "should not coerce %s as a date",
+      const basedOnQuery = await createQuery({
-      async notDateStr => {
+        fields: {
+          sql: "SELECT name FROM test_table WHERE id = 1",
+        },
+      })
+
+      await config.api.datasource.update({
+        ...datasource,
+        config: {
+          ...datasource.config,
+          dynamicVariables: [
+            {
+              queryId: basedOnQuery._id!,
+              name: "foo",
+              value: "{{ data[0].name }}",
+            },
+          ],
+        },
+      })
+
+      await config.api.query.delete(basedOnQuery)
+
+      const preview = await config.api.query.previewQuery({
+        datasourceId: datasource._id!,
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT '{{ foo }}' as foo`,
+        },
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      })
+
+      expect(preview.schema).toEqual({
+        foo: {
+          name: "foo",
+          type: "string",
+        },
+      })
+
+      // TODO: is this the correct behaviour? To return an empty string when the
+      // underlying query has been deleted?
+      expect(preview.rows).toEqual([
+        {
+          foo: "",
+        },
+      ])
+    })
+  })
+
+  describe("query verbs", () => {
+    describe("create", () => {
+      it("should be able to insert with bindings", async () => {
         const query = await createQuery({
           fields: {
-            sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
+            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
           },
           parameters: [
             {
-              name: "name",
+              name: "foo",
-              default: "",
+              default: "bar",
             },
           ],
           queryVerb: "create",
@@ -192,210 +448,306 @@ describe.each([
 
         const result = await config.api.query.execute(query._id!, {
           parameters: {
-            name: notDateStr,
+            foo: "baz",
           },
         })
 
-        expect(result.data).toEqual([{ created: true }])
+        expect(result.data).toEqual([
+          {
+            created: true,
+          },
+        ])
 
         const rows = await rawQuery(
-          `SELECT * FROM test_table WHERE name = '${notDateStr}'`
+          "SELECT * FROM test_table WHERE name = 'baz'"
         )
         expect(rows).toHaveLength(1)
-      }
-    )
-  })
-
-  describe("read", () => {
-    it("should execute a query", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "SELECT * FROM test_table ORDER BY id",
-        },
       })
 
-      const result = await config.api.query.execute(query._id!)
+      it("should not allow handlebars as parameters", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+          },
+          parameters: [
+            {
+              name: "foo",
+              default: "bar",
+            },
+          ],
+          queryVerb: "create",
+        })
 
-      expect(result.data).toEqual([
+        await config.api.query.execute(
-        {
+          query._id!,
-          id: 1,
+          {
-          name: "one",
+            parameters: {
-          birthday: null,
+              foo: "{{ 'test' }}",
-        },
+            },
-        {
+          },
-          id: 2,
+          {
-          name: "two",
+            status: 400,
-          birthday: null,
+            body: {
-        },
+              message:
-        {
+                "Parameter 'foo' input contains a handlebars binding - this is not allowed.",
-          id: 3,
+            },
-          name: "three",
+          }
-          birthday: null,
+        )
-        },
+      })
-        {
+
-          id: 4,
+      it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
-          name: "four",
+        "should coerce %s into a date",
-          birthday: null,
+        async datetimeStr => {
-        },
+          const date = new Date(datetimeStr)
-        {
+          const query = await createQuery({
-          id: 5,
+            fields: {
-          name: "five",
+              sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
-          birthday: null,
+            },
-        },
+            parameters: [
-      ])
+              {
+                name: "birthday",
+                default: "",
+              },
+            ],
+            queryVerb: "create",
+          })
+
+          const result = await config.api.query.execute(query._id!, {
+            parameters: { birthday: datetimeStr },
+          })
+
+          expect(result.data).toEqual([{ created: true }])
+
+          const rows = await rawQuery(
+            `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'`
+          )
+          expect(rows).toHaveLength(1)
+        }
+      )
+
+      it.each(["2021,02,05", "202205-1500"])(
+        "should not coerce %s as a date",
+        async notDateStr => {
+          const query = await createQuery({
+            fields: {
+              sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
+            },
+            parameters: [
+              {
+                name: "name",
+                default: "",
+              },
+            ],
+            queryVerb: "create",
+          })
+
+          const result = await config.api.query.execute(query._id!, {
+            parameters: {
+              name: notDateStr,
+            },
+          })
+
+          expect(result.data).toEqual([{ created: true }])
+
+          const rows = await rawQuery(
+            `SELECT * FROM test_table WHERE name = '${notDateStr}'`
+          )
+          expect(rows).toHaveLength(1)
+        }
+      )
     })
 
-    it("should be able to transform a query", async () => {
+    describe("read", () => {
-      const query = await createQuery({
+      it("should execute a query", async () => {
-        fields: {
+        const query = await createQuery({
-          sql: "SELECT * FROM test_table WHERE id = 1",
+          fields: {
-        },
+            sql: "SELECT * FROM test_table ORDER BY id",
-        transformer: `
+          },
+        })
+
+        const result = await config.api.query.execute(query._id!)
+
+        expect(result.data).toEqual([
+          {
+            id: 1,
+            name: "one",
+            birthday: null,
+          },
+          {
+            id: 2,
+            name: "two",
+            birthday: null,
+          },
+          {
+            id: 3,
+            name: "three",
+            birthday: null,
+          },
+          {
+            id: 4,
+            name: "four",
+            birthday: null,
+          },
+          {
+            id: 5,
+            name: "five",
+            birthday: null,
+          },
+        ])
+      })
+
+      it("should be able to transform a query", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "SELECT * FROM test_table WHERE id = 1",
+          },
+          transformer: `
         data[0].id = data[0].id + 1;
         return data;
       `,
-      })
+        })
 
-      const result = await config.api.query.execute(query._id!)
+        const result = await config.api.query.execute(query._id!)
 
-      expect(result.data).toEqual([
+        expect(result.data).toEqual([
-        {
-          id: 2,
-          name: "one",
-          birthday: null,
-        },
-      ])
-    })
-
-    it("should coerce numeric bindings", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "SELECT * FROM test_table WHERE id = {{ id }}",
-        },
-        parameters: [
           {
-            name: "id",
+            id: 2,
-            default: "",
+            name: "one",
+            birthday: null,
           },
-        ],
+        ])
       })
 
-      const result = await config.api.query.execute(query._id!, {
+      it("should coerce numeric bindings", async () => {
-        parameters: {
+        const query = await createQuery({
-          id: "1",
+          fields: {
-        },
+            sql: "SELECT * FROM test_table WHERE id = {{ id }}",
-      })
+          },
+          parameters: [
+            {
+              name: "id",
+              default: "",
+            },
+          ],
+        })
 
-      expect(result.data).toEqual([
+        const result = await config.api.query.execute(query._id!, {
-        {
+          parameters: {
-          id: 1,
+            id: "1",
-          name: "one",
+          },
-          birthday: null,
+        })
-        },
-      ])
-    })
-  })
 
-  describe("update", () => {
+        expect(result.data).toEqual([
-    it("should be able to update rows", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
-        },
-        parameters: [
           {
-            name: "id",
+            id: 1,
-            default: "",
+            name: "one",
+            birthday: null,
           },
+        ])
+      })
+    })
+
+    describe("update", () => {
+      it("should be able to update rows", async () => {
+        const query = await createQuery({
+          fields: {
+            sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+          },
+          parameters: [
+            {
+              name: "id",
+              default: "",
+            },
+            {
+              name: "name",
+              default: "updated",
+            },
+          ],
+          queryVerb: "update",
+        })
+
+        const result = await config.api.query.execute(query._id!, {
+          parameters: {
+            id: "1",
+            name: "foo",
+          },
+        })
+
+        expect(result.data).toEqual([
           {
-            name: "name",
+            updated: true,
-            default: "updated",
           },
-        ],
+        ])
-        queryVerb: "update",
+
+        const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+        expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }])
       })
 
-      const result = await config.api.query.execute(query._id!, {
+      it("should be able to execute an update that updates no rows", async () => {
-        parameters: {
+        const query = await createQuery({
-          id: "1",
+          fields: {
-          name: "foo",
+            sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
-        },
+          },
-      })
+          queryVerb: "update",
+        })
 
-      expect(result.data).toEqual([
+        const result = await config.api.query.execute(query._id!)
-        {
-          updated: true,
-        },
-      ])
 
-      const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+        expect(result.data).toEqual([
-      expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }])
-    })
-
-    it("should be able to execute an update that updates no rows", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
-        },
-        queryVerb: "update",
-      })
-
-      const result = await config.api.query.execute(query._id!)
-
-      expect(result.data).toEqual([
-        {
-          updated: true,
-        },
-      ])
-    })
-
-    it("should be able to execute a delete that deletes no rows", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "DELETE FROM test_table WHERE id = 100",
-        },
-        queryVerb: "delete",
-      })
-
-      const result = await config.api.query.execute(query._id!)
-
-      expect(result.data).toEqual([
-        {
-          deleted: true,
-        },
-      ])
-    })
-  })
-
-  describe("delete", () => {
-    it("should be able to delete rows", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "DELETE FROM test_table WHERE id = {{ id }}",
-        },
-        parameters: [
           {
-            name: "id",
+            updated: true,
-            default: "",
           },
-        ],
+        ])
-        queryVerb: "delete",
       })
 
-      const result = await config.api.query.execute(query._id!, {
+      it("should be able to execute a delete that deletes no rows", async () => {
-        parameters: {
+        const query = await createQuery({
-          id: "1",
+          fields: {
-        },
+            sql: "DELETE FROM test_table WHERE id = 100",
+          },
+          queryVerb: "delete",
+        })
+
+        const result = await config.api.query.execute(query._id!)
+
+        expect(result.data).toEqual([
+          {
+            deleted: true,
+          },
+        ])
       })
+    })
 
-      expect(result.data).toEqual([
+    describe("delete", () => {
-        {
+      it("should be able to delete rows", async () => {
-          deleted: true,
+        const query = await createQuery({
-        },
+          fields: {
-      ])
+            sql: "DELETE FROM test_table WHERE id = {{ id }}",
+          },
+          parameters: [
+            {
+              name: "id",
+              default: "",
+            },
+          ],
+          queryVerb: "delete",
+        })
 
-      const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+        const result = await config.api.query.execute(query._id!, {
-      expect(rows).toHaveLength(0)
+          parameters: {
+            id: "1",
+          },
+        })
+
+        expect(result.data).toEqual([
+          {
+            deleted: true,
+          },
+        ])
+
+        const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+        expect(rows).toHaveLength(0)
+      })
     })
   })
 })

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -2,6 +2,7 @@ import { Datasource, Query } from "@budibase/types"
 import * as setup from "../utilities"
 import { databaseTestProviders } from "../../../../integrations/tests/utils"
 import { MongoClient, type Collection, BSON } from "mongodb"
+import { generator } from "@budibase/backend-core/tests"
 
 jest.unmock("mongodb")
 
@@ -33,30 +34,30 @@ describe("/queries", () => {
     ) {
       combinedQuery.fields.extra.collection = collection
     }
-    return await config.api.query.create(combinedQuery)
+    return await config.api.query.save(combinedQuery)
   }
 
-  async function withClient(
+  async function withClient<T>(
-    callback: (client: MongoClient) => Promise<void>
+    callback: (client: MongoClient) => Promise<T>
-  ): Promise<void> {
+  ): Promise<T> {
     const ds = await databaseTestProviders.mongodb.datasource()
     const client = new MongoClient(ds.config!.connectionString)
     await client.connect()
     try {
-      await callback(client)
+      return await callback(client)
     } finally {
       await client.close()
     }
   }
 
-  async function withCollection(
+  async function withCollection<T>(
-    callback: (collection: Collection) => Promise<void>
+    callback: (collection: Collection) => Promise<T>
-  ): Promise<void> {
+  ): Promise<T> {
-    await withClient(async client => {
+    return await withClient(async client => {
       const db = client.db(
         (await databaseTestProviders.mongodb.datasource()).config!.db
       )
-      await callback(db.collection(collection))
+      return await callback(db.collection(collection))
     })
   }
 
@@ -85,309 +86,453 @@ describe("/queries", () => {
   })
 
   afterEach(async () => {
-    await withCollection(async collection => {
+    await withCollection(collection => collection.drop())
-      await collection.drop()
-    })
   })
 
-  it("should execute a count query", async () => {
+  describe.only("preview", () => {
-    const query = await createQuery({
+    it("should generate a nested schema with an empty array", async () => {
-      fields: {
+      const name = generator.guid()
-        json: {},
+      await withCollection(
-        extra: {
+        async collection => await collection.insertOne({ name, nested: [] })
-          actionType: "count",
+      )
+
+      const preview = await config.api.query.previewQuery({
+        name: "New Query",
+        datasourceId: datasource._id!,
+        fields: {
+          json: {
+            name: { $eq: name },
+          },
+          extra: {
+            collection,
+            actionType: "findOne",
+          },
         },
-      },
+        schema: {},
+        queryVerb: "read",
+        parameters: [],
+        transformer: "return data",
+        readable: true,
+      })
+
+      expect(preview).toEqual({
+        nestedSchemaFields: {},
+        rows: [{ _id: expect.any(String), name, nested: [] }],
+        schema: {
+          _id: {
+            type: "string",
+            name: "_id",
+          },
+          name: {
+            type: "string",
+            name: "name",
+          },
+          nested: {
+            type: "array",
+            name: "nested",
+          },
+        },
+      })
     })
 
-    const result = await config.api.query.execute(query._id!)
+    it("should generate a nested schema based on all of the nested items", async () => {
+      const name = generator.guid()
+      const item = {
+        name,
+        contacts: [
+          {
+            address: "123 Lane",
+          },
+          {
+            address: "456 Drive",
+          },
+          {
+            postcode: "BT1 12N",
+            lat: 54.59,
+            long: -5.92,
+          },
+          {
+            city: "Belfast",
+          },
+          {
+            address: "789 Avenue",
+            phoneNumber: "0800-999-5555",
+          },
+          {
+            name: "Name",
+            isActive: false,
+          },
+        ],
+      }
 
-    expect(result.data).toEqual([{ value: 5 }])
+      await withCollection(collection => collection.insertOne(item))
-  })
 
-  it("should execute a count query with a transformer", async () => {
+      const preview = await config.api.query.previewQuery({
-    const query = await createQuery({
+        name: "New Query",
-      fields: {
+        datasourceId: datasource._id!,
-        json: {},
+        fields: {
-        extra: {
+          json: {
-          actionType: "count",
+            name: { $eq: name },
+          },
+          extra: {
+            collection,
+            actionType: "findOne",
+          },
         },
-      },
+        schema: {},
-      transformer: "return data + 1",
+        queryVerb: "read",
-    })
+        parameters: [],
+        transformer: "return data",
+        readable: true,
+      })
 
-    const result = await config.api.query.execute(query._id!)
+      expect(preview).toEqual({
-
+        nestedSchemaFields: {
-    expect(result.data).toEqual([{ value: 6 }])
+          contacts: {
-  })
+            address: {
-
+              type: "string",
-  it("should execute a find query", async () => {
+              name: "address",
-    const query = await createQuery({
+            },
-      fields: {
+            postcode: {
-        json: {},
+              type: "string",
-        extra: {
+              name: "postcode",
-          actionType: "find",
+            },
+            lat: {
+              type: "number",
+              name: "lat",
+            },
+            long: {
+              type: "number",
+              name: "long",
+            },
+            city: {
+              type: "string",
+              name: "city",
+            },
+            phoneNumber: {
+              type: "string",
+              name: "phoneNumber",
+            },
+            name: {
+              type: "string",
+              name: "name",
+            },
+            isActive: {
+              type: "boolean",
+              name: "isActive",
+            },
+          },
         },
-      },
+        rows: [{ ...item, _id: expect.any(String) }],
-    })
+        schema: {
-
+          _id: { type: "string", name: "_id" },
-    const result = await config.api.query.execute(query._id!)
+          name: { type: "string", name: "name" },
-
+          contacts: { type: "json", name: "contacts", subtype: "array" },
-    expect(result.data).toEqual([
-      { _id: expectValidId, name: "one" },
-      { _id: expectValidId, name: "two" },
-      { _id: expectValidId, name: "three" },
-      { _id: expectValidId, name: "four" },
-      { _id: expectValidId, name: "five" },
-    ])
-  })
-
-  it("should execute a findOne query", async () => {
-    const query = await createQuery({
-      fields: {
-        json: {},
-        extra: {
-          actionType: "findOne",
         },
-      },
-    })
-
-    const result = await config.api.query.execute(query._id!)
-
-    expect(result.data).toEqual([{ _id: expectValidId, name: "one" }])
-  })
-
-  it("should execute a findOneAndUpdate query", async () => {
-    const query = await createQuery({
-      fields: {
-        json: {
-          filter: { name: { $eq: "one" } },
-          update: { $set: { name: "newName" } },
-        },
-        extra: {
-          actionType: "findOneAndUpdate",
-        },
-      },
-    })
-
-    const result = await config.api.query.execute(query._id!)
-
-    expect(result.data).toEqual([
-      {
-        lastErrorObject: { n: 1, updatedExisting: true },
-        ok: 1,
-        value: { _id: expectValidId, name: "one" },
-      },
-    ])
-
-    await withCollection(async collection => {
-      expect(await collection.countDocuments()).toBe(5)
-
-      const doc = await collection.findOne({ name: { $eq: "newName" } })
-      expect(doc).toEqual({
-        _id: expectValidBsonObjectId,
-        name: "newName",
       })
     })
   })
 
-  it("should execute a distinct query", async () => {
+  describe("execute", () => {
-    const query = await createQuery({
+    it("a count query", async () => {
-      fields: {
+      const query = await createQuery({
-        json: "name",
+        fields: {
-        extra: {
+          json: {},
-          actionType: "distinct",
+          extra: {
+            actionType: "count",
+          },
         },
-      },
-    })
-
-    const result = await config.api.query.execute(query._id!)
-    const values = result.data.map(o => o.value).sort()
-    expect(values).toEqual(["five", "four", "one", "three", "two"])
-  })
-
-  it("should execute a create query with parameters", async () => {
-    const query = await createQuery({
-      fields: {
-        json: { foo: "{{ foo }}" },
-        extra: {
-          actionType: "insertOne",
-        },
-      },
-      queryVerb: "create",
-      parameters: [
-        {
-          name: "foo",
-          default: "default",
-        },
-      ],
-    })
-
-    const result = await config.api.query.execute(query._id!, {
-      parameters: { foo: "bar" },
-    })
-
-    expect(result.data).toEqual([
-      {
-        acknowledged: true,
-        insertedId: expectValidId,
-      },
-    ])
-
-    await withCollection(async collection => {
-      const doc = await collection.findOne({ foo: { $eq: "bar" } })
-      expect(doc).toEqual({
-        _id: expectValidBsonObjectId,
-        foo: "bar",
-      })
-    })
-  })
-
-  it("should execute a delete query with parameters", async () => {
-    const query = await createQuery({
-      fields: {
-        json: { name: { $eq: "{{ name }}" } },
-        extra: {
-          actionType: "deleteOne",
-        },
-      },
-      queryVerb: "delete",
-      parameters: [
-        {
-          name: "name",
-          default: "",
-        },
-      ],
-    })
-
-    const result = await config.api.query.execute(query._id!, {
-      parameters: { name: "one" },
-    })
-
-    expect(result.data).toEqual([
-      {
-        acknowledged: true,
-        deletedCount: 1,
-      },
-    ])
-
-    await withCollection(async collection => {
-      const doc = await collection.findOne({ name: { $eq: "one" } })
-      expect(doc).toBeNull()
-    })
-  })
-
-  it("should execute an update query with parameters", async () => {
-    const query = await createQuery({
-      fields: {
-        json: {
-          filter: { name: { $eq: "{{ name }}" } },
-          update: { $set: { name: "{{ newName }}" } },
-        },
-        extra: {
-          actionType: "updateOne",
-        },
-      },
-      queryVerb: "update",
-      parameters: [
-        {
-          name: "name",
-          default: "",
-        },
-        {
-          name: "newName",
-          default: "",
-        },
-      ],
-    })
-
-    const result = await config.api.query.execute(query._id!, {
-      parameters: { name: "one", newName: "newOne" },
-    })
-
-    expect(result.data).toEqual([
-      {
-        acknowledged: true,
-        matchedCount: 1,
-        modifiedCount: 1,
-        upsertedCount: 0,
-        upsertedId: null,
-      },
-    ])
-
-    await withCollection(async collection => {
-      const doc = await collection.findOne({ name: { $eq: "newOne" } })
-      expect(doc).toEqual({
-        _id: expectValidBsonObjectId,
-        name: "newOne",
       })
 
-      const oldDoc = await collection.findOne({ name: { $eq: "one" } })
+      const result = await config.api.query.execute(query._id!)
-      expect(oldDoc).toBeNull()
-    })
-  })
 
-  it("should be able to delete all records", async () => {
+      expect(result.data).toEqual([{ value: 5 }])
-    const query = await createQuery({
+    })
-      fields: {
+
-        json: {},
+    it("a count query with a transformer", async () => {
-        extra: {
+      const query = await createQuery({
-          actionType: "deleteMany",
+        fields: {
+          json: {},
+          extra: {
+            actionType: "count",
+          },
         },
-      },
+        transformer: "return data + 1",
-      queryVerb: "delete",
+      })
+
+      const result = await config.api.query.execute(query._id!)
+
+      expect(result.data).toEqual([{ value: 6 }])
     })
 
-    const result = await config.api.query.execute(query._id!)
+    it("a find query", async () => {
-
+      const query = await createQuery({
-    expect(result.data).toEqual([
+        fields: {
-      {
+          json: {},
-        acknowledged: true,
+          extra: {
-        deletedCount: 5,
+            actionType: "find",
-      },
+          },
-    ])
-
-    await withCollection(async collection => {
-      const docs = await collection.find().toArray()
-      expect(docs).toHaveLength(0)
-    })
-  })
-
-  it("should be able to update all documents", async () => {
-    const query = await createQuery({
-      fields: {
-        json: {
-          filter: {},
-          update: { $set: { name: "newName" } },
         },
-        extra: {
+      })
-          actionType: "updateMany",
+
-        },
+      const result = await config.api.query.execute(query._id!)
-      },
+
-      queryVerb: "update",
+      expect(result.data).toEqual([
+        { _id: expectValidId, name: "one" },
+        { _id: expectValidId, name: "two" },
+        { _id: expectValidId, name: "three" },
+        { _id: expectValidId, name: "four" },
+        { _id: expectValidId, name: "five" },
+      ])
     })
 
-    const result = await config.api.query.execute(query._id!)
+    it("a findOne query", async () => {
+      const query = await createQuery({
+        fields: {
+          json: {},
+          extra: {
+            actionType: "findOne",
+          },
+        },
+      })
 
-    expect(result.data).toEqual([
+      const result = await config.api.query.execute(query._id!)
-      {
-        acknowledged: true,
-        matchedCount: 5,
-        modifiedCount: 5,
-        upsertedCount: 0,
-        upsertedId: null,
-      },
-    ])
 
-    await withCollection(async collection => {
+      expect(result.data).toEqual([{ _id: expectValidId, name: "one" }])
-      const docs = await collection.find().toArray()
+    })
-      expect(docs).toHaveLength(5)
+
-      for (const doc of docs) {
+    it("a findOneAndUpdate query", async () => {
+      const query = await createQuery({
+        fields: {
+          json: {
+            filter: { name: { $eq: "one" } },
+            update: { $set: { name: "newName" } },
+          },
+          extra: {
+            actionType: "findOneAndUpdate",
+          },
+        },
+      })
+
+      const result = await config.api.query.execute(query._id!)
+
+      expect(result.data).toEqual([
+        {
+          lastErrorObject: { n: 1, updatedExisting: true },
+          ok: 1,
+          value: { _id: expectValidId, name: "one" },
+        },
+      ])
+
+      await withCollection(async collection => {
+        expect(await collection.countDocuments()).toBe(5)
+
+        const doc = await collection.findOne({ name: { $eq: "newName" } })
         expect(doc).toEqual({
           _id: expectValidBsonObjectId,
           name: "newName",
         })
-      }
+      })
+    })
+
+    it("a distinct query", async () => {
+      const query = await createQuery({
+        fields: {
+          json: "name",
+          extra: {
+            actionType: "distinct",
+          },
+        },
+      })
+
+      const result = await config.api.query.execute(query._id!)
+      const values = result.data.map(o => o.value).sort()
+      expect(values).toEqual(["five", "four", "one", "three", "two"])
+    })
+
+    it("a create query with parameters", async () => {
+      const query = await createQuery({
+        fields: {
+          json: { foo: "{{ foo }}" },
+          extra: {
+            actionType: "insertOne",
+          },
+        },
+        queryVerb: "create",
+        parameters: [
+          {
+            name: "foo",
+            default: "default",
+          },
+        ],
+      })
+
+      const result = await config.api.query.execute(query._id!, {
+        parameters: { foo: "bar" },
+      })
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          insertedId: expectValidId,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const doc = await collection.findOne({ foo: { $eq: "bar" } })
+        expect(doc).toEqual({
+          _id: expectValidBsonObjectId,
+          foo: "bar",
+        })
+      })
+    })
+
+    it("a delete query with parameters", async () => {
+      const query = await createQuery({
+        fields: {
+          json: { name: { $eq: "{{ name }}" } },
+          extra: {
+            actionType: "deleteOne",
+          },
+        },
+        queryVerb: "delete",
+        parameters: [
+          {
+            name: "name",
+            default: "",
+          },
+        ],
+      })
+
+      const result = await config.api.query.execute(query._id!, {
+        parameters: { name: "one" },
+      })
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          deletedCount: 1,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const doc = await collection.findOne({ name: { $eq: "one" } })
+        expect(doc).toBeNull()
+      })
+    })
+
+    it("an update query with parameters", async () => {
+      const query = await createQuery({
+        fields: {
+          json: {
+            filter: { name: { $eq: "{{ name }}" } },
+            update: { $set: { name: "{{ newName }}" } },
+          },
+          extra: {
+            actionType: "updateOne",
+          },
+        },
+        queryVerb: "update",
+        parameters: [
+          {
+            name: "name",
+            default: "",
+          },
+          {
+            name: "newName",
+            default: "",
+          },
+        ],
+      })
+
+      const result = await config.api.query.execute(query._id!, {
+        parameters: { name: "one", newName: "newOne" },
+      })
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          matchedCount: 1,
+          modifiedCount: 1,
+          upsertedCount: 0,
+          upsertedId: null,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const doc = await collection.findOne({ name: { $eq: "newOne" } })
+        expect(doc).toEqual({
+          _id: expectValidBsonObjectId,
+          name: "newOne",
+        })
+
+        const oldDoc = await collection.findOne({ name: { $eq: "one" } })
+        expect(oldDoc).toBeNull()
+      })
+    })
+
+    it("should be able to delete all records", async () => {
+      const query = await createQuery({
+        fields: {
+          json: {},
+          extra: {
+            actionType: "deleteMany",
+          },
+        },
+        queryVerb: "delete",
+      })
+
+      const result = await config.api.query.execute(query._id!)
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          deletedCount: 5,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const docs = await collection.find().toArray()
+        expect(docs).toHaveLength(0)
+      })
+    })
+
+    it("should be able to update all documents", async () => {
+      const query = await createQuery({
+        fields: {
+          json: {
+            filter: {},
+            update: { $set: { name: "newName" } },
+          },
+          extra: {
+            actionType: "updateMany",
+          },
+        },
+        queryVerb: "update",
+      })
+
+      const result = await config.api.query.execute(query._id!)
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          matchedCount: 5,
+          modifiedCount: 5,
+          upsertedCount: 0,
+          upsertedId: null,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const docs = await collection.find().toArray()
+        expect(docs).toHaveLength(5)
+        for (const doc of docs) {
+          expect(doc).toEqual({
+            _id: expectValidBsonObjectId,
+            name: "newName",
+          })
+        }
+      })
     })
   })
 })

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -55,165 +55,15 @@ describe("/queries", () => {
     await setupTest()
   })
 
-  const createQuery = async (query: Query) => {
+  it("should apply authorization to endpoint", async () => {
-    return request
+    await checkBuilderEndpoint({
-      .post(`/api/queries`)
+      config,
-      .send(query)
+      method: "GET",
-      .set(config.defaultHeaders())
+      url: `/api/datasources`,
-      .expect("Content-Type", /json/)
-      .expect(200)
-  }
-
-  describe("create", () => {
-    it("should create a new query", async () => {
-      const { _id } = await config.createDatasource()
-      const query = basicQuery(_id)
-      jest.clearAllMocks()
-      const res = await createQuery(query)
-
-      expect((res as any).res.statusMessage).toEqual(
-        `Query ${query.name} saved successfully.`
-      )
-      expect(res.body).toEqual({
-        _rev: res.body._rev,
-        _id: res.body._id,
-        ...query,
-        createdAt: new Date().toISOString(),
-        updatedAt: new Date().toISOString(),
-      })
-      expect(events.query.created).toHaveBeenCalledTimes(1)
-      expect(events.query.updated).not.toHaveBeenCalled()
-    })
-  })
-
-  describe("update", () => {
-    it("should update query", async () => {
-      const { _id } = await config.createDatasource()
-      const query = basicQuery(_id)
-      const res = await createQuery(query)
-      jest.clearAllMocks()
-      query._id = res.body._id
-      query._rev = res.body._rev
-      await createQuery(query)
-
-      expect((res as any).res.statusMessage).toEqual(
-        `Query ${query.name} saved successfully.`
-      )
-      expect(res.body).toEqual({
-        _rev: res.body._rev,
-        _id: res.body._id,
-        ...query,
-        createdAt: new Date().toISOString(),
-        updatedAt: new Date().toISOString(),
-      })
-      expect(events.query.created).not.toHaveBeenCalled()
-      expect(events.query.updated).toHaveBeenCalledTimes(1)
-    })
-  })
-
-  describe("fetch", () => {
-    beforeEach(async () => {
-      await setupTest()
-    })
-
-    it("returns all the queries from the server", async () => {
-      const res = await request
-        .get(`/api/queries`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      const queries = res.body
-      expect(queries).toEqual([
-        {
-          _rev: query._rev,
-          _id: query._id,
-          createdAt: new Date().toISOString(),
-          ...basicQuery(datasource._id),
-          updatedAt: new Date().toISOString(),
-          readable: true,
-        },
-      ])
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/datasources`,
-      })
-    })
-  })
-
-  describe("find", () => {
-    it("should find a query in builder", async () => {
-      const query = await config.createQuery()
-      const res = await request
-        .get(`/api/queries/${query._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toEqual(query._id)
-    })
-
-    it("should find a query in cloud", async () => {
-      await config.withEnv({ SELF_HOSTED: "true" }, async () => {
-        const query = await config.createQuery()
-        const res = await request
-          .get(`/api/queries/${query._id}`)
-          .set(await config.defaultHeaders())
-          .expect(200)
-          .expect("Content-Type", /json/)
-        expect(res.body.fields).toBeDefined()
-        expect(res.body.parameters).toBeDefined()
-        expect(res.body.schema).toBeDefined()
-      })
-    })
-
-    it("should remove sensitive info for prod apps", async () => {
-      // Mock isProdAppID to pretend we are using a prod app
-      mockIsProdAppID.mockClear()
-      mockIsProdAppID.mockImplementation(() => true)
-
-      const query = await config.createQuery()
-      const res = await request
-        .get(`/api/queries/${query._id}`)
-        .set(await config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toEqual(query._id)
-      expect(res.body.fields).toBeUndefined()
-      expect(res.body.parameters).toBeUndefined()
-      expect(res.body.schema).toBeDefined()
-
-      // Reset isProdAppID mock
-      expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1)
-      mockIsProdAppID.mockImplementation(() => false)
     })
   })
 
   describe("destroy", () => {
-    beforeEach(async () => {
-      await setupTest()
-    })
-
-    it("deletes a query and returns a success message", async () => {
-      await request
-        .delete(`/api/queries/${query._id}/${query._rev}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-
-      const res = await request
-        .get(`/api/queries`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      expect(res.body).toEqual([])
-      expect(events.query.deleted).toHaveBeenCalledTimes(1)
-      expect(events.query.deleted).toHaveBeenCalledWith(datasource, query)
-    })
-
     it("should apply authorization to endpoint", async () => {
       const query = await config.createQuery()
       await checkBuilderEndpoint({
@@ -225,32 +75,6 @@ describe("/queries", () => {
   })
 
   describe("preview", () => {
-    it("should be able to preview the query", async () => {
-      const queryPreview: QueryPreview = {
-        datasourceId: datasource._id,
-        queryVerb: "read",
-        fields: {},
-        parameters: [],
-        transformer: "return data",
-        name: datasource.name!,
-        schema: {},
-        readable: true,
-      }
-      const responseBody = await config.api.query.previewQuery(queryPreview)
-      // these responses come from the mock
-      expect(responseBody.schema).toEqual({
-        a: { type: "string", name: "a" },
-        b: { type: "number", name: "b" },
-      })
-      expect(responseBody.rows.length).toEqual(1)
-      expect(events.query.previewed).toHaveBeenCalledTimes(1)
-      delete datasource.config
-      expect(events.query.previewed).toHaveBeenCalledWith(
-        datasource,
-        queryPreview
-      )
-    })
-
     it("should apply authorization to endpoint", async () => {
       await checkBuilderEndpoint({
         config,
@@ -258,129 +82,6 @@ describe("/queries", () => {
         url: `/api/queries/preview`,
       })
     })
-
-    it("should not error when trying to generate a nested schema for an empty array", async () => {
-      const queryPreview: QueryPreview = {
-        datasourceId: datasource._id,
-        parameters: [],
-        fields: {},
-        queryVerb: "read",
-        name: datasource.name!,
-        transformer: "return data",
-        schema: {},
-        readable: true,
-      }
-      const rows = [
-        {
-          contacts: [],
-        },
-      ]
-      pg.queryMock.mockImplementation(() => ({
-        rows,
-      }))
-
-      const responseBody = await config.api.query.previewQuery(queryPreview)
-      expect(responseBody).toEqual({
-        nestedSchemaFields: {},
-        rows,
-        schema: {
-          contacts: { type: "array", name: "contacts" },
-        },
-      })
-      expect(responseBody.rows.length).toEqual(1)
-      delete datasource.config
-    })
-
-    it("should generate a nested schema based on all the nested items", async () => {
-      const queryPreview: QueryPreview = {
-        datasourceId: datasource._id,
-        parameters: [],
-        fields: {},
-        queryVerb: "read",
-        name: datasource.name!,
-        transformer: "return data",
-        schema: {},
-        readable: true,
-      }
-      const rows = [
-        {
-          contacts: [
-            {
-              address: "123 Lane",
-            },
-            {
-              address: "456 Drive",
-            },
-            {
-              postcode: "BT1 12N",
-              lat: 54.59,
-              long: -5.92,
-            },
-            {
-              city: "Belfast",
-            },
-            {
-              address: "789 Avenue",
-              phoneNumber: "0800-999-5555",
-            },
-            {
-              name: "Name",
-              isActive: false,
-            },
-          ],
-        },
-      ]
-
-      pg.queryMock.mockImplementation(() => ({
-        rows,
-      }))
-
-      const responseBody = await config.api.query.previewQuery(queryPreview)
-      expect(responseBody).toEqual({
-        nestedSchemaFields: {
-          contacts: {
-            address: {
-              type: "string",
-              name: "address",
-            },
-            postcode: {
-              type: "string",
-              name: "postcode",
-            },
-            lat: {
-              type: "number",
-              name: "lat",
-            },
-            long: {
-              type: "number",
-              name: "long",
-            },
-            city: {
-              type: "string",
-              name: "city",
-            },
-            phoneNumber: {
-              type: "string",
-              name: "phoneNumber",
-            },
-            name: {
-              type: "string",
-              name: "name",
-            },
-            isActive: {
-              type: "boolean",
-              name: "isActive",
-            },
-          },
-        },
-        rows,
-        schema: {
-          contacts: { type: "json", name: "contacts", subtype: "array" },
-        },
-      })
-      expect(responseBody.rows.length).toEqual(1)
-      delete datasource.config
-    })
   })
 
   describe("execute", () => {
@@ -412,21 +113,6 @@ describe("/queries", () => {
         },
       })
     })
-
-    it("shouldn't allow handlebars to be passed as parameters", async () => {
-      const res = await request
-        .post(`/api/queries/${query._id}`)
-        .send({
-          parameters: {
-            a: "{{ 'test' }}",
-          },
-        })
-        .set(config.defaultHeaders())
-        .expect(400)
-      expect(res.body.message).toEqual(
-        "Parameter 'a' input contains a handlebars binding - this is not allowed."
-      )
-    })
   })
 
   describe("variables", () => {
@@ -444,40 +130,6 @@ describe("/queries", () => {
       return await config.api.query.previewQuery(queryPreview)
     }
 
-    it("should work with static variables", async () => {
-      const datasource = await config.restDatasource({
-        staticVariables: {
-          variable: "google",
-          variable2: "1",
-        },
-      })
-      const responseBody = await preview(datasource, {
-        path: "www.{{ variable }}.com",
-        queryString: "test={{ variable2 }}",
-      })
-      // these responses come from the mock
-      expect(responseBody.schema).toEqual({
-        opts: { type: "json", name: "opts" },
-        url: { type: "string", name: "url" },
-        value: { type: "string", name: "value" },
-      })
-      expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1")
-    })
-
-    it("should work with dynamic variables", async () => {
-      const { datasource } = await config.dynamicVariableDatasource()
-      const responseBody = await preview(datasource, {
-        path: "www.google.com",
-        queryString: "test={{ variable3 }}",
-      })
-      expect(responseBody.schema).toEqual({
-        opts: { type: "json", name: "opts" },
-        url: { type: "string", name: "url" },
-        value: { type: "string", name: "value" },
-      })
-      expect(responseBody.rows[0].url).toContain("doctype%20html")
-    })
-
     it("check that it automatically retries on fail with cached dynamics", async () => {
       const { datasource, query: base } =
         await config.dynamicVariableDatasource()
@@ -503,29 +155,6 @@ describe("/queries", () => {
       })
       expect(responseBody.rows[0].fails).toEqual(1)
     })
-
-    it("deletes variables when linked query is deleted", async () => {
-      const { datasource, query: base } =
-        await config.dynamicVariableDatasource()
-      // preview once to cache
-      await preview(datasource, {
-        path: "www.google.com",
-        queryString: "test={{ variable3 }}",
-      })
-      // check its in cache
-      let contents = await checkCacheForDynamicVariable(base._id!, "variable3")
-      expect(contents.rows.length).toEqual(1)
-
-      // delete the query
-      await request
-        .delete(`/api/queries/${base._id}/${base._rev}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-
-      // check variables no longer in cache
-      contents = await checkCacheForDynamicVariable(base._id!, "variable3")
-      expect(contents).toBe(null)
-    })
   })
 
   describe("Current User Request Mapping", () => {

packages/server/src/tests/utilities/api/datasource.ts

@@ -45,4 +45,17 @@ export class DatasourceAPI extends TestAPI {
       expectations,
     })
   }
+
+  delete = async (datasource: Datasource, expectations?: Expectations) => {
+    return await this._delete(
+      `/api/datasources/${datasource._id!}/${datasource._rev!}`,
+      { expectations }
+    )
+  }
+
+  get = async (id: string, expectations?: Expectations) => {
+    return await this._get<Datasource>(`/api/datasources/${id}`, {
+      expectations,
+    })
+  }
 }

packages/server/src/tests/utilities/api/query.ts

@@ -5,28 +5,58 @@ import {
   PreviewQueryRequest,
   PreviewQueryResponse,
 } from "@budibase/types"
-import { TestAPI } from "./base"
+import { Expectations, TestAPI } from "./base"
+import { constants } from "@budibase/backend-core"
 
 export class QueryAPI extends TestAPI {
-  create = async (body: Query): Promise<Query> => {
+  save = async (body: Query, expectations?: Expectations): Promise<Query> => {
-    return await this._post<Query>(`/api/queries`, { body })
+    return await this._post<Query>(`/api/queries`, { body, expectations })
   }
 
   execute = async (
     queryId: string,
-    body?: ExecuteQueryRequest
+    body?: ExecuteQueryRequest,
+    expectations?: Expectations
   ): Promise<ExecuteQueryResponse> => {
     return await this._post<ExecuteQueryResponse>(
       `/api/v2/queries/${queryId}`,
       {
         body,
+        expectations,
       }
     )
   }
 
-  previewQuery = async (queryPreview: PreviewQueryRequest) => {
+  previewQuery = async (
+    queryPreview: PreviewQueryRequest,
+    expectations?: Expectations
+  ) => {
     return await this._post<PreviewQueryResponse>(`/api/queries/preview`, {
       body: queryPreview,
+      expectations,
     })
   }
+
+  delete = async (query: Query, expectations?: Expectations) => {
+    return await this._delete(`/api/queries/${query._id!}/${query._rev!}`, {
+      expectations,
+    })
+  }
+
+  get = async (queryId: string, expectations?: Expectations) => {
+    return await this._get<Query>(`/api/queries/${queryId}`, { expectations })
+  }
+
+  getProd = async (queryId: string, expectations?: Expectations) => {
+    return await this._get<Query>(`/api/queries/${queryId}`, {
+      expectations,
+      headers: {
+        [constants.Header.APP_ID]: this.config.getProdAppId(),
+      },
+    })
+  }
+
+  fetch = async (expectations?: Expectations) => {
+    return await this._get<Query[]>(`/api/queries`, { expectations })
+  }
 }