From 1c135654599584e2f2065c6665d34a4fddf9cafc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 21 Mar 2024 18:16:52 +0000 Subject: [PATCH 01/19] Making really good progress removing the pg mocks. More to do, though. --- packages/server/__mocks__/pg.ts | 25 - packages/server/jest.config.ts | 2 + packages/server/scripts/test.sh | 2 +- .../routes/tests/queries/generic-sql.spec.ts | 818 +++++++++++++----- .../api/routes/tests/queries/mongodb.spec.ts | 709 +++++++++------ .../routes/tests/queries/query.seq.spec.ts | 381 +------- .../src/tests/utilities/api/datasource.ts | 13 + .../server/src/tests/utilities/api/query.ts | 40 +- 8 files changed, 1068 insertions(+), 922 deletions(-) delete mode 100644 packages/server/__mocks__/pg.ts diff --git a/packages/server/__mocks__/pg.ts b/packages/server/__mocks__/pg.ts deleted file mode 100644 index 50a7c7349e..0000000000 --- a/packages/server/__mocks__/pg.ts +++ /dev/null @@ -1,25 +0,0 @@ -const query = jest.fn(() => ({ - rows: [ - { - a: "string", - b: 1, - }, - ], -})) - -class Client { - query = query - end = jest.fn(cb => { - if (cb) cb() - }) - connect = jest.fn() - release = jest.fn() -} - -const on = jest.fn() - -module.exports = { - Client, - queryMock: query, - on, -} diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index 6c6d6a20d3..f3f8ebce02 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -60,6 +60,8 @@ const config: Config.InitialOptions = { "!src/db/views/staticViews.*", "!src/**/*.spec.{js,ts}", "!src/tests/**/*.{js,ts}", + // The use of coverage in the JS runner bundles breaks tests + "!src/jsRunner/bundles/**/*.{js,ts}", ], coverageReporters: ["lcov", "json", "clover"], } diff --git a/packages/server/scripts/test.sh b/packages/server/scripts/test.sh index 3ecf8bb794..53677a68e0 100644 --- a/packages/server/scripts/test.sh +++ b/packages/server/scripts/test.sh @@ -10,5 +10,5 @@ else # --maxWorkers performs better in development export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS" echo "jest --coverage --maxWorkers=2 --forceExit $@" - jest --coverage --maxWorkers=2 --forceExit $@ + jest --maxWorkers=2 --forceExit $@ fi \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 1fc0ecb382..4d97e07ed7 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -1,9 +1,11 @@ -import { Datasource, Query, SourceName } from "@budibase/types" +import { Datasource, Query, QueryPreview, SourceName } from "@budibase/types" import * as setup from "../utilities" import { databaseTestProviders } from "../../../../integrations/tests/utils" import pg from "pg" import mysql from "mysql2/promise" import mssql from "mssql" +import { Expectations } from "src/tests/utilities/api/base" +import { events } from "@budibase/backend-core" jest.unmock("pg") @@ -40,7 +42,10 @@ describe.each([ const config = setup.getConfig() let datasource: Datasource - async function createQuery(query: Partial): Promise { + async function createQuery( + query: Partial, + expectations?: Expectations + ): Promise { const defaultQuery: Query = { datasourceId: datasource._id!, name: "New Query", @@ -51,17 +56,16 @@ describe.each([ transformer: "return data", readable: true, } - return await config.api.query.create({ ...defaultQuery, ...query }) + return await config.api.query.save( + { ...defaultQuery, ...query }, + expectations + ) } async function rawQuery(sql: string): Promise { - // We re-fetch the datasource here because the one returned by - // config.api.datasource.create has the password field blanked out, and we - // need the password to connect to the database. - const ds = await dsProvider.datasource() - switch (ds.source) { + switch (datasource.source) { case SourceName.POSTGRES: { - const client = new pg.Client(ds.config!) + const client = new pg.Client(datasource.config!) await client.connect() try { const { rows } = await client.query(sql) @@ -71,7 +75,7 @@ describe.each([ } } case SourceName.MYSQL: { - const con = await mysql.createConnection(ds.config!) + const con = await mysql.createConnection(datasource.config!) try { const [rows] = await con.query(sql) return rows @@ -80,7 +84,9 @@ describe.each([ } } case SourceName.SQL_SERVER: { - const pool = new mssql.ConnectionPool(ds.config! as mssql.config) + const pool = new mssql.ConnectionPool( + datasource.config! as mssql.config + ) const client = await pool.connect() try { const { recordset } = await client.query(sql) @@ -94,17 +100,26 @@ describe.each([ beforeAll(async () => { await config.init() - datasource = await config.api.datasource.create( - await dsProvider.datasource() - ) }) beforeEach(async () => { + const datasourceRequest = await dsProvider.datasource() + datasource = await config.api.datasource.create(datasourceRequest) + + // The Datasource API does not return the password, but we need + // it later to connect to the underlying database, so we fill it + // back in here. + datasource.config!.password = datasourceRequest.config!.password + await rawQuery(createTableSQL[datasource.source]) await rawQuery(insertSQL) + + jest.clearAllMocks() }) afterEach(async () => { + const ds = await config.api.datasource.get(datasource._id!) + config.api.datasource.delete(ds) await rawQuery(dropTableSQL) }) @@ -113,78 +128,319 @@ describe.each([ setup.afterAll() }) - describe("create", () => { - it("should be able to insert with bindings", async () => { - const query = await createQuery({ - fields: { - sql: "INSERT INTO test_table (name) VALUES ({{ foo }})", - }, - parameters: [ - { - name: "foo", - default: "bar", + describe("query admin", () => { + describe("create", () => { + it("should be able to create a query", async () => { + const query = await createQuery({ + name: "New Query", + fields: { + sql: "SELECT * FROM test_table", }, - ], - queryVerb: "create", + }) + + expect(query).toMatchObject({ + datasourceId: datasource._id!, + name: "New Query", + parameters: [], + fields: { + sql: "SELECT * FROM test_table", + }, + schema: {}, + queryVerb: "read", + transformer: "return data", + readable: true, + createdAt: expect.any(String), + updatedAt: expect.any(String), + }) + + expect(events.query.created).toHaveBeenCalledTimes(1) + expect(events.query.updated).not.toHaveBeenCalled() }) - - const result = await config.api.query.execute(query._id!, { - parameters: { - foo: "baz", - }, - }) - - expect(result.data).toEqual([ - { - created: true, - }, - ]) - - const rows = await rawQuery("SELECT * FROM test_table WHERE name = 'baz'") - expect(rows).toHaveLength(1) }) - it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])( - "should coerce %s into a date", - async datetimeStr => { - const date = new Date(datetimeStr) + describe("update", () => { + it("should be able to update a query", async () => { const query = await createQuery({ fields: { - sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`, + sql: "SELECT * FROM test_table", }, - parameters: [ + }) + + jest.clearAllMocks() + + const updatedQuery = await config.api.query.save({ + ...query, + name: "Updated Query", + fields: { + sql: "SELECT * FROM test_table WHERE id = 1", + }, + }) + + expect(updatedQuery).toMatchObject({ + datasourceId: datasource._id!, + name: "Updated Query", + parameters: [], + fields: { + sql: "SELECT * FROM test_table WHERE id = 1", + }, + schema: {}, + queryVerb: "read", + transformer: "return data", + readable: true, + }) + + expect(events.query.created).not.toHaveBeenCalled() + expect(events.query.updated).toHaveBeenCalledTimes(1) + }) + }) + + describe("delete", () => { + it("should be able to delete a query", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table", + }, + }) + + await config.api.query.delete(query) + await config.api.query.get(query._id!, { status: 404 }) + + const queries = await config.api.query.fetch() + expect(queries).not.toContainEqual(query) + + expect(events.query.deleted).toHaveBeenCalledTimes(1) + expect(events.query.deleted).toHaveBeenCalledWith(datasource, query) + }) + }) + + describe("read", () => { + it("should be able to list queries", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table", + }, + }) + + const queries = await config.api.query.fetch() + expect(queries).toContainEqual(query) + }) + + it("should strip sensitive fields for prod apps", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table", + }, + }) + + await config.publish() + const prodQuery = await config.api.query.getProd(query._id!) + + expect(prodQuery._id).toEqual(query._id) + expect(prodQuery.fields).toBeUndefined() + expect(prodQuery.parameters).toBeUndefined() + expect(prodQuery.schema).toBeDefined() + }) + }) + }) + + describe("preview", () => { + it("should be able to preview a query", async () => { + const request: QueryPreview = { + datasourceId: datasource._id!, + queryVerb: "read", + fields: { + sql: `SELECT * FROM test_table WHERE id = 1`, + }, + parameters: [], + transformer: "return data", + name: datasource.name!, + schema: {}, + readable: true, + } + const response = await config.api.query.previewQuery(request) + expect(response.schema).toEqual({ + birthday: { + name: "birthday", + type: "string", + }, + id: { + name: "id", + type: "number", + }, + name: { + name: "name", + type: "string", + }, + }) + expect(response.rows).toEqual([ + { + birthday: null, + id: 1, + name: "one", + }, + ]) + expect(events.query.previewed).toHaveBeenCalledTimes(1) + + const dsWithoutConfig = { ...datasource } + delete dsWithoutConfig.config + expect(events.query.previewed).toHaveBeenCalledWith( + dsWithoutConfig, + request + ) + }) + + it("should work with static variables", async () => { + await config.api.datasource.update({ + ...datasource, + config: { + ...datasource.config, + staticVariables: { + foo: "bar", + }, + }, + }) + + const request: QueryPreview = { + datasourceId: datasource._id!, + queryVerb: "read", + fields: { + sql: `SELECT '{{ foo }}' as foo`, + }, + parameters: [], + transformer: "return data", + name: datasource.name!, + schema: {}, + readable: true, + } + + const response = await config.api.query.previewQuery(request) + + expect(response.schema).toEqual({ + foo: { + name: "foo", + type: "string", + }, + }) + + expect(response.rows).toEqual([ + { + foo: "bar", + }, + ]) + }) + + it("should work with dynamic variables", async () => { + const basedOnQuery = await createQuery({ + fields: { + sql: "SELECT name FROM test_table WHERE id = 1", + }, + }) + + await config.api.datasource.update({ + ...datasource, + config: { + ...datasource.config, + dynamicVariables: [ { - name: "birthday", - default: "", + queryId: basedOnQuery._id!, + name: "foo", + value: "{{ data[0].name }}", }, ], - queryVerb: "create", - }) + }, + }) - const result = await config.api.query.execute(query._id!, { - parameters: { birthday: datetimeStr }, - }) + const preview = await config.api.query.previewQuery({ + datasourceId: datasource._id!, + queryVerb: "read", + fields: { + sql: `SELECT '{{ foo }}' as foo`, + }, + parameters: [], + transformer: "return data", + name: datasource.name!, + schema: {}, + readable: true, + }) - expect(result.data).toEqual([{ created: true }]) + expect(preview.schema).toEqual({ + foo: { + name: "foo", + type: "string", + }, + }) - const rows = await rawQuery( - `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'` - ) - expect(rows).toHaveLength(1) - } - ) + expect(preview.rows).toEqual([ + { + foo: "one", + }, + ]) + }) - it.each(["2021,02,05", "202205-1500"])( - "should not coerce %s as a date", - async notDateStr => { + it("should handle the dynamic base query being deleted", async () => { + const basedOnQuery = await createQuery({ + fields: { + sql: "SELECT name FROM test_table WHERE id = 1", + }, + }) + + await config.api.datasource.update({ + ...datasource, + config: { + ...datasource.config, + dynamicVariables: [ + { + queryId: basedOnQuery._id!, + name: "foo", + value: "{{ data[0].name }}", + }, + ], + }, + }) + + await config.api.query.delete(basedOnQuery) + + const preview = await config.api.query.previewQuery({ + datasourceId: datasource._id!, + queryVerb: "read", + fields: { + sql: `SELECT '{{ foo }}' as foo`, + }, + parameters: [], + transformer: "return data", + name: datasource.name!, + schema: {}, + readable: true, + }) + + expect(preview.schema).toEqual({ + foo: { + name: "foo", + type: "string", + }, + }) + + // TODO: is this the correct behaviour? To return an empty string when the + // underlying query has been deleted? + expect(preview.rows).toEqual([ + { + foo: "", + }, + ]) + }) + }) + + describe("query verbs", () => { + describe("create", () => { + it("should be able to insert with bindings", async () => { const query = await createQuery({ fields: { - sql: "INSERT INTO test_table (name) VALUES ({{ name }})", + sql: "INSERT INTO test_table (name) VALUES ({{ foo }})", }, parameters: [ { - name: "name", - default: "", + name: "foo", + default: "bar", }, ], queryVerb: "create", @@ -192,210 +448,306 @@ describe.each([ const result = await config.api.query.execute(query._id!, { parameters: { - name: notDateStr, + foo: "baz", }, }) - expect(result.data).toEqual([{ created: true }]) + expect(result.data).toEqual([ + { + created: true, + }, + ]) const rows = await rawQuery( - `SELECT * FROM test_table WHERE name = '${notDateStr}'` + "SELECT * FROM test_table WHERE name = 'baz'" ) expect(rows).toHaveLength(1) - } - ) - }) - - describe("read", () => { - it("should execute a query", async () => { - const query = await createQuery({ - fields: { - sql: "SELECT * FROM test_table ORDER BY id", - }, }) - const result = await config.api.query.execute(query._id!) + it("should not allow handlebars as parameters", async () => { + const query = await createQuery({ + fields: { + sql: "INSERT INTO test_table (name) VALUES ({{ foo }})", + }, + parameters: [ + { + name: "foo", + default: "bar", + }, + ], + queryVerb: "create", + }) - expect(result.data).toEqual([ - { - id: 1, - name: "one", - birthday: null, - }, - { - id: 2, - name: "two", - birthday: null, - }, - { - id: 3, - name: "three", - birthday: null, - }, - { - id: 4, - name: "four", - birthday: null, - }, - { - id: 5, - name: "five", - birthday: null, - }, - ]) + await config.api.query.execute( + query._id!, + { + parameters: { + foo: "{{ 'test' }}", + }, + }, + { + status: 400, + body: { + message: + "Parameter 'foo' input contains a handlebars binding - this is not allowed.", + }, + } + ) + }) + + it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])( + "should coerce %s into a date", + async datetimeStr => { + const date = new Date(datetimeStr) + const query = await createQuery({ + fields: { + sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`, + }, + parameters: [ + { + name: "birthday", + default: "", + }, + ], + queryVerb: "create", + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { birthday: datetimeStr }, + }) + + expect(result.data).toEqual([{ created: true }]) + + const rows = await rawQuery( + `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'` + ) + expect(rows).toHaveLength(1) + } + ) + + it.each(["2021,02,05", "202205-1500"])( + "should not coerce %s as a date", + async notDateStr => { + const query = await createQuery({ + fields: { + sql: "INSERT INTO test_table (name) VALUES ({{ name }})", + }, + parameters: [ + { + name: "name", + default: "", + }, + ], + queryVerb: "create", + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { + name: notDateStr, + }, + }) + + expect(result.data).toEqual([{ created: true }]) + + const rows = await rawQuery( + `SELECT * FROM test_table WHERE name = '${notDateStr}'` + ) + expect(rows).toHaveLength(1) + } + ) }) - it("should be able to transform a query", async () => { - const query = await createQuery({ - fields: { - sql: "SELECT * FROM test_table WHERE id = 1", - }, - transformer: ` + describe("read", () => { + it("should execute a query", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table ORDER BY id", + }, + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + id: 1, + name: "one", + birthday: null, + }, + { + id: 2, + name: "two", + birthday: null, + }, + { + id: 3, + name: "three", + birthday: null, + }, + { + id: 4, + name: "four", + birthday: null, + }, + { + id: 5, + name: "five", + birthday: null, + }, + ]) + }) + + it("should be able to transform a query", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table WHERE id = 1", + }, + transformer: ` data[0].id = data[0].id + 1; return data; `, - }) + }) - const result = await config.api.query.execute(query._id!) + const result = await config.api.query.execute(query._id!) - expect(result.data).toEqual([ - { - id: 2, - name: "one", - birthday: null, - }, - ]) - }) - - it("should coerce numeric bindings", async () => { - const query = await createQuery({ - fields: { - sql: "SELECT * FROM test_table WHERE id = {{ id }}", - }, - parameters: [ + expect(result.data).toEqual([ { - name: "id", - default: "", + id: 2, + name: "one", + birthday: null, }, - ], + ]) }) - const result = await config.api.query.execute(query._id!, { - parameters: { - id: "1", - }, - }) + it("should coerce numeric bindings", async () => { + const query = await createQuery({ + fields: { + sql: "SELECT * FROM test_table WHERE id = {{ id }}", + }, + parameters: [ + { + name: "id", + default: "", + }, + ], + }) - expect(result.data).toEqual([ - { - id: 1, - name: "one", - birthday: null, - }, - ]) - }) - }) + const result = await config.api.query.execute(query._id!, { + parameters: { + id: "1", + }, + }) - describe("update", () => { - it("should be able to update rows", async () => { - const query = await createQuery({ - fields: { - sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}", - }, - parameters: [ + expect(result.data).toEqual([ { - name: "id", - default: "", + id: 1, + name: "one", + birthday: null, }, + ]) + }) + }) + + describe("update", () => { + it("should be able to update rows", async () => { + const query = await createQuery({ + fields: { + sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}", + }, + parameters: [ + { + name: "id", + default: "", + }, + { + name: "name", + default: "updated", + }, + ], + queryVerb: "update", + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { + id: "1", + name: "foo", + }, + }) + + expect(result.data).toEqual([ { - name: "name", - default: "updated", + updated: true, }, - ], - queryVerb: "update", + ]) + + const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") + expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }]) }) - const result = await config.api.query.execute(query._id!, { - parameters: { - id: "1", - name: "foo", - }, - }) + it("should be able to execute an update that updates no rows", async () => { + const query = await createQuery({ + fields: { + sql: "UPDATE test_table SET name = 'updated' WHERE id = 100", + }, + queryVerb: "update", + }) - expect(result.data).toEqual([ - { - updated: true, - }, - ]) + const result = await config.api.query.execute(query._id!) - const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") - expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }]) - }) - - it("should be able to execute an update that updates no rows", async () => { - const query = await createQuery({ - fields: { - sql: "UPDATE test_table SET name = 'updated' WHERE id = 100", - }, - queryVerb: "update", - }) - - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([ - { - updated: true, - }, - ]) - }) - - it("should be able to execute a delete that deletes no rows", async () => { - const query = await createQuery({ - fields: { - sql: "DELETE FROM test_table WHERE id = 100", - }, - queryVerb: "delete", - }) - - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([ - { - deleted: true, - }, - ]) - }) - }) - - describe("delete", () => { - it("should be able to delete rows", async () => { - const query = await createQuery({ - fields: { - sql: "DELETE FROM test_table WHERE id = {{ id }}", - }, - parameters: [ + expect(result.data).toEqual([ { - name: "id", - default: "", + updated: true, }, - ], - queryVerb: "delete", + ]) }) - const result = await config.api.query.execute(query._id!, { - parameters: { - id: "1", - }, + it("should be able to execute a delete that deletes no rows", async () => { + const query = await createQuery({ + fields: { + sql: "DELETE FROM test_table WHERE id = 100", + }, + queryVerb: "delete", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + deleted: true, + }, + ]) }) + }) - expect(result.data).toEqual([ - { - deleted: true, - }, - ]) + describe("delete", () => { + it("should be able to delete rows", async () => { + const query = await createQuery({ + fields: { + sql: "DELETE FROM test_table WHERE id = {{ id }}", + }, + parameters: [ + { + name: "id", + default: "", + }, + ], + queryVerb: "delete", + }) - const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") - expect(rows).toHaveLength(0) + const result = await config.api.query.execute(query._id!, { + parameters: { + id: "1", + }, + }) + + expect(result.data).toEqual([ + { + deleted: true, + }, + ]) + + const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") + expect(rows).toHaveLength(0) + }) }) }) }) diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index b61f905bef..6ab3dec80d 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -2,6 +2,7 @@ import { Datasource, Query } from "@budibase/types" import * as setup from "../utilities" import { databaseTestProviders } from "../../../../integrations/tests/utils" import { MongoClient, type Collection, BSON } from "mongodb" +import { generator } from "@budibase/backend-core/tests" jest.unmock("mongodb") @@ -33,30 +34,30 @@ describe("/queries", () => { ) { combinedQuery.fields.extra.collection = collection } - return await config.api.query.create(combinedQuery) + return await config.api.query.save(combinedQuery) } - async function withClient( - callback: (client: MongoClient) => Promise - ): Promise { + async function withClient( + callback: (client: MongoClient) => Promise + ): Promise { const ds = await databaseTestProviders.mongodb.datasource() const client = new MongoClient(ds.config!.connectionString) await client.connect() try { - await callback(client) + return await callback(client) } finally { await client.close() } } - async function withCollection( - callback: (collection: Collection) => Promise - ): Promise { - await withClient(async client => { + async function withCollection( + callback: (collection: Collection) => Promise + ): Promise { + return await withClient(async client => { const db = client.db( (await databaseTestProviders.mongodb.datasource()).config!.db ) - await callback(db.collection(collection)) + return await callback(db.collection(collection)) }) } @@ -85,309 +86,453 @@ describe("/queries", () => { }) afterEach(async () => { - await withCollection(async collection => { - await collection.drop() - }) + await withCollection(collection => collection.drop()) }) - it("should execute a count query", async () => { - const query = await createQuery({ - fields: { - json: {}, - extra: { - actionType: "count", + describe.only("preview", () => { + it("should generate a nested schema with an empty array", async () => { + const name = generator.guid() + await withCollection( + async collection => await collection.insertOne({ name, nested: [] }) + ) + + const preview = await config.api.query.previewQuery({ + name: "New Query", + datasourceId: datasource._id!, + fields: { + json: { + name: { $eq: name }, + }, + extra: { + collection, + actionType: "findOne", + }, }, - }, + schema: {}, + queryVerb: "read", + parameters: [], + transformer: "return data", + readable: true, + }) + + expect(preview).toEqual({ + nestedSchemaFields: {}, + rows: [{ _id: expect.any(String), name, nested: [] }], + schema: { + _id: { + type: "string", + name: "_id", + }, + name: { + type: "string", + name: "name", + }, + nested: { + type: "array", + name: "nested", + }, + }, + }) }) - const result = await config.api.query.execute(query._id!) + it("should generate a nested schema based on all of the nested items", async () => { + const name = generator.guid() + const item = { + name, + contacts: [ + { + address: "123 Lane", + }, + { + address: "456 Drive", + }, + { + postcode: "BT1 12N", + lat: 54.59, + long: -5.92, + }, + { + city: "Belfast", + }, + { + address: "789 Avenue", + phoneNumber: "0800-999-5555", + }, + { + name: "Name", + isActive: false, + }, + ], + } - expect(result.data).toEqual([{ value: 5 }]) - }) + await withCollection(collection => collection.insertOne(item)) - it("should execute a count query with a transformer", async () => { - const query = await createQuery({ - fields: { - json: {}, - extra: { - actionType: "count", + const preview = await config.api.query.previewQuery({ + name: "New Query", + datasourceId: datasource._id!, + fields: { + json: { + name: { $eq: name }, + }, + extra: { + collection, + actionType: "findOne", + }, }, - }, - transformer: "return data + 1", - }) + schema: {}, + queryVerb: "read", + parameters: [], + transformer: "return data", + readable: true, + }) - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([{ value: 6 }]) - }) - - it("should execute a find query", async () => { - const query = await createQuery({ - fields: { - json: {}, - extra: { - actionType: "find", + expect(preview).toEqual({ + nestedSchemaFields: { + contacts: { + address: { + type: "string", + name: "address", + }, + postcode: { + type: "string", + name: "postcode", + }, + lat: { + type: "number", + name: "lat", + }, + long: { + type: "number", + name: "long", + }, + city: { + type: "string", + name: "city", + }, + phoneNumber: { + type: "string", + name: "phoneNumber", + }, + name: { + type: "string", + name: "name", + }, + isActive: { + type: "boolean", + name: "isActive", + }, + }, }, - }, - }) - - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([ - { _id: expectValidId, name: "one" }, - { _id: expectValidId, name: "two" }, - { _id: expectValidId, name: "three" }, - { _id: expectValidId, name: "four" }, - { _id: expectValidId, name: "five" }, - ]) - }) - - it("should execute a findOne query", async () => { - const query = await createQuery({ - fields: { - json: {}, - extra: { - actionType: "findOne", + rows: [{ ...item, _id: expect.any(String) }], + schema: { + _id: { type: "string", name: "_id" }, + name: { type: "string", name: "name" }, + contacts: { type: "json", name: "contacts", subtype: "array" }, }, - }, - }) - - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([{ _id: expectValidId, name: "one" }]) - }) - - it("should execute a findOneAndUpdate query", async () => { - const query = await createQuery({ - fields: { - json: { - filter: { name: { $eq: "one" } }, - update: { $set: { name: "newName" } }, - }, - extra: { - actionType: "findOneAndUpdate", - }, - }, - }) - - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([ - { - lastErrorObject: { n: 1, updatedExisting: true }, - ok: 1, - value: { _id: expectValidId, name: "one" }, - }, - ]) - - await withCollection(async collection => { - expect(await collection.countDocuments()).toBe(5) - - const doc = await collection.findOne({ name: { $eq: "newName" } }) - expect(doc).toEqual({ - _id: expectValidBsonObjectId, - name: "newName", }) }) }) - it("should execute a distinct query", async () => { - const query = await createQuery({ - fields: { - json: "name", - extra: { - actionType: "distinct", + describe("execute", () => { + it("a count query", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "count", + }, }, - }, - }) - - const result = await config.api.query.execute(query._id!) - const values = result.data.map(o => o.value).sort() - expect(values).toEqual(["five", "four", "one", "three", "two"]) - }) - - it("should execute a create query with parameters", async () => { - const query = await createQuery({ - fields: { - json: { foo: "{{ foo }}" }, - extra: { - actionType: "insertOne", - }, - }, - queryVerb: "create", - parameters: [ - { - name: "foo", - default: "default", - }, - ], - }) - - const result = await config.api.query.execute(query._id!, { - parameters: { foo: "bar" }, - }) - - expect(result.data).toEqual([ - { - acknowledged: true, - insertedId: expectValidId, - }, - ]) - - await withCollection(async collection => { - const doc = await collection.findOne({ foo: { $eq: "bar" } }) - expect(doc).toEqual({ - _id: expectValidBsonObjectId, - foo: "bar", - }) - }) - }) - - it("should execute a delete query with parameters", async () => { - const query = await createQuery({ - fields: { - json: { name: { $eq: "{{ name }}" } }, - extra: { - actionType: "deleteOne", - }, - }, - queryVerb: "delete", - parameters: [ - { - name: "name", - default: "", - }, - ], - }) - - const result = await config.api.query.execute(query._id!, { - parameters: { name: "one" }, - }) - - expect(result.data).toEqual([ - { - acknowledged: true, - deletedCount: 1, - }, - ]) - - await withCollection(async collection => { - const doc = await collection.findOne({ name: { $eq: "one" } }) - expect(doc).toBeNull() - }) - }) - - it("should execute an update query with parameters", async () => { - const query = await createQuery({ - fields: { - json: { - filter: { name: { $eq: "{{ name }}" } }, - update: { $set: { name: "{{ newName }}" } }, - }, - extra: { - actionType: "updateOne", - }, - }, - queryVerb: "update", - parameters: [ - { - name: "name", - default: "", - }, - { - name: "newName", - default: "", - }, - ], - }) - - const result = await config.api.query.execute(query._id!, { - parameters: { name: "one", newName: "newOne" }, - }) - - expect(result.data).toEqual([ - { - acknowledged: true, - matchedCount: 1, - modifiedCount: 1, - upsertedCount: 0, - upsertedId: null, - }, - ]) - - await withCollection(async collection => { - const doc = await collection.findOne({ name: { $eq: "newOne" } }) - expect(doc).toEqual({ - _id: expectValidBsonObjectId, - name: "newOne", }) - const oldDoc = await collection.findOne({ name: { $eq: "one" } }) - expect(oldDoc).toBeNull() - }) - }) + const result = await config.api.query.execute(query._id!) - it("should be able to delete all records", async () => { - const query = await createQuery({ - fields: { - json: {}, - extra: { - actionType: "deleteMany", + expect(result.data).toEqual([{ value: 5 }]) + }) + + it("a count query with a transformer", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "count", + }, }, - }, - queryVerb: "delete", + transformer: "return data + 1", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([{ value: 6 }]) }) - const result = await config.api.query.execute(query._id!) - - expect(result.data).toEqual([ - { - acknowledged: true, - deletedCount: 5, - }, - ]) - - await withCollection(async collection => { - const docs = await collection.find().toArray() - expect(docs).toHaveLength(0) - }) - }) - - it("should be able to update all documents", async () => { - const query = await createQuery({ - fields: { - json: { - filter: {}, - update: { $set: { name: "newName" } }, + it("a find query", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "find", + }, }, - extra: { - actionType: "updateMany", - }, - }, - queryVerb: "update", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { _id: expectValidId, name: "one" }, + { _id: expectValidId, name: "two" }, + { _id: expectValidId, name: "three" }, + { _id: expectValidId, name: "four" }, + { _id: expectValidId, name: "five" }, + ]) }) - const result = await config.api.query.execute(query._id!) + it("a findOne query", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "findOne", + }, + }, + }) - expect(result.data).toEqual([ - { - acknowledged: true, - matchedCount: 5, - modifiedCount: 5, - upsertedCount: 0, - upsertedId: null, - }, - ]) + const result = await config.api.query.execute(query._id!) - await withCollection(async collection => { - const docs = await collection.find().toArray() - expect(docs).toHaveLength(5) - for (const doc of docs) { + expect(result.data).toEqual([{ _id: expectValidId, name: "one" }]) + }) + + it("a findOneAndUpdate query", async () => { + const query = await createQuery({ + fields: { + json: { + filter: { name: { $eq: "one" } }, + update: { $set: { name: "newName" } }, + }, + extra: { + actionType: "findOneAndUpdate", + }, + }, + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + lastErrorObject: { n: 1, updatedExisting: true }, + ok: 1, + value: { _id: expectValidId, name: "one" }, + }, + ]) + + await withCollection(async collection => { + expect(await collection.countDocuments()).toBe(5) + + const doc = await collection.findOne({ name: { $eq: "newName" } }) expect(doc).toEqual({ _id: expectValidBsonObjectId, name: "newName", }) - } + }) + }) + + it("a distinct query", async () => { + const query = await createQuery({ + fields: { + json: "name", + extra: { + actionType: "distinct", + }, + }, + }) + + const result = await config.api.query.execute(query._id!) + const values = result.data.map(o => o.value).sort() + expect(values).toEqual(["five", "four", "one", "three", "two"]) + }) + + it("a create query with parameters", async () => { + const query = await createQuery({ + fields: { + json: { foo: "{{ foo }}" }, + extra: { + actionType: "insertOne", + }, + }, + queryVerb: "create", + parameters: [ + { + name: "foo", + default: "default", + }, + ], + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { foo: "bar" }, + }) + + expect(result.data).toEqual([ + { + acknowledged: true, + insertedId: expectValidId, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ foo: { $eq: "bar" } }) + expect(doc).toEqual({ + _id: expectValidBsonObjectId, + foo: "bar", + }) + }) + }) + + it("a delete query with parameters", async () => { + const query = await createQuery({ + fields: { + json: { name: { $eq: "{{ name }}" } }, + extra: { + actionType: "deleteOne", + }, + }, + queryVerb: "delete", + parameters: [ + { + name: "name", + default: "", + }, + ], + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { name: "one" }, + }) + + expect(result.data).toEqual([ + { + acknowledged: true, + deletedCount: 1, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ name: { $eq: "one" } }) + expect(doc).toBeNull() + }) + }) + + it("an update query with parameters", async () => { + const query = await createQuery({ + fields: { + json: { + filter: { name: { $eq: "{{ name }}" } }, + update: { $set: { name: "{{ newName }}" } }, + }, + extra: { + actionType: "updateOne", + }, + }, + queryVerb: "update", + parameters: [ + { + name: "name", + default: "", + }, + { + name: "newName", + default: "", + }, + ], + }) + + const result = await config.api.query.execute(query._id!, { + parameters: { name: "one", newName: "newOne" }, + }) + + expect(result.data).toEqual([ + { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + upsertedCount: 0, + upsertedId: null, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ name: { $eq: "newOne" } }) + expect(doc).toEqual({ + _id: expectValidBsonObjectId, + name: "newOne", + }) + + const oldDoc = await collection.findOne({ name: { $eq: "one" } }) + expect(oldDoc).toBeNull() + }) + }) + + it("should be able to delete all records", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "deleteMany", + }, + }, + queryVerb: "delete", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + acknowledged: true, + deletedCount: 5, + }, + ]) + + await withCollection(async collection => { + const docs = await collection.find().toArray() + expect(docs).toHaveLength(0) + }) + }) + + it("should be able to update all documents", async () => { + const query = await createQuery({ + fields: { + json: { + filter: {}, + update: { $set: { name: "newName" } }, + }, + extra: { + actionType: "updateMany", + }, + }, + queryVerb: "update", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + acknowledged: true, + matchedCount: 5, + modifiedCount: 5, + upsertedCount: 0, + upsertedId: null, + }, + ]) + + await withCollection(async collection => { + const docs = await collection.find().toArray() + expect(docs).toHaveLength(5) + for (const doc of docs) { + expect(doc).toEqual({ + _id: expectValidBsonObjectId, + name: "newName", + }) + } + }) }) }) }) diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts index 10b90eafb1..d8b06c001d 100644 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts @@ -55,165 +55,15 @@ describe("/queries", () => { await setupTest() }) - const createQuery = async (query: Query) => { - return request - .post(`/api/queries`) - .send(query) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - } - - describe("create", () => { - it("should create a new query", async () => { - const { _id } = await config.createDatasource() - const query = basicQuery(_id) - jest.clearAllMocks() - const res = await createQuery(query) - - expect((res as any).res.statusMessage).toEqual( - `Query ${query.name} saved successfully.` - ) - expect(res.body).toEqual({ - _rev: res.body._rev, - _id: res.body._id, - ...query, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - expect(events.query.created).toHaveBeenCalledTimes(1) - expect(events.query.updated).not.toHaveBeenCalled() - }) - }) - - describe("update", () => { - it("should update query", async () => { - const { _id } = await config.createDatasource() - const query = basicQuery(_id) - const res = await createQuery(query) - jest.clearAllMocks() - query._id = res.body._id - query._rev = res.body._rev - await createQuery(query) - - expect((res as any).res.statusMessage).toEqual( - `Query ${query.name} saved successfully.` - ) - expect(res.body).toEqual({ - _rev: res.body._rev, - _id: res.body._id, - ...query, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - expect(events.query.created).not.toHaveBeenCalled() - expect(events.query.updated).toHaveBeenCalledTimes(1) - }) - }) - - describe("fetch", () => { - beforeEach(async () => { - await setupTest() - }) - - it("returns all the queries from the server", async () => { - const res = await request - .get(`/api/queries`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - const queries = res.body - expect(queries).toEqual([ - { - _rev: query._rev, - _id: query._id, - createdAt: new Date().toISOString(), - ...basicQuery(datasource._id), - updatedAt: new Date().toISOString(), - readable: true, - }, - ]) - }) - - it("should apply authorization to endpoint", async () => { - await checkBuilderEndpoint({ - config, - method: "GET", - url: `/api/datasources`, - }) - }) - }) - - describe("find", () => { - it("should find a query in builder", async () => { - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body._id).toEqual(query._id) - }) - - it("should find a query in cloud", async () => { - await config.withEnv({ SELF_HOSTED: "true" }, async () => { - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(await config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.fields).toBeDefined() - expect(res.body.parameters).toBeDefined() - expect(res.body.schema).toBeDefined() - }) - }) - - it("should remove sensitive info for prod apps", async () => { - // Mock isProdAppID to pretend we are using a prod app - mockIsProdAppID.mockClear() - mockIsProdAppID.mockImplementation(() => true) - - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(await config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body._id).toEqual(query._id) - expect(res.body.fields).toBeUndefined() - expect(res.body.parameters).toBeUndefined() - expect(res.body.schema).toBeDefined() - - // Reset isProdAppID mock - expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1) - mockIsProdAppID.mockImplementation(() => false) + it("should apply authorization to endpoint", async () => { + await checkBuilderEndpoint({ + config, + method: "GET", + url: `/api/datasources`, }) }) describe("destroy", () => { - beforeEach(async () => { - await setupTest() - }) - - it("deletes a query and returns a success message", async () => { - await request - .delete(`/api/queries/${query._id}/${query._rev}`) - .set(config.defaultHeaders()) - .expect(200) - - const res = await request - .get(`/api/queries`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - expect(res.body).toEqual([]) - expect(events.query.deleted).toHaveBeenCalledTimes(1) - expect(events.query.deleted).toHaveBeenCalledWith(datasource, query) - }) - it("should apply authorization to endpoint", async () => { const query = await config.createQuery() await checkBuilderEndpoint({ @@ -225,32 +75,6 @@ describe("/queries", () => { }) describe("preview", () => { - it("should be able to preview the query", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - queryVerb: "read", - fields: {}, - parameters: [], - transformer: "return data", - name: datasource.name!, - schema: {}, - readable: true, - } - const responseBody = await config.api.query.previewQuery(queryPreview) - // these responses come from the mock - expect(responseBody.schema).toEqual({ - a: { type: "string", name: "a" }, - b: { type: "number", name: "b" }, - }) - expect(responseBody.rows.length).toEqual(1) - expect(events.query.previewed).toHaveBeenCalledTimes(1) - delete datasource.config - expect(events.query.previewed).toHaveBeenCalledWith( - datasource, - queryPreview - ) - }) - it("should apply authorization to endpoint", async () => { await checkBuilderEndpoint({ config, @@ -258,129 +82,6 @@ describe("/queries", () => { url: `/api/queries/preview`, }) }) - - it("should not error when trying to generate a nested schema for an empty array", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [], - }, - ] - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: {}, - rows, - schema: { - contacts: { type: "array", name: "contacts" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) - - it("should generate a nested schema based on all the nested items", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [ - { - address: "123 Lane", - }, - { - address: "456 Drive", - }, - { - postcode: "BT1 12N", - lat: 54.59, - long: -5.92, - }, - { - city: "Belfast", - }, - { - address: "789 Avenue", - phoneNumber: "0800-999-5555", - }, - { - name: "Name", - isActive: false, - }, - ], - }, - ] - - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: { - contacts: { - address: { - type: "string", - name: "address", - }, - postcode: { - type: "string", - name: "postcode", - }, - lat: { - type: "number", - name: "lat", - }, - long: { - type: "number", - name: "long", - }, - city: { - type: "string", - name: "city", - }, - phoneNumber: { - type: "string", - name: "phoneNumber", - }, - name: { - type: "string", - name: "name", - }, - isActive: { - type: "boolean", - name: "isActive", - }, - }, - }, - rows, - schema: { - contacts: { type: "json", name: "contacts", subtype: "array" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) }) describe("execute", () => { @@ -412,21 +113,6 @@ describe("/queries", () => { }, }) }) - - it("shouldn't allow handlebars to be passed as parameters", async () => { - const res = await request - .post(`/api/queries/${query._id}`) - .send({ - parameters: { - a: "{{ 'test' }}", - }, - }) - .set(config.defaultHeaders()) - .expect(400) - expect(res.body.message).toEqual( - "Parameter 'a' input contains a handlebars binding - this is not allowed." - ) - }) }) describe("variables", () => { @@ -444,40 +130,6 @@ describe("/queries", () => { return await config.api.query.previewQuery(queryPreview) } - it("should work with static variables", async () => { - const datasource = await config.restDatasource({ - staticVariables: { - variable: "google", - variable2: "1", - }, - }) - const responseBody = await preview(datasource, { - path: "www.{{ variable }}.com", - queryString: "test={{ variable2 }}", - }) - // these responses come from the mock - expect(responseBody.schema).toEqual({ - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - value: { type: "string", name: "value" }, - }) - expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1") - }) - - it("should work with dynamic variables", async () => { - const { datasource } = await config.dynamicVariableDatasource() - const responseBody = await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - expect(responseBody.schema).toEqual({ - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - value: { type: "string", name: "value" }, - }) - expect(responseBody.rows[0].url).toContain("doctype%20html") - }) - it("check that it automatically retries on fail with cached dynamics", async () => { const { datasource, query: base } = await config.dynamicVariableDatasource() @@ -503,29 +155,6 @@ describe("/queries", () => { }) expect(responseBody.rows[0].fails).toEqual(1) }) - - it("deletes variables when linked query is deleted", async () => { - const { datasource, query: base } = - await config.dynamicVariableDatasource() - // preview once to cache - await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - // check its in cache - let contents = await checkCacheForDynamicVariable(base._id!, "variable3") - expect(contents.rows.length).toEqual(1) - - // delete the query - await request - .delete(`/api/queries/${base._id}/${base._rev}`) - .set(config.defaultHeaders()) - .expect(200) - - // check variables no longer in cache - contents = await checkCacheForDynamicVariable(base._id!, "variable3") - expect(contents).toBe(null) - }) }) describe("Current User Request Mapping", () => { diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 06aa9b4e1e..fd2a74a409 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -45,4 +45,17 @@ export class DatasourceAPI extends TestAPI { expectations, }) } + + delete = async (datasource: Datasource, expectations?: Expectations) => { + return await this._delete( + `/api/datasources/${datasource._id!}/${datasource._rev!}`, + { expectations } + ) + } + + get = async (id: string, expectations?: Expectations) => { + return await this._get(`/api/datasources/${id}`, { + expectations, + }) + } } diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index 32866314ff..aba4058832 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -5,28 +5,58 @@ import { PreviewQueryRequest, PreviewQueryResponse, } from "@budibase/types" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" +import { constants } from "@budibase/backend-core" export class QueryAPI extends TestAPI { - create = async (body: Query): Promise => { - return await this._post(`/api/queries`, { body }) + save = async (body: Query, expectations?: Expectations): Promise => { + return await this._post(`/api/queries`, { body, expectations }) } execute = async ( queryId: string, - body?: ExecuteQueryRequest + body?: ExecuteQueryRequest, + expectations?: Expectations ): Promise => { return await this._post( `/api/v2/queries/${queryId}`, { body, + expectations, } ) } - previewQuery = async (queryPreview: PreviewQueryRequest) => { + previewQuery = async ( + queryPreview: PreviewQueryRequest, + expectations?: Expectations + ) => { return await this._post(`/api/queries/preview`, { body: queryPreview, + expectations, }) } + + delete = async (query: Query, expectations?: Expectations) => { + return await this._delete(`/api/queries/${query._id!}/${query._rev!}`, { + expectations, + }) + } + + get = async (queryId: string, expectations?: Expectations) => { + return await this._get(`/api/queries/${queryId}`, { expectations }) + } + + getProd = async (queryId: string, expectations?: Expectations) => { + return await this._get(`/api/queries/${queryId}`, { + expectations, + headers: { + [constants.Header.APP_ID]: this.config.getProdAppId(), + }, + }) + } + + fetch = async (expectations?: Expectations) => { + return await this._get(`/api/queries`, { expectations }) + } } From 08fecca9f289db9a8f57b0e45f85ebe939cfef5a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Mar 2024 12:00:26 +0000 Subject: [PATCH 02/19] Delete query.seq.spec.ts --- packages/server/package.json | 1 + .../server/src/api/controllers/datasource.ts | 4 +- .../server/src/api/controllers/query/index.ts | 4 +- .../src/api/routes/tests/datasource.spec.ts | 61 +-- .../routes/tests/queries/generic-sql.spec.ts | 42 +- .../api/routes/tests/queries/mongodb.spec.ts | 4 +- .../routes/tests/queries/permissions.spec.ts | 47 ++ .../routes/tests/queries/query.seq.spec.ts | 400 ----------------- .../src/api/routes/tests/queries/rest.spec.ts | 406 ++++++++++++++++++ .../src/tests/utilities/api/datasource.ts | 8 + .../server/src/tests/utilities/api/query.ts | 2 +- packages/server/src/threads/utils.ts | 26 +- yarn.lock | 179 ++++++-- 13 files changed, 691 insertions(+), 493 deletions(-) create mode 100644 packages/server/src/api/routes/tests/queries/permissions.spec.ts delete mode 100644 packages/server/src/api/routes/tests/queries/query.seq.spec.ts create mode 100644 packages/server/src/api/routes/tests/queries/rest.spec.ts diff --git a/packages/server/package.json b/packages/server/package.json index 8e66f3d7e0..182b29bb61 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -144,6 +144,7 @@ "jest-openapi": "0.14.2", "jest-runner": "29.7.0", "jest-serial-runner": "1.2.1", + "nock": "^13.5.4", "nodemon": "2.0.15", "openapi-typescript": "5.2.0", "path-to-regexp": "6.2.0", diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 0f17c5a2f5..243d0a17a0 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -1,6 +1,6 @@ import { getQueryParams, getTableParams } from "../../db/utils" import { getIntegration } from "../../integrations" -import { invalidateDynamicVariables } from "../../threads/utils" +import { invalidateCachedVariable } from "../../threads/utils" import { context, db as dbCore, events } from "@budibase/backend-core" import { BuildSchemaFromSourceRequest, @@ -121,7 +121,7 @@ async function invalidateVariables( } }) } - await invalidateDynamicVariables(toInvalidate) + await invalidateCachedVariable(toInvalidate) } export async function update( diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 0dba20dacd..d4d1711d42 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -2,7 +2,7 @@ import { generateQueryID } from "../../../db/utils" import { Thread, ThreadType } from "../../../threads" import { save as saveDatasource } from "../datasource" import { RestImporter } from "./import" -import { invalidateDynamicVariables } from "../../../threads/utils" +import { invalidateCachedVariable } from "../../../threads/utils" import env from "../../../environment" import { events, context, utils, constants } from "@budibase/backend-core" import sdk from "../../../sdk" @@ -401,7 +401,7 @@ const removeDynamicVariables = async (queryId: string) => { const variablesToDelete = dynamicVariables!.filter( (dv: any) => dv.queryId === queryId ) - await invalidateDynamicVariables(variablesToDelete) + await invalidateCachedVariable(variablesToDelete) } } diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index cbd830aee5..0066be2a64 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -1,18 +1,16 @@ -jest.mock("pg") import * as setup from "./utilities" import { checkBuilderEndpoint } from "./utilities/TestFunctions" -import { checkCacheForDynamicVariable } from "../../../threads/utils" +import { getCachedVariable } from "../../../threads/utils" import { context, events } from "@budibase/backend-core" import sdk from "../../../sdk" import tk from "timekeeper" import { mocks } from "@budibase/backend-core/tests" -import { QueryPreview } from "@budibase/types" +import { QueryPreview, SourceName } from "@budibase/types" tk.freeze(mocks.date.MOCK_DATE) let { basicDatasource } = setup.structures -const pg = require("pg") describe("/datasources", () => { let request = setup.getRequest() @@ -42,6 +40,23 @@ describe("/datasources", () => { expect(res.body.errors).toEqual({}) expect(events.datasource.created).toHaveBeenCalledTimes(1) }) + + it("should fail if the datasource is invalid", async () => { + await config.api.datasource.create( + { + name: "Test", + type: "test", + source: "invalid" as SourceName, + config: {}, + }, + { + status: 500, + body: { + message: "No datasource implementation found.", + }, + } + ) + }) }) describe("update", () => { @@ -74,7 +89,7 @@ describe("/datasources", () => { schema: {}, readable: true, } - return config.api.query.previewQuery(queryPreview) + return config.api.query.preview(queryPreview) } it("should invalidate changed or removed variables", async () => { @@ -85,10 +100,7 @@ describe("/datasources", () => { queryString: "test={{ variable3 }}", }) // check variables in cache - let contents = await checkCacheForDynamicVariable( - query._id!, - "variable3" - ) + let contents = await getCachedVariable(query._id!, "variable3") expect(contents.rows.length).toEqual(1) // update the datasource to remove the variables @@ -102,7 +114,7 @@ describe("/datasources", () => { expect(res.body.errors).toBeUndefined() // check variables no longer in cache - contents = await checkCacheForDynamicVariable(query._id!, "variable3") + contents = await getCachedVariable(query._id!, "variable3") expect(contents).toBe(null) }) }) @@ -149,35 +161,6 @@ describe("/datasources", () => { }) }) - describe("query", () => { - it("should be able to query a pg datasource", async () => { - const res = await request - .post(`/api/datasources/query`) - .send({ - endpoint: { - datasourceId: datasource._id, - operation: "READ", - // table name below - entityId: "users", - }, - resource: { - fields: ["users.name", "users.age"], - }, - filters: { - string: { - name: "John", - }, - }, - }) - .set(config.defaultHeaders()) - .expect(200) - // this is mock data, can't test it - expect(res.body).toBeDefined() - const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"` - expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000]) - }) - }) - describe("destroy", () => { beforeAll(setupTest) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 4d97e07ed7..0e93fb6c98 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -1,4 +1,10 @@ -import { Datasource, Query, QueryPreview, SourceName } from "@budibase/types" +import { + Datasource, + Operation, + Query, + QueryPreview, + SourceName, +} from "@budibase/types" import * as setup from "../utilities" import { databaseTestProviders } from "../../../../integrations/tests/utils" import pg from "pg" @@ -6,6 +12,7 @@ import mysql from "mysql2/promise" import mssql from "mssql" import { Expectations } from "src/tests/utilities/api/base" import { events } from "@budibase/backend-core" +import { getCachedVariable } from "../../../../../src/threads/utils" jest.unmock("pg") @@ -257,7 +264,7 @@ describe.each([ schema: {}, readable: true, } - const response = await config.api.query.previewQuery(request) + const response = await config.api.query.preview(request) expect(response.schema).toEqual({ birthday: { name: "birthday", @@ -313,7 +320,7 @@ describe.each([ readable: true, } - const response = await config.api.query.previewQuery(request) + const response = await config.api.query.preview(request) expect(response.schema).toEqual({ foo: { @@ -350,7 +357,7 @@ describe.each([ }, }) - const preview = await config.api.query.previewQuery({ + const preview = await config.api.query.preview({ datasourceId: datasource._id!, queryVerb: "read", fields: { @@ -400,7 +407,7 @@ describe.each([ await config.api.query.delete(basedOnQuery) - const preview = await config.api.query.previewQuery({ + const preview = await config.api.query.preview({ datasourceId: datasource._id!, queryVerb: "read", fields: { @@ -750,4 +757,29 @@ describe.each([ }) }) }) + + describe("query through datasource", () => { + it("should be able to query a pg datasource", async () => { + const res = await config.api.datasource.query({ + endpoint: { + datasourceId: datasource._id!, + operation: Operation.READ, + entityId: "test_table", + }, + resource: { + fields: ["id", "name"], + }, + filters: { + string: { + name: "two", + }, + }, + }) + expect(res).toHaveLength(1) + expect(res[0]).toEqual({ + id: 2, + name: "two", + }) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index 6ab3dec80d..aaeb93afc2 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -96,7 +96,7 @@ describe("/queries", () => { async collection => await collection.insertOne({ name, nested: [] }) ) - const preview = await config.api.query.previewQuery({ + const preview = await config.api.query.preview({ name: "New Query", datasourceId: datasource._id!, fields: { @@ -167,7 +167,7 @@ describe("/queries", () => { await withCollection(collection => collection.insertOne(item)) - const preview = await config.api.query.previewQuery({ + const preview = await config.api.query.preview({ name: "New Query", datasourceId: datasource._id!, fields: { diff --git a/packages/server/src/api/routes/tests/queries/permissions.spec.ts b/packages/server/src/api/routes/tests/queries/permissions.spec.ts new file mode 100644 index 0000000000..a0b342e64d --- /dev/null +++ b/packages/server/src/api/routes/tests/queries/permissions.spec.ts @@ -0,0 +1,47 @@ +import * as setup from "../utilities" +import { checkBuilderEndpoint } from "../utilities/TestFunctions" +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" +import { Datasource, Query, SourceName } from "@budibase/types" + +describe("query permissions", () => { + let config: TestConfiguration + let datasource: Datasource + let query: Query + + beforeAll(async () => { + config = setup.getConfig() + await config.init() + datasource = await config.api.datasource.create({ + name: "test datasource", + type: "test", + source: SourceName.REST, + config: {}, + }) + query = await config.api.query.save({ + name: "test query", + datasourceId: datasource._id!, + parameters: [], + fields: {}, + transformer: "", + schema: {}, + readable: true, + queryVerb: "read", + }) + }) + + it("delete should require builder", async () => { + await checkBuilderEndpoint({ + config, + method: "DELETE", + url: `/api/queries/${query._id}/${query._rev}`, + }) + }) + + it("preview should require builder", async () => { + await checkBuilderEndpoint({ + config, + method: "POST", + url: `/api/queries/preview`, + }) + }) +}) diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts deleted file mode 100644 index d8b06c001d..0000000000 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ /dev/null @@ -1,400 +0,0 @@ -import tk from "timekeeper" - -const pg = require("pg") - -// Mock out postgres for this -jest.mock("pg") -jest.mock("node-fetch") - -// Mock isProdAppID to we can later mock the implementation and pretend we are -// using prod app IDs -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - isProdAppID: jest.fn(), - }, - } -}) -import * as setup from "../utilities" -import { checkBuilderEndpoint } from "../utilities/TestFunctions" -import { checkCacheForDynamicVariable } from "../../../../threads/utils" - -const { basicQuery, basicDatasource } = setup.structures -import { events, db as dbCore } from "@budibase/backend-core" -import { - Datasource, - Query, - SourceName, - QueryPreview, - QueryParameter, -} from "@budibase/types" - -tk.freeze(Date.now()) - -const mockIsProdAppID = dbCore.isProdAppID as jest.MockedFunction< - typeof dbCore.isProdAppID -> - -describe("/queries", () => { - let request = setup.getRequest() - let config = setup.getConfig() - let datasource: Datasource & Required>, query: Query - - afterAll(setup.afterAll) - - const setupTest = async () => { - await config.init() - datasource = await config.createDatasource() - query = await config.createQuery() - } - - beforeAll(async () => { - await setupTest() - }) - - it("should apply authorization to endpoint", async () => { - await checkBuilderEndpoint({ - config, - method: "GET", - url: `/api/datasources`, - }) - }) - - describe("destroy", () => { - it("should apply authorization to endpoint", async () => { - const query = await config.createQuery() - await checkBuilderEndpoint({ - config, - method: "DELETE", - url: `/api/queries/${query._id}/${query._rev}`, - }) - }) - }) - - describe("preview", () => { - it("should apply authorization to endpoint", async () => { - await checkBuilderEndpoint({ - config, - method: "POST", - url: `/api/queries/preview`, - }) - }) - }) - - describe("execute", () => { - beforeEach(async () => { - await setupTest() - }) - - it("should be able to execute the query", async () => { - const res = await request - .post(`/api/queries/${query._id}`) - .send({ - parameters: {}, - }) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body.length).toEqual(1) - }) - - it("should fail with invalid integration type", async () => { - const datasource: Datasource = { - ...basicDatasource().datasource, - source: "INVALID_INTEGRATION" as SourceName, - } - await config.api.datasource.create(datasource, { - status: 500, - body: { - message: "No datasource implementation found.", - }, - }) - }) - }) - - describe("variables", () => { - async function preview(datasource: Datasource, fields: any) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: [], - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) - } - - it("check that it automatically retries on fail with cached dynamics", async () => { - const { datasource, query: base } = - await config.dynamicVariableDatasource() - // preview once to cache - await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - // check its in cache - const contents = await checkCacheForDynamicVariable( - base._id!, - "variable3" - ) - expect(contents.rows.length).toEqual(1) - const responseBody = await preview(datasource, { - path: "www.failonce.com", - queryString: "test={{ variable3 }}", - }) - expect(responseBody.schema).toEqual({ - fails: { type: "number", name: "fails" }, - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - }) - expect(responseBody.rows[0].fails).toEqual(1) - }) - }) - - describe("Current User Request Mapping", () => { - async function previewGet( - datasource: Datasource, - fields: any, - params: QueryParameter[] - ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) - } - - async function previewPost( - datasource: Datasource, - fields: any, - params: QueryParameter[] - ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, - fields, - queryVerb: "create", - name: datasource.name!, - transformer: null, - schema: {}, - readable: false, - } - return await config.api.query.previewQuery(queryPreview) - } - - it("should parse global and query level header mappings", async () => { - const userDetails = config.getUserDetails() - - const datasource = await config.restDatasource({ - defaultHeaders: { - test: "headerVal", - emailHdr: "{{[user].[email]}}", - }, - }) - const responseBody = await previewGet( - datasource, - { - path: "www.google.com", - queryString: "email={{[user].[email]}}", - headers: { - queryHdr: "{{[user].[firstName]}}", - secondHdr: "1234", - }, - }, - [] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - expect(parsedRequest.opts.headers).toEqual({ - test: "headerVal", - emailHdr: userDetails.email, - queryHdr: userDetails.firstName, - secondHdr: "1234", - }) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?email=" + userDetails.email.replace("@", "%40") - ) - }) - - it("should bind the current user to query parameters", async () => { - const userDetails = config.getUserDetails() - - const datasource = await config.restDatasource() - - const responseBody = await previewGet( - datasource, - { - path: "www.google.com", - queryString: - "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", - }, - [ - { name: "myEmail", default: "{{[user].[email]}}" }, - { name: "myName", default: "{{[user].[firstName]}}" }, - { name: "testParam", default: "1234" }, - ] - ) - - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?test=" + - userDetails.email.replace("@", "%40") + - "&testName=" + - userDetails.firstName + - "&testParam=1234" - ) - }) - - it("should bind the current user the request body - plain text", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", - bodyType: "text", - }, - [{ name: "testParam", default: "1234" }] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - expect(parsedRequest.opts.body).toEqual( - `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234` - ) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - json", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "json", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}` - expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - xml", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - " {{[user].[email]}} {{testParam}} " + - "{{userId}} testing ", - bodyType: "xml", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userId", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - const test = ` ${userDetails.email} 1234 ${userDetails.firstName} testing ` - - expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - form-data", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "form", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - - const emailData = parsedRequest.opts.body._streams[1] - expect(emailData).toEqual(userDetails.email) - - const queryCodeData = parsedRequest.opts.body._streams[4] - expect(queryCodeData).toEqual("1234") - - const userRef = parsedRequest.opts.body._streams[7] - expect(userRef).toEqual(userDetails.firstName) - - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - encoded", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "encoded", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - const parsedRequest = JSON.parse(responseBody.extra.raw) - - expect(parsedRequest.opts.body.email).toEqual(userDetails.email) - expect(parsedRequest.opts.body.queryCode).toEqual("1234") - expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName) - }) - }) -}) diff --git a/packages/server/src/api/routes/tests/queries/rest.spec.ts b/packages/server/src/api/routes/tests/queries/rest.spec.ts new file mode 100644 index 0000000000..5c41583244 --- /dev/null +++ b/packages/server/src/api/routes/tests/queries/rest.spec.ts @@ -0,0 +1,406 @@ +import * as setup from "../utilities" +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" +import { Datasource, SourceName } from "@budibase/types" +import { getCachedVariable } from "../../../../threads/utils" +import nock from "nock" +import { generator } from "@budibase/backend-core/tests" + +jest.unmock("node-fetch") + +describe("rest", () => { + let config: TestConfiguration + let datasource: Datasource + + async function createQuery(fields: any) { + return await config.api.query.save({ + name: "test query", + datasourceId: datasource._id!, + parameters: [], + fields, + transformer: "", + schema: {}, + readable: true, + queryVerb: "read", + }) + } + + beforeAll(async () => { + config = setup.getConfig() + await config.init() + datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: {}, + }) + }) + + afterEach(() => { + nock.cleanAll() + }) + + it("should automatically retry on fail with cached dynamics", async () => { + const basedOnQuery = await createQuery({ + path: "one.example.com", + }) + + let cached = await getCachedVariable(basedOnQuery._id!, "foo") + expect(cached).toBeNull() + + await config.api.datasource.update({ + ...datasource, + config: { + ...datasource.config, + dynamicVariables: [ + { + queryId: basedOnQuery._id!, + name: "foo", + value: "{{ data[0].name }}", + }, + ], + }, + }) + + cached = await getCachedVariable(basedOnQuery._id!, "foo") + expect(cached).toBeNull() + + nock("http://one.example.com") + .get("/") + .reply(200, [{ name: "one" }]) + nock("http://two.example.com").get("/?test=one").reply(500) + nock("http://two.example.com") + .get("/?test=one") + .reply(200, [{ name: "two" }]) + + const res = await config.api.query.preview({ + datasourceId: datasource._id!, + name: "test query", + parameters: [], + queryVerb: "read", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "two.example.com", + queryString: "test={{ foo }}", + }, + }) + expect(res.schema).toEqual({ + name: { type: "string", name: "name" }, + }) + + cached = await getCachedVariable(basedOnQuery._id!, "foo") + expect(cached.rows.length).toEqual(1) + expect(cached.rows[0].name).toEqual("one") + }) + + it("should parse global and query level header mappings", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com", { + reqheaders: { + test: "headerVal", + emailhdr: user.email, + queryhdr: user.firstName!, + secondhdr: "1234", + }, + }) + .get("/?email=" + user.email.replace("@", "%40")) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [], + queryVerb: "read", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + queryString: "email={{[user].[email]}}", + headers: { + queryHdr: "{{[user].[firstName]}}", + secondHdr: "1234", + }, + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to query params", async () => { + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .get( + "/?test=" + + user.email.replace("@", "%40") + + "&testName=" + + user.firstName + + "&testParam=1234" + ) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [ + { name: "myEmail", default: "{{[user].[email]}}" }, + { name: "myName", default: "{{[user].[firstName]}}" }, + { name: "testParam", default: "1234" }, + ], + queryVerb: "read", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + queryString: + "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to the request body - plain text", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + method: "POST", + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .post( + "/?testParam=1234", + "This is plain text and this is my email: " + + user.email + + ". This is a test param: 1234" + ) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [{ name: "testParam", default: "1234" }], + queryVerb: "create", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + bodyType: "text", + queryString: "&testParam={{testParam}}", + requestBody: + "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to the request body - json", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + method: "POST", + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .post("/?testParam=1234", { + email: user.email, + queryCode: 1234, + userRef: user.firstName, + }) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [ + { name: "testParam", default: "1234" }, + { name: "userRef", default: "{{[user].[firstName]}}" }, + ], + queryVerb: "create", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + bodyType: "json", + queryString: "&testParam={{testParam}}", + requestBody: + '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to the request body - xml", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + method: "POST", + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .post( + "/?testParam=1234", + ` ${user.email} 1234 ${user.firstName} testing ` + ) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [ + { name: "testParam", default: "1234" }, + { name: "userId", default: "{{[user].[firstName]}}" }, + ], + queryVerb: "create", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + bodyType: "xml", + queryString: "&testParam={{testParam}}", + requestBody: + " {{[user].[email]}} {{testParam}} " + + "{{userId}} testing ", + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to the request body - form-data", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + method: "POST", + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .post("/?testParam=1234", body => { + return ( + body.includes('name="email"\r\n\r\n' + user.email + "\r\n") && + body.includes('name="queryCode"\r\n\r\n1234\r\n') && + body.includes('name="userRef"\r\n\r\n' + user.firstName + "\r\n") + ) + }) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [ + { name: "testParam", default: "1234" }, + { name: "userRef", default: "{{[user].[firstName]}}" }, + ], + queryVerb: "create", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + bodyType: "form", + queryString: "&testParam={{testParam}}", + requestBody: + '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', + }, + }) + + expect(mock.isDone()).toEqual(true) + }) + + it("should bind the current user to the request body - encoded", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: { + method: "POST", + defaultHeaders: { + test: "headerVal", + emailHdr: "{{[user].[email]}}", + }, + }, + }) + + const user = config.getUserDetails() + const mock = nock("http://www.example.com") + .post("/?testParam=1234", { + email: user.email, + queryCode: 1234, + userRef: user.firstName, + }) + .reply(200, {}) + + await config.api.query.preview({ + datasourceId: datasource._id!, + name: generator.guid(), + parameters: [ + { name: "testParam", default: "1234" }, + { name: "userRef", default: "{{[user].[firstName]}}" }, + ], + queryVerb: "create", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + bodyType: "encoded", + queryString: "&testParam={{testParam}}", + requestBody: + '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', + }, + }) + + expect(mock.isDone()).toEqual(true) + }) +}) diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index fd2a74a409..0296f58f7d 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -4,6 +4,7 @@ import { CreateDatasourceResponse, UpdateDatasourceResponse, UpdateDatasourceRequest, + QueryJson, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -58,4 +59,11 @@ export class DatasourceAPI extends TestAPI { expectations, }) } + + query = async (query: QueryJson, expectations?: Expectations) => { + return await this._post(`/api/datasources/query`, { + body: query, + expectations, + }) + } } diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index aba4058832..2d5f7970cd 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -27,7 +27,7 @@ export class QueryAPI extends TestAPI { ) } - previewQuery = async ( + preview = async ( queryPreview: PreviewQueryRequest, expectations?: Expectations ) => { diff --git a/packages/server/src/threads/utils.ts b/packages/server/src/threads/utils.ts index cd547cacae..46752975a2 100644 --- a/packages/server/src/threads/utils.ts +++ b/packages/server/src/threads/utils.ts @@ -5,7 +5,7 @@ import { redis, db as dbCore } from "@budibase/backend-core" import * as jsRunner from "../jsRunner" const VARIABLE_TTL_SECONDS = 3600 -let client: any +let client: redis.Client | null = null async function getClient() { if (!client) { @@ -36,23 +36,15 @@ export function threadSetup() { db.init() } -export async function checkCacheForDynamicVariable( - queryId: string, - variable: string -) { - const cache = await getClient() - return cache.get(makeVariableKey(queryId, variable)) +export async function getCachedVariable(queryId: string, variable: string) { + return (await getClient()).get(makeVariableKey(queryId, variable)) } -export async function invalidateDynamicVariables(cachedVars: QueryVariable[]) { +export async function invalidateCachedVariable(vars: QueryVariable[]) { const cache = await getClient() - let promises = [] - for (let variable of cachedVars) { - promises.push( - cache.delete(makeVariableKey(variable.queryId, variable.name)) - ) - } - await Promise.all(promises) + await Promise.all( + vars.map(v => cache.delete(makeVariableKey(v.queryId, v.name))) + ) } export async function storeDynamicVariable( @@ -93,7 +85,7 @@ export default { hasExtraData, formatResponse, storeDynamicVariable, - invalidateDynamicVariables, - checkCacheForDynamicVariable, + invalidateDynamicVariables: invalidateCachedVariable, + checkCacheForDynamicVariable: getCachedVariable, threadSetup, } diff --git a/yarn.lock b/yarn.lock index d035d65ba1..601bbf4641 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7265,7 +7265,37 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.21.4, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1, axios@^0.21.4: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^0.26.0: + version "0.26.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" + integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== + dependencies: + follow-redirects "^1.14.8" + +axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11255,6 +11285,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0, follow-redirects@^1.14.4, follow-redirects@^1.14.8: + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== + follow-redirects@^1.15.0: version "1.15.2" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" @@ -12396,7 +12431,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13346,11 +13386,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -15945,7 +15980,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16113,6 +16148,15 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +nock@^13.5.4: + version "13.5.4" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.5.4.tgz#8918f0addc70a63736170fef7106a9721e0dc479" + integrity sha512-yAyTfdeNJGGBFxWdzSKCBYxs5FxLbCg5X5Q4ets974hcQzG1+qCxvIyOo4j2Ry6MUlhWVMX4OoYDefAIIwupjw== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + propagate "^2.0.0" + node-abi@^3.3.0: version "3.54.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.54.0.tgz#f6386f7548817acac6434c6cba02999c9aebcc69" @@ -16145,13 +16189,25 @@ node-duration@^1.0.4: resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17301,7 +17357,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18486,6 +18550,11 @@ promzard@^1.0.0: dependencies: read "^2.0.0" +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + proper-lockfile@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" @@ -18596,7 +18665,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19621,6 +19690,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" + integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19702,13 +19776,40 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@~2.3.1, semver@~7.0.0: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + +semver@~7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" + integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21311,7 +21412,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21321,6 +21422,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21797,14 +21906,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22575,10 +22676,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22588,6 +22712,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 5e6a9d2176405ea50681384040e602ea6838d33c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Mar 2024 14:41:09 +0000 Subject: [PATCH 03/19] Convert executeQuery.spec.js to TypeScript and make it hit real databases. --- .../routes/tests/queries/generic-sql.spec.ts | 1 - .../automations/tests/executeQuery.spec.js | 39 -------- .../automations/tests/executeQuery.spec.ts | 92 +++++++++++++++++++ 3 files changed, 92 insertions(+), 40 deletions(-) delete mode 100644 packages/server/src/automations/tests/executeQuery.spec.js create mode 100644 packages/server/src/automations/tests/executeQuery.spec.ts diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 0e93fb6c98..3ce96bd7d7 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -12,7 +12,6 @@ import mysql from "mysql2/promise" import mssql from "mssql" import { Expectations } from "src/tests/utilities/api/base" import { events } from "@budibase/backend-core" -import { getCachedVariable } from "../../../../../src/threads/utils" jest.unmock("pg") diff --git a/packages/server/src/automations/tests/executeQuery.spec.js b/packages/server/src/automations/tests/executeQuery.spec.js deleted file mode 100644 index 3b691f48ea..0000000000 --- a/packages/server/src/automations/tests/executeQuery.spec.js +++ /dev/null @@ -1,39 +0,0 @@ -const setup = require("./utilities") - -describe("test the execute query action", () => { - let query - let config = setup.getConfig() - - beforeAll(async () => { - await config.init() - - await config.createDatasource() - query = await config.createQuery() - }) - - afterAll(setup.afterAll) - - it("should be able to execute a query", async () => { - let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { - query: { queryId: query._id }, - }) - expect(res.response).toEqual([{ a: "string", b: 1 }]) - expect(res.success).toEqual(true) - }) - - it("should handle a null query value", async () => { - let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { - query: null, - }) - expect(res.response.message).toEqual("Invalid inputs") - expect(res.success).toEqual(false) - }) - - it("should handle an error executing a query", async () => { - let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { - query: { queryId: "wrong_id" }, - }) - expect(res.response).toEqual("Error: missing") - expect(res.success).toEqual(false) - }) -}) diff --git a/packages/server/src/automations/tests/executeQuery.spec.ts b/packages/server/src/automations/tests/executeQuery.spec.ts new file mode 100644 index 0000000000..1d25a31b92 --- /dev/null +++ b/packages/server/src/automations/tests/executeQuery.spec.ts @@ -0,0 +1,92 @@ +import { Datasource, Query, SourceName } from "@budibase/types" +import * as setup from "./utilities" +import { databaseTestProviders } from "../../integrations/tests/utils" +import knex, { Knex } from "knex" +import { generator } from "@budibase/backend-core/tests" + +function getKnexClientName(source: SourceName) { + switch (source) { + case SourceName.MYSQL: + return "mysql2" + case SourceName.SQL_SERVER: + return "mssql" + case SourceName.POSTGRES: + return "pg" + } + throw new Error(`Unsupported source: ${source}`) +} + +describe.each([ + ["postgres", databaseTestProviders.postgres], + ["mysql", databaseTestProviders.mysql], + ["mssql", databaseTestProviders.mssql], + ["mariadb", databaseTestProviders.mariadb], +])("execute query action (%s)", (__, dsProvider) => { + let tableName: string + let client: Knex + let datasource: Datasource + let query: Query + let config = setup.getConfig() + + beforeAll(async () => { + await config.init() + + const ds = await dsProvider.datasource() + datasource = await config.api.datasource.create(ds) + client = knex({ + client: getKnexClientName(ds.source), + connection: ds.config, + }) + }) + + beforeEach(async () => { + tableName = generator.guid() + await client.schema.createTable(tableName, table => { + table.string("a") + table.integer("b") + }) + await client(tableName).insert({ a: "string", b: 1 }) + query = await config.api.query.save({ + name: "test query", + datasourceId: datasource._id!, + parameters: [], + fields: { + sql: client(tableName).select("*").toSQL().toNative().sql, + }, + transformer: "", + schema: {}, + readable: true, + queryVerb: "read", + }) + }) + + afterEach(async () => { + await client.schema.dropTable(tableName) + }) + + afterAll(setup.afterAll) + + it("should be able to execute a query", async () => { + let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { + query: { queryId: query._id }, + }) + expect(res.response).toEqual([{ a: "string", b: 1 }]) + expect(res.success).toEqual(true) + }) + + it("should handle a null query value", async () => { + let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { + query: null, + }) + expect(res.response.message).toEqual("Invalid inputs") + expect(res.success).toEqual(false) + }) + + it("should handle an error executing a query", async () => { + let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, { + query: { queryId: "wrong_id" }, + }) + expect(res.response).toEqual("Error: missing") + expect(res.success).toEqual(false) + }) +}) From a9aa409d7c976c8b5fd8ec40c5f205f3b3d889b4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Mar 2024 15:01:40 +0000 Subject: [PATCH 04/19] Fix up tests. --- packages/server/jest.config.ts | 6 ------ packages/server/package.json | 1 - .../api/routes/tests/queries/generic-sql.spec.ts | 2 -- packages/server/src/api/routes/tests/row.spec.ts | 1 - .../server/src/api/routes/tests/viewV2.spec.ts | 1 - .../server/src/integration-test/postgres.spec.ts | 1 - .../server/src/integrations/tests/utils/index.ts | 2 -- .../server/src/sdk/users/tests/utils.spec.ts | 16 ++++++++++++---- 8 files changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index f3f8ebce02..db0625241d 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -42,12 +42,6 @@ if (fs.existsSync("../pro/src")) { const config: Config.InitialOptions = { projects: [ - { - ...baseConfig, - displayName: "sequential test", - testMatch: ["/**/*.seq.spec.[jt]s"], - runner: "jest-serial-runner", - }, { ...baseConfig, testMatch: ["/**/!(*.seq).spec.[jt]s"], diff --git a/packages/server/package.json b/packages/server/package.json index 182b29bb61..27f34e3607 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -143,7 +143,6 @@ "jest": "29.7.0", "jest-openapi": "0.14.2", "jest-runner": "29.7.0", - "jest-serial-runner": "1.2.1", "nock": "^13.5.4", "nodemon": "2.0.15", "openapi-typescript": "5.2.0", diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 3ce96bd7d7..060699450c 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -13,8 +13,6 @@ import mssql from "mssql" import { Expectations } from "src/tests/utilities/api/base" import { events } from "@budibase/backend-core" -jest.unmock("pg") - const createTableSQL: Record = { [SourceName.POSTGRES]: ` CREATE TABLE test_table ( diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f638f2c4bf..7f8a28ad99 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -30,7 +30,6 @@ const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) jest.unmock("mssql") -jest.unmock("pg") describe.each([ ["internal", undefined], diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index f9d213a26b..b99edc2264 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -26,7 +26,6 @@ import { quotas } from "@budibase/pro" import { roles } from "@budibase/backend-core" jest.unmock("mssql") -jest.unmock("pg") describe.each([ ["internal", undefined], diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 107c4ade1e..bb6882e0e1 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -23,7 +23,6 @@ fetch.mockSearch() const config = setup.getConfig()! -jest.unmock("pg") jest.mock("../websockets") describe("postgres integrations", () => { diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index b2be3df4e0..7faa84099b 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -1,5 +1,3 @@ -jest.unmock("pg") - import { Datasource } from "@budibase/types" import * as postgres from "./postgres" import * as mongodb from "./mongodb" diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 6f1c5afd3d..0aa112094d 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -35,11 +35,20 @@ describe("syncGlobalUsers", () => { builder: { global: true }, }) await config.doInContext(config.appId, async () => { - expect(await rawUserMetadata()).toHaveLength(1) + let metadata = await rawUserMetadata() + expect(metadata).not.toContainEqual( + expect.objectContaining({ + _id: db.generateUserMetadataID(user1._id!), + }) + ) + expect(metadata).not.toContainEqual( + expect.objectContaining({ + _id: db.generateUserMetadataID(user2._id!), + }) + ) await syncGlobalUsers() - const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(3) + metadata = await rawUserMetadata() expect(metadata).toContainEqual( expect.objectContaining({ _id: db.generateUserMetadataID(user1._id!), @@ -62,7 +71,6 @@ describe("syncGlobalUsers", () => { await syncGlobalUsers() const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(1) expect(metadata).not.toContainEqual( expect.objectContaining({ _id: db.generateUserMetadataID(user._id!), From a925132fdff9fc8905cdc8de33f4dba56b751bb3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Mar 2024 15:00:16 +0000 Subject: [PATCH 05/19] Fix broken tests. --- .../routes/tests/queries/generic-sql.spec.ts | 26 ++- yarn.lock | 165 +++--------------- 2 files changed, 35 insertions(+), 156 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index ecae874913..f8e54e064e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -278,22 +278,20 @@ describe.each([ name: "name", type: "string", }, + number: { + name: "number", + type: "string", + }, }) expect(response.rows).toEqual([ { birthday: null, id: 1, name: "one", + number: null, }, ]) expect(events.query.previewed).toHaveBeenCalledTimes(1) - - const dsWithoutConfig = { ...datasource } - delete dsWithoutConfig.config - expect(events.query.previewed).toHaveBeenCalledWith( - dsWithoutConfig, - request - ) }) it("should work with static variables", async () => { @@ -427,11 +425,9 @@ describe.each([ }, }) - // TODO: is this the correct behaviour? To return an empty string when the - // underlying query has been deleted? expect(preview.rows).toEqual([ { - foo: "", + foo: datasource.source === SourceName.SQL_SERVER ? "" : null, }, ]) }) @@ -626,6 +622,7 @@ describe.each([ id: 2, name: "one", birthday: null, + number: null, }, ]) }) @@ -693,7 +690,9 @@ describe.each([ ]) const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") - expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }]) + expect(rows).toEqual([ + { id: 1, name: "foo", birthday: null, number: null }, + ]) }) it("should be able to execute an update that updates no rows", async () => { @@ -786,11 +785,6 @@ describe.each([ id: 2, name: "two", }) - - const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") - expect(rows).toEqual([ - { id: 1, name: "foo", birthday: null, number: null }, - ]) }) it("should be able to execute an update that updates no rows", async () => { diff --git a/yarn.lock b/yarn.lock index 601bbf4641..9e1dcae301 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7265,37 +7265,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0: - version "0.24.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" - integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== - dependencies: - follow-redirects "^1.14.4" - -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1, axios@^0.21.4: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^0.26.0: - version "0.26.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" - integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== - dependencies: - follow-redirects "^1.14.8" - -axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.21.4, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11285,11 +11255,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.14.4, follow-redirects@^1.14.8: - version "1.15.6" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" - integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== - follow-redirects@^1.15.0: version "1.15.2" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" @@ -12431,12 +12396,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13386,6 +13346,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -15980,7 +15945,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16189,25 +16154,13 @@ node-duration@^1.0.4: resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA== -node-fetch@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9, node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17357,15 +17310,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18665,7 +18610,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19690,11 +19635,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" - integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19776,40 +19716,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@~2.3.1, semver@~7.0.0: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4: - version "7.6.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" - integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== - dependencies: - lru-cache "^6.0.0" - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - -semver@~7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" - integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21412,7 +21325,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21422,14 +21335,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21906,6 +21811,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22676,33 +22589,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22712,11 +22602,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 110e20c0cee0660ccfb62a32448e6cc548bce1f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 15:30:15 +0100 Subject: [PATCH 06/19] Update submodules. --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index f5b467b6b1..c51458d26f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a +Subproject commit c51458d26f6bded1984ba9ac5d2948480ac809ce diff --git a/packages/pro b/packages/pro index dd748e045f..f8e8f87bd5 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit dd748e045ffdbc6662c5d2b76075f01d65a96a2f +Subproject commit f8e8f87bd52081e1303a5ae92c432ea5b38f3bb4 From 193d40f86cdc87f9a87f123a73e9bc7454b9df80 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 15:40:10 +0100 Subject: [PATCH 07/19] Fix some tests. --- .../routes/tests/queries/query.seq.spec.ts | 774 ------------------ .../automations/tests/executeQuery.spec.ts | 18 +- 2 files changed, 10 insertions(+), 782 deletions(-) delete mode 100644 packages/server/src/api/routes/tests/queries/query.seq.spec.ts diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts deleted file mode 100644 index 4c25a762b8..0000000000 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ /dev/null @@ -1,774 +0,0 @@ -import tk from "timekeeper" - -const pg = require("pg") - -// Mock out postgres for this -jest.mock("pg") -jest.mock("node-fetch") - -// Mock isProdAppID to we can later mock the implementation and pretend we are -// using prod app IDs -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - isProdAppID: jest.fn(), - }, - } -}) -import * as setup from "../utilities" -import { checkBuilderEndpoint } from "../utilities/TestFunctions" -import { checkCacheForDynamicVariable } from "../../../../threads/utils" - -const { basicQuery, basicDatasource } = setup.structures -import { events, db as dbCore } from "@budibase/backend-core" -import { - Datasource, - Query, - SourceName, - QueryPreview, - QueryParameter, -} from "@budibase/types" - -tk.freeze(Date.now()) - -const mockIsProdAppID = dbCore.isProdAppID as jest.MockedFunction< - typeof dbCore.isProdAppID -> - -describe("/queries", () => { - let request = setup.getRequest() - let config = setup.getConfig() - let datasource: Datasource & Required>, query: Query - - afterAll(setup.afterAll) - - const setupTest = async () => { - await config.init() - datasource = await config.createDatasource() - query = await config.createQuery() - } - - beforeAll(async () => { - await setupTest() - }) - - const createQuery = async (query: Query) => { - return request - .post(`/api/queries`) - .send(query) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - } - - describe("create", () => { - it("should create a new query", async () => { - const { _id } = await config.createDatasource() - const query = basicQuery(_id) - jest.clearAllMocks() - const res = await createQuery(query) - - expect((res as any).res.statusMessage).toEqual( - `Query ${query.name} saved successfully.` - ) - expect(res.body).toEqual({ - _rev: res.body._rev, - _id: res.body._id, - ...query, - nullDefaultSupport: true, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - expect(events.query.created).toHaveBeenCalledTimes(1) - expect(events.query.updated).not.toHaveBeenCalled() - }) - }) - - describe("update", () => { - it("should update query", async () => { - const { _id } = await config.createDatasource() - const query = basicQuery(_id) - const res = await createQuery(query) - jest.clearAllMocks() - query._id = res.body._id - query._rev = res.body._rev - await createQuery(query) - - expect((res as any).res.statusMessage).toEqual( - `Query ${query.name} saved successfully.` - ) - expect(res.body).toEqual({ - _rev: res.body._rev, - _id: res.body._id, - ...query, - nullDefaultSupport: true, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - expect(events.query.created).not.toHaveBeenCalled() - expect(events.query.updated).toHaveBeenCalledTimes(1) - }) - }) - - describe("fetch", () => { - beforeEach(async () => { - await setupTest() - }) - - it("returns all the queries from the server", async () => { - const res = await request - .get(`/api/queries`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - const queries = res.body - expect(queries).toEqual([ - { - _rev: query._rev, - _id: query._id, - createdAt: new Date().toISOString(), - ...basicQuery(datasource._id), - nullDefaultSupport: true, - updatedAt: new Date().toISOString(), - readable: true, - }, - ]) - }) - - it("should apply authorization to endpoint", async () => { - await checkBuilderEndpoint({ - config, - method: "GET", - url: `/api/datasources`, - }) - }) - }) - - describe("find", () => { - it("should find a query in builder", async () => { - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body._id).toEqual(query._id) - }) - - it("should find a query in cloud", async () => { - await config.withEnv({ SELF_HOSTED: "true" }, async () => { - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(await config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.fields).toBeDefined() - expect(res.body.parameters).toBeDefined() - expect(res.body.schema).toBeDefined() - }) - }) - - it("should remove sensitive info for prod apps", async () => { - // Mock isProdAppID to pretend we are using a prod app - mockIsProdAppID.mockClear() - mockIsProdAppID.mockImplementation(() => true) - - const query = await config.createQuery() - const res = await request - .get(`/api/queries/${query._id}`) - .set(await config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body._id).toEqual(query._id) - expect(res.body.fields).toBeUndefined() - expect(res.body.parameters).toBeUndefined() - expect(res.body.schema).toBeDefined() - - // Reset isProdAppID mock - expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1) - mockIsProdAppID.mockImplementation(() => false) - }) - }) - - describe("destroy", () => { - beforeEach(async () => { - await setupTest() - }) - - it("deletes a query and returns a success message", async () => { - await request - .delete(`/api/queries/${query._id}/${query._rev}`) - .set(config.defaultHeaders()) - .expect(200) - - const res = await request - .get(`/api/queries`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - expect(res.body).toEqual([]) - expect(events.query.deleted).toHaveBeenCalledTimes(1) - expect(events.query.deleted).toHaveBeenCalledWith(datasource, query) - }) - - it("should apply authorization to endpoint", async () => { - const query = await config.createQuery() - await checkBuilderEndpoint({ - config, - method: "DELETE", - url: `/api/queries/${query._id}/${query._rev}`, - }) - }) - }) - - describe("preview", () => { - it("should be able to preview the query", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - queryVerb: "read", - fields: {}, - parameters: [], - transformer: "return data", - name: datasource.name!, - schema: {}, - readable: true, - } - const responseBody = await config.api.query.previewQuery(queryPreview) - // these responses come from the mock - expect(responseBody.schema).toEqual({ - a: { type: "string", name: "a" }, - b: { type: "number", name: "b" }, - }) - expect(responseBody.rows.length).toEqual(1) - expect(events.query.previewed).toHaveBeenCalledTimes(1) - delete datasource.config - expect(events.query.previewed).toHaveBeenCalledWith(datasource, { - ...queryPreview, - nullDefaultSupport: true, - }) - }) - - it("should apply authorization to endpoint", async () => { - await checkBuilderEndpoint({ - config, - method: "POST", - url: `/api/queries/preview`, - }) - }) - - it("should not error when trying to generate a nested schema for an empty array", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [], - }, - ] - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: {}, - rows, - schema: { - contacts: { type: "array", name: "contacts" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) - - it("should generate a nested schema based on all the nested items", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [ - { - address: "123 Lane", - }, - { - address: "456 Drive", - }, - { - postcode: "BT1 12N", - lat: 54.59, - long: -5.92, - }, - { - city: "Belfast", - }, - { - address: "789 Avenue", - phoneNumber: "0800-999-5555", - }, - { - name: "Name", - isActive: false, - }, - ], - }, - ] - - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: { - contacts: { - address: { - type: "string", - name: "address", - }, - postcode: { - type: "string", - name: "postcode", - }, - lat: { - type: "number", - name: "lat", - }, - long: { - type: "number", - name: "long", - }, - city: { - type: "string", - name: "city", - }, - phoneNumber: { - type: "string", - name: "phoneNumber", - }, - name: { - type: "string", - name: "name", - }, - isActive: { - type: "boolean", - name: "isActive", - }, - }, - }, - rows, - schema: { - contacts: { type: "json", name: "contacts", subtype: "array" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) - }) - - describe("execute", () => { - beforeEach(async () => { - await setupTest() - }) - - it("should be able to execute the query", async () => { - const res = await request - .post(`/api/queries/${query._id}`) - .send({ - parameters: {}, - }) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body.length).toEqual(1) - }) - - it("should fail with invalid integration type", async () => { - const datasource: Datasource = { - ...basicDatasource().datasource, - source: "INVALID_INTEGRATION" as SourceName, - } - await config.api.datasource.create(datasource, { - status: 500, - body: { - message: "No datasource implementation found.", - }, - }) - }) - - it("shouldn't allow handlebars to be passed as parameters", async () => { - const res = await request - .post(`/api/queries/${query._id}`) - .send({ - parameters: { - a: "{{ 'test' }}", - }, - }) - .set(config.defaultHeaders()) - .expect(400) - expect(res.body.message).toEqual( - "Parameter 'a' input contains a handlebars binding - this is not allowed." - ) - }) - }) - - describe("variables", () => { - async function preview(datasource: Datasource, fields: any) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: [], - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) - } - - it("should work with static variables", async () => { - const datasource = await config.restDatasource({ - staticVariables: { - variable: "google", - variable2: "1", - }, - }) - const responseBody = await preview(datasource, { - path: "www.{{ variable }}.com", - queryString: "test={{ variable2 }}", - }) - // these responses come from the mock - expect(responseBody.schema).toEqual({ - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - value: { type: "string", name: "value" }, - }) - expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1") - }) - - it("should work with dynamic variables", async () => { - const { datasource } = await config.dynamicVariableDatasource() - const responseBody = await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - expect(responseBody.schema).toEqual({ - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - value: { type: "string", name: "value" }, - }) - expect(responseBody.rows[0].url).toContain("doctype%20html") - }) - - it("check that it automatically retries on fail with cached dynamics", async () => { - const { datasource, query: base } = - await config.dynamicVariableDatasource() - // preview once to cache - await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - // check its in cache - const contents = await checkCacheForDynamicVariable( - base._id!, - "variable3" - ) - expect(contents.rows.length).toEqual(1) - const responseBody = await preview(datasource, { - path: "www.failonce.com", - queryString: "test={{ variable3 }}", - }) - expect(responseBody.schema).toEqual({ - fails: { type: "number", name: "fails" }, - opts: { type: "json", name: "opts" }, - url: { type: "string", name: "url" }, - }) - expect(responseBody.rows[0].fails).toEqual(1) - }) - - it("deletes variables when linked query is deleted", async () => { - const { datasource, query: base } = - await config.dynamicVariableDatasource() - // preview once to cache - await preview(datasource, { - path: "www.google.com", - queryString: "test={{ variable3 }}", - }) - // check its in cache - let contents = await checkCacheForDynamicVariable(base._id!, "variable3") - expect(contents.rows.length).toEqual(1) - - // delete the query - await request - .delete(`/api/queries/${base._id}/${base._rev}`) - .set(config.defaultHeaders()) - .expect(200) - - // check variables no longer in cache - contents = await checkCacheForDynamicVariable(base._id!, "variable3") - expect(contents).toBe(null) - }) - }) - - describe("Current User Request Mapping", () => { - async function previewGet( - datasource: Datasource, - fields: any, - params: QueryParameter[] - ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) - } - - async function previewPost( - datasource: Datasource, - fields: any, - params: QueryParameter[] - ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, - fields, - queryVerb: "create", - name: datasource.name!, - transformer: null, - schema: {}, - readable: false, - } - return await config.api.query.previewQuery(queryPreview) - } - - it("should parse global and query level header mappings", async () => { - const userDetails = config.getUserDetails() - - const datasource = await config.restDatasource({ - defaultHeaders: { - test: "headerVal", - emailHdr: "{{[user].[email]}}", - }, - }) - const responseBody = await previewGet( - datasource, - { - path: "www.google.com", - queryString: "email={{[user].[email]}}", - headers: { - queryHdr: "{{[user].[firstName]}}", - secondHdr: "1234", - }, - }, - [] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - expect(parsedRequest.opts.headers).toEqual({ - test: "headerVal", - emailHdr: userDetails.email, - queryHdr: userDetails.firstName, - secondHdr: "1234", - }) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?email=" + userDetails.email.replace("@", "%40") - ) - }) - - it("should bind the current user to query parameters", async () => { - const userDetails = config.getUserDetails() - - const datasource = await config.restDatasource() - - const responseBody = await previewGet( - datasource, - { - path: "www.google.com", - queryString: - "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", - }, - [ - { name: "myEmail", default: "{{[user].[email]}}" }, - { name: "myName", default: "{{[user].[firstName]}}" }, - { name: "testParam", default: "1234" }, - ] - ) - - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?test=" + - userDetails.email.replace("@", "%40") + - "&testName=" + - userDetails.firstName + - "&testParam=1234" - ) - }) - - it("should bind the current user the request body - plain text", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", - bodyType: "text", - }, - [{ name: "testParam", default: "1234" }] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - expect(parsedRequest.opts.body).toEqual( - `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234` - ) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - json", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "json", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}` - expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - xml", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - " {{[user].[email]}} {{testParam}} " + - "{{userId}} testing ", - bodyType: "xml", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userId", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - const test = ` ${userDetails.email} 1234 ${userDetails.firstName} testing ` - - expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - form-data", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "form", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - - const parsedRequest = JSON.parse(responseBody.extra.raw) - - const emailData = parsedRequest.opts.body._streams[1] - expect(emailData).toEqual(userDetails.email) - - const queryCodeData = parsedRequest.opts.body._streams[4] - expect(queryCodeData).toEqual("1234") - - const userRef = parsedRequest.opts.body._streams[7] - expect(userRef).toEqual(userDetails.firstName) - - expect(responseBody.rows[0].url).toEqual( - "http://www.google.com?testParam=1234" - ) - }) - - it("should bind the current user the request body - encoded", async () => { - const userDetails = config.getUserDetails() - const datasource = await config.restDatasource() - - const responseBody = await previewPost( - datasource, - { - path: "www.google.com", - queryString: "testParam={{testParam}}", - requestBody: - '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', - bodyType: "encoded", - }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] - ) - const parsedRequest = JSON.parse(responseBody.extra.raw) - - expect(parsedRequest.opts.body.email).toEqual(userDetails.email) - expect(parsedRequest.opts.body.queryCode).toEqual("1234") - expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName) - }) - }) -}) diff --git a/packages/server/src/automations/tests/executeQuery.spec.ts b/packages/server/src/automations/tests/executeQuery.spec.ts index 1d25a31b92..996e44af79 100644 --- a/packages/server/src/automations/tests/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/executeQuery.spec.ts @@ -1,6 +1,6 @@ import { Datasource, Query, SourceName } from "@budibase/types" import * as setup from "./utilities" -import { databaseTestProviders } from "../../integrations/tests/utils" +import { DatabaseName, getDatasource } from "../../integrations/tests/utils" import knex, { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" @@ -16,12 +16,14 @@ function getKnexClientName(source: SourceName) { throw new Error(`Unsupported source: ${source}`) } -describe.each([ - ["postgres", databaseTestProviders.postgres], - ["mysql", databaseTestProviders.mysql], - ["mssql", databaseTestProviders.mssql], - ["mariadb", databaseTestProviders.mariadb], -])("execute query action (%s)", (__, dsProvider) => { +describe.each( + [ + DatabaseName.POSTGRES, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + ].map(name => [name, getDatasource(name)]) +)("execute query action (%s)", (_, dsProvider) => { let tableName: string let client: Knex let datasource: Datasource @@ -31,7 +33,7 @@ describe.each([ beforeAll(async () => { await config.init() - const ds = await dsProvider.datasource() + const ds = await dsProvider datasource = await config.api.datasource.create(ds) client = knex({ client: getKnexClientName(ds.source), From 6d41b890db6b1053acf8952f043e4cdcb747a8b3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 16:09:12 +0100 Subject: [PATCH 08/19] Fix test that doesn't work if you run it with reuse containers. --- .../server/src/sdk/tests/attachments.spec.ts | 103 ++++++++---------- 1 file changed, 45 insertions(+), 58 deletions(-) diff --git a/packages/server/src/sdk/tests/attachments.spec.ts b/packages/server/src/sdk/tests/attachments.spec.ts index 0fd43ac5a8..c1736e6f8e 100644 --- a/packages/server/src/sdk/tests/attachments.spec.ts +++ b/packages/server/src/sdk/tests/attachments.spec.ts @@ -1,4 +1,12 @@ -import newid from "../../db/newid" +import TestConfig from "../../tests/utilities/TestConfiguration" +import { db as dbCore } from "@budibase/backend-core" +import sdk from "../index" +import { + FieldType, + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, +} from "@budibase/types" +import { FIND_LIMIT } from "../app/rows/attachments" const attachment = { size: 73479, @@ -8,69 +16,48 @@ const attachment = { key: "app_bbb/attachments/a.png", } -const row = { - _id: "ro_ta_aaa", - photo: [attachment], - otherCol: "string", -} - -const table = { - _id: "ta_aaa", - name: "photos", - schema: { - photo: { - type: "attachment", - name: "photo", - }, - otherCol: { - type: "string", - name: "otherCol", - }, - }, -} - -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - directCouchFind: jest.fn(), - }, - } -}) - -import { db as dbCore } from "@budibase/backend-core" -import sdk from "../index" - describe("should be able to re-write attachment URLs", () => { + const config = new TestConfig() + + beforeAll(async () => { + await config.init() + }) + it("should update URLs on a number of rows over the limit", async () => { - const db = dbCore.getDB("app_aaa") - await db.put(table) - const limit = 30 - let rows = [] - for (let i = 0; i < limit; i++) { - const rowToWrite = { - ...row, - _id: `${row._id}_${newid()}`, - } - const { rev } = await db.put(rowToWrite) - rows.push({ - ...rowToWrite, - _rev: rev, + const table = await config.api.table.save({ + name: "photos", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + photo: { + type: FieldType.ATTACHMENT, + name: "photo", + }, + otherCol: { + type: FieldType.STRING, + name: "otherCol", + }, + }, + }) + + for (let i = 0; i < FIND_LIMIT * 4; i++) { + await config.api.row.save(table._id!, { + photo: [attachment], + otherCol: "string", }) } - dbCore.directCouchFind - // @ts-ignore - .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" }) - .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" }) + const db = dbCore.getDB(config.getAppId()) await sdk.backups.updateAttachmentColumns(db.name, db) - const finalRows = await sdk.rows.getAllInternalRows(db.name) - for (let rowToCheck of finalRows) { - expect(rowToCheck.otherCol).toBe(row.otherCol) - expect(rowToCheck.photo[0].url).toBe("") - expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`) + + const rows = (await sdk.rows.getAllInternalRows(db.name)).filter( + row => row.tableId === table._id + ) + for (const row of rows) { + expect(row.otherCol).toBe("string") + expect(row.photo[0].url).toBe("") + expect(row.photo[0].key).toBe(`${db.name}/attachments/a.png`) } }) }) From afa757f21ac23c7f0e57ac9945464ce7b321c091 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 16:24:19 +0100 Subject: [PATCH 09/19] Remove catch-all error handler in preview endpoint to help debug tests. --- .../server/src/api/controllers/query/index.ts | 72 +++++++++---------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 8ccd3daec9..02a2bf25b0 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -281,48 +281,44 @@ export async function preview( return { previewSchema, nestedSchemaFields } } - try { - const inputs: QueryEvent = { - appId: ctx.appId, - queryVerb: query.queryVerb, - fields: query.fields, - parameters: enrichParameters(query), - transformer: query.transformer, - schema: query.schema, - nullDefaultSupport: query.nullDefaultSupport, - queryId, - datasource, - // have to pass down to the thread runner - can't put into context now - environmentVariables: envVars, - ctx: { - user: ctx.user, - auth: { ...authConfigCtx }, - }, - } + const inputs: QueryEvent = { + appId: ctx.appId, + queryVerb: query.queryVerb, + fields: query.fields, + parameters: enrichParameters(query), + transformer: query.transformer, + schema: query.schema, + nullDefaultSupport: query.nullDefaultSupport, + queryId, + datasource, + // have to pass down to the thread runner - can't put into context now + environmentVariables: envVars, + ctx: { + user: ctx.user, + auth: { ...authConfigCtx }, + }, + } - const { rows, keys, info, extra } = await Runner.run(inputs) - const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys) + const { rows, keys, info, extra } = await Runner.run(inputs) + const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys) - // if existing schema, update to include any previous schema keys - if (existingSchema) { - for (let key of Object.keys(previewSchema)) { - if (existingSchema[key]) { - previewSchema[key] = existingSchema[key] - } + // if existing schema, update to include any previous schema keys + if (existingSchema) { + for (let key of Object.keys(previewSchema)) { + if (existingSchema[key]) { + previewSchema[key] = existingSchema[key] } } - // remove configuration before sending event - delete datasource.config - await events.query.previewed(datasource, ctx.request.body) - ctx.body = { - rows, - nestedSchemaFields, - schema: previewSchema, - info, - extra, - } - } catch (err: any) { - ctx.throw(400, err) + } + // remove configuration before sending event + delete datasource.config + await events.query.previewed(datasource, ctx.request.body) + ctx.body = { + rows, + nestedSchemaFields, + schema: previewSchema, + info, + extra, } } From 94084db77d60e6fd121953a3d1087048e94229de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 16:36:01 +0100 Subject: [PATCH 10/19] Exclude isolated-vm.ts from coverage. --- packages/server/jest.config.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index 428ffd5761..d0c184248c 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -56,6 +56,10 @@ const config: Config.InitialOptions = { "!src/tests/**/*.{js,ts}", // The use of coverage in the JS runner bundles breaks tests "!src/jsRunner/bundles/**/*.{js,ts}", + // We have a polyfill for the TextDecoder class in here that gets + // injected into the vm for deserializing BSON. If it gets coveraged + // it breaks the tests. + "!src/jsRunner/vm/isolated-vm.ts", ], coverageReporters: ["lcov", "json", "clover"], } From 51af154215faf465bc1dc4f3f06dceefa757d0cf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 16:49:42 +0100 Subject: [PATCH 11/19] Add --coverage back to local tests for packages/server --- packages/server/scripts/test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/server/scripts/test.sh b/packages/server/scripts/test.sh index 688cfbc35f..4b456e4731 100644 --- a/packages/server/scripts/test.sh +++ b/packages/server/scripts/test.sh @@ -4,11 +4,9 @@ set -e if [[ -n $CI ]] then export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS" - echo "jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@" jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@ else # --maxWorkers performs better in development export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS" - echo "jest --coverage --maxWorkers=2 --forceExit $@" - jest --maxWorkers=2 --forceExit $@ + jest --coverage --maxWorkers=2 --forceExit $@ fi \ No newline at end of file From a8ae8391864fc4fe9be1f888af63429df520d03d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Apr 2024 17:03:10 +0100 Subject: [PATCH 12/19] Tidy up function rename. --- packages/server/src/threads/query.ts | 4 ++-- packages/server/src/threads/utils.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 97e7a05cf7..54322b1156 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -167,7 +167,7 @@ class QueryRunner { this.hasRerun = true } - await threadUtils.invalidateDynamicVariables(this.cachedVariables) + await threadUtils.invalidateCachedVariable(this.cachedVariables) return this.execute() } @@ -254,7 +254,7 @@ class QueryRunner { let { parameters } = this const queryId = variable.queryId, name = variable.name - let value = await threadUtils.checkCacheForDynamicVariable(queryId, name) + let value = await threadUtils.getCachedVariable(queryId, name) if (!value) { value = this.queryResponse[queryId] ? this.queryResponse[queryId] diff --git a/packages/server/src/threads/utils.ts b/packages/server/src/threads/utils.ts index 46752975a2..bf0d8f2231 100644 --- a/packages/server/src/threads/utils.ts +++ b/packages/server/src/threads/utils.ts @@ -85,7 +85,7 @@ export default { hasExtraData, formatResponse, storeDynamicVariable, - invalidateDynamicVariables: invalidateCachedVariable, - checkCacheForDynamicVariable: getCachedVariable, + invalidateCachedVariable, + getCachedVariable, threadSetup, } From b6431e57ffa8eb450b43007200871a2e5623f17b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Apr 2024 10:28:19 +0100 Subject: [PATCH 13/19] Attempt to make JS timeout tests more consistent. --- packages/server/src/api/routes/tests/row.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 92573f9652..1010a8ed55 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1295,7 +1295,7 @@ describe.each([ describe("Formula JS protection", () => { it("should time out JS execution if a single cell takes too long", async () => { - await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => { + await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => { const js = Buffer.from( ` let i = 0; @@ -1332,11 +1332,11 @@ describe.each([ }) }) - it("should time out JS execution if a multiple cells take too long", async () => { + it.only("should time out JS execution if a multiple cells take too long", async () => { await config.withEnv( { - JS_PER_INVOCATION_TIMEOUT_MS: 20, - JS_PER_REQUEST_TIMEOUT_MS: 40, + JS_PER_INVOCATION_TIMEOUT_MS: 40, + JS_PER_REQUEST_TIMEOUT_MS: 80, }, async () => { const js = Buffer.from( From a1a06b5d186623d765a14b643b3fbc5679bd48e8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Apr 2024 13:52:57 +0200 Subject: [PATCH 14/19] Fix import --- packages/server/src/utilities/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index 85dfdd3506..34113759ed 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -54,7 +54,7 @@ export function validate(rows: Rows, schema: TableSchema): ValidationResults { type: columnType, subtype: columnSubtype, autocolumn: isAutoColumn, - } = schema[columnName] + } = schema[columnName] || {} // If the column had an invalid value we don't want to override it if (results.schemaValidation[columnName] === false) { From 5a82de01dd31c2fde07172d154ed41a1234b4f12 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Apr 2024 13:54:45 +0200 Subject: [PATCH 15/19] Display "Attachment" on UI --- .../backend/TableNavigator/ExistingTableDataImport.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index eb1e7bc7ff..efbfd26565 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -49,7 +49,10 @@ label: "Long Form Text", value: FIELDS.LONGFORM.type, }, - + { + label: "Attachment", + value: FIELDS.ATTACHMENT.type, + }, { label: "User", value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`, From 60f782d33d5fd0bde71cfe9489656d7af544f8fb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Apr 2024 14:14:51 +0200 Subject: [PATCH 16/19] Update account-portal --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 011fa3c175..532c4db35c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 011fa3c175ae0a1bbbb0f6e1341ba0154bca5c76 +Subproject commit 532c4db35cecd346b5c24f0b89ab7b397a122a36 From d0a5ae6ae3b990bb1a2588a32143077b9a6a4dca Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Apr 2024 15:27:50 +0100 Subject: [PATCH 17/19] Respond to Mike's feedback. --- packages/server/jest.config.ts | 9 +++------ packages/server/package.json | 2 +- packages/server/src/api/controllers/query/index.ts | 9 ++++++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index d0c184248c..6341c8e5bd 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -54,12 +54,9 @@ const config: Config.InitialOptions = { "!src/db/views/staticViews.*", "!src/**/*.spec.{js,ts}", "!src/tests/**/*.{js,ts}", - // The use of coverage in the JS runner bundles breaks tests - "!src/jsRunner/bundles/**/*.{js,ts}", - // We have a polyfill for the TextDecoder class in here that gets - // injected into the vm for deserializing BSON. If it gets coveraged - // it breaks the tests. - "!src/jsRunner/vm/isolated-vm.ts", + // The use of coverage in the JS runner breaks tests by inserting + // coverage functions into code that will run inside of the isolate. + "!src/jsRunner/**/*.{js,ts}", ], coverageReporters: ["lcov", "json", "clover"], } diff --git a/packages/server/package.json b/packages/server/package.json index d138577090..4d1df4d734 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -143,7 +143,7 @@ "jest": "29.7.0", "jest-openapi": "0.14.2", "jest-runner": "29.7.0", - "nock": "^13.5.4", + "nock": "13.5.4", "nodemon": "2.0.15", "openapi-typescript": "5.2.0", "path-to-regexp": "6.2.0", diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 02a2bf25b0..b52cea553f 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -299,7 +299,14 @@ export async function preview( }, } - const { rows, keys, info, extra } = await Runner.run(inputs) + let queryResponse: QueryResponse + try { + queryResponse = await Runner.run(inputs) + } catch (err: any) { + ctx.throw(400, err) + } + + const { rows, keys, info, extra } = queryResponse const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys) // if existing schema, update to include any previous schema keys From d5a04d2569821a3ba84cfe33b68e7c32a74ea33a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Apr 2024 15:29:09 +0100 Subject: [PATCH 18/19] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index c51458d26f..532c4db35c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c51458d26f6bded1984ba9ac5d2948480ac809ce +Subproject commit 532c4db35cecd346b5c24f0b89ab7b397a122a36 From db6aec725b2d862b3d6151d1c686aeabefe09ed5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Apr 2024 15:32:05 +0100 Subject: [PATCH 19/19] Removed focused test. --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 1010a8ed55..f9e05c5bd8 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1332,7 +1332,7 @@ describe.each([ }) }) - it.only("should time out JS execution if a multiple cells take too long", async () => { + it("should time out JS execution if a multiple cells take too long", async () => { await config.withEnv( { JS_PER_INVOCATION_TIMEOUT_MS: 40,