From d4068609ca01f35ada171e7e3bc78b79846c8e8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 7 Jun 2024 14:03:23 +0100 Subject: [PATCH 1/4] Update CouchDB image from 3.2.1 to 3.3.3. --- charts/budibase/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 58bd889857..4e80be7322 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -641,7 +641,7 @@ couchdb: # @ignore repository: budibase/couchdb # @ignore - tag: v3.2.1 + tag: v3.3.3 # @ignore pullPolicy: Always From f2e3789ad2490b66b3598025e1cbcd8556cf6d29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 7 Jun 2024 14:24:39 +0100 Subject: [PATCH 2/4] Rename rename test from mysql.spec.ts, it's covered in table.spec.ts --- .../server/src/integration-test/mysql.spec.ts | 59 +------------------ 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 8cf4fb8212..04a2541ad6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -4,13 +4,7 @@ import { MakeRequestResponse, } from "../api/routes/public/tests/utils" import * as setup from "../api/routes/tests/utilities" -import { - Datasource, - FieldType, - Table, - TableRequest, - TableSourceType, -} from "@budibase/types" +import { Datasource, FieldType, Table, TableSourceType } from "@budibase/types" import { DatabaseName, getDatasource, @@ -231,57 +225,6 @@ describe("mysql integrations", () => { }) }) - describe("POST /api/tables/", () => { - it("will rename a column", async () => { - await makeRequest("post", "/api/tables/", primaryMySqlTable) - - let renameColumnOnTable: TableRequest = { - ...primaryMySqlTable, - schema: { - id: { - name: "id", - type: FieldType.AUTO, - autocolumn: true, - externalType: "unsigned integer", - }, - name: { - name: "name", - type: FieldType.STRING, - externalType: "text", - }, - description: { - name: "description", - type: FieldType.STRING, - externalType: "text", - }, - age: { - name: "age", - type: FieldType.NUMBER, - externalType: "float(8,2)", - }, - }, - } - - const response = await makeRequest( - "post", - "/api/tables/", - renameColumnOnTable - ) - - const ds = ( - await makeRequest("post", `/api/datasources/${datasource._id}/schema`) - ).body.datasource - - expect(response.status).toEqual(200) - expect(Object.keys(ds.entities![primaryMySqlTable.name].schema)).toEqual([ - "id", - "name", - "description", - "age", - ]) - }) - }) - describe("POST /api/datasources/:datasourceId/schema", () => { let tableName: string From c07d73beaf99ed8ea5d38f2a1a794ebf08b6e9f8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 7 Jun 2024 16:35:46 +0100 Subject: [PATCH 3/4] Modernise datasource.spec.ts. --- .../__snapshots__/datasource.spec.ts.snap | 91 --- .../src/api/routes/tests/datasource.spec.ts | 571 +++++++++--------- .../src/tests/utilities/api/datasource.ts | 4 + 3 files changed, 288 insertions(+), 378 deletions(-) delete mode 100644 packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap diff --git a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap deleted file mode 100644 index 57d79db24b..0000000000 --- a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap +++ /dev/null @@ -1,91 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`/datasources fetch returns all the datasources from the server 1`] = ` -[ - { - "config": {}, - "entities": [ - { - "_id": "ta_users", - "_rev": "1-73b7912e6cbdd3d696febc60f3715844", - "createdAt": "2020-01-01T00:00:00.000Z", - "name": "Users", - "primaryDisplay": "email", - "schema": { - "email": { - "constraints": { - "email": true, - "length": { - "maximum": "", - }, - "presence": true, - "type": "string", - }, - "name": "email", - "type": "string", - }, - "firstName": { - "constraints": { - "presence": false, - "type": "string", - }, - "name": "firstName", - "type": "string", - }, - "lastName": { - "constraints": { - "presence": false, - "type": "string", - }, - "name": "lastName", - "type": "string", - }, - "roleId": { - "constraints": { - "inclusion": [ - "ADMIN", - "POWER", - "BASIC", - "PUBLIC", - ], - "presence": false, - "type": "string", - }, - "name": "roleId", - "type": "options", - }, - "status": { - "constraints": { - "inclusion": [ - "active", - "inactive", - ], - "presence": false, - "type": "string", - }, - "name": "status", - "type": "options", - }, - }, - "sourceId": "bb_internal", - "sourceType": "internal", - "type": "table", - "updatedAt": "2020-01-01T00:00:00.000Z", - "views": {}, - }, - ], - "name": "Budibase DB", - "source": "BUDIBASE", - "type": "budibase", - }, - { - "config": {}, - "createdAt": "2020-01-01T00:00:00.000Z", - "isSQL": true, - "name": "Test", - "source": "POSTGRES", - "type": "datasource", - "updatedAt": "2020-01-01T00:00:00.000Z", - }, -] -`; diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index f2cea90675..349a2be9fd 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -4,14 +4,12 @@ import { getCachedVariable } from "../../../threads/utils" import { context, events } from "@budibase/backend-core" import sdk from "../../../sdk" -import tk from "timekeeper" -import { mocks } from "@budibase/backend-core/tests" +import { generator } from "@budibase/backend-core/tests" import { Datasource, FieldSchema, BBReferenceFieldSubType, FieldType, - QueryPreview, RelationshipType, SourceName, Table, @@ -21,36 +19,34 @@ import { import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" -tk.freeze(mocks.date.MOCK_DATE) - -let { basicDatasource } = setup.structures - describe("/datasources", () => { - let request = setup.getRequest() - let config = setup.getConfig() - let datasource: any + const config = setup.getConfig() + let datasource: Datasource + beforeAll(async () => { + await config.init() + }) afterAll(setup.afterAll) - async function setupTest() { - await config.init() - datasource = await config.createDatasource() + beforeEach(async () => { + datasource = await config.api.datasource.create({ + type: "datasource", + name: "Test", + source: SourceName.POSTGRES, + config: {}, + }) jest.clearAllMocks() - } - - beforeAll(setupTest) + }) describe("create", () => { it("should create a new datasource", async () => { - const res = await request - .post(`/api/datasources`) - .send(basicDatasource()) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - expect(res.body.datasource.name).toEqual("Test") - expect(res.body.errors).toEqual({}) + const ds = await config.api.datasource.create({ + type: "datasource", + name: "Test", + source: SourceName.POSTGRES, + config: {}, + }) + expect(ds.name).toEqual("Test") expect(events.datasource.created).toHaveBeenCalledTimes(1) }) @@ -72,88 +68,71 @@ describe("/datasources", () => { }) }) - describe("update", () => { - it("should update an existing datasource", async () => { - datasource.name = "Updated Test" - const res = await request - .put(`/api/datasources/${datasource._id}`) - .send(datasource) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) + describe("dynamic variables", () => { + it("should invalidate changed or removed variables", async () => { + let datasource = await config.api.datasource.create({ + type: "datasource", + name: "Rest", + source: SourceName.REST, + config: {}, + }) - expect(res.body.datasource.name).toEqual("Updated Test") - expect(res.body.errors).toBeUndefined() - expect(events.datasource.updated).toHaveBeenCalledTimes(1) - }) + const query = await config.api.query.save({ + datasourceId: datasource._id!, + fields: { + path: "www.google.com", + }, + parameters: [], + transformer: null, + queryVerb: "read", + name: datasource.name!, + schema: {}, + readable: true, + }) - describe("dynamic variables", () => { - async function preview( - datasource: any, - fields: { path: string; queryString: string } - ) { - const queryPreview: QueryPreview = { - fields, - datasourceId: datasource._id, - parameters: [], - transformer: null, - queryVerb: "read", - name: datasource.name, - schema: {}, - readable: true, - } - return config.api.query.preview(queryPreview) - } + datasource = await config.api.datasource.update({ + ...datasource, + config: { + dynamicVariables: [ + { + queryId: query._id, + name: "variable3", + value: "{{ data.0.[value] }}", + }, + ], + }, + }) - it("should invalidate changed or removed variables", async () => { - const { datasource, query } = await config.dynamicVariableDatasource() - // preview once to cache variables - await preview(datasource, { + // preview once to cache variables + await config.api.query.preview({ + fields: { path: "www.example.com", queryString: "test={{ variable3 }}", - }) - // check variables in cache - let contents = await getCachedVariable(query._id!, "variable3") - expect(contents.rows.length).toEqual(1) - - // update the datasource to remove the variables - datasource.config!.dynamicVariables = [] - const res = await request - .put(`/api/datasources/${datasource._id}`) - .send(datasource) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body.errors).toBeUndefined() - - // check variables no longer in cache - contents = await getCachedVariable(query._id!, "variable3") - expect(contents).toBe(null) + }, + datasourceId: datasource._id!, + parameters: [], + transformer: null, + queryVerb: "read", + name: datasource.name!, + schema: {}, + readable: true, }) + + // check variables in cache + let contents = await getCachedVariable(query._id!, "variable3") + expect(contents.rows.length).toEqual(1) + + // update the datasource to remove the variables + datasource.config!.dynamicVariables = [] + await config.api.datasource.update(datasource) + + // check variables no longer in cache + contents = await getCachedVariable(query._id!, "variable3") + expect(contents).toBe(null) }) }) - describe("fetch", () => { - beforeAll(setupTest) - - it("returns all the datasources from the server", async () => { - const res = await request - .get(`/api/datasources`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - const datasources = res.body - - // remove non-deterministic fields - for (let source of datasources) { - delete source._id - delete source._rev - } - - expect(datasources).toMatchSnapshot() - }) - + describe("permissions", () => { it("should apply authorization to endpoint", async () => { await checkBuilderEndpoint({ config, @@ -161,41 +140,8 @@ describe("/datasources", () => { url: `/api/datasources`, }) }) - }) - describe("find", () => { - it("should be able to find a datasource", async () => { - const res = await request - .get(`/api/datasources/${datasource._id}`) - .set(config.defaultHeaders()) - .expect(200) - expect(res.body._rev).toBeDefined() - expect(res.body._id).toEqual(datasource._id) - }) - }) - - describe("destroy", () => { - beforeAll(setupTest) - - it("deletes queries for the datasource after deletion and returns a success message", async () => { - await config.createQuery() - - await request - .delete(`/api/datasources/${datasource._id}/${datasource._rev}`) - .set(config.defaultHeaders()) - .expect(200) - - const res = await request - .get(`/api/datasources`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - expect(res.body.length).toEqual(1) - expect(events.datasource.deleted).toHaveBeenCalledTimes(1) - }) - - it("should apply authorization to endpoint", async () => { + it("should apply authorization to delete endpoint", async () => { await checkBuilderEndpoint({ config, method: "DELETE", @@ -204,175 +150,226 @@ describe("/datasources", () => { }) }) - describe("check secret replacement", () => { - async function makeDatasource() { - datasource = basicDatasource() - datasource.datasource.config.password = "testing" - const res = await request - .post(`/api/datasources`) - .send(datasource) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return res.body.datasource - } - - it("should save a datasource with password", async () => { - const datasource = await makeDatasource() - expect(datasource.config.password).toBe("--secret-value--") - }) - - it("should not the password on update with the --secret-value--", async () => { - const datasource = await makeDatasource() - await request - .put(`/api/datasources/${datasource._id}`) - .send(datasource) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - await context.doInAppContext(config.getAppId(), async () => { - const dbDatasource: any = await sdk.datasources.get(datasource._id) - expect(dbDatasource.config.password).toBe("testing") - }) - }) - }) - describe.each([ [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - ])("fetch schema (%s)", (_, dsProvider) => { - beforeAll(async () => { + ])("%s", (_, dsProvider) => { + beforeEach(async () => { datasource = await config.api.datasource.create(await dsProvider) }) - it("fetching schema will not drop tables or columns", async () => { - const datasourceId = datasource!._id! + describe("get", () => { + it("should be able to get a datasource", async () => { + const ds = await config.api.datasource.get(datasource._id!) + expect(ds._id).toEqual(datasource._id) + expect(ds._rev).toBeDefined() + }) - const simpleTable = await config.api.table.save( - tableForDatasource(datasource, { - name: "simple", - schema: { - name: { - name: "name", - type: FieldType.STRING, + it("should not return database password", async () => { + const ds = await config.api.datasource.get(datasource._id!) + expect(ds.config!.password).toBe("--secret-value--") + }) + }) + + describe("list", () => { + it("returns all the datasources", async () => { + const datasources = await config.api.datasource.fetch() + expect(datasources).toContainEqual(expect.objectContaining(datasource)) + }) + }) + + describe("put", () => { + it("should update an existing datasource", async () => { + const newName = generator.guid() + datasource.name = newName + const updatedDs = await config.api.datasource.update(datasource) + expect(updatedDs.name).toEqual(newName) + expect(events.datasource.updated).toHaveBeenCalledTimes(1) + }) + + it("should not overwrite database password with --secret-value--", async () => { + const password = await context.doInAppContext( + config.getAppId(), + async () => { + const ds = await sdk.datasources.get(datasource._id!) + return ds.config!.password + } + ) + + expect(password).not.toBe("--secret-value--") + + const ds = await config.api.datasource.get(datasource._id!) + expect(ds.config!.password).toBe("--secret-value--") + + await config.api.datasource.update( + await config.api.datasource.get(datasource._id!) + ) + + const newPassword = await context.doInAppContext( + config.getAppId(), + async () => { + const ds = await sdk.datasources.get(datasource._id!) + return ds.config!.password + } + ) + + expect(newPassword).not.toBe("--secret-value--") + expect(newPassword).toBe(password) + }) + }) + + describe("destroy", () => { + it("deletes queries for the datasource after deletion and returns a success message", async () => { + await config.api.query.save({ + datasourceId: datasource._id!, + name: "Test Query", + parameters: [], + fields: {}, + schema: {}, + queryVerb: "read", + transformer: null, + readable: true, + }) + + await config.api.datasource.delete(datasource) + const datasources = await config.api.datasource.fetch() + expect(datasources).not.toContainEqual( + expect.objectContaining(datasource) + ) + expect(events.datasource.deleted).toHaveBeenCalledTimes(1) + }) + }) + + describe("schema", () => { + it("fetching schema will not drop tables or columns", async () => { + const datasourceId = datasource!._id! + + const simpleTable = await config.api.table.save( + tableForDatasource(datasource, { + name: "simple", + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const fullSchema: { + [type in SupportedSqlTypes]: FieldSchema & { type: type } + } = { + [FieldType.STRING]: { + name: "string", + type: FieldType.STRING, + constraints: { + presence: true, }, }, - }) - ) - - const fullSchema: { - [type in SupportedSqlTypes]: FieldSchema & { type: type } - } = { - [FieldType.STRING]: { - name: "string", - type: FieldType.STRING, - constraints: { - presence: true, + [FieldType.LONGFORM]: { + name: "longform", + type: FieldType.LONGFORM, }, - }, - [FieldType.LONGFORM]: { - name: "longform", - type: FieldType.LONGFORM, - }, - [FieldType.OPTIONS]: { - name: "options", - type: FieldType.OPTIONS, - constraints: { - presence: { allowEmpty: false }, - }, - }, - [FieldType.NUMBER]: { - name: "number", - type: FieldType.NUMBER, - }, - [FieldType.BOOLEAN]: { - name: "boolean", - type: FieldType.BOOLEAN, - }, - [FieldType.ARRAY]: { - name: "array", - type: FieldType.ARRAY, - }, - [FieldType.DATETIME]: { - name: "datetime", - type: FieldType.DATETIME, - dateOnly: true, - timeOnly: false, - }, - [FieldType.LINK]: { - name: "link", - type: FieldType.LINK, - tableId: simpleTable._id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "link", - }, - [FieldType.FORMULA]: { - name: "formula", - type: FieldType.FORMULA, - formula: "any formula", - }, - [FieldType.BARCODEQR]: { - name: "barcodeqr", - type: FieldType.BARCODEQR, - }, - [FieldType.BIGINT]: { - name: "bigint", - type: FieldType.BIGINT, - }, - [FieldType.BB_REFERENCE]: { - name: "bb_reference", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, - [FieldType.BB_REFERENCE_SINGLE]: { - name: "bb_reference_single", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - } - - await config.api.table.save( - tableForDatasource(datasource, { - name: "full", - schema: fullSchema, - }) - ) - - const persisted = await config.api.datasource.get(datasourceId) - await config.api.datasource.fetchSchema(datasourceId) - - const updated = await config.api.datasource.get(datasourceId) - const expected: Datasource = { - ...persisted, - entities: - persisted?.entities && - Object.entries(persisted.entities).reduce>( - (acc, [tableName, table]) => { - acc[tableName] = { - ...table, - primaryDisplay: expect.not.stringMatching( - new RegExp(`^${table.primaryDisplay || ""}$`) - ), - schema: Object.entries(table.schema).reduce( - (acc, [fieldName, field]) => { - acc[fieldName] = expect.objectContaining({ - ...field, - }) - return acc - }, - {} - ), - } - return acc + [FieldType.OPTIONS]: { + name: "options", + type: FieldType.OPTIONS, + constraints: { + presence: { allowEmpty: false }, }, - {} - ), + }, + [FieldType.NUMBER]: { + name: "number", + type: FieldType.NUMBER, + }, + [FieldType.BOOLEAN]: { + name: "boolean", + type: FieldType.BOOLEAN, + }, + [FieldType.ARRAY]: { + name: "array", + type: FieldType.ARRAY, + }, + [FieldType.DATETIME]: { + name: "datetime", + type: FieldType.DATETIME, + dateOnly: true, + timeOnly: false, + }, + [FieldType.LINK]: { + name: "link", + type: FieldType.LINK, + tableId: simpleTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + }, + [FieldType.FORMULA]: { + name: "formula", + type: FieldType.FORMULA, + formula: "any formula", + }, + [FieldType.BARCODEQR]: { + name: "barcodeqr", + type: FieldType.BARCODEQR, + }, + [FieldType.BIGINT]: { + name: "bigint", + type: FieldType.BIGINT, + }, + [FieldType.BB_REFERENCE]: { + name: "bb_reference", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + [FieldType.BB_REFERENCE_SINGLE]: { + name: "bb_reference_single", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + } - _rev: expect.any(String), - } - expect(updated).toEqual(expected) + await config.api.table.save( + tableForDatasource(datasource, { + name: "full", + schema: fullSchema, + }) + ) + + const persisted = await config.api.datasource.get(datasourceId) + await config.api.datasource.fetchSchema(datasourceId) + + const updated = await config.api.datasource.get(datasourceId) + const expected: Datasource = { + ...persisted, + entities: + persisted?.entities && + Object.entries(persisted.entities).reduce>( + (acc, [tableName, table]) => { + acc[tableName] = { + ...table, + primaryDisplay: expect.not.stringMatching( + new RegExp(`^${table.primaryDisplay || ""}$`) + ), + schema: Object.entries(table.schema).reduce( + (acc, [fieldName, field]) => { + acc[fieldName] = expect.objectContaining({ + ...field, + }) + return acc + }, + {} + ), + } + return acc + }, + {} + ), + + _rev: expect.any(String), + } + expect(updated).toEqual(expected) + }) }) }) }) diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index bb4c74093c..780a7fc7a3 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -61,6 +61,10 @@ export class DatasourceAPI extends TestAPI { }) } + fetch = async (expectations?: Expectations) => { + return await this._get(`/api/datasources`, { expectations }) + } + query = async ( query: Omit & Partial>, expectations?: Expectations From d68232037158d49834391b9d4376a8fd2b4c757b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 7 Jun 2024 16:40:25 +0100 Subject: [PATCH 4/4] Fix last test. --- packages/server/src/api/routes/tests/datasource.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 349a2be9fd..2d433edb4f 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -367,6 +367,7 @@ describe("/datasources", () => { ), _rev: expect.any(String), + updatedAt: expect.any(String), } expect(updated).toEqual(expected) })