From 3aec3df36df5d74e78befe368dc466bf7106b271 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 23 Dec 2022 16:03:02 +0000 Subject: [PATCH 01/31] Improvements on apps and tables --- .../TestConfiguration/applications.ts | 23 +++++++ .../internal-api/TestConfiguration/rows.ts | 20 +++++- .../src/config/internal-api/fixtures/rows.ts | 24 ++++++++ .../applications/applications.spec.ts | 9 +++ .../tests/internal-api/tables/tables.spec.ts | 61 ++++++++++++++++++- 5 files changed, 134 insertions(+), 3 deletions(-) diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 1cfd025974..1cb9033984 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -153,4 +153,27 @@ export default class AppApi { expect(response).toHaveStatusCode(204) return [response] } + + async unlock(appId: string): Promise<[Response, responseMessage]> { + const response = await this.api.del(`/dev/${appId}/lock`) + const json = await response.json() + expect(response).toHaveStatusCode(200) + expect(json.message).toEqual("Lock released successfully.") + return [response, json] + } + + async updateIcon(appId: string): Promise<[Response, Application]> { + const body = { + icon: { + name: "ConversionFunnel", + color: "var(--spectrum-global-color-red-400)" + } + } + const response = await this.api.put(`/applications/${appId}`, { body }) + const json = await response.json() + expect(response).toHaveStatusCode(200) + expect(json.icon.name).toEqual(body.icon.name) + expect(json.icon.color).toEqual(body.icon.color) + return [response, json] + } } diff --git a/qa-core/src/config/internal-api/TestConfiguration/rows.ts b/qa-core/src/config/internal-api/TestConfiguration/rows.ts index 39ba01f467..435be85d56 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/rows.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/rows.ts @@ -15,7 +15,7 @@ export default class RowsApi { const json = await response.json() if (this.rowAdded) { expect(response).toHaveStatusCode(200) - expect(json.length).toEqual(1) + expect(json.length).toBeGreaterThanOrEqual(1) } return [response, json] } @@ -36,4 +36,22 @@ export default class RowsApi { expect(response).toHaveStatusCode(200) return [response, json] } + + async searchSinglePage(tableId: string, body: any): Promise<[Response, Row[]]> { + const response = await this.api.post(`/${tableId}/search`, { body }) + const json = await response.json() + expect(response).toHaveStatusCode(200) + expect(json.rows.length).toBeLessThanOrEqual(9) + expect(json.hasNextPage).toEqual(false) + return [response, json.rows] + } + + async searchMultiPage(tableId: string, body: any): Promise<[Response, Row[]]> { + const response = await this.api.post(`/${tableId}/search`, { body }) + const json = await response.json() + expect(response).toHaveStatusCode(200) + expect(json.hasNextPage).toEqual(true) + expect(json.rows.length).toEqual(10) + return [response, json.rows] + } } diff --git a/qa-core/src/config/internal-api/fixtures/rows.ts b/qa-core/src/config/internal-api/fixtures/rows.ts index 90f6350dcf..fbb965c215 100644 --- a/qa-core/src/config/internal-api/fixtures/rows.ts +++ b/qa-core/src/config/internal-api/fixtures/rows.ts @@ -6,3 +6,27 @@ export const generateNewRowForTable = (tableId: string): Row => { tableId: tableId, } } + +export const searchBody = (primaryDisplay: string): any => { + return { + bookmark: null, + limit: 10, + paginate: true, + query: { + contains: {}, + containsAny: {}, + empty: {}, + equal: {}, + fuzzy: {}, + notContains: {}, + notEmpty: {}, + notEqual: {}, + oneOf: {}, + range: {}, + string: {}, + }, + sort: primaryDisplay, + sortOrder: "ascending", + sortType: "string" + } +} \ No newline at end of file diff --git a/qa-core/src/tests/internal-api/applications/applications.spec.ts b/qa-core/src/tests/internal-api/applications/applications.spec.ts index 7d889b7e87..4edcd34ca4 100644 --- a/qa-core/src/tests/internal-api/applications/applications.spec.ts +++ b/qa-core/src/tests/internal-api/applications/applications.spec.ts @@ -104,6 +104,14 @@ describe("Internal API - Application creation, update, publish and delete", () = }) }) + it("Update the icon and color of an application", async () => { + const app = await config.applications.create(generateApp()) + + config.applications.api.appId = app.appId + + await config.applications.updateIcon(app.appId) + }) + it("POST - Revert Changes without changes", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId @@ -124,6 +132,7 @@ describe("Internal API - Application creation, update, publish and delete", () = // // Revert the app to published state await config.applications.revertPublished(app.appId) + await config.applications.unlock(app.appId) // Check screen is removed await config.applications.getRoutes() }) diff --git a/qa-core/src/tests/internal-api/tables/tables.spec.ts b/qa-core/src/tests/internal-api/tables/tables.spec.ts index 6b2d2240e5..41acf42f92 100644 --- a/qa-core/src/tests/internal-api/tables/tables.spec.ts +++ b/qa-core/src/tests/internal-api/tables/tables.spec.ts @@ -6,9 +6,9 @@ import { generateTable, generateNewColumnForTable, } from "../../../config/internal-api/fixtures/table" -import { generateNewRowForTable } from "../../../config/internal-api/fixtures/rows" +import { generateNewRowForTable, searchBody } from "../../../config/internal-api/fixtures/rows" -describe("Internal API - Application creation, update, publish and delete", () => { +describe("Internal API - Table Operations", () => { const api = new InternalAPIClient() const config = new TestConfiguration(api) @@ -86,4 +86,61 @@ describe("Internal API - Application creation, update, publish and delete", () = //Table was deleted await config.tables.getAll(2) }) + + it("Search and pagination", async () => { + // create the app + const appName = generator.word() + const app = await createAppFromTemplate() + config.applications.api.appId = app.appId + + // Get current tables: expect 2 in this template + await config.tables.getAll(2) + + // Add new table + const [createdTableResponse, createdTableData] = await config.tables.save( + generateTable() + ) + + //Table was added + await config.tables.getAll(3) + + //Get information about the table + await config.tables.getTableById(createdTableData._id) + + //Add Column to table + const newColumn = generateNewColumnForTable(createdTableData) + const [addColumnResponse, addColumnData] = await config.tables.save( + newColumn, + true + ) + + //Add Row to table + let newRow = generateNewRowForTable(addColumnData._id) + await config.rows.add(addColumnData._id, newRow) + + //Search single row + await config.rows.searchSinglePage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + + //Add 10 more rows + for (let i = 0; i < 10; i++) { + let newRow = generateNewRowForTable(addColumnData._id) + await config.rows.add(addColumnData._id, newRow) + } + + //Search multiple rows + const [allRowsResponse, allRowsJson] = await config.rows.searchMultiPage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + + //Delete Rows from table + const rowToDelete = { + rows: [allRowsJson], + } + const [deleteRowResponse, deleteRowData] = await config.rows.delete( + addColumnData._id, + rowToDelete + ) + + //Search single row + await config.rows.searchSinglePage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + + }) }) From 125a06517d3f790fd4a96261a6d177272fab7280 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 28 Dec 2022 15:46:01 +0000 Subject: [PATCH 02/31] "Edit multiple rows" --- qa-core/src/tests/internal-api/tables/tables.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa-core/src/tests/internal-api/tables/tables.spec.ts b/qa-core/src/tests/internal-api/tables/tables.spec.ts index 41acf42f92..f93ec0438b 100644 --- a/qa-core/src/tests/internal-api/tables/tables.spec.ts +++ b/qa-core/src/tests/internal-api/tables/tables.spec.ts @@ -127,7 +127,7 @@ describe("Internal API - Table Operations", () => { await config.rows.add(addColumnData._id, newRow) } - //Search multiple rows + //Search rows with pagination const [allRowsResponse, allRowsJson] = await config.rows.searchMultiPage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) //Delete Rows from table @@ -135,7 +135,7 @@ describe("Internal API - Table Operations", () => { rows: [allRowsJson], } const [deleteRowResponse, deleteRowData] = await config.rows.delete( - addColumnData._id, + createdTableData._id, rowToDelete ) From 3fc6dd62f7f8a7a8de4940c39c6338f38360f8f4 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 2 Jan 2023 10:06:05 +0000 Subject: [PATCH 03/31] Add test for table pagination --- qa-core/src/config/internal-api/TestConfiguration/rows.ts | 5 ++--- qa-core/src/tests/internal-api/tables/tables.spec.ts | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/qa-core/src/config/internal-api/TestConfiguration/rows.ts b/qa-core/src/config/internal-api/TestConfiguration/rows.ts index 435be85d56..3514dd1b35 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/rows.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/rows.ts @@ -37,16 +37,15 @@ export default class RowsApi { return [response, json] } - async searchSinglePage(tableId: string, body: any): Promise<[Response, Row[]]> { + async searchNoPagination(tableId: string, body: any): Promise<[Response, Row[]]> { const response = await this.api.post(`/${tableId}/search`, { body }) const json = await response.json() expect(response).toHaveStatusCode(200) - expect(json.rows.length).toBeLessThanOrEqual(9) expect(json.hasNextPage).toEqual(false) return [response, json.rows] } - async searchMultiPage(tableId: string, body: any): Promise<[Response, Row[]]> { + async searchWithPagination(tableId: string, body: any): Promise<[Response, Row[]]> { const response = await this.api.post(`/${tableId}/search`, { body }) const json = await response.json() expect(response).toHaveStatusCode(200) diff --git a/qa-core/src/tests/internal-api/tables/tables.spec.ts b/qa-core/src/tests/internal-api/tables/tables.spec.ts index f93ec0438b..c6dd4ebf3d 100644 --- a/qa-core/src/tests/internal-api/tables/tables.spec.ts +++ b/qa-core/src/tests/internal-api/tables/tables.spec.ts @@ -31,7 +31,7 @@ describe("Internal API - Table Operations", () => { }) } - it("Operations on Tables", async () => { + it("Create and delete table, columns and rows", async () => { // create the app const appName = generator.word() const app = await createAppFromTemplate() @@ -119,7 +119,7 @@ describe("Internal API - Table Operations", () => { await config.rows.add(addColumnData._id, newRow) //Search single row - await config.rows.searchSinglePage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + await config.rows.searchNoPagination(createdTableData._id, searchBody(createdTableData.primaryDisplay)) //Add 10 more rows for (let i = 0; i < 10; i++) { @@ -128,7 +128,7 @@ describe("Internal API - Table Operations", () => { } //Search rows with pagination - const [allRowsResponse, allRowsJson] = await config.rows.searchMultiPage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + const [allRowsResponse, allRowsJson] = await config.rows.searchWithPagination(createdTableData._id, searchBody(createdTableData.primaryDisplay)) //Delete Rows from table const rowToDelete = { @@ -140,7 +140,7 @@ describe("Internal API - Table Operations", () => { ) //Search single row - await config.rows.searchSinglePage(createdTableData._id, searchBody(createdTableData.primaryDisplay)) + await config.rows.searchWithPagination(createdTableData._id, searchBody(createdTableData.primaryDisplay)) }) }) From 41c51cb8340e601c1e7cbcbbb4aeb8a0e470ecc8 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 2 Jan 2023 10:09:55 +0000 Subject: [PATCH 04/31] Improve naming and comments --- .../TestConfiguration/applications.ts | 2 +- .../applications/applications.spec.ts | 14 +++--- .../internal-api/screens/screens.spec.ts | 6 +-- .../userManagement/appSpecificRoles.spec.ts | 49 +++++++++++++++---- .../userManagement/userManagement.spec.ts | 5 ++ 5 files changed, 55 insertions(+), 21 deletions(-) diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 1cb9033984..a874ef0cb8 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -117,7 +117,7 @@ export default class AppApi { return [response, json] } - async update( + async rename( appId: string, oldName: string, body: any diff --git a/qa-core/src/tests/internal-api/applications/applications.spec.ts b/qa-core/src/tests/internal-api/applications/applications.spec.ts index 4edcd34ca4..e0e07e3fba 100644 --- a/qa-core/src/tests/internal-api/applications/applications.spec.ts +++ b/qa-core/src/tests/internal-api/applications/applications.spec.ts @@ -67,7 +67,7 @@ describe("Internal API - Application creation, update, publish and delete", () = await config.applications.unpublish(app.appId) }) - it("POST - Sync application before deployment", async () => { + it("Sync application before deployment", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId @@ -79,7 +79,7 @@ describe("Internal API - Application creation, update, publish and delete", () = }) }) - it("POST - Sync application after deployment", async () => { + it("Sync application after deployment", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId @@ -94,12 +94,12 @@ describe("Internal API - Application creation, update, publish and delete", () = }) }) - it("PUT - Update an application", async () => { + it("Rename an application", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId - await config.applications.update(app.appId, app.name, { + await config.applications.rename(app.appId, app.name, { name: generator.word(), }) }) @@ -112,14 +112,14 @@ describe("Internal API - Application creation, update, publish and delete", () = await config.applications.updateIcon(app.appId) }) - it("POST - Revert Changes without changes", async () => { + it("Revert Changes without changes", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId await config.applications.revertUnpublished(app.appId) }) - it("POST - Revert Changes", async () => { + it("Revert Changes", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId @@ -137,7 +137,7 @@ describe("Internal API - Application creation, update, publish and delete", () = await config.applications.getRoutes() }) - it("DELETE - Delete an application", async () => { + it("Delete an application", async () => { const app = await config.applications.create(generateApp()) await config.applications.delete(app.appId) diff --git a/qa-core/src/tests/internal-api/screens/screens.spec.ts b/qa-core/src/tests/internal-api/screens/screens.spec.ts index 1d2a21a8c7..2e4292afa8 100644 --- a/qa-core/src/tests/internal-api/screens/screens.spec.ts +++ b/qa-core/src/tests/internal-api/screens/screens.spec.ts @@ -18,7 +18,7 @@ describe("Internal API - /screens endpoints", () => { await config.afterAll() }) - it("POST - Create a screen with each role type", async () => { + it("Create a screen with each role type", async () => { // Create app const app = await appConfig.applications.create(generateApp()) @@ -32,7 +32,7 @@ describe("Internal API - /screens endpoints", () => { } }) - it("GET - Fetch screens", async () => { + it("Get screens", async () => { // Create app const app = await appConfig.applications.create(generateApp()) @@ -44,7 +44,7 @@ describe("Internal API - /screens endpoints", () => { await appConfig.applications.getRoutes(true) }) - it("DELETE - Delete a screen", async () => { + it("Delete a screen", async () => { // Create app const app = await appConfig.applications.create(generateApp()) diff --git a/qa-core/src/tests/internal-api/userManagement/appSpecificRoles.spec.ts b/qa-core/src/tests/internal-api/userManagement/appSpecificRoles.spec.ts index 2447a31558..fe985094fd 100644 --- a/qa-core/src/tests/internal-api/userManagement/appSpecificRoles.spec.ts +++ b/qa-core/src/tests/internal-api/userManagement/appSpecificRoles.spec.ts @@ -22,15 +22,21 @@ describe("Internal API - App Specific Roles & Permissions", () => { }) it("Add BASIC user to app", async () => { + // Create a user with BASIC role and check if it was created successfully const appUser = generateUser() expect(appUser[0].builder?.global).toEqual(false) expect(appUser[0].admin?.global).toEqual(false) + + // Add the user to the tenant. const [createUserResponse, createUserJson] = await config.users.addMultiple(appUser) const app = await config.applications.create(appFromTemplate()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -39,6 +45,7 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("BASIC") @@ -46,18 +53,19 @@ describe("Internal API - App Specific Roles & Permissions", () => { }) it("Add ADMIN user to app", async () => { + // Create a user with ADMIN role and check if it was created successfully const adminUser = generateUser(1, "admin") expect(adminUser[0].builder?.global).toEqual(true) expect(adminUser[0].admin?.global).toEqual(true) const [createUserResponse, createUserJson] = await config.users.addMultiple(adminUser) - //const app = await config.applications.create(generateApp()) - //config.applications.api.appId = app.appId - const app = await config.applications.create(appFromTemplate()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -66,28 +74,26 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("ADMIN") - // publish app - await config.applications.publish(app.appId) - // check published app renders - config.applications.api.appId = db.getProdAppID(app.appId!) - await config.applications.canRender() - }) it("Add POWER user to app", async () => { + // Create a user with POWER role and check if it was created successfully const powerUser = generateUser(1, 'developer') expect(powerUser[0].builder?.global).toEqual(true) - const [createUserResponse, createUserJson] = await config.users.addMultiple(powerUser) const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -96,6 +102,7 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("POWER") @@ -104,6 +111,7 @@ describe("Internal API - App Specific Roles & Permissions", () => { describe("Check Access for default roles", () => { it("Check Table access for app user", async () => { + // Create a user with BASIC role and check if it was created successfully const appUser = generateUser() expect(appUser[0].builder?.global).toEqual(false) expect(appUser[0].admin?.global).toEqual(false) @@ -112,7 +120,10 @@ describe("Internal API - App Specific Roles & Permissions", () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -121,13 +132,17 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("BASIC") + // Create a table const [createdTableResponse, createdTableData] = await config.tables.save( generateTable() ) + + // Login with the user created and try to create a column await config.login(appUser[0].email, appUser[0].password) const newColumn = generateNewColumnForTable(createdTableData) await config.tables.forbiddenSave( @@ -136,6 +151,7 @@ describe("Internal API - App Specific Roles & Permissions", () => { }) it("Check Table access for developer", async () => { + // Create a user with POWER role and check if it was created successfully const developer = generateUser(1, 'developer') expect(developer[0].builder?.global).toEqual(true) @@ -144,7 +160,10 @@ describe("Internal API - App Specific Roles & Permissions", () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -153,13 +172,17 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("POWER") + // Create a table const [createdTableResponse, createdTableData] = await config.tables.save( generateTable() ) + + // Login with the user created and try to create a column await config.login(developer[0].email, developer[0].password) const newColumn = generateNewColumnForTable(createdTableData) const [addColumnResponse, addColumnData] = await config.tables.save( @@ -169,6 +192,7 @@ describe("Internal API - App Specific Roles & Permissions", () => { }) it("Check Table access for admin", async () => { + // Create a user with ADMIN role and check if it was created successfully const adminUser = generateUser(1, "admin") expect(adminUser[0].builder?.global).toEqual(true) expect(adminUser[0].admin?.global).toEqual(true) @@ -177,7 +201,10 @@ describe("Internal API - App Specific Roles & Permissions", () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId + // Get all the information from the create user const [userInfoResponse, userInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) + + // Create the body with the information from the user and add the role to the app const body: User = { ...userInfoJson, roles: { @@ -186,10 +213,12 @@ describe("Internal API - App Specific Roles & Permissions", () => { } await config.users.updateInfo(body) + // Get the user information again and check if the role was added const [changedUserInfoResponse, changedUserInfoJson] = await config.users.getInfo(createUserJson.created.successful[0]._id) expect(changedUserInfoJson.roles[app.appId]).toBeDefined() expect(changedUserInfoJson.roles[app.appId]).toEqual("ADMIN") + // Login with the created user and create a table await config.login(adminUser[0].email, adminUser[0].password) const [createdTableResponse, createdTableData] = await config.tables.save( generateTable() diff --git a/qa-core/src/tests/internal-api/userManagement/userManagement.spec.ts b/qa-core/src/tests/internal-api/userManagement/userManagement.spec.ts index 32820b8b7f..2290106731 100644 --- a/qa-core/src/tests/internal-api/userManagement/userManagement.spec.ts +++ b/qa-core/src/tests/internal-api/userManagement/userManagement.spec.ts @@ -18,9 +18,13 @@ describe("Internal API - User Management & Permissions", () => { }) it("Add Users with different roles", async () => { + // Get all users await config.users.search() + + // Get all roles await config.users.getRoles() + // Add users with each role const admin = generateUser(1, "admin") expect(admin[0].builder?.global).toEqual(true) expect(admin[0].admin?.global).toEqual(true) @@ -34,6 +38,7 @@ describe("Internal API - User Management & Permissions", () => { await config.users.addMultiple(userList) + // Check users are added const [allUsersResponse, allUsersJson] = await config.users.getAll() expect(allUsersJson.length).toBeGreaterThan(0) From 24f8f3a7cbfe5d5aa30103f96e8491f757cac4f3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 12 Jan 2023 15:38:22 +0000 Subject: [PATCH 05/31] Fix currentapp middleware to allow app_ parameters --- packages/server/src/middleware/currentapp.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index 2cd11aa438..593e96adcb 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -25,6 +25,7 @@ export default async (ctx: BBContext, next: any) => { if (!appCookie && !requestAppId) { return next() } + // check the app exists referenced in cookie if (appCookie) { const appId = appCookie.appId @@ -51,7 +52,7 @@ export default async (ctx: BBContext, next: any) => { let appId: string | undefined, roleId = roles.BUILTIN_ROLE_IDS.PUBLIC - if (!ctx.user) { + if (!ctx.user?._id) { // not logged in, try to set a cookie for public apps appId = requestAppId } else if (requestAppId != null) { @@ -96,7 +97,7 @@ export default async (ctx: BBContext, next: any) => { // need to judge this only based on the request app ID, if ( env.MULTI_TENANCY && - ctx.user && + ctx.user?._id && requestAppId && !tenancy.isUserInAppTenant(requestAppId, ctx.user) ) { From 09b4533cc8fb14f5da53e37e92cf0c57567b58b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 12 Jan 2023 16:26:46 +0000 Subject: [PATCH 06/31] Add endpoint to deactivate user from app on delete --- packages/server/src/api/controllers/user.ts | 32 +++++++++++++++++++++ packages/server/src/api/routes/user.ts | 5 ++++ 2 files changed, 37 insertions(+) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index df64ffc7d0..f37af55ee0 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -173,3 +173,35 @@ export async function getFlags(ctx: BBContext) { } ctx.body = doc } + +export async function removeUserFromApp(ctx: BBContext) { + const { id: userId, prodAppId } = ctx.params + + const devAppId = dbCore.getDevelopmentAppID(prodAppId) + for (let appId of [prodAppId, devAppId]) { + if (!(await dbCore.dbExists(appId))) { + continue + } + await context.doInAppContext(appId, async () => { + const db = context.getAppDB() + const metadataId = generateUserMetadataID(userId) + let metadata + try { + metadata = await db.get(metadataId) + } catch (err) { + return + } + + let combined = { + ...metadata, + status: constants.UserStatus.INACTIVE, + metadata: rolesCore.BUILTIN_ROLE_IDS.PUBLIC, + } + + await db.put(combined) + }) + } + ctx.body = { + message: `User ${userId} deleted from ${prodAppId} and ${"devapp"}.`, + } +} diff --git a/packages/server/src/api/routes/user.ts b/packages/server/src/api/routes/user.ts index 14deb111e6..556954fd77 100644 --- a/packages/server/src/api/routes/user.ts +++ b/packages/server/src/api/routes/user.ts @@ -47,5 +47,10 @@ router authorized(PermissionType.USER, PermissionLevel.READ), controller.getFlags ) + .delete( + "/api/users/metadata/:id/app/:prodAppId", + authorized(PermissionType.USER, PermissionLevel.WRITE), + controller.removeUserFromApp + ) export default router From 09570e26f6e0ccf87c4f325c51ce04650aea470b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 12 Jan 2023 16:28:02 +0000 Subject: [PATCH 07/31] Remove user within the app on deletion --- packages/worker/src/sdk/users/users.ts | 20 ++++++++++++++++++++ packages/worker/src/utilities/appService.ts | 11 +++++++++++ 2 files changed, 31 insertions(+) diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index f3117b63ab..cd3f0622d4 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -188,6 +188,10 @@ const validateUniqueUser = async (email: string, tenantId: string) => { } } +function instanceOfUser(user: User | ThirdPartyUser): user is User { + return !!(user as User).roles +} + export const save = async ( user: User | ThirdPartyUser, opts: SaveUserOpts = {} @@ -257,6 +261,17 @@ export const save = async ( } } + let appsToRemove: string[] = [] + if (dbUser && instanceOfUser(user)) { + const newRoles = Object.keys(user.roles) + const existingRoles = Object.keys(dbUser.roles) + + appsToRemove = existingRoles.filter(r => !newRoles.includes(r)) + if (appsToRemove.length) { + console.log("Deleting access to apps", { appsToRemove }) + } + } + try { // save the user to db let response = await db.put(builtUser) @@ -265,6 +280,11 @@ export const save = async ( await eventHelpers.handleSaveEvents(builtUser, dbUser) await addTenant(tenantId, _id, email) await cache.user.invalidateUser(response.id) + + for (const appId of appsToRemove) { + await apps.removeUserFromApp(_id, appId) + } + // let server know to sync user await apps.syncUserInApps(_id) diff --git a/packages/worker/src/utilities/appService.ts b/packages/worker/src/utilities/appService.ts index a0c4314f65..95a90aebc0 100644 --- a/packages/worker/src/utilities/appService.ts +++ b/packages/worker/src/utilities/appService.ts @@ -30,3 +30,14 @@ export async function syncUserInApps(userId: string) { throw "Unable to sync user." } } + +export async function removeUserFromApp(userId: string, appId: string) { + const response = await makeAppRequest( + `/api/users/metadata/${userId}/app/${appId}`, + "DELETE", + undefined + ) + if (response && response.status !== 200) { + throw "Unable to delete user from app." + } +} From 34cd26781bd0f59adf20f686b6b09aeac0bd25c5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 13 Jan 2023 10:26:05 +0000 Subject: [PATCH 08/31] Delete instead of deactivating --- packages/server/src/api/controllers/user.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index f37af55ee0..5458d97f76 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -189,16 +189,11 @@ export async function removeUserFromApp(ctx: BBContext) { try { metadata = await db.get(metadataId) } catch (err) { + console.warn(`User cannot be found in the app`, { userId, appId }) return } - let combined = { - ...metadata, - status: constants.UserStatus.INACTIVE, - metadata: rolesCore.BUILTIN_ROLE_IDS.PUBLIC, - } - - await db.put(combined) + await db.remove(metadata) }) } ctx.body = { From 5477cf420aeb11361060c04164c97b88773cc49b Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 13 Jan 2023 11:22:59 +0000 Subject: [PATCH 09/31] Allow primary keys to be foreign key (#9331) --- .../backend/Datasources/CreateEditRelationship.svelte | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 4defcbafab..ec39cc6d71 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -340,9 +340,7 @@ {:else if isManyToOne && toTable} + + +{#if fileName && Object.keys(validation).length === 0} +

No valid fields, try another file

+{:else if rows.length > 0 && !error} +
+ {#each Object.keys(validation) as name} +
+ {name} + -
-{#if fields.length} +{#if rows.length > 0 && !error}
- {#each fields as columnName} + {#each Object.values(schema) as column}
- {columnName} + {column.name} -
- {/if} -{:else if hasValidated} -
- +
- - + +
diff --git a/packages/builder/src/components/backend/TableNavigator/utils.js b/packages/builder/src/components/backend/TableNavigator/utils.js new file mode 100644 index 0000000000..658f037912 --- /dev/null +++ b/packages/builder/src/components/backend/TableNavigator/utils.js @@ -0,0 +1,71 @@ +import { API } from "api" +import { FIELDS } from "constants/backend" + +const BYTES_IN_MB = 1000000 +const FILE_SIZE_LIMIT = BYTES_IN_MB * 5 + +const getDefaultSchema = rows => { + const newSchema = {} + + rows.forEach(row => { + Object.keys(row).forEach(column => { + newSchema[column] = { + name: column, + type: "string", + constraints: FIELDS["STRING"].constraints, + } + }) + }) + + return newSchema +} + +export const parseFile = e => { + return new Promise((resolve, reject) => { + const file = Array.from(e.target.files)[0] + + if (file.size >= FILE_SIZE_LIMIT) { + reject("file too large") + return + } + + let reader = new FileReader() + + const resolveRows = (rows, schema = null) => { + resolve({ + rows, + schema: schema ?? getDefaultSchema(rows), + fileName: file.name, + fileType: file.type, + }) + } + + reader.addEventListener("load", function (e) { + const fileData = e.target.result + + if (file.type === "text/csv") { + API.csvToJson(fileData) + .then(rows => { + resolveRows(rows) + }) + .catch(() => { + reject("can't convert csv to json") + }) + } else if (file.type === "application/json") { + const parsedFileData = JSON.parse(fileData) + + if (Array.isArray(parsedFileData)) { + resolveRows(parsedFileData) + } else if (typeof parsedFileData === "object") { + resolveRows(parsedFileData.rows, parsedFileData.schema) + } else { + reject("invalid json format") + } + } else { + reject("invalid file type") + } + }) + + reader.readAsText(file) + }) +} diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js index 279d3ba6fb..00f0e9ff2e 100644 --- a/packages/frontend-core/src/api/tables.js +++ b/packages/frontend-core/src/api/tables.js @@ -64,32 +64,22 @@ export const buildTableEndpoints = API => ({ * @param tableId the table ID to import to * @param data the data import object */ - importTableData: async ({ tableId, data }) => { + importTableData: async ({ tableId, rows }) => { return await API.post({ url: `/api/tables/${tableId}/import`, body: { - dataImport: data, + rows, }, }) }, - - /** - * Validates a candidate CSV to be imported for a certain table. - * @param tableId the table ID to import to - * @param csvString the CSV contents as a string - * @param schema the proposed schema - */ - validateTableCSV: async ({ tableId, csvString, schema }) => { + csvToJson: async csvString => { return await API.post({ - url: "/api/tables/csv/validate", + url: "/api/convert/csvToJson", body: { csvString, - schema, - tableId, }, }) }, - /** * Gets a list o tables. */ @@ -120,4 +110,22 @@ export const buildTableEndpoints = API => ({ url: `/api/tables/${tableId}/${tableRev}`, }) }, + validateNewTableImport: async ({ rows, schema }) => { + return await API.post({ + url: "/api/tables/validateNewTableImport", + body: { + rows, + schema, + }, + }) + }, + validateExistingTableImport: async ({ rows, tableId }) => { + return await API.post({ + url: "/api/tables/validateExistingTableImport", + body: { + rows, + tableId, + }, + }) + }, }) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 9bfdff24ea..963eb6b954 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -27,7 +27,7 @@ import { import { cloneDeep } from "lodash/fp" import { context, db as dbCore } from "@budibase/backend-core" import { finaliseRow, updateRelatedFormula } from "./staticFormula" -import * as exporters from "../view/exporters" +import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" import { Ctx, @@ -412,14 +412,15 @@ export async function exportRows(ctx: Ctx) { rows = result } - let headers = Object.keys(rows[0]) - // @ts-ignore - const exporter = exporters[format] - const filename = `export.${format}` - - // send down the file - ctx.attachment(filename) - return apiFileReturn(exporter(headers, rows)) + if (format === Format.CSV) { + ctx.attachment("export.csv") + return apiFileReturn(csv(Object.keys(rows[0]), rows)) + } else if (format === Format.JSON) { + ctx.attachment("export.json") + return apiFileReturn(json(rows)) + } else { + throw "Format not recognised" + } } export async function fetchEnrichedRow(ctx: Ctx) { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 8bb8886479..dbe09f59c1 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -10,9 +10,9 @@ import { } from "./utils" import { FieldTypes, RelationshipTypes } from "../../../constants" import { makeExternalQuery } from "../../../integrations/base/query" -import * as csvParser from "../../../utilities/csvParser" import { handleRequest } from "../row/external" import { events, context } from "@budibase/backend-core" +import { parse, isRows, isSchema } from "../../../utilities/schema" import { Datasource, Table, @@ -197,7 +197,7 @@ export async function save(ctx: BBContext) { const table: TableRequest = ctx.request.body const renamed = table?._rename // can't do this right now - delete table.dataImport + delete table.rows const datasourceId = getDatasourceId(ctx.request.body)! // table doesn't exist already, note that it is created if (!table._id) { @@ -338,17 +338,17 @@ export async function destroy(ctx: BBContext) { export async function bulkImport(ctx: BBContext) { const table = await sdk.tables.getTable(ctx.params.tableId) - const { dataImport } = ctx.request.body - if (!dataImport || !dataImport.schema || !dataImport.csvString) { + const { rows }: { rows: unknown } = ctx.request.body + const schema: unknown = table.schema + + if (!rows || !isRows(rows) || !isSchema(schema)) { ctx.throw(400, "Provided data import information is invalid.") } - const rows = await csvParser.transform({ - ...dataImport, - existingTable: table, - }) + + const parsedRows = await parse(rows, schema) await handleRequest(Operation.BULK_CREATE, table._id!, { - rows, + rows: parsedRows, }) - await events.rows.imported(table, "csv", rows.length) + await events.rows.imported(table, parsedRows.length) return table } diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 0df974adc4..aa6dfde536 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -1,11 +1,16 @@ import * as internal from "./internal" import * as external from "./external" -import * as csvParser from "../../../utilities/csvParser" +import { + validate as validateSchema, + isSchema, + isRows, +} from "../../../utilities/schema" import { isExternalTable, isSQL } from "../../../integrations/utils" import { getDatasourceParams } from "../../../db/utils" import { context, events } from "@budibase/backend-core" import { Table, BBContext } from "@budibase/types" import sdk from "../../../sdk" +import csv from "csvtojson" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -56,16 +61,16 @@ export async function find(ctx: BBContext) { export async function save(ctx: BBContext) { const appId = ctx.appId const table = ctx.request.body - const importFormat = - table.dataImport && table.dataImport.csvString ? "csv" : undefined + const isImport = table.rows + const savedTable = await pickApi({ table }).save(ctx) if (!table._id) { await events.table.created(savedTable) } else { await events.table.updated(savedTable) } - if (importFormat) { - await events.table.imported(savedTable, importFormat) + if (isImport) { + await events.table.imported(savedTable) } ctx.status = 200 ctx.message = `Table ${table.name} saved successfully.` @@ -96,19 +101,43 @@ export async function bulkImport(ctx: BBContext) { ctx.body = { message: `Bulk rows created.` } } -export async function validateCSVSchema(ctx: BBContext) { - // tableId being specified means its an import to an existing table - const { csvString, schema = {}, tableId } = ctx.request.body - let existingTable - if (tableId) { - existingTable = await sdk.tables.getTable(tableId) - } - let result: Record | undefined = await csvParser.parse( - csvString, - schema - ) - if (existingTable) { - result = csvParser.updateSchema({ schema: result, existingTable }) - } - ctx.body = { schema: result } +export async function csvToJson(ctx: BBContext) { + const { csvString } = ctx.request.body + + const result = await csv().fromString(csvString) + + ctx.status = 200 + ctx.body = result +} + +export async function validateNewTableImport(ctx: BBContext) { + const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body + + if (isRows(rows) && isSchema(schema)) { + ctx.status = 200 + ctx.body = validateSchema(rows, schema) + } else { + ctx.status = 422 + } +} + +export async function validateExistingTableImport(ctx: BBContext) { + const { rows, tableId }: { rows: unknown; tableId: unknown } = + ctx.request.body + + let schema = null + if (tableId) { + const table = await sdk.tables.getTable(tableId) + schema = table.schema + } else { + ctx.status = 422 + return + } + + if (tableId && isRows(rows) && isSchema(schema)) { + ctx.status = 200 + ctx.body = validateSchema(rows, schema) + } else { + ctx.status = 422 + } } diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 3229134bf0..628932bba1 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -35,7 +35,7 @@ function checkAutoColumns(table: Table, oldTable: Table) { export async function save(ctx: any) { const db = context.getAppDB() - const { dataImport, ...rest } = ctx.request.body + const { rows, ...rest } = ctx.request.body let tableToSave = { type: "table", _id: generateTableID(), @@ -61,7 +61,7 @@ export async function save(ctx: any) { const tableSaveFunctions = new TableSaveFunctions({ user: ctx.user, oldTable, - dataImport, + importRows: rows, }) tableToSave = await tableSaveFunctions.before(tableToSave) @@ -185,7 +185,7 @@ export async function destroy(ctx: any) { export async function bulkImport(ctx: any) { const table = await sdk.tables.getTable(ctx.params.tableId) - const { dataImport } = ctx.request.body - await handleDataImport(ctx.user, table, dataImport) + const { rows } = ctx.request.body + await handleDataImport(ctx.user, table, rows) return table } diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index b672561325..30e79f6ee4 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -1,4 +1,4 @@ -import { transform } from "../../../utilities/csvParser" +import { parse, isSchema, isRows } from "../../../utilities/schema" import { getRowParams, generateRowID, InternalTables } from "../../../db/utils" import { isEqual } from "lodash" import { AutoFieldSubTypes, FieldTypes } from "../../../constants" @@ -128,24 +128,23 @@ export function importToRows(data: any, table: any, user: any = {}) { return finalData } -export async function handleDataImport(user: any, table: any, dataImport: any) { - if (!dataImport || !dataImport.csvString) { +export async function handleDataImport(user: any, table: any, rows: any) { + const schema: unknown = table.schema + + if (!rows || !isRows(rows) || !isSchema(schema)) { return table } const db = context.getAppDB() - // Populate the table with rows imported from CSV in a bulk update - const data = await transform({ - ...dataImport, - existingTable: table, - }) + const data = parse(rows, schema) let finalData: any = importToRows(data, table, user) await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), { tableId: table._id, }) - await events.rows.imported(table, "csv", finalData.length) + + await events.rows.imported(table, finalData.length) return table } @@ -210,14 +209,14 @@ class TableSaveFunctions { db: any user: any oldTable: any - dataImport: any + importRows: any rows: any - constructor({ user, oldTable, dataImport }: any) { + constructor({ user, oldTable, importRows }: any) { this.db = context.getAppDB() this.user = user this.oldTable = oldTable - this.dataImport = dataImport + this.importRows = importRows // any rows that need updated this.rows = [] } @@ -241,7 +240,7 @@ class TableSaveFunctions { // after saving async after(table: any) { table = await handleSearchIndexes(table) - table = await handleDataImport(this.user, table, this.dataImport) + table = await handleDataImport(this.user, table, this.importRows) return table } diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index eec6e69641..4d927bca27 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -1,4 +1,4 @@ -import { Row } from "@budibase/types" +import { Row, TableSchema } from "@budibase/types" export function csv(headers: string[], rows: Row[]) { let csv = headers.map(key => `"${key}"`).join(",") @@ -18,11 +18,26 @@ export function csv(headers: string[], rows: Row[]) { return csv } -export function json(headers: string[], rows: Row[]) { +export function json(rows: Row[]) { return JSON.stringify(rows, undefined, 2) } -export const ExportFormats = { - CSV: "csv", - JSON: "json", +export function jsonWithSchema(schema: TableSchema, rows: Row[]) { + const newSchema: TableSchema = {} + Object.values(schema).forEach(column => { + if (!column.autocolumn) { + newSchema[column.name] = column + } + }) + return JSON.stringify({ schema: newSchema, rows }, undefined, 2) +} + +export enum Format { + CSV = "csv", + JSON = "json", + JSON_WITH_SCHEMA = "jsonWithSchema", +} + +export function isFormat(format: any): format is Format { + return Object.values(Format).includes(format as Format) } diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index 932823d172..7bb803d035 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -1,6 +1,6 @@ import viewTemplate from "./viewBuilder" import { apiFileReturn } from "../../../utilities/fileSystem" -import * as exporters from "./exporters" +import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters" import { deleteView, getView, getViews, saveView } from "./utils" import { fetchView } from "../row" import { FieldTypes } from "../../../constants" @@ -127,9 +127,13 @@ export async function exportView(ctx: BBContext) { const viewName = decodeURIComponent(ctx.query.view as string) const view = await getView(viewName) - const format = ctx.query.format as string - if (!format || !Object.values(exporters.ExportFormats).includes(format)) { - ctx.throw(400, "Format must be specified, either csv or json") + const format = ctx.query.format as unknown + + if (!isFormat(format)) { + ctx.throw( + 400, + "Format must be specified, either csv, json or jsonWithSchema" + ) } if (view) { @@ -171,7 +175,7 @@ export async function exportView(ctx: BBContext) { }) // make sure no "undefined" entries appear in the CSV - if (format === exporters.ExportFormats.CSV) { + if (format === Format.CSV) { const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { for (let row of rows) { @@ -182,13 +186,18 @@ export async function exportView(ctx: BBContext) { } } - // Export part - let headers = Object.keys(schema) - const exporter = format === "csv" ? exporters.csv : exporters.json - const filename = `${viewName}.${format}` - // send down the file - ctx.attachment(filename) - ctx.body = apiFileReturn(exporter(headers, rows)) + if (format === Format.CSV) { + ctx.attachment(`${viewName}.csv`) + ctx.body = apiFileReturn(csv(Object.keys(schema), rows)) + } else if (format === Format.JSON) { + ctx.attachment(`${viewName}.json`) + ctx.body = apiFileReturn(json(rows)) + } else if (format === Format.JSON_WITH_SCHEMA) { + ctx.attachment(`${viewName}.json`) + ctx.body = apiFileReturn(jsonWithSchema(schema, rows)) + } else { + throw "Format not recognised" + } if (viewName.startsWith(DocumentType.TABLE)) { await events.table.exported(table, format as TableExportFormat) diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts index 23b1f4e988..7ffa5acb3e 100644 --- a/packages/server/src/api/routes/table.ts +++ b/packages/server/src/api/routes/table.ts @@ -67,10 +67,7 @@ router * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix * the rows in the table. This functionality is only available for internal tables. - * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the - * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call - * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating - * or for external tables. + * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided. * * @apiParamExample {json} Example: * { @@ -99,15 +96,7 @@ router * "old": "columnName", * "updated": "newColumnName", * }, - * "dataImport": { - * "csvString": "column\nvalue", - * "primaryDisplay": "column", - * "schema": { - * "column": { - * "type": "string" - * } - * } - * } + * "rows": [] * } * * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and @@ -121,30 +110,20 @@ router tableValidator(), tableController.save ) - /** - * @api {post} /api/tables/csv/validate Validate a CSV for a table - * @apiName Validate a CSV for a table - * @apiGroup tables - * @apiPermission builder - * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and - * converted into a Budibase schema; this endpoint does this. - * - * @apiParam (Body) {string} csvString The CSV which is to be validated as a string. - * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the - * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can - * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g. - * using a number instead of strings. - * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and - * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When - * importing a CSV to an existing table only fields that are present on the table can be imported. - * - * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for - * the CSV - this will be in the same format used for table schema.s - */ .post( - "/api/tables/csv/validate", + "/api/convert/csvToJson", authorized(BUILDER), - tableController.validateCSVSchema + tableController.csvToJson + ) + .post( + "/api/tables/validateNewTableImport", + authorized(BUILDER), + tableController.validateNewTableImport + ) + .post( + "/api/tables/validateExistingTableImport", + authorized(BUILDER), + tableController.validateExistingTableImport ) /** * @api {post} /api/tables/:tableId/:revId Delete a table @@ -177,9 +156,7 @@ router * * @apiParam {string} tableId The ID of the table which the data should be imported to. * - * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with - * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be - * turned into rows. + * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored. * * @apiSuccess {string} message A message stating that the data was imported successfully. */ diff --git a/packages/server/src/api/routes/tests/misc.spec.js b/packages/server/src/api/routes/tests/misc.spec.js index 21dba8f085..2451953df1 100644 --- a/packages/server/src/api/routes/tests/misc.spec.js +++ b/packages/server/src/api/routes/tests/misc.spec.js @@ -42,7 +42,7 @@ describe("run misc tests", () => { }) describe("test table utilities", () => { - it("should be able to import a CSV", async () => { + it("should be able to import data", async () => { return config.doInContext(null, async () => { const table = await config.createTable({ name: "table", @@ -75,17 +75,11 @@ describe("run misc tests", () => { }, }, }) - const dataImport = { - csvString: "a,b,c,d\n1,2,3,4", - schema: {}, - } - for (let col of ["a", "b", "c", "d"]) { - dataImport.schema[col] = { type: "string" } - } + await tableUtils.handleDataImport( { userId: "test" }, table, - dataImport + [{ a: '1', b: '2', c: '3', d: '4'}] ) const rows = await config.getRows() expect(rows[0].a).toEqual("1") @@ -94,4 +88,4 @@ describe("run misc tests", () => { }) }) }) -}) \ No newline at end of file +}) diff --git a/packages/server/src/api/routes/tests/table.spec.js b/packages/server/src/api/routes/tests/table.spec.js index b4fd354b9d..521b64e0c0 100644 --- a/packages/server/src/api/routes/tests/table.spec.js +++ b/packages/server/src/api/routes/tests/table.spec.js @@ -43,21 +43,18 @@ describe("/tables", () => { expect(events.table.created).toBeCalledWith(res.body) }) - it("creates a table via data import CSV", async () => { + it("creates a table via data import", async () => { const table = basicTable() - table.dataImport = { - csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"", - } - table.dataImport.schema = table.schema + table.rows = [{ name: 'test-name', description: 'test-desc' }] const res = await createTable(table) expect(events.table.created).toBeCalledTimes(1) expect(events.table.created).toBeCalledWith(res.body) expect(events.table.imported).toBeCalledTimes(1) - expect(events.table.imported).toBeCalledWith(res.body, "csv") + expect(events.table.imported).toBeCalledWith(res.body) expect(events.rows.imported).toBeCalledTimes(1) - expect(events.rows.imported).toBeCalledWith(res.body, "csv", 1) + expect(events.rows.imported).toBeCalledWith(res.body, 1) }) it("should apply authorization to endpoint", async () => { @@ -155,11 +152,10 @@ describe("/tables", () => { it("imports rows successfully", async () => { const table = await config.createTable() const importRequest = { - dataImport: { - csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"", - schema: table.schema - } + schema: table.schema, + rows: [{ name: 'test-name', description: 'test-desc' }] } + jest.clearAllMocks() await request @@ -171,7 +167,7 @@ describe("/tables", () => { expect(events.table.created).not.toHaveBeenCalled() expect(events.rows.imported).toBeCalledTimes(1) - expect(events.rows.imported).toBeCalledWith(table, "csv", 1) + expect(events.rows.imported).toBeCalledWith(table, 1) }) }) @@ -206,24 +202,6 @@ describe("/tables", () => { }) }) - describe("validate csv", () => { - it("should be able to validate a CSV layout", async () => { - const res = await request - .post(`/api/tables/csv/validate`) - .send({ - csvString: "a,b,c,d\n1,2,3,4" - }) - .set(config.defaultHeaders()) - .expect('Content-Type', /json/) - .expect(200) - expect(res.body.schema).toBeDefined() - expect(res.body.schema.a).toEqual({ - type: "string", - success: true, - }) - }) - }) - describe("indexing", () => { it("should be able to create a table with indexes", async () => { await context.doInAppContext(appId, async () => { diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index b6def14d70..4da680edd1 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -18,7 +18,7 @@ export function tableValidator() { schema: Joi.object().required(), name: Joi.string().required(), views: Joi.object(), - dataImport: Joi.object(), + rows: Joi.array(), }).unknown(true)) } diff --git a/packages/server/src/utilities/csvParser.ts b/packages/server/src/utilities/csvParser.ts deleted file mode 100644 index 0c138abc3e..0000000000 --- a/packages/server/src/utilities/csvParser.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { FieldSchema, Table } from "@budibase/types" -import csv from "csvtojson" -import { FieldTypes } from "../constants" - -type CsvParseOpts = { - schema?: { [key: string]: any } - existingTable: Table - csvString?: string -} - -const VALIDATORS: any = { - [FieldTypes.STRING]: () => true, - [FieldTypes.OPTIONS]: () => true, - [FieldTypes.BARCODEQR]: () => true, - [FieldTypes.NUMBER]: (attribute?: string) => { - // allow not to be present - if (!attribute) { - return true - } - return !isNaN(Number(attribute)) - }, - [FieldTypes.DATETIME]: (attribute?: string) => { - // allow not to be present - if (!attribute) { - return true - } - return !isNaN(new Date(attribute).getTime()) - }, -} - -const PARSERS: any = { - [FieldTypes.NUMBER]: (attribute?: string) => { - if (!attribute) { - return attribute - } - return Number(attribute) - }, - [FieldTypes.DATETIME]: (attribute?: string) => { - if (!attribute) { - return attribute - } - return new Date(attribute).toISOString() - }, -} - -export function parse(csvString: string, parsers: any): Record { - const result = csv().fromString(csvString) - - const schema: Record = {} - - return new Promise((resolve, reject) => { - result.on("header", headers => { - for (let header of headers) { - schema[header] = { - type: parsers[header] ? parsers[header].type : "string", - success: true, - } - } - }) - result.subscribe(row => { - // For each CSV row parse all the columns that need parsed - for (let key of Object.keys(parsers)) { - if (!schema[key] || schema[key].success) { - // get the validator for the column type - const validator = VALIDATORS[parsers[key].type] - - try { - // allow null/undefined values - schema[key].success = !row[key] || validator(row[key]) - } catch (err) { - schema[key].success = false - } - } - } - }) - result.on("done", error => { - if (error) { - console.error(error) - reject(error) - } - - resolve(schema) - }) - }) -} - -export function updateSchema({ - schema, - existingTable, -}: { - schema?: Record - existingTable?: Table -}) { - if (!schema) { - return schema - } - const finalSchema: Record = {} - const schemaKeyMap: Record = {} - Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key)) - for (let [key, field] of Object.entries(existingTable?.schema || {})) { - const lcKey = key.toLowerCase() - const foundKey: string = schemaKeyMap[lcKey] - if (foundKey) { - finalSchema[key] = schema[foundKey] - finalSchema[key].type = field.type - } - } - return finalSchema -} - -export async function transform({ - schema, - csvString, - existingTable, -}: CsvParseOpts) { - if (!schema || !csvString) { - throw new Error("Unable to transform CSV without schema") - } - const colParser: any = {} - - // make sure the table has all the columns required for import - if (existingTable) { - schema = updateSchema({ schema, existingTable }) - } - - for (let [key, field] of Object.entries(schema || {})) { - // don't import data to auto columns - if (!field.autocolumn) { - colParser[key] = PARSERS[field.type] || field.type - } - } - - try { - const data = await csv({ colParser }).fromString(csvString) - const schemaKeyMap: any = {} - Object.keys(schema || {}).forEach( - key => (schemaKeyMap[key.toLowerCase()] = key) - ) - for (let element of data) { - if (!data) { - continue - } - for (let key of Object.keys(element)) { - const mappedKey = schemaKeyMap[key.toLowerCase()] - // isn't a column in the table, remove it - if (mappedKey == null) { - delete element[key] - } - // casing is different, fix it in row - else if (key !== mappedKey) { - element[mappedKey] = element[key] - delete element[key] - } - } - } - return data - } catch (err) { - console.error(`Error transforming CSV to JSON for data import`, err) - throw err - } -} diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts new file mode 100644 index 0000000000..f3d98d35c9 --- /dev/null +++ b/packages/server/src/utilities/schema.ts @@ -0,0 +1,141 @@ +import { FieldTypes } from "../constants" + +interface SchemaColumn { + readonly name: string + readonly type: FieldTypes + readonly autocolumn?: boolean +} + +interface Schema { + readonly [index: string]: SchemaColumn +} + +interface Row { + [index: string]: any +} + +type Rows = Array + +interface SchemaValidation { + [index: string]: boolean +} + +interface ValidationResults { + schemaValidation: SchemaValidation + allValid: boolean + invalidColumns: Array +} + +const PARSERS: any = { + [FieldTypes.NUMBER]: (attribute?: string) => { + if (!attribute) { + return attribute + } + return Number(attribute) + }, + [FieldTypes.DATETIME]: (attribute?: string) => { + if (!attribute) { + return attribute + } + return new Date(attribute).toISOString() + }, +} + +export function isSchema(schema: any): schema is Schema { + return ( + typeof schema === "object" && + Object.values(schema).every(rawColumn => { + const column = rawColumn as SchemaColumn + + return ( + column !== null && + typeof column === "object" && + typeof column.type === "string" && + Object.values(FieldTypes).includes(column.type as FieldTypes) + ) + }) + ) +} + +export function isRows(rows: any): rows is Rows { + return Array.isArray(rows) && rows.every(row => typeof row === "object") +} + +export function validate(rows: Rows, schema: Schema): ValidationResults { + const results: ValidationResults = { + schemaValidation: {}, + allValid: false, + invalidColumns: [], + } + + rows.forEach(row => { + Object.entries(row).forEach(([columnName, columnData]) => { + const columnType = schema[columnName]?.type + const isAutoColumn = schema[columnName]?.autocolumn + + // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array + if (typeof columnType !== "string") { + results.invalidColumns.push(columnName) + } else if ( + // If there's no data for this field don't bother with further checks + // If the field is already marked as invalid there's no need for further checks + results.schemaValidation[columnName] === false || + columnData == null || + isAutoColumn + ) { + return + } else if ( + columnType === FieldTypes.NUMBER && + isNaN(Number(columnData)) + ) { + // If provided must be a valid number + results.schemaValidation[columnName] = false + } else if ( + // If provided must be a valid date + columnType === FieldTypes.DATETIME && + isNaN(new Date(columnData).getTime()) + ) { + results.schemaValidation[columnName] = false + } else { + results.schemaValidation[columnName] = true + } + }) + }) + + results.allValid = + Object.values(results.schemaValidation).length > 0 && + Object.values(results.schemaValidation).every(column => column) + + // Select unique values + results.invalidColumns = [...new Set(results.invalidColumns)] + return results +} + +export function parse(rows: Rows, schema: Schema): Rows { + return rows.map(row => { + const parsedRow: Row = {} + + Object.entries(row).forEach(([columnName, columnData]) => { + if (!(columnName in schema) || schema[columnName]?.autocolumn) { + // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case + return + } + + const columnType = schema[columnName].type + + if (columnType === FieldTypes.NUMBER) { + // If provided must be a valid number + parsedRow[columnName] = columnData ? Number(columnData) : columnData + } else if (columnType === FieldTypes.DATETIME) { + // If provided must be a valid date + parsedRow[columnName] = columnData + ? new Date(columnData).toISOString() + : columnData + } else { + parsedRow[columnName] = columnData + } + }) + + return parsedRow + }) +} diff --git a/packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap b/packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap deleted file mode 100644 index 07f73ba2ef..0000000000 --- a/packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap +++ /dev/null @@ -1,15 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = ` -Array [ - Object { - "Age": 4324, - }, - Object { - "Age": 34, - }, - Object { - "Age": 23423, - }, -] -`; diff --git a/packages/server/src/utilities/tests/csvParser.spec.js b/packages/server/src/utilities/tests/csvParser.spec.js deleted file mode 100644 index 2a997e6f3a..0000000000 --- a/packages/server/src/utilities/tests/csvParser.spec.js +++ /dev/null @@ -1,112 +0,0 @@ -const { readFileSync } = require("../fileSystem") -const csvParser = require("../csvParser") - -const CSV_PATH = __dirname + "/test.csv" - -const SCHEMAS = { - VALID: { - Age: { - type: "number", - }, - }, - INVALID: { - Address: { - type: "number", - }, - Age: { - type: "number", - }, - }, - IGNORE: { - Address: { - type: "omit", - }, - Age: { - type: "omit", - }, - Name: { - type: "string", - }, - }, - BROKEN: { - Address: { - type: "datetime", - }, - }, -} - -describe("CSV Parser", () => { - const csvString = readFileSync(CSV_PATH, "utf8") - - describe("parsing", () => { - it("returns status and types for a valid CSV transformation", async () => { - expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({ - Address: { - success: true, - type: "string", - }, - Age: { - success: true, - type: "number", - }, - Name: { - success: true, - type: "string", - }, - }) - }) - - it("returns status and types for an invalid CSV transformation", async () => { - expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({ - Address: { - success: false, - type: "number", - }, - Age: { - success: true, - type: "number", - }, - Name: { - success: true, - type: "string", - }, - }) - }) - }) - - describe("transformation", () => { - it("transforms a CSV file into JSON", async () => { - expect( - await csvParser.transform({ - schema: SCHEMAS.VALID, - csvString, - }) - ).toMatchSnapshot() - }) - - it("transforms a CSV file into JSON ignoring certain fields", async () => { - expect( - await csvParser.transform({ - schema: SCHEMAS.IGNORE, - csvString, - }) - ).toEqual([ - { - Name: "Bertå", - }, - { - Name: "Ernie", - }, - { - Name: "Big Bird", - }, - ]) - }) - - it("throws an error on invalid schema", async () => { - await expect( - csvParser.transform({ schema: SCHEMAS.BROKEN, csvString }) - ).rejects.toThrow() - }) - }) -}) diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts index 23d77c5ad5..8223ce29c6 100644 --- a/packages/types/src/documents/app/table.ts +++ b/packages/types/src/documents/app/table.ts @@ -69,7 +69,7 @@ export interface Table extends Document { constrained?: string[] sql?: boolean indexes?: { [key: string]: any } - dataImport?: { [key: string]: any } + rows?: { [key: string]: any } } export interface TableRequest extends Table { diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 10c5dfc6b0..2c197e7738 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -185,6 +185,4 @@ export interface BaseEvent { hosting?: Hosting } -export type RowImportFormat = "csv" export type TableExportFormat = "json" | "csv" -export type TableImportFormat = "csv" diff --git a/packages/types/src/sdk/events/rows.ts b/packages/types/src/sdk/events/rows.ts index 871658e6c9..c5736fec4e 100644 --- a/packages/types/src/sdk/events/rows.ts +++ b/packages/types/src/sdk/events/rows.ts @@ -1,8 +1,7 @@ -import { BaseEvent, RowImportFormat } from "./event" +import { BaseEvent } from "./event" export interface RowsImportedEvent extends BaseEvent { tableId: string - format: RowImportFormat count: number } diff --git a/packages/types/src/sdk/events/table.ts b/packages/types/src/sdk/events/table.ts index 2f9ca1c93e..da3c7edf47 100644 --- a/packages/types/src/sdk/events/table.ts +++ b/packages/types/src/sdk/events/table.ts @@ -1,4 +1,4 @@ -import { BaseEvent, TableExportFormat, TableImportFormat } from "./event" +import { BaseEvent, TableExportFormat } from "./event" export interface TableCreatedEvent extends BaseEvent { tableId: string @@ -19,5 +19,4 @@ export interface TableExportedEvent extends BaseEvent { export interface TableImportedEvent extends BaseEvent { tableId: string - format: TableImportFormat } diff --git a/qa-core/src/config/internal-api/fixtures/table.ts b/qa-core/src/config/internal-api/fixtures/table.ts index 5060a405bb..3de950fd9f 100644 --- a/qa-core/src/config/internal-api/fixtures/table.ts +++ b/qa-core/src/config/internal-api/fixtures/table.ts @@ -6,10 +6,6 @@ export const generateTable = (): Table => { schema: {}, sourceId: "bb_internal", type: "internal", - dataImport: { - valid: true, - schema: {}, - }, } } From 3dc1f80abf9c2908d0045582b6484da441dc86ed Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 17 Jan 2023 15:18:15 +0000 Subject: [PATCH 29/31] v2.2.12-alpha.21 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 670546b6fc..4e6affc904 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 273067d8fb..54d31fe088 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -23,7 +23,7 @@ }, "dependencies": { "@budibase/nano": "10.1.1", - "@budibase/types": "2.2.12-alpha.20", + "@budibase/types": "2.2.12-alpha.21", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index ab64320758..dd92a9101e 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.2.12-alpha.20", + "@budibase/string-templates": "2.2.12-alpha.21", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", "@spectrum-css/avatar": "3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 40439197e5..f1e0fd203e 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.2.12-alpha.20", - "@budibase/client": "2.2.12-alpha.20", - "@budibase/frontend-core": "2.2.12-alpha.20", - "@budibase/string-templates": "2.2.12-alpha.20", + "@budibase/bbui": "2.2.12-alpha.21", + "@budibase/client": "2.2.12-alpha.21", + "@budibase/frontend-core": "2.2.12-alpha.21", + "@budibase/string-templates": "2.2.12-alpha.21", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index bb964251b7..be7e5ce760 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.2.12-alpha.20", - "@budibase/string-templates": "2.2.12-alpha.20", - "@budibase/types": "2.2.12-alpha.20", + "@budibase/backend-core": "2.2.12-alpha.21", + "@budibase/string-templates": "2.2.12-alpha.21", + "@budibase/types": "2.2.12-alpha.21", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 262c023108..ba04e055a1 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.2.12-alpha.20", - "@budibase/frontend-core": "2.2.12-alpha.20", - "@budibase/string-templates": "2.2.12-alpha.20", + "@budibase/bbui": "2.2.12-alpha.21", + "@budibase/frontend-core": "2.2.12-alpha.21", + "@budibase/string-templates": "2.2.12-alpha.21", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 9469ea47dd..ec8ab4fe43 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.2.12-alpha.20", + "@budibase/bbui": "2.2.12-alpha.21", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index d1ffbe0e0f..91644678a9 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 059c8fe4f1..616f27348c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -43,11 +43,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.2.12-alpha.20", - "@budibase/client": "2.2.12-alpha.20", + "@budibase/backend-core": "2.2.12-alpha.21", + "@budibase/client": "2.2.12-alpha.21", "@budibase/pro": "2.2.12-alpha.20", - "@budibase/string-templates": "2.2.12-alpha.20", - "@budibase/types": "2.2.12-alpha.20", + "@budibase/string-templates": "2.2.12-alpha.21", + "@budibase/types": "2.2.12-alpha.21", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 9f7440920a..576900d69d 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 7171f77de0..4c5637e128 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index b074c6ba50..777d3aa968 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.2.12-alpha.20", + "version": "2.2.12-alpha.21", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.2.12-alpha.20", + "@budibase/backend-core": "2.2.12-alpha.21", "@budibase/pro": "2.2.12-alpha.20", - "@budibase/string-templates": "2.2.12-alpha.20", - "@budibase/types": "2.2.12-alpha.20", + "@budibase/string-templates": "2.2.12-alpha.21", + "@budibase/types": "2.2.12-alpha.21", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From acafdf52bf2d37b7c60b7f4e57fed52b55961386 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 17 Jan 2023 15:21:53 +0000 Subject: [PATCH 30/31] Update pro version to 2.2.12-alpha.21 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 616f27348c..8210cf4cba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -45,7 +45,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "2.2.12-alpha.21", "@budibase/client": "2.2.12-alpha.21", - "@budibase/pro": "2.2.12-alpha.20", + "@budibase/pro": "2.2.12-alpha.21", "@budibase/string-templates": "2.2.12-alpha.21", "@budibase/types": "2.2.12-alpha.21", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 61a4a65e84..2b670f6f90 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1273,13 +1273,13 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.20.tgz#aac6b995d0c53c599eff1f069bbde6426f383a36" - integrity sha512-ta9gEa3fklCQHUybJk8wxQM0VawV+IN7P4yVH4kZU3mr9nAS42rhvb+xoqx4vB8HIa6312uj0UgUaYWacK15Rw== +"@budibase/backend-core@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.21.tgz#99844f641cddb99ca6b6abd6a8af7d990e6b92e1" + integrity sha512-4ZFcLTRtApF1aCE2CJrgMO44O0rkQq9bmyBxXxMUMjmv83lsue8rpWV00G5Z7fCg2C/Z2VdzeOJU97OumT3dmQ== dependencies: "@budibase/nano" "10.1.1" - "@budibase/types" "2.2.12-alpha.20" + "@budibase/types" "2.2.12-alpha.21" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-cloudfront-sign "2.2.0" @@ -1374,13 +1374,13 @@ qs "^6.11.0" tough-cookie "^4.1.2" -"@budibase/pro@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.20.tgz#3eb45b7ddef7e5c2190851a8f021b6638d01ef90" - integrity sha512-j169PLqdCxR6oPR6nevdVrwHDTH1SilKhtA7GSrP3ANPGTK3G9fxVBvx+KH4IaGikTSZnCaGUP6p376EcOMMCg== +"@budibase/pro@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.21.tgz#5419fd78ac68ed9feddc4af52da8a79ff874cbfc" + integrity sha512-Fvm4ruWP9V514PJMqO9n2T4CQ2DGJpXDbK/hRSRooGyrAfR3lDJC6TB1boa4WYC+1YACF+f757adLPcNvUNCnA== dependencies: - "@budibase/backend-core" "2.2.12-alpha.20" - "@budibase/types" "2.2.12-alpha.20" + "@budibase/backend-core" "2.2.12-alpha.21" + "@budibase/types" "2.2.12-alpha.21" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1405,10 +1405,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.20.tgz#d708eb74df555a7649873247662b6d6be1054bfe" - integrity sha512-HQjCgYjgd8NLkykMy5oQA48VS03Py/GbDqnIuUT70fhmX8mgORctvvK7AgNsSLTrv2uRFuvZOcL8VE8MyJBhKA== +"@budibase/types@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.21.tgz#9344fae4504cf5d9a3a92ffb94434f988389c9a9" + integrity sha512-F+UMqKvrYqHYtJUcmvT9kRBFtPRPGBhbktJgYHa8D3X+CK/nf93UYN8j9nTeXme7iOA+i/cmg+zALFuHyZi45Q== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 777d3aa968..1b4aedb664 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "2.2.12-alpha.21", - "@budibase/pro": "2.2.12-alpha.20", + "@budibase/pro": "2.2.12-alpha.21", "@budibase/string-templates": "2.2.12-alpha.21", "@budibase/types": "2.2.12-alpha.21", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 8833c9eda9..e38ee5dfff 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -470,13 +470,13 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.20.tgz#aac6b995d0c53c599eff1f069bbde6426f383a36" - integrity sha512-ta9gEa3fklCQHUybJk8wxQM0VawV+IN7P4yVH4kZU3mr9nAS42rhvb+xoqx4vB8HIa6312uj0UgUaYWacK15Rw== +"@budibase/backend-core@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.21.tgz#99844f641cddb99ca6b6abd6a8af7d990e6b92e1" + integrity sha512-4ZFcLTRtApF1aCE2CJrgMO44O0rkQq9bmyBxXxMUMjmv83lsue8rpWV00G5Z7fCg2C/Z2VdzeOJU97OumT3dmQ== dependencies: "@budibase/nano" "10.1.1" - "@budibase/types" "2.2.12-alpha.20" + "@budibase/types" "2.2.12-alpha.21" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-cloudfront-sign "2.2.0" @@ -521,23 +521,23 @@ qs "^6.11.0" tough-cookie "^4.1.2" -"@budibase/pro@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.20.tgz#3eb45b7ddef7e5c2190851a8f021b6638d01ef90" - integrity sha512-j169PLqdCxR6oPR6nevdVrwHDTH1SilKhtA7GSrP3ANPGTK3G9fxVBvx+KH4IaGikTSZnCaGUP6p376EcOMMCg== +"@budibase/pro@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.21.tgz#5419fd78ac68ed9feddc4af52da8a79ff874cbfc" + integrity sha512-Fvm4ruWP9V514PJMqO9n2T4CQ2DGJpXDbK/hRSRooGyrAfR3lDJC6TB1boa4WYC+1YACF+f757adLPcNvUNCnA== dependencies: - "@budibase/backend-core" "2.2.12-alpha.20" - "@budibase/types" "2.2.12-alpha.20" + "@budibase/backend-core" "2.2.12-alpha.21" + "@budibase/types" "2.2.12-alpha.21" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" jsonwebtoken "8.5.1" node-fetch "^2.6.1" -"@budibase/types@2.2.12-alpha.20": - version "2.2.12-alpha.20" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.20.tgz#d708eb74df555a7649873247662b6d6be1054bfe" - integrity sha512-HQjCgYjgd8NLkykMy5oQA48VS03Py/GbDqnIuUT70fhmX8mgORctvvK7AgNsSLTrv2uRFuvZOcL8VE8MyJBhKA== +"@budibase/types@2.2.12-alpha.21": + version "2.2.12-alpha.21" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.21.tgz#9344fae4504cf5d9a3a92ffb94434f988389c9a9" + integrity sha512-F+UMqKvrYqHYtJUcmvT9kRBFtPRPGBhbktJgYHa8D3X+CK/nf93UYN8j9nTeXme7iOA+i/cmg+zALFuHyZi45Q== "@cspotcode/source-map-support@^0.8.0": version "0.8.1" From cbdd85225dc3337aa961647288fe9e10dacc1c61 Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Wed, 18 Jan 2023 12:00:08 +0000 Subject: [PATCH 31/31] Fix Table Creation With No Import (#9373) --- .../components/backend/TableNavigator/TableDataImport.svelte | 2 +- .../backend/TableNavigator/modals/CreateTableModal.svelte | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index a056ea3256..3dbe145084 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -14,7 +14,7 @@ export let rows = [] export let schema = {} - export let allValid = false + export let allValid = true export let displayColumn = null const typeOptions = [ diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index 9ad0fb726e..175705732f 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -33,7 +33,7 @@ let autoColumns = getAutoColumnInformation() let schema = {} let rows = [] - let allValid = false + let allValid = true let displayColumn = null function getAutoColumns() { @@ -99,7 +99,7 @@ title="Create Table" confirmText="Create" onConfirm={saveTable} - disabled={error || !name || !allValid} + disabled={error || !name || (rows.length && !allValid)} >