diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index b84fd54fae..e060db8370 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -25,13 +25,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -46,13 +46,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -70,13 +70,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -96,13 +96,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -119,7 +119,7 @@ jobs: test-pro: runs-on: ubuntu-latest - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -140,13 +140,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -166,7 +166,7 @@ jobs: check-pro-submodule: runs-on: ubuntu-latest - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -190,6 +190,8 @@ jobs: base_commit=$(git rev-parse origin/develop) fi + echo "target_branch=$branch" + echo "target_branch=$branch" >> "$GITHUB_OUTPUT" echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" @@ -204,7 +206,7 @@ jobs: const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the develop branch.'); + console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.'); console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') process.exit(1); } else { diff --git a/.github/workflows/check_unreleased_changes.yml b/.github/workflows/check_unreleased_changes.yml new file mode 100644 index 0000000000..d558330545 --- /dev/null +++ b/.github/workflows/check_unreleased_changes.yml @@ -0,0 +1,29 @@ +name: check_unreleased_changes + +on: + pull_request: + branches: + - master + +jobs: + check_unreleased: + runs-on: ubuntu-latest + steps: + - name: Check for unreleased changes + env: + REPO: "Budibase/budibase" + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ + "https://api.github.com/repos/$REPO/releases/latest" | \ + jq -r .published_at) + COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ + "https://api.github.com/repos/$REPO/commits/master" | \ + jq -r .commit.committer.date) + RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s") + COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s") + if (( COMMIT_SECONDS > RELEASE_SECONDS )); then + echo "There are unreleased changes. Please release these changes before merging." + exit 1 + fi + echo "No unreleased changes detected." diff --git a/lerna.json b/lerna.json index 9ae6b7b4f8..dc7cabb9fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.8.29-alpha.18", + "version": "2.8.32-alpha.7", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 0ab933d925..6b57fe3d18 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -108,7 +108,13 @@ /****************************************************/ const getInputData = (testData, blockInputs) => { - let newInputData = cloneDeep(testData || blockInputs) + // Test data is not cloned for reactivity + let newInputData = testData || cloneDeep(blockInputs) + + // Ensures the app action fields are populated + if (block.event === "app:trigger" && !newInputData?.fields) { + newInputData = cloneDeep(blockInputs) + } /** * TODO - Remove after November 2023 diff --git a/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte index a2db33306c..4f64edb34d 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte @@ -2,14 +2,15 @@ import { Button, Layout } from "@budibase/bbui" import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte" import Panel from "components/design/Panel.svelte" - import { isActive, goto, redirect } from "@roxi/routify" + import { isActive, redirect, goto, params } from "@roxi/routify" import BetaButton from "./_components/BetaButton.svelte" import { datasources } from "stores/backend" $: { // If we ever don't have any data other than the users table, prompt the // user to add some - if (!$datasources.hasData) { + // Don't redirect if setting up google sheets, or we lose the query parameter + if (!$datasources.hasData && !$params["?continue_google_setup"]) { $redirect("./new") } } diff --git a/packages/pro b/packages/pro index cf3bef2aad..ecee8071eb 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit cf3bef2aad9c739111b306fd0712397adc363f81 +Subproject commit ecee8071ebe0f98a5bb19646954e373264be210d diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index e875a545ca..c5556479fe 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -317,6 +317,11 @@ async function performAppCreate(ctx: UserCtx) { } }) + // Keep existing validation setting + if (!existing.features?.componentValidation) { + newApplication.features!.componentValidation = false + } + // Migrate navigation settings and screens if required if (existing) { const navigation = await migrateAppNavigation() diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 7f6f494621..695e626630 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -11,6 +11,9 @@ import { Row, PatchRowRequest, PatchRowResponse, + SearchRowResponse, + SearchRowRequest, + SearchParams, } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" @@ -197,10 +200,10 @@ export async function destroy(ctx: UserCtx) { ctx.body = response } -export async function search(ctx: any) { +export async function search(ctx: Ctx) { const tableId = utils.getTableId(ctx) - const searchParams = { + const searchParams: SearchParams = { ...ctx.request.body, tableId, } diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index c38b7fe56e..c3b716ef88 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -1,14 +1,18 @@ import { quotas } from "@budibase/pro" import { UserCtx, - SearchResponse, - SortOrder, - SortType, ViewV2, + SearchRowResponse, + SearchViewRowRequest, + RequiredKeys, + SearchParams, } from "@budibase/types" +import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" -export async function searchView(ctx: UserCtx) { +export async function searchView( + ctx: UserCtx +) { const { viewId } = ctx.params const view = await sdk.views.get(viewId) @@ -29,49 +33,35 @@ export async function searchView(ctx: UserCtx) { undefined ctx.status = 200 - const result = await quotas.addQuery( - () => - sdk.rows.search({ - tableId: view.tableId, - query: view.query || {}, - fields: viewFields, - ...getSortOptions(ctx, view), - }), - { - datasourceId: view.tableId, - } - ) + + const { body } = ctx.request + const query = dataFilters.buildLuceneQuery(view.query || []) + + const searchOptions: RequiredKeys & + RequiredKeys> = { + tableId: view.tableId, + query, + fields: viewFields, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + } + + const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), { + datasourceId: view.tableId, + }) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } -function getSortOptions( - ctx: UserCtx, - view: ViewV2 -): - | { - sort: string - sortOrder?: SortOrder - sortType?: SortType - } - | undefined { - const { sort_column, sort_order, sort_type } = ctx.query - if (Array.isArray(sort_column)) { - ctx.throw(400, "sort_column cannot be an array") - } - if (Array.isArray(sort_order)) { - ctx.throw(400, "sort_order cannot be an array") - } - if (Array.isArray(sort_type)) { - ctx.throw(400, "sort_type cannot be an array") - } - - if (sort_column) { +function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { + if (request.sort) { return { - sort: sort_column, - sortOrder: sort_order as SortOrder, - sortType: sort_type as SortType, + sort: request.sort, + sortOrder: request.sortOrder, + sortType: request.sortType, } } if (view.sort) { @@ -82,5 +72,9 @@ function getSortOptions( } } - return + return { + sort: undefined, + sortOrder: undefined, + sortType: undefined, + } } diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index 3227df98ed..55906c2ffe 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -266,7 +266,9 @@ router authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.exportRows ) - .get( + +router + .post( "/api/v2/views/:viewId/search", authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.views.searchView diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 86c41b8503..5e1616340f 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -796,252 +796,6 @@ describe("/rows", () => { }) }) - describe("view search", () => { - function userTable(): Table { - return { - name: "user", - type: "user", - schema: { - name: { - type: FieldType.STRING, - name: "name", - constraints: { type: "string" }, - }, - age: { - type: FieldType.NUMBER, - name: "age", - constraints: {}, - }, - }, - } - } - - it("returns table rows from view", async () => { - const table = await config.createTable(userTable()) - const rows = [] - for (let i = 0; i < 10; i++) { - rows.push(await config.createRow({ tableId: table._id })) - } - - const createViewResponse = await config.api.viewV2.create() - const response = await config.api.viewV2.search(createViewResponse.id) - - expect(response.body.rows).toHaveLength(10) - expect(response.body).toEqual({ - rows: expect.arrayContaining(rows.map(expect.objectContaining)), - }) - }) - - it("searching respects the view filters", async () => { - const table = await config.createTable(userTable()) - const expectedRows = [] - for (let i = 0; i < 10; i++) - await config.createRow({ - tableId: table._id, - name: generator.name(), - age: generator.integer({ min: 10, max: 30 }), - }) - - for (let i = 0; i < 5; i++) - expectedRows.push( - await config.createRow({ - tableId: table._id, - name: generator.name(), - age: 40, - }) - ) - - const createViewResponse = await config.api.viewV2.create({ - query: { equal: { age: 40 } }, - }) - - const response = await config.api.viewV2.search(createViewResponse.id) - - expect(response.body.rows).toHaveLength(5) - expect(response.body).toEqual({ - rows: expect.arrayContaining(expectedRows.map(expect.objectContaining)), - }) - }) - - const sortTestOptions: [ - { - field: string - order?: SortOrder - type?: SortType - }, - string[] - ][] = [ - [ - { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - type: SortType.STRING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - type: SortType.number, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - type: SortType.number, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - ] - - it.each(sortTestOptions)( - "allow sorting (%s)", - async (sortParams, expected) => { - await config.createTable(userTable()) - const users = [ - { name: "Alice", age: 25 }, - { name: "Bob", age: 30 }, - { name: "Charly", age: 27 }, - { name: "Danny", age: 15 }, - ] - for (const user of users) { - await config.createRow({ - tableId: config.table!._id, - ...user, - }) - } - - const createViewResponse = await config.api.viewV2.create({ - sort: sortParams, - }) - - const response = await config.api.viewV2.search(createViewResponse.id) - - expect(response.body.rows).toHaveLength(4) - expect(response.body).toEqual({ - rows: expected.map(name => expect.objectContaining({ name })), - }) - } - ) - - it.each(sortTestOptions)( - "allow override the default view sorting (%s)", - async (sortParams, expected) => { - await config.createTable(userTable()) - const users = [ - { name: "Alice", age: 25 }, - { name: "Bob", age: 30 }, - { name: "Charly", age: 27 }, - { name: "Danny", age: 15 }, - ] - for (const user of users) { - await config.createRow({ - tableId: config.table!._id, - ...user, - }) - } - - const createViewResponse = await config.api.viewV2.create({ - sort: { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - }) - - const response = await config.api.viewV2.search(createViewResponse.id, { - sort: { - column: sortParams.field, - order: sortParams.order, - type: sortParams.type, - }, - }) - - expect(response.body.rows).toHaveLength(4) - expect(response.body).toEqual({ - rows: expected.map(name => expect.objectContaining({ name })), - }) - } - ) - - it("when schema is defined, defined columns and row attributes are returned", async () => { - const table = await config.createTable(userTable()) - const rows = [] - for (let i = 0; i < 10; i++) { - rows.push( - await config.createRow({ - tableId: table._id, - name: generator.name(), - age: generator.age(), - }) - ) - } - - const view = await config.api.viewV2.create({ - schema: { name: {} }, - }) - const response = await config.api.viewV2.search(view.id) - - expect(response.body.rows).toHaveLength(10) - expect(response.body.rows).toEqual( - expect.arrayContaining( - rows.map(r => ({ - ...expectAnyInternalColsAttributes, - _viewId: view.id, - name: r.name, - })) - ) - ) - }) - - it("views without data can be returned", async () => { - const table = await config.createTable(userTable()) - - const createViewResponse = await config.api.viewV2.create() - const response = await config.api.viewV2.search(createViewResponse.id) - - expect(response.body.rows).toHaveLength(0) - }) - }) - describe("view 2.0", () => { function userTable(): Table { return { @@ -1244,5 +998,327 @@ describe("/rows", () => { await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 }) }) }) + + describe("view search", () => { + function userTable(): Table { + return { + name: "user", + type: "user", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { type: "string" }, + }, + age: { + type: FieldType.NUMBER, + name: "age", + constraints: {}, + }, + }, + } + } + + it("returns table rows from view", async () => { + const table = await config.createTable(userTable()) + const rows = [] + for (let i = 0; i < 10; i++) { + rows.push(await config.createRow({ tableId: table._id })) + } + + const createViewResponse = await config.api.viewV2.create() + const response = await config.api.viewV2.search(createViewResponse.id) + + expect(response.body.rows).toHaveLength(10) + expect(response.body).toEqual({ + rows: expect.arrayContaining(rows.map(expect.objectContaining)), + }) + }) + + it("searching respects the view filters", async () => { + const table = await config.createTable(userTable()) + const expectedRows = [] + for (let i = 0; i < 10; i++) + await config.createRow({ + tableId: table._id, + name: generator.name(), + age: generator.integer({ min: 10, max: 30 }), + }) + + for (let i = 0; i < 5; i++) + expectedRows.push( + await config.createRow({ + tableId: table._id, + name: generator.name(), + age: 40, + }) + ) + + const createViewResponse = await config.api.viewV2.create({ + query: [{ operator: "equal", field: "age", value: 40 }], + }) + + const response = await config.api.viewV2.search(createViewResponse.id) + + expect(response.body.rows).toHaveLength(5) + expect(response.body).toEqual({ + rows: expect.arrayContaining( + expectedRows.map(expect.objectContaining) + ), + }) + }) + + const sortTestOptions: [ + { + field: string + order?: SortOrder + type?: SortType + }, + string[] + ][] = [ + [ + { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + type: SortType.STRING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + type: SortType.number, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + type: SortType.number, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + ] + + it.each(sortTestOptions)( + "allow sorting (%s)", + async (sortParams, expected) => { + await config.createTable(userTable()) + const users = [ + { name: "Alice", age: 25 }, + { name: "Bob", age: 30 }, + { name: "Charly", age: 27 }, + { name: "Danny", age: 15 }, + ] + for (const user of users) { + await config.createRow({ + tableId: config.table!._id, + ...user, + }) + } + + const createViewResponse = await config.api.viewV2.create({ + sort: sortParams, + }) + + const response = await config.api.viewV2.search(createViewResponse.id) + + expect(response.body.rows).toHaveLength(4) + expect(response.body).toEqual({ + rows: expected.map(name => expect.objectContaining({ name })), + }) + } + ) + + it.each(sortTestOptions)( + "allow override the default view sorting (%s)", + async (sortParams, expected) => { + await config.createTable(userTable()) + const users = [ + { name: "Alice", age: 25 }, + { name: "Bob", age: 30 }, + { name: "Charly", age: 27 }, + { name: "Danny", age: 15 }, + ] + for (const user of users) { + await config.createRow({ + tableId: config.table!._id, + ...user, + }) + } + + const createViewResponse = await config.api.viewV2.create({ + sort: { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + }) + + const response = await config.api.viewV2.search( + createViewResponse.id, + { + sort: sortParams.field, + sortOrder: sortParams.order, + sortType: sortParams.type, + } + ) + + expect(response.body.rows).toHaveLength(4) + expect(response.body).toEqual({ + rows: expected.map(name => expect.objectContaining({ name })), + }) + } + ) + + it("when schema is defined, defined columns and row attributes are returned", async () => { + const table = await config.createTable(userTable()) + const rows = [] + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + tableId: table._id, + name: generator.name(), + age: generator.age(), + }) + ) + } + + const view = await config.api.viewV2.create({ + schema: { name: {} }, + }) + const response = await config.api.viewV2.search(view.id) + + expect(response.body.rows).toHaveLength(10) + expect(response.body.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyInternalColsAttributes, + _viewId: view.id, + name: r.name, + })) + ) + ) + }) + + it("views without data can be returned", async () => { + const table = await config.createTable(userTable()) + + const createViewResponse = await config.api.viewV2.create() + const response = await config.api.viewV2.search(createViewResponse.id) + + expect(response.body.rows).toHaveLength(0) + }) + + it("respects the limit parameter", async () => { + const table = await config.createTable(userTable()) + const rows = [] + for (let i = 0; i < 10; i++) { + rows.push(await config.createRow({ tableId: table._id })) + } + const limit = generator.integer({ min: 1, max: 8 }) + + const createViewResponse = await config.api.viewV2.create() + const response = await config.api.viewV2.search(createViewResponse.id, { + limit, + }) + + expect(response.body.rows).toHaveLength(limit) + }) + + it("can handle pagination", async () => { + const table = await config.createTable(userTable()) + const rows = [] + for (let i = 0; i < 10; i++) { + rows.push(await config.createRow({ tableId: table._id })) + } + // rows.sort((a, b) => (a._id! > b._id! ? 1 : -1)) + + const createViewResponse = await config.api.viewV2.create() + const allRows = (await config.api.viewV2.search(createViewResponse.id)) + .body.rows + + const firstPageResponse = await config.api.viewV2.search( + createViewResponse.id, + { + paginate: true, + limit: 4, + } + ) + expect(firstPageResponse.body).toEqual({ + rows: expect.arrayContaining(allRows.slice(0, 4)), + totalRows: 10, + hasNextPage: true, + bookmark: expect.any(String), + }) + + const secondPageResponse = await config.api.viewV2.search( + createViewResponse.id, + { + paginate: true, + limit: 4, + bookmark: firstPageResponse.body.bookmark, + } + ) + expect(secondPageResponse.body).toEqual({ + rows: expect.arrayContaining(allRows.slice(4, 8)), + totalRows: 10, + hasNextPage: true, + bookmark: expect.any(String), + }) + + const lastPageResponse = await config.api.viewV2.search( + createViewResponse.id, + { + paginate: true, + limit: 4, + bookmark: secondPageResponse.body.bookmark, + } + ) + expect(lastPageResponse.body).toEqual({ + rows: expect.arrayContaining(allRows.slice(8)), + totalRows: 10, + hasNextPage: false, + bookmark: expect.any(String), + }) + }) + }) }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index fc97e657c5..e30bc2c0b1 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -62,7 +62,7 @@ describe("/v2/views", () => { name: generator.name(), tableId: config.table!._id!, primaryDisplay: generator.word(), - query: { allOr: false, equal: { field: "value" } }, + query: [{ operator: "equal", field: "field", value: "value" }], sort: { field: "fieldToSort", order: SortOrder.DESCENDING, @@ -190,7 +190,7 @@ describe("/v2/views", () => { const tableId = config.table!._id! await config.api.viewV2.update({ ...view, - query: { equal: { newField: "thatValue" } }, + query: [{ operator: "equal", field: "newField", value: "thatValue" }], }) expect(await config.api.table.get(tableId)).toEqual({ @@ -198,7 +198,9 @@ describe("/v2/views", () => { views: { [view.name]: { ...view, - query: { equal: { newField: "thatValue" } }, + query: [ + { operator: "equal", field: "newField", value: "thatValue" }, + ], schema: expect.anything(), }, }, @@ -216,7 +218,13 @@ describe("/v2/views", () => { tableId, name: view.name, primaryDisplay: generator.word(), - query: { equal: { [generator.word()]: generator.word() } }, + query: [ + { + operator: "equal", + field: generator.word(), + value: generator.word(), + }, + ], sort: { field: generator.word(), order: SortOrder.DESCENDING, @@ -285,7 +293,7 @@ describe("/v2/views", () => { { ...view, tableId: generator.guid(), - query: { equal: { newField: "thatValue" } }, + query: [{ operator: "equal", field: "newField", value: "thatValue" }], }, { expectStatus: 404 } ) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index a523d828f3..4861f473ea 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,23 +1,9 @@ -import { SearchFilters, SortOrder, SortType } from "@budibase/types" +import { SearchFilters, SearchParams } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" import { Format } from "../../../api/controllers/view/exporters" -export interface SearchParams { - tableId: string - paginate?: boolean - query: SearchFilters - bookmark?: string - limit?: number - sort?: string - sortOrder?: SortOrder - sortType?: SortType - version?: string - disableEscaping?: boolean - fields?: string[] -} - export interface ViewParams { calculation: string group: string diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 159f3d84fd..447d1d7d16 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -6,6 +6,7 @@ import { IncludeRelationship, Row, SearchFilters, + SearchParams, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" @@ -13,7 +14,7 @@ import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" -import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import { ExportRowsParams, ExportRowsResult } from "../search" import { HTTPError, db } from "@budibase/backend-core" import pick from "lodash/pick" diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index e7f0aadfd6..dfc5b7dcab 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -12,7 +12,7 @@ import { } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Database, Row, Table } from "@budibase/types" +import { Database, Row, Table, SearchParams } from "@budibase/types" import { cleanExportRows } from "../utils" import { Format, @@ -28,7 +28,7 @@ import { getFromMemoryDoc, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" -import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import { ExportRowsParams, ExportRowsResult } from "../search" import pick from "lodash/pick" export async function search(options: SearchParams) { diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index b24557fd4f..b3bddfbc97 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -1,8 +1,15 @@ import { GenericContainer } from "testcontainers" -import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types" +import { + Datasource, + FieldType, + Row, + SourceName, + Table, + SearchParams, +} from "@budibase/types" + import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" -import { SearchParams } from "../../search" import { search } from "../external" import { expectAnyExternalColsAttributes, diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index a58c368cea..b3e98a1149 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -1,6 +1,5 @@ -import { FieldType, Row, Table } from "@budibase/types" +import { FieldType, Row, Table, SearchParams } from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" -import { SearchParams } from "../../search" import { search } from "../internal" import { expectAnyInternalColsAttributes, diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index 5ad2b2d3d7..813d2ebfd1 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -1,13 +1,12 @@ import { CreateViewRequest, - SortOrder, - SortType, UpdateViewRequest, DeleteRowRequest, PatchRowRequest, PatchRowResponse, Row, ViewV2, + SearchViewRowRequest, } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -81,31 +80,12 @@ export class ViewV2API extends TestAPI { search = async ( viewId: string, - options?: { - sort: { - column: string - order?: SortOrder - type?: SortType - } - }, + params?: SearchViewRowRequest, { expectStatus } = { expectStatus: 200 } ) => { - const qs: [string, any][] = [] - if (options?.sort.column) { - qs.push(["sort_column", options.sort.column]) - } - if (options?.sort.order) { - qs.push(["sort_order", options.sort.order]) - } - if (options?.sort.type) { - qs.push(["sort_type", options.sort.type]) - } - let url = `/api/v2/views/${viewId}/search` - if (qs.length) { - url += "?" + qs.map(q => q.join("=")).join("&") - } return this.request - .get(url) + .post(`/api/v2/views/${viewId}/search`) + .send(params) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) .expect(expectStatus) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 54e0df92df..8739db1b40 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -1,4 +1,12 @@ -import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types" +import { + Datasource, + FieldType, + SearchFilter, + SearchQuery, + SearchQueryFields, + SortDirection, + SortType, +} from "@budibase/types" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet } from "./helpers" @@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [ OperatorOptions.NotEquals.value, OperatorOptions.Contains.value, OperatorOptions.NotContains.value, -] as (keyof QueryFields)[] +] as (keyof SearchQueryFields)[] /** * Removes any fields that contain empty strings that would cause inconsistent * behaviour with how backend tables are filtered (no value means no filter). */ -const cleanupQuery = (query: Query) => { +const cleanupQuery = (query: SearchQuery) => { if (!query) { return query } @@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => { } } -type Filter = { - operator: keyof Query - field: string - type: any - value: any - externalType: keyof typeof SqlNumberTypeRangeMap -} - -type Query = QueryFields & QueryConfig -type QueryFields = { - string?: { - [key: string]: string - } - fuzzy?: { - [key: string]: string - } - range?: { - [key: string]: { - high: number | string - low: number | string - } - } - equal?: { - [key: string]: any - } - notEqual?: { - [key: string]: any - } - empty?: { - [key: string]: any - } - notEmpty?: { - [key: string]: any - } - oneOf?: { - [key: string]: any[] - } - contains?: { - [key: string]: any[] - } - notContains?: { - [key: string]: any[] - } - containsAny?: { - [key: string]: any[] - } -} - -type QueryConfig = { - allOr?: boolean -} - -type QueryFieldsType = keyof QueryFields - /** * Builds a lucene JSON query from the filter structure generated in the builder * @param filter the builder filter structure */ -export const buildLuceneQuery = (filter: Filter[]) => { - let query: Query = { +export const buildLuceneQuery = (filter: SearchFilter[]) => { + let query: SearchQuery = { string: {}, fuzzy: {}, range: {}, @@ -227,9 +181,13 @@ export const buildLuceneQuery = (filter: Filter[]) => { } if (operator.startsWith("range") && query.range) { const minint = - SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.min || Number.MIN_SAFE_INTEGER const maxint = - SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.max || Number.MAX_SAFE_INTEGER if (!query.range[field]) { query.range[field] = { low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", @@ -275,7 +233,7 @@ export const buildLuceneQuery = (filter: Filter[]) => { * @param docs the data * @param query the JSON lucene query */ -export const runLuceneQuery = (docs: any[], query?: Query) => { +export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { if (!docs || !Array.isArray(docs)) { return [] } @@ -289,7 +247,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => { // Iterates over a set of filters and evaluates a fail function against a doc const match = ( - type: QueryFieldsType, + type: keyof SearchQueryFields, failFn: (docValue: any, testValue: any) => boolean ) => (doc: any) => { @@ -456,7 +414,7 @@ export const luceneLimit = (docs: any[], limit: string) => { return docs.slice(0, numLimit) } -export const hasFilters = (query?: Query) => { +export const hasFilters = (query?: SearchQuery) => { if (!query) { return false } diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index fedb8ec146..f1890ef777 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,3 +1,4 @@ +import { SearchParams } from "../../../sdk" import { Row } from "../../../documents" export interface PatchRowRequest extends Row { @@ -8,6 +9,14 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} -export interface SearchResponse { +export interface SearchRowRequest extends Omit {} + +export interface SearchViewRowRequest + extends Pick< + SearchRowRequest, + "sort" | "sortOrder" | "sortType" | "limit" | "bookmark" | "paginate" + > {} + +export interface SearchRowResponse { rows: any[] } diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 0e0527eb7f..cba1e04f9a 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -8,3 +8,4 @@ export * from "./system" export * from "./app" export * from "./global" export * from "./pagination" +export * from "./searchFilter" diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts new file mode 100644 index 0000000000..1b5948e50c --- /dev/null +++ b/packages/types/src/api/web/searchFilter.ts @@ -0,0 +1,51 @@ +import { FieldType } from "../../documents" + +export type SearchFilter = { + operator: keyof SearchQuery + field: string + type?: FieldType + value: any + externalType?: string +} + +export type SearchQuery = { + allOr?: boolean + string?: { + [key: string]: string + } + fuzzy?: { + [key: string]: string + } + range?: { + [key: string]: { + high: number | string + low: number | string + } + } + equal?: { + [key: string]: any + } + notEqual?: { + [key: string]: any + } + empty?: { + [key: string]: any + } + notEmpty?: { + [key: string]: any + } + oneOf?: { + [key: string]: any[] + } + contains?: { + [key: string]: any[] + } + notContains?: { + [key: string]: any[] + } + containsAny?: { + [key: string]: any[] + } +} + +export type SearchQueryFields = Omit diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 3fe8b4a500..204b995337 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,6 +1,5 @@ -import { SortOrder, SortType } from "../../api" -import { SearchFilters } from "../../sdk" -import { TableSchema, UIFieldMetadata } from "./table" +import { SearchFilter, SortOrder, SortType } from "../../api" +import { UIFieldMetadata } from "./table" export interface View { name: string @@ -20,7 +19,7 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilters + query?: SearchFilter[] sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index e4b5778ed9..0eab2ba556 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -19,3 +19,4 @@ export * from "./user" export * from "./cli" export * from "./websocket" export * from "./permissions" +export * from "./row" diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts new file mode 100644 index 0000000000..7e4b89e0f7 --- /dev/null +++ b/packages/types/src/sdk/row.ts @@ -0,0 +1,16 @@ +import { SortOrder, SortType } from "../api" +import { SearchFilters } from "./search" + +export interface SearchParams { + tableId: string + paginate?: boolean + query: SearchFilters + bookmark?: string + limit?: number + sort?: string + sortOrder?: SortOrder + sortType?: SortType + version?: string + disableEscaping?: boolean + fields?: string[] +}