Merge master.

hosting/proxy/nginx.prod.conf

@@ -61,7 +61,7 @@ http {
     set $csp_img "img-src http: https: data: blob:";
     set $csp_manifest "manifest-src 'self'";
     set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; 
-    set $csp_worker "worker-src 'none'";
+    set $csp_worker "worker-src blob:";
 
     error_page 502 503 504 /error.html;
     location = /error.html {

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.26.4",
+  "version": "2.27.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte

@@ -4,13 +4,14 @@
   export let max
   export let hideArrows = false
   export let width
+  export let type = "number"
 
   $: style = width ? `width:${width}px;` : ""
 </script>
 
 <input
   class:hide-arrows={hideArrows}
-  type="number"
+  {type}
   {style}
   {value}
   {min}
@@ -51,4 +52,7 @@
   input.hide-arrows {
     -moz-appearance: textfield;
   }
+  input[type="time"]::-webkit-calendar-picker-indicator {
+    display: none;
+  }
 </style>

packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte

@@ -1,5 +1,4 @@
 <script>
-  import { cleanInput } from "./utils"
   import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
@@ -8,39 +7,26 @@
 
   const dispatch = createEventDispatcher()
 
-  $: displayValue = value || dayjs()
+  $: displayValue = value?.format("HH:mm")
 
-  const handleHourChange = e => {
+  const handleChange = e => {
-    dispatch("change", displayValue.hour(parseInt(e.target.value)))
+    if (!e.target.value) {
+      dispatch("change", undefined)
+      return
     }
 
-  const handleMinuteChange = e => {
+    const [hour, minute] = e.target.value.split(":").map(x => parseInt(x))
-    dispatch("change", displayValue.minute(parseInt(e.target.value)))
+    dispatch("change", (value || dayjs()).hour(hour).minute(minute))
   }
-
-  const cleanHour = cleanInput({ max: 23, pad: 2, fallback: "00" })
-  const cleanMinute = cleanInput({ max: 59, pad: 2, fallback: "00" })
 </script>
 
 <div class="time-picker">
   <NumberInput
     hideArrows
-    value={displayValue.hour().toString().padStart(2, "0")}
+    type={"time"}
-    min={0}
+    value={displayValue}
-    max={23}
+    on:input={handleChange}
-    width={20}
+    on:change={handleChange}
-    on:input={cleanHour}
-    on:change={handleHourChange}
-  />
-  <span>:</span>
-  <NumberInput
-    hideArrows
-    value={displayValue.minute().toString().padStart(2, "0")}
-    min={0}
-    max={59}
-    width={20}
-    on:input={cleanMinute}
-    on:change={handleMinuteChange}
   />
 </div>
 
@@ -50,10 +36,4 @@
     flex-direction: row;
     align-items: center;
   }
-  .time-picker span {
-    font-weight: bold;
-    font-size: 18px;
-    z-index: 0;
-    margin-bottom: 1px;
-  }
 </style>

packages/bbui/src/helpers.js

@@ -166,7 +166,7 @@ export const stringifyDate = (
   const offsetForTimezone = (enableTime && ignoreTimezones) || timeOnly
   if (offsetForTimezone) {
     // Ensure we use the correct offset for the date
-    const referenceDate = timeOnly ? new Date() : value.toDate()
+    const referenceDate = value.toDate()
     const offset = referenceDate.getTimezoneOffset() * 60000
     return new Date(value.valueOf() - offset).toISOString().slice(0, -1)
   }
@@ -177,7 +177,7 @@ export const stringifyDate = (
     const year = value.year()
     const month = `${value.month() + 1}`.padStart(2, "0")
     const day = `${value.date()}`.padStart(2, "0")
-    return `${year}-${month}-${day}T00:00:00.000`
+    return `${year}-${month}-${day}`
   }
 
   // Otherwise use a normal ISO string with time and timezone

packages/builder/src/components/backend/DataTable/formula.js

@@ -56,7 +56,7 @@ export function getBindings({
       )
     }
     const field = Object.values(FIELDS).find(
-      field => field.type === schema.type && field.subtype === schema.subtype
+      field => field.type === schema.type
     )
 
     const label = path == null ? column : `${path}.0.${column}`

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -237,8 +237,13 @@
 
   const onChangeJSValue = e => {
     jsValue = encodeJSBinding(e.detail)
+    if (!e.detail?.trim()) {
+      // Don't bother saving empty values as JS
+      updateValue(null)
+    } else {
       updateValue(jsValue)
     }
+  }
 
   onMount(() => {
     // Set the initial mode appropriately

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -4,7 +4,6 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "dataBinding"
-
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/DropdownMenu.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { screenStore, componentStore } from "stores/builder"
+  import { screenStore, componentStore, navigationStore } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import {
     ActionMenu,
@@ -12,6 +12,7 @@
   import ScreenDetailsModal from "components/design/ScreenDetailsModal.svelte"
   import sanitizeUrl from "helpers/sanitizeUrl"
   import { makeComponentUnique } from "helpers/components"
+  import { capitalise } from "helpers"
 
   export let screenId
 
@@ -48,6 +49,13 @@
     try {
       // Create the screen
       await screenStore.save(duplicateScreen)
+
+      // Add new screen to navigation
+      await navigationStore.saveLink(
+        duplicateScreen.routing.route,
+        capitalise(duplicateScreen.routing.route.split("/")[1]),
+        duplicateScreen.routing.roleId
+      )
     } catch (error) {
       notifications.error("Error duplicating screen")
     }

packages/builder/src/stores/builder/components.js

@@ -20,7 +20,7 @@ import {
   previewStore,
   tables,
   componentTreeNodesStore,
-} from "stores/builder/index"
+} from "stores/builder"
 import { buildFormSchema, getSchemaForDatasource } from "dataBinding"
 import {
   BUDIBASE_INTERNAL_DB_ID,
@@ -30,6 +30,7 @@ import {
 } from "constants/backend"
 import BudiStore from "../BudiStore"
 import { Utils } from "@budibase/frontend-core"
+import { FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -296,6 +297,80 @@ export class ComponentStore extends BudiStore {
         }
       }
     })
+
+    // Add default bindings to card blocks
+    if (component._component.endsWith("/cardsblock")) {
+      // Only proceed if the card is empty, i.e. we just changed datasource or
+      // just created the card
+      const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"]
+      if (cardKeys.every(key => !component[key]) && !component.cardImageURL) {
+        const { _id, dataSource } = component
+        if (dataSource) {
+          const { schema, table } = getSchemaForDatasource(screen, dataSource)
+
+          // Finds fields by types from the schema of the configured datasource
+          const findFieldTypes = fieldTypes => {
+            if (!Array.isArray(fieldTypes)) {
+              fieldTypes = [fieldTypes]
+            }
+            return Object.entries(schema || {})
+              .filter(([name, fieldSchema]) => {
+                return (
+                  fieldTypes.includes(fieldSchema.type) &&
+                  !fieldSchema.autoColumn &&
+                  name !== table?.primaryDisplay &&
+                  !name.startsWith("_")
+                )
+              })
+              .map(([name]) => name)
+          }
+
+          // Inserts a card binding for a certain setting
+          const addBinding = (key, fallback, ...parts) => {
+            if (parts.some(x => x == null)) {
+              component[key] = fallback
+            } else {
+              parts.unshift(`${_id}-repeater`)
+              component[key] = `{{ ${parts.map(safe).join(".")} }}`
+            }
+          }
+
+          // Extract good field candidates to prefill our cards with.
+          // Use the primary display as the best field, if it exists.
+          const shortFields = [
+            ...findFieldTypes(FieldType.STRING),
+            ...findFieldTypes(FieldType.OPTIONS),
+            ...findFieldTypes(FieldType.ARRAY),
+            ...findFieldTypes(FieldType.NUMBER),
+          ]
+          const longFields = findFieldTypes(FieldType.LONGFORM)
+          if (schema?.[table?.primaryDisplay]) {
+            shortFields.unshift(table.primaryDisplay)
+          }
+
+          // Fill title and subtitle with short fields
+          addBinding("cardTitle", "Title", shortFields[0])
+          addBinding("cardSubtitle", "Subtitle", shortFields[1])
+
+          // Fill description with a long field if possible
+          const longField = longFields[0] ?? shortFields[2]
+          addBinding("cardDescription", "Description", longField)
+
+          // Attempt to fill the image setting.
+          // Check single attachment fields first.
+          let imgField = findFieldTypes(FieldType.ATTACHMENT_SINGLE)[0]
+          if (imgField) {
+            addBinding("cardImageURL", null, imgField, "url")
+          } else {
+            // Then try multi-attachment fields if no single ones exist
+            imgField = findFieldTypes(FieldType.ATTACHMENTS)[0]
+            if (imgField) {
+              addBinding("cardImageURL", null, imgField, 0, "url")
+            }
+          }
+        }
+      }
+    }
   }
 
   /**
@@ -324,21 +399,21 @@ export class ComponentStore extends BudiStore {
       ...presetProps,
     }
 
-    // Enrich empty settings
+    // Standard post processing
     this.enrichEmptySettings(instance, {
       parent,
       screen: get(selectedScreen),
       useDefaultValues: true,
     })
-
-    // Migrate nested component settings
     this.migrateSettings(instance)
 
-    // Add any extra properties the component needs
+    // Custom post processing for creation only
     let extras = {}
     if (definition.hasChildren) {
       extras._children = []
     }
+
+    // Add step name to form steps
     if (componentName.endsWith("/formstep")) {
       const parentForm = findClosestMatchingComponent(
         get(selectedScreen).props,
@@ -351,6 +426,7 @@ export class ComponentStore extends BudiStore {
       extras.step = formSteps.length + 1
       extras._instanceName = `Step ${formSteps.length + 1}`
     }
+
     return {
       ...cloneDeep(instance),
       ...extras,
@@ -463,7 +539,6 @@ export class ComponentStore extends BudiStore {
     if (!componentId || !screenId) {
       const state = get(this.store)
       componentId = componentId || state.selectedComponentId
-
       const screenState = get(screenStore)
       screenId = screenId || screenState.selectedScreenId
     }
@@ -471,7 +546,6 @@ export class ComponentStore extends BudiStore {
       return
     }
     const patchScreen = screen => {
-      // findComponent looks in the tree not comp.settings[0]
       let component = findComponent(screen.props, componentId)
       if (!component) {
         return false
@@ -480,7 +554,7 @@ export class ComponentStore extends BudiStore {
       // Mutates the fetched component with updates
       const patchResult = patchFn(component, screen)
 
-      // Mutates the component with any required settings updates
+      // Post processing
       const migrated = this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this

packages/builder/src/stores/builder/tests/component.test.js

@@ -23,6 +23,7 @@ import {
   DB_TYPE_EXTERNAL,
   DEFAULT_BB_DATASOURCE_ID,
 } from "constants/backend"
+import { makePropSafe as safe } from "@budibase/string-templates"
 
 // Could move to fixtures
 const COMP_PREFIX = "@budibase/standard-components"
@@ -360,8 +361,30 @@ describe("Component store", () => {
       resourceId: internalTableDoc._id,
       type: "table",
     })
+
+    return comp
   }
 
+  it("enrichEmptySettings - initialise cards blocks with correct fields", async ctx => {
+    const comp = enrichSettingsDS("cardsblock", ctx)
+    const expectBinding = (setting, ...parts) => {
+      expect(comp[setting]).toStrictEqual(
+        `{{ ${safe(`${comp._id}-repeater`)}.${parts.map(safe).join(".")} }}`
+      )
+    }
+    expectBinding("cardTitle", internalTableDoc.schema.MediaTitle.name)
+    expectBinding("cardSubtitle", internalTableDoc.schema.MediaVersion.name)
+    expectBinding(
+      "cardDescription",
+      internalTableDoc.schema.MediaDescription.name
+    )
+    expectBinding(
+      "cardImageURL",
+      internalTableDoc.schema.MediaImage.name,
+      "url"
+    )
+  })
+
   it("enrichEmptySettings - set default datasource for 'table' setting type", async ctx => {
     enrichSettingsDS("formblock", ctx)
   })

packages/builder/src/stores/builder/tests/fixtures/index.js

@@ -8,6 +8,7 @@ import {
   DB_TYPE_EXTERNAL,
   DEFAULT_BB_DATASOURCE_ID,
 } from "constants/backend"
+import { FieldType } from "@budibase/types"
 
 const getDocId = () => {
   return v4().replace(/-/g, "")
@@ -45,6 +46,52 @@ export const COMPONENT_DEFINITIONS = {
       },
     ],
   },
+  cardsblock: {
+    block: true,
+    name: "Cards Block",
+    settings: [
+      {
+        type: "dataSource",
+        label: "Data",
+        key: "dataSource",
+        required: true,
+      },
+      {
+        section: true,
+        name: "Cards",
+        settings: [
+          {
+            type: "text",
+            key: "cardTitle",
+            label: "Title",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardSubtitle",
+            label: "Subtitle",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardDescription",
+            label: "Description",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardImageURL",
+            label: "Image URL",
+            nested: true,
+            resetOn: "dataSource",
+          },
+        ],
+      },
+    ],
+  },
   container: {
     name: "Container",
   },
@@ -262,14 +309,23 @@ export const internalTableDoc = {
   name: "Media",
   sourceId: BUDIBASE_INTERNAL_DB_ID,
   sourceType: DB_TYPE_INTERNAL,
+  primaryDisplay: "MediaTitle",
   schema: {
     MediaTitle: {
       name: "MediaTitle",
-      type: "string",
+      type: FieldType.STRING,
     },
     MediaVersion: {
       name: "MediaVersion",
-      type: "string",
+      type: FieldType.STRING,
+    },
+    MediaDescription: {
+      name: "MediaDescription",
+      type: FieldType.LONGFORM,
+    },
+    MediaImage: {
+      name: "MediaImage",
+      type: FieldType.ATTACHMENT_SINGLE,
     },
   },
 }

packages/client/manifest.json

@@ -6243,27 +6243,28 @@
             "key": "cardTitle",
             "label": "Title",
             "nested": true,
-            "defaultValue": "Title"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardSubtitle",
             "label": "Subtitle",
             "nested": true,
-            "defaultValue": "Subtitle"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardDescription",
             "label": "Description",
             "nested": true,
-            "defaultValue": "Description"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardImageURL",
             "label": "Image URL",
-            "nested": true
+            "nested": true,
+            "resetOn": "dataSource"
           },
           {
             "type": "boolean",

packages/client/src/components/app/NavItem.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import active from "svelte-spa-router/active"
   import { Icon } from "@budibase/bbui"
 
@@ -13,8 +13,6 @@
   export let navStateStore
 
   const dispatch = createEventDispatcher()
-  const sdk = getContext("sdk")
-  const { linkable } = sdk
 
   let renderKey
 
@@ -46,10 +44,9 @@
       styled
     -->
     <a
-      href={url}
+      href="#{url}"
       on:click={onClickLink}
       use:active={url}
-      use:linkable
       class:active={false}
     >
       {text}
@@ -73,10 +70,9 @@
           {#each subLinks || [] as subLink}
             {#if subLink.internalLink}
               <a
-                href={subLink.url}
+                href="#{subLink.url}"
                 on:click={onClickLink}
                 use:active={subLink.url}
-                use:linkable
               >
                 {subLink.text}
               </a>

packages/client/src/utils/buttonActions.js

@@ -238,7 +238,13 @@ const triggerAutomationHandler = async action => {
   }
 }
 const navigationHandler = action => {
-  const { url, peek, externalNewTab } = action.parameters
+  let { url, peek, externalNewTab, type } = action.parameters
+
+  // Ensure in-app navigation starts with a slash
+  if (type === "screen" && url && !url.startsWith("/")) {
+    url = `/${url}`
+  }
+
   routeStore.actions.navigate(url, peek, externalNewTab)
   closeSidePanelHandler()
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,3 +1,4 @@
+import dayjs from "dayjs"
 import {
   AutoFieldSubType,
   AutoReason,
@@ -285,13 +286,10 @@ export class ExternalRequest<T extends Operation> {
       // parse floats/numbers
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
-      }
+      } else if (field.type === FieldType.LINK) {
-      // if its not a link then just copy it over
+        const { tableName: linkTableName } = breakExternalTableId(
-      if (field.type !== FieldType.LINK) {
+          field?.tableId
-        newRow[key] = row[key]
+        )
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field?.tableId)
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -306,7 +304,8 @@ export class ExternalRequest<T extends Operation> {
             if (typeof row[key] === "string") {
               id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
             }
-          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+            newRow[field.foreignKey || linkTablePrimary] =
+              breakRowIdField(id)[0]
           } else {
             // Removing from both new and row, as we don't know if it has already been processed
             row[field.foreignKey || linkTablePrimary] = null
@@ -345,6 +344,16 @@ export class ExternalRequest<T extends Operation> {
             })
           }
         }
+      } else if (
+        field.type === FieldType.DATETIME &&
+        field.timeOnly &&
+        row[key] &&
+        dayjs(row[key]).isValid()
+      ) {
+        newRow[key] = dayjs(row[key]).format("HH:mm")
+      } else {
+        newRow[key] = row[key]
+      }
     }
     // we return the relationships that may need to be created in the through table
     // we do this so that if the ID is generated by the DB it can be inserted

packages/server/src/api/routes/tests/row.spec.ts

@@ -485,6 +485,25 @@ describe.each([
         )
         expect(response.message).toBe("Cannot create new user entry.")
       })
+
+    it("should not mis-parse date string out of JSON", async () => {
+      const table = await config.api.table.save(
+        saveTableRequest({
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+            },
+          },
+        })
+      )
+
+      const row = await config.api.row.save(table._id!, {
+        name: `{ "foo": "2023-01-26T11:48:57.000Z" }`,
+      })
+
+      expect(row.name).toEqual(`{ "foo": "2023-01-26T11:48:57.000Z" }`)
+    })
   })
 
   describe("get", () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -16,6 +16,8 @@ import {
   Table,
   TableSchema,
   User,
+  Row,
+  RelationshipType,
 } from "@budibase/types"
 import _ from "lodash"
 import tk from "timekeeper"
@@ -72,19 +74,20 @@ describe.each([
   })
 
   async function createTable(schema: TableSchema) {
-    table = await config.api.table.save(
+    return await config.api.table.save(
       tableForDatasource(datasource, { schema })
     )
   }
 
   async function createRows(rows: Record<string, any>[]) {
-    await config.api.row.bulkImport(table._id!, { rows })
+    // Shuffling to avoid false positives given a fixed order
+    await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) })
   }
 
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private findRow(expectedRow: any, foundRows: any[]) {
+    private popRow(expectedRow: any, foundRows: any[]) {
       const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
       if (!row) {
         const fields = Object.keys(expectedRow)
@@ -97,6 +100,9 @@ describe.each([
           )} in ${JSON.stringify(searchedObjects)}`
         )
       }
+
+      // Ensuring the same row is not matched twice
+      foundRows.splice(foundRows.indexOf(row), 1)
       return row
     }
 
@@ -113,9 +119,9 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expectedRows.map((expectedRow: any) =>
-          expect.objectContaining(this.findRow(expectedRow, foundRows))
+          expect.objectContaining(this.popRow(expectedRow, foundRows))
         )
       )
     }
@@ -132,10 +138,10 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )
@@ -151,10 +157,10 @@ describe.each([
       })
 
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )
@@ -185,7 +191,7 @@ describe.each([
 
   describe("boolean", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
       })
       await createRows([{ isTrue: true }, { isTrue: false }])
@@ -315,7 +321,7 @@ describe.each([
           })
       )
 
-      await createTable({
+      table = await createTable({
         name: { name: "name", type: FieldType.STRING },
         appointment: { name: "appointment", type: FieldType.DATETIME },
         single_user: {
@@ -591,7 +597,7 @@ describe.each([
 
   describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         name: { name: "name", type: FieldType.STRING },
       })
       await createRows([{ name: "foo" }, { name: "bar" }])
@@ -629,6 +635,19 @@ describe.each([
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { name: "none" } }).toFindNothing())
+
+      it("works as an or condition", () =>
+        expectQuery({
+          allOr: true,
+          equal: { name: "foo" },
+          oneOf: { name: ["bar"] },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
+
+      it("can have multiple values for same column", () =>
+        expectQuery({
+          allOr: true,
+          equal: { "1:name": "foo", "2:name": "bar" },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
     })
 
     describe("notEqual", () => {
@@ -663,6 +682,21 @@ describe.each([
         expectQuery({ fuzzy: { name: "none" } }).toFindNothing())
     })
 
+    describe("string", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ string: { name: "fo" } }).toContainExactly([
+          { name: "foo" },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ string: { name: "none" } }).toFindNothing())
+
+      it("is case-insensitive", () =>
+        expectQuery({ string: { name: "FO" } }).toContainExactly([
+          { name: "foo" },
+        ]))
+    })
+
     describe("range", () => {
       it("successfully finds multiple rows", () =>
         expectQuery({
@@ -683,6 +717,20 @@ describe.each([
         expectQuery({
           range: { name: { low: "g", high: "h" } },
         }).toFindNothing())
+
+      !isLucene &&
+        it("ignores low if it's an empty object", () =>
+          expectQuery({
+            // @ts-ignore
+            range: { name: { low: {}, high: "z" } },
+          }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
+
+      !isLucene &&
+        it("ignores high if it's an empty object", () =>
+          expectQuery({
+            // @ts-ignore
+            range: { name: { low: "a", high: {} } },
+          }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
     })
 
     describe("empty", () => {
@@ -747,7 +795,7 @@ describe.each([
 
   describe("numbers", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         age: { name: "age", type: FieldType.NUMBER },
       })
       await createRows([{ age: 1 }, { age: 10 }])
@@ -856,7 +904,7 @@ describe.each([
     const JAN_10TH = "2020-01-10T00:00:00.000Z"
 
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         dob: { name: "dob", type: FieldType.DATETIME },
       })
 
@@ -966,9 +1014,162 @@ describe.each([
     })
   })
 
+  !isInternal &&
+    describe("datetime - time only", () => {
+      const T_1000 = "10:00"
+      const T_1045 = "10:45"
+      const T_1200 = "12:00"
+      const T_1530 = "15:30"
+      const T_0000 = "00:00"
+
+      const UNEXISTING_TIME = "10:01"
+
+      const NULL_TIME__ID = `null_time__id`
+
+      beforeAll(async () => {
+        table = await createTable({
+          timeid: { name: "timeid", type: FieldType.STRING },
+          time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
+        })
+
+        await createRows([
+          { timeid: NULL_TIME__ID, time: null },
+          { time: T_1000 },
+          { time: T_1045 },
+          { time: T_1200 },
+          { time: T_1530 },
+          { time: T_0000 },
+        ])
+      })
+
+      describe("equal", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ equal: { time: T_1000 } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing())
+      })
+
+      describe("notEqual", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+            { time: "00:00:00" },
+          ]))
+
+        it("return all when requesting non-existing", () =>
+          expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly(
+            [
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+              { time: "00:00:00" },
+            ]
+          ))
+      })
+
+      describe("oneOf", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing())
+      })
+
+      describe("range", () => {
+        it("successfully finds a row", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1045 } },
+          }).toContainExactly([{ time: "10:45:00" }]))
+
+        it("successfully finds multiple rows", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1530 } },
+          }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("successfully finds no rows", () =>
+          expectQuery({
+            range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } },
+          }).toFindNothing())
+      })
+
+      describe("sort", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatchExactly([
+            { timeid: NULL_TIME__ID },
+            { time: "00:00:00" },
+            { time: "10:00:00" },
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatchExactly([
+            { time: "15:30:00" },
+            { time: "12:00:00" },
+            { time: "10:45:00" },
+            { time: "10:00:00" },
+            { time: "00:00:00" },
+            { timeid: NULL_TIME__ID },
+          ]))
+
+        describe("sortType STRING", () => {
+          it("sorts ascending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.ASCENDING,
+            }).toMatchExactly([
+              { timeid: NULL_TIME__ID },
+              { time: "00:00:00" },
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+            ]))
+
+          it("sorts descending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.DESCENDING,
+            }).toMatchExactly([
+              { time: "15:30:00" },
+              { time: "12:00:00" },
+              { time: "10:45:00" },
+              { time: "10:00:00" },
+              { time: "00:00:00" },
+              { timeid: NULL_TIME__ID },
+            ]))
+        })
+      })
+    })
+
   describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         numbers: {
           name: "numbers",
           type: FieldType.ARRAY,
@@ -1048,7 +1249,7 @@ describe.each([
     const BIG = "9223372036854775807"
 
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         num: { name: "num", type: FieldType.BIGINT },
       })
       await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@@ -1139,7 +1340,7 @@ describe.each([
   isInternal &&
     describe("auto", () => {
       beforeAll(async () => {
-        await createTable({
+        table = await createTable({
           auto: {
             name: "auto",
             type: FieldType.AUTO,
@@ -1266,7 +1467,76 @@ describe.each([
               { auto: 2 },
               { auto: 1 },
             ]))
+
+          // This is important for pagination. The order of results must always
+          // be stable or pagination will break. We don't want the user to need
+          // to specify an order for pagination to work.
+          it("is stable without a sort specified", async () => {
+            let { rows } = await config.api.row.search(table._id!, {
+              tableId: table._id!,
+              query: {},
             })
+
+            for (let i = 0; i < 10; i++) {
+              const response = await config.api.row.search(table._id!, {
+                tableId: table._id!,
+                limit: 1,
+                query: {},
+              })
+              expect(response.rows).toEqual(rows)
+            }
+          })
+        })
+
+      // TODO(samwho): fix for SQS
+      !isSqs &&
+        describe("pagination", () => {
+          it("should paginate through all rows", async () => {
+            // @ts-ignore
+            let bookmark: string | number = undefined
+            let rows: Row[] = []
+
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+              const response = await config.api.row.search(table._id!, {
+                tableId: table._id!,
+                limit: 3,
+                query: {},
+                bookmark,
+                paginate: true,
+              })
+
+              rows.push(...response.rows)
+
+              if (!response.bookmark || !response.hasNextPage) {
+                break
+              }
+              bookmark = response.bookmark
+            }
+
+            expect(rows).toHaveLength(10)
+            expect(rows.map(row => row.auto)).toEqual(
+              expect.arrayContaining([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+            )
+          })
+        })
+    })
+
+  describe("field name 1:name", () => {
+    beforeAll(async () => {
+      table = await createTable({
+        "1:name": { name: "1:name", type: FieldType.STRING },
+      })
+      await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
+    })
+
+    it("successfully finds a row", () =>
+      expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([
+        { "1:name": "bar" },
+      ]))
+
+    it("fails to find nonexistent row", () =>
+      expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing())
   })
 
   describe("user", () => {
@@ -1277,7 +1547,7 @@ describe.each([
       user1 = await config.createUser({ _id: `us_${utils.newid()}` })
       user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      await createTable({
+      table = await createTable({
         user: {
           name: "user",
           type: FieldType.BB_REFERENCE_SINGLE,
@@ -1349,7 +1619,7 @@ describe.each([
       user1 = await config.createUser({ _id: `us_${utils.newid()}` })
       user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      await createTable({
+      table = await createTable({
         users: {
           name: "users",
           type: FieldType.BB_REFERENCE,
@@ -1425,4 +1695,51 @@ describe.each([
         }).toFindNothing())
     })
   })
+
+  // This will never work for Lucene.
+  !isLucene &&
+    describe("relations", () => {
+      let otherTable: Table
+      let rows: Row[]
+
+      beforeAll(async () => {
+        otherTable = await createTable({
+          one: { name: "one", type: FieldType.STRING },
+        })
+        table = await createTable({
+          two: { name: "two", type: FieldType.STRING },
+          other: {
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            name: "other",
+            fieldName: "other",
+            tableId: otherTable._id!,
+            constraints: {
+              type: "array",
+            },
+          },
+        })
+
+        rows = await Promise.all([
+          config.api.row.save(otherTable._id!, { one: "foo" }),
+          config.api.row.save(otherTable._id!, { one: "bar" }),
+        ])
+
+        await Promise.all([
+          config.api.row.save(table._id!, {
+            two: "foo",
+            other: [rows[0]._id],
+          }),
+          config.api.row.save(table._id!, {
+            two: "bar",
+            other: [rows[1]._id],
+          }),
+        ])
+      })
+
+      it("can search through relations", () =>
+        expectQuery({
+          equal: { [`${otherTable.name}.one`]: "foo" },
+        }).toContainExactly([{ two: "foo", other: [{ _id: rows[0]._id }] }]))
+    })
 })

packages/server/src/api/routes/tests/table.spec.ts

@@ -52,14 +52,24 @@ describe.each([
       jest.clearAllMocks()
     })
 
-    it("creates a table successfully", async () => {
+    it.each([
-      const name = generator.guid()
+      "alphanum",
+      "with spaces",
+      "with-dashes",
+      "with_underscores",
+      'with "double quotes"',
+      "with 'single quotes'",
+      "with `backticks`",
+    ])("creates a table with name: %s", async name => {
       const table = await config.api.table.save(
         tableForDatasource(datasource, { name })
       )
       expect(table.name).toEqual(name)
       expect(events.table.created).toHaveBeenCalledTimes(1)
       expect(events.table.created).toHaveBeenCalledWith(table)
+
+      const res = await config.api.table.get(table._id!)
+      expect(res.name).toEqual(name)
     })
 
     it("creates a table via data import", async () => {

packages/server/src/integrations/base/sql.ts

@@ -122,11 +122,8 @@ function generateSelectStatement(
     const fieldNames = field.split(/\./g)
     const tableName = fieldNames[0]
     const columnName = fieldNames[1]
-    if (
+    const columnSchema = schema?.[columnName]
-      columnName &&
+    if (columnSchema && knex.client.config.client === SqlClient.POSTGRES) {
-      schema?.[columnName] &&
-      knex.client.config.client === SqlClient.POSTGRES
-    ) {
       const externalType = schema[columnName].externalType
       if (externalType?.includes("money")) {
         return knex.raw(
@@ -134,6 +131,14 @@ function generateSelectStatement(
         )
       }
     }
+    if (
+      knex.client.config.client === SqlClient.MS_SQL &&
+      columnSchema?.type === FieldType.DATETIME &&
+      columnSchema.timeOnly
+    ) {
+      // Time gets returned as timestamp from mssql, not matching the expected HH:mm format
+      return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+    }
     return `${field} as ${field}`
   })
 }
@@ -402,7 +407,13 @@ class InternalBuilder {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
           value.direction === SortDirection.ASCENDING ? "asc" : "desc"
-        query = query.orderBy(`${aliased}.${key}`, direction)
+        let nulls
+        if (this.client === SqlClient.POSTGRES) {
+          // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
+          nulls = value.direction === SortDirection.ASCENDING ? "first" : "last"
+        }
+
+        query = query.orderBy(`${aliased}.${key}`, direction, nulls)
       }
     } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
       // @ts-ignore
@@ -653,12 +664,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    */
   _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
-    const config: { client: string; useNullAsDefault?: boolean } = {
+    const config: Knex.Config = {
       client: sqlClient,
     }
     if (sqlClient === SqlClient.SQL_LITE) {
       config.useNullAsDefault = true
     }
+
     const client = knex(config)
     let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)

packages/server/src/integrations/base/sqlTable.ts

@@ -79,9 +79,13 @@ function generateSchema(
         schema.boolean(key)
         break
       case FieldType.DATETIME:
+        if (!column.timeOnly) {
           schema.datetime(key, {
             useTz: !column.ignoreTimezones,
           })
+        } else {
+          schema.time(key)
+        }
         break
       case FieldType.ARRAY:
       case FieldType.BB_REFERENCE:

packages/server/src/integrations/tests/sql.spec.ts

@@ -56,48 +56,6 @@ function generateReadJson({
   }
 }
 
-function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
-  return {
-    endpoint: endpoint(table, "CREATE"),
-    meta: {
-      table: TABLE,
-    },
-    body,
-  }
-}
-
-function generateUpdateJson({
-  table = TABLE_NAME,
-  body = {},
-  filters = {},
-  meta = {},
-}: {
-  table: string
-  body?: any
-  filters?: any
-  meta?: any
-}): QueryJson {
-  if (!meta.table) {
-    meta.table = TABLE
-  }
-  return {
-    endpoint: endpoint(table, "UPDATE"),
-    filters,
-    body,
-    meta,
-  }
-}
-
-function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
-  return {
-    endpoint: endpoint(table, "DELETE"),
-    meta: {
-      table: TABLE,
-    },
-    filters,
-  }
-}
-
 function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
   return {
     endpoint: {
@@ -178,353 +136,6 @@ describe("SQL query builder", () => {
     sql = new Sql(client, limit)
   })
 
-  it("should test a basic read", () => {
-    const query = sql._query(generateReadJson())
-    expect(query).toEqual({
-      bindings: [limit],
-      sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should test a read with specific columns", () => {
-    const nameProp = `${TABLE_NAME}.name`,
-      ageProp = `${TABLE_NAME}.age`
-    const query = sql._query(
-      generateReadJson({
-        fields: [nameProp, ageProp],
-      })
-    )
-    expect(query).toEqual({
-      bindings: [limit],
-      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should test a where string starts with read", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          string: {
-            name: "John",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["John%", limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should test a where range read", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            age: {
-              low: 2,
-              high: 10,
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [2, 10, limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should test for multiple IDs with OR", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          equal: {
-            age: 10,
-            name: "John",
-          },
-          allOr: true,
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10, "John", limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should allow filtering on a related field", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          equal: {
-            age: 10,
-            "task.name": "task 1",
-          },
-        },
-      })
-    )
-    // order of bindings changes because relationship filters occur outside inner query
-    expect(query).toEqual({
-      bindings: [10, limit, "task 1"],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`,
-    })
-  })
-
-  it("should test an create statement", () => {
-    const query = sql._query(
-      generateCreateJson(TABLE_NAME, {
-        name: "Michael",
-        age: 45,
-      })
-    )
-    expect(query).toEqual({
-      bindings: [45, "Michael"],
-      sql: `insert into "${TABLE_NAME}" ("age", "name") values ($1, $2) returning *`,
-    })
-  })
-
-  it("should test an update statement", () => {
-    const query = sql._query(
-      generateUpdateJson({
-        table: TABLE_NAME,
-        body: {
-          name: "John",
-        },
-        filters: {
-          equal: {
-            id: 1001,
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["John", 1001],
-      sql: `update "${TABLE_NAME}" set "name" = $1 where "${TABLE_NAME}"."id" = $2 returning *`,
-    })
-  })
-
-  it("should test a delete statement", () => {
-    const query = sql._query(
-      generateDeleteJson(TABLE_NAME, {
-        equal: {
-          id: 1001,
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [1001],
-      sql: `delete from "${TABLE_NAME}" where "${TABLE_NAME}"."id" = $1 returning *`,
-    })
-  })
-
-  it("should work with MS-SQL", () => {
-    const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson())
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`,
-    })
-  })
-
-  it("should work with MySQL", () => {
-    const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson())
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``,
-    })
-  })
-
-  it("should use greater than when only low range specified", () => {
-    const date = new Date()
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            property: {
-              low: date,
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [date, limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should use less than when only high range specified", () => {
-    const date = new Date()
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            property: {
-              high: date,
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [date, limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should use AND like expression for MS-SQL when filter is contains", () => {
-    const query = new Sql(SqlClient.MS_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          contains: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10, "%20%", "%25%", `%"john"%`, `%"mary"%`],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER([${TABLE_NAME}].[age]) LIKE @p1 AND LOWER([${TABLE_NAME}].[age]) LIKE @p2) and (LOWER([${TABLE_NAME}].[name]) LIKE @p3 AND LOWER([${TABLE_NAME}].[name]) LIKE @p4)) as [${TABLE_NAME}]`,
-    })
-  })
-
-  it("should use JSON_CONTAINS expression for MySQL when filter is contains", () => {
-    const query = new Sql(SqlClient.MY_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          contains: {
-            age: [20],
-            name: ["John"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``,
-    })
-  })
-
-  it("should use jsonb operator expression for PostgreSQL when filter is contains", () => {
-    const query = new Sql(SqlClient.POSTGRES, 10)._query(
-      generateReadJson({
-        filters: {
-          contains: {
-            age: [20],
-            name: ["John"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age"::jsonb @> '[20]' and "${TABLE_NAME}"."name"::jsonb @> '["John"]' limit $1) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should use NOT like expression for MS-SQL when filter is notContains", () => {
-    const query = new Sql(SqlClient.MS_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          notContains: {
-            age: [20],
-            name: ["John"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10, "%20%", `%"john"%`],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where NOT (LOWER([${TABLE_NAME}].[age]) LIKE @p1) and NOT (LOWER([${TABLE_NAME}].[name]) LIKE @p2)) as [${TABLE_NAME}]`,
-    })
-  })
-
-  it("should use NOT JSON_CONTAINS expression for MySQL when filter is notContains", () => {
-    const query = new Sql(SqlClient.MY_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          notContains: {
-            age: [20],
-            name: ["John"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` where NOT JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and NOT JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``,
-    })
-  })
-
-  it("should use jsonb operator NOT expression for PostgreSQL when filter is notContains", () => {
-    const query = new Sql(SqlClient.POSTGRES, 10)._query(
-      generateReadJson({
-        filters: {
-          notContains: {
-            age: [20],
-            name: ["John"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from "${TABLE_NAME}" where NOT "${TABLE_NAME}"."age"::jsonb @> '[20]' and NOT "${TABLE_NAME}"."name"::jsonb @> '["John"]' limit $1) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should use OR like expression for MS-SQL when filter is containsAny", () => {
-    const query = new Sql(SqlClient.MS_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          containsAny: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10, "%20%", "%25%", `%"john"%`, `%"mary"%`],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER([${TABLE_NAME}].[age]) LIKE @p1 OR LOWER([${TABLE_NAME}].[age]) LIKE @p2) and (LOWER([${TABLE_NAME}].[name]) LIKE @p3 OR LOWER([${TABLE_NAME}].[name]) LIKE @p4)) as [${TABLE_NAME}]`,
-    })
-  })
-
-  it("should use JSON_OVERLAPS expression for MySQL when filter is containsAny", () => {
-    const query = new Sql(SqlClient.MY_SQL, 10)._query(
-      generateReadJson({
-        filters: {
-          containsAny: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_OVERLAPS(${TABLE_NAME}.age, '[20,25]') and JSON_OVERLAPS(${TABLE_NAME}.name, '["John","Mary"]') limit ?) as \`${TABLE_NAME}\``,
-    })
-  })
-
-  it("should use ?| operator expression for PostgreSQL when filter is containsAny", () => {
-    const query = new Sql(SqlClient.POSTGRES, 10)._query(
-      generateReadJson({
-        filters: {
-          containsAny: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age"::jsonb ?| array [20,25] and "${TABLE_NAME}"."name"::jsonb ?| array ['John','Mary'] limit $1) as "${TABLE_NAME}"`,
-    })
-  })
-
   it("should add the schema to the LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
@@ -551,80 +162,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should handle table names with dashes when performing a LIKE in MySQL", () => {
-    const tableName = "Table-Name-With-Dashes"
-    const query = new Sql(SqlClient.MY_SQL, limit)._query(
-      generateReadJson({
-        table: tableName,
-        filters: {
-          string: {
-            name: "John",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["john%", limit],
-      sql: `select * from (select * from \`${tableName}\` where LOWER(\`${tableName}\`.\`name\`) LIKE ? limit ?) as \`${tableName}\``,
-    })
-  })
-
-  it("should handle table names with dashes when performing a LIKE in SQL Server", () => {
-    const tableName = "Table-Name-With-Dashes"
-    const query = new Sql(SqlClient.MS_SQL, limit)._query(
-      generateReadJson({
-        table: tableName,
-        filters: {
-          string: {
-            name: "John",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [limit, "john%"],
-      sql: `select * from (select top (@p0) * from [${tableName}] where LOWER([${tableName}].[name]) LIKE @p1) as [${tableName}]`,
-    })
-  })
-
-  it("should ignore high range value if it is an empty object", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            dob: {
-              low: "2000-01-01 00:00:00",
-              high: {},
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["2000-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should ignore low range value if it is an empty object", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            dob: {
-              low: {},
-              high: "2010-01-01 00:00:00",
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["2010-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
   it("should lowercase the values for Oracle LIKE statements", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({
@@ -669,139 +206,4 @@ describe("SQL query builder", () => {
       sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`,
     })
   })
-
-  it("should sort SQL Server tables by the primary key if no sort data is provided", () => {
-    let query = new Sql(SqlClient.MS_SQL, limit)._query(
-      generateReadJson({
-        sort: {},
-        paginate: {
-          limit: 10,
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select top (@p0) * from [test] order by [test].[id] asc) as [test]`,
-    })
-  })
-
-  it("should not parse JSON string as Date", () => {
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(
-      generateCreateJson(TABLE_NAME, {
-        name: '{ "created_at":"2023-09-09T03:21:06.024Z" }',
-      })
-    )
-    expect(query).toEqual({
-      bindings: ['{ "created_at":"2023-09-09T03:21:06.024Z" }'],
-      sql: `insert into "test" ("name") values ($1) returning *`,
-    })
-  })
-
-  it("should parse and trim valid string as Date", () => {
-    const dateObj = new Date("2023-09-09T03:21:06.024Z")
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(
-      generateCreateJson(TABLE_NAME, {
-        name: " 2023-09-09T03:21:06.024Z ",
-      })
-    )
-    expect(query).toEqual({
-      bindings: [dateObj],
-      sql: `insert into "test" ("name") values ($1) returning *`,
-    })
-  })
-
-  it("should be able to rename column for MySQL", () => {
-    const table: Table = {
-      type: "table",
-      sourceType: TableSourceType.EXTERNAL,
-      name: TABLE_NAME,
-      schema: {
-        first_name: {
-          type: FieldType.STRING,
-          name: "first_name",
-          externalType: "varchar(45)",
-        },
-      },
-      sourceId: "SOURCE_ID",
-    }
-    const oldTable: Table = {
-      ...table,
-      schema: {
-        name: {
-          type: FieldType.STRING,
-          name: "name",
-          externalType: "varchar(45)",
-        },
-      },
-    }
-    const query = new Sql(SqlClient.MY_SQL, limit)._query({
-      table,
-      endpoint: {
-        datasourceId: "MySQL",
-        operation: Operation.UPDATE_TABLE,
-        entityId: TABLE_NAME,
-      },
-      meta: {
-        table: oldTable,
-        tables: { [oldTable.name]: oldTable },
-        renamed: {
-          old: "name",
-          updated: "first_name",
-        },
-      },
-    })
-    expect(query).toEqual({
-      bindings: [],
-      sql: `alter table \`${TABLE_NAME}\` rename column \`name\` to \`first_name\`;`,
-    })
-  })
-
-  it("should be able to delete a column", () => {
-    const table: Table = {
-      type: "table",
-      sourceType: TableSourceType.EXTERNAL,
-      name: TABLE_NAME,
-      schema: {
-        first_name: {
-          type: FieldType.STRING,
-          name: "first_name",
-          externalType: "varchar(45)",
-        },
-      },
-      sourceId: "SOURCE_ID",
-    }
-    const oldTable: Table = {
-      ...table,
-      schema: {
-        first_name: {
-          type: FieldType.STRING,
-          name: "first_name",
-          externalType: "varchar(45)",
-        },
-        last_name: {
-          type: FieldType.STRING,
-          name: "last_name",
-          externalType: "varchar(45)",
-        },
-      },
-    }
-    const query = sql._query({
-      table,
-      endpoint: {
-        datasourceId: "Postgres",
-        operation: Operation.UPDATE_TABLE,
-        entityId: TABLE_NAME,
-      },
-      meta: {
-        table: oldTable,
-        tables: [oldTable],
-      },
-    })
-    expect(query).toEqual([
-      {
-        bindings: [],
-        sql: `alter table "${TABLE_NAME}" drop column "last_name"`,
-      },
-    ])
-  })
 })

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -61,9 +61,9 @@ describe("Captures of real examples", () => {
               "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" 
+              from (select * from "persons" as "a" order by "a"."firstname" asc nulls first limit $1) as "a" 
               left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" 
-              order by "a"."firstname" asc limit $2`),
+              order by "a"."firstname" asc nulls first limit $2`),
       })
     })
 
@@ -75,10 +75,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
-              order by "a"."productname" asc limit $3`),
+              order by "a"."productname" asc nulls first limit $3`),
       })
     })
 
@@ -90,10 +90,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" 
-              order by "a"."productname" asc limit $2`),
+              order by "a"."productname" asc nulls first limit $2`),
       })
     })
 
@@ -138,11 +138,11 @@ describe("Captures of real examples", () => {
              "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
              "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
              from (select * from "tasks" as "a" where not "a"."completed" = $1 
-             order by "a"."taskname" asc limit $2) as "a" 
+             order by "a"."taskname" asc nulls first limit $2) as "a" 
              left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
              left join "products" as "b" on "b"."productid" = "d"."productid" 
              left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
-             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`),
+             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`),
       })
     })
   })

packages/server/src/integrations/utils/utils.ts

@@ -71,7 +71,11 @@ const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
 }
 
 const SQL_DATE_ONLY_TYPES = ["date"]
-const SQL_TIME_ONLY_TYPES = ["time"]
+const SQL_TIME_ONLY_TYPES = [
+  "time",
+  "time without time zone",
+  "time with time zone",
+]
 
 const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
   varchar: FieldType.STRING,

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -26,6 +26,7 @@ import {
 } from "../../../../db/utils"
 import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
+import pick from "lodash/pick"
 
 function buildInternalFieldList(
   table: Table,
@@ -54,8 +55,8 @@ function buildInternalFieldList(
   return fieldList
 }
 
-function tableInFilter(name: string) {
+function tableNameInFieldRegex(tableName: string) {
-  return `:${name}.`
+  return new RegExp(`^${tableName}.|:${tableName}.`, "g")
 }
 
 function cleanupFilters(filters: SearchFilters, tables: Table[]) {
@@ -71,15 +72,13 @@ function cleanupFilters(filters: SearchFilters, tables: Table[]) {
       // relationship, switch to table ID
       const tableRelated = tables.find(
         table =>
-          table.originalName && key.includes(tableInFilter(table.originalName))
+          table.originalName &&
+          key.match(tableNameInFieldRegex(table.originalName))
       )
       if (tableRelated && tableRelated.originalName) {
-        filter[
+        // only replace the first, not replaceAll
-          key.replace(
+        filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
-            tableInFilter(tableRelated.originalName),
+          filter[key]
-            tableInFilter(tableRelated._id!)
-          )
-        ] = filter[key]
         delete filter[key]
       }
     }
@@ -187,13 +186,19 @@ export async function search(
       }
     )
 
-    return {
+    const output = {
-      // final row processing for response
       rows: await outputProcessing<Row[]>(table, processed, {
         preserveLinks: true,
         squash: true,
       }),
     }
+
+    if (options.fields) {
+      const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
+      output.rows = output.rows.map((r: any) => pick(r, fields))
+    }
+
+    return output
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
     if (err.status === 404 && err.message?.includes(SQLITE_DESIGN_DOC_ID)) {

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -1,166 +0,0 @@
-import { GenericContainer } from "testcontainers"
-
-import {
-  Datasource,
-  FieldType,
-  Row,
-  SourceName,
-  Table,
-  RowSearchParams,
-  TableSourceType,
-} from "@budibase/types"
-
-import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { search } from "../external"
-import {
-  expectAnyExternalColsAttributes,
-  generator,
-} from "@budibase/backend-core/tests"
-
-describe("external search", () => {
-  const config = new TestConfiguration()
-
-  let externalDatasource: Datasource, tableData: Table
-  const rows: Row[] = []
-
-  beforeAll(async () => {
-    const container = await new GenericContainer("mysql:8.3")
-      .withExposedPorts(3306)
-      .withEnvironment({
-        MYSQL_ROOT_PASSWORD: "admin",
-        MYSQL_DATABASE: "db",
-        MYSQL_USER: "user",
-        MYSQL_PASSWORD: "password",
-      })
-      .start()
-
-    const host = container.getHost()
-    const port = container.getMappedPort(3306)
-
-    await config.init()
-
-    externalDatasource = await config.createDatasource({
-      datasource: {
-        type: "datasource",
-        name: "Test",
-        source: SourceName.MYSQL,
-        plus: true,
-        config: {
-          host,
-          port,
-          user: "user",
-          database: "db",
-          password: "password",
-          rejectUnauthorized: true,
-        },
-      },
-    })
-
-    tableData = {
-      name: generator.word(),
-      type: "table",
-      primary: ["id"],
-      sourceId: externalDatasource._id!,
-      sourceType: TableSourceType.EXTERNAL,
-      schema: {
-        id: {
-          name: "id",
-          type: FieldType.AUTO,
-          autocolumn: true,
-        },
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-        },
-        surname: {
-          name: "surname",
-          type: FieldType.STRING,
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-        },
-        address: {
-          name: "address",
-          type: FieldType.STRING,
-        },
-      },
-    }
-
-    const table = await config.createExternalTable({
-      ...tableData,
-      sourceId: externalDatasource._id,
-    })
-    for (let i = 0; i < 10; i++) {
-      rows.push(
-        await config.createRow({
-          tableId: table._id,
-          name: generator.first(),
-          surname: generator.last(),
-          age: generator.age(),
-          address: generator.address(),
-        })
-      )
-    }
-  })
-
-  it("default search returns all the data", async () => {
-    await config.doInContext(config.appId, async () => {
-      const tableId = config.table!._id!
-
-      const searchParams: RowSearchParams = {
-        tableId,
-        query: {},
-      }
-      const result = await search(searchParams, config.table!)
-
-      expect(result.rows).toHaveLength(10)
-      expect(result.rows).toEqual(
-        expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
-      )
-    })
-  })
-
-  it("querying by fields will always return data attribute columns", async () => {
-    await config.doInContext(config.appId, async () => {
-      const tableId = config.table!._id!
-
-      const searchParams: RowSearchParams = {
-        tableId,
-        query: {},
-        fields: ["name", "age"],
-      }
-      const result = await search(searchParams, config.table!)
-
-      expect(result.rows).toHaveLength(10)
-      expect(result.rows).toEqual(
-        expect.arrayContaining(
-          rows.map(r => ({
-            ...expectAnyExternalColsAttributes,
-            name: r.name,
-            age: r.age,
-          }))
-        )
-      )
-    })
-  })
-
-  it("will decode _id in oneOf query", async () => {
-    await config.doInContext(config.appId, async () => {
-      const tableId = config.table!._id!
-
-      const searchParams: RowSearchParams = {
-        tableId,
-        query: {
-          oneOf: {
-            _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
-          },
-        },
-      }
-      const result = await search(searchParams, config.table!)
-
-      expect(result.rows).toHaveLength(3)
-      expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])
-    })
-  })
-})

packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts

@@ -1,249 +0,0 @@
-const nodeFetch = require("node-fetch")
-
-nodeFetch.mockSearch()
-import * as search from "../utils"
-import { RowSearchParams, SortOrder, SortType } from "@budibase/types"
-
-// this will be mocked out for _search endpoint
-const PARAMS: RowSearchParams = {
-  query: {},
-  tableId: "ta_12345679abcdef",
-  version: "1",
-  bookmark: undefined,
-  sort: undefined,
-  sortOrder: SortOrder.ASCENDING,
-  sortType: SortType.STRING,
-}
-
-function checkLucene(resp: any, expected: any, params = PARAMS) {
-  const query = resp.rows[0].query
-  const json = JSON.parse(query)
-  if (PARAMS.sort) {
-    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-  }
-  if (PARAMS.bookmark) {
-    expect(json.bookmark).toBe(PARAMS.bookmark)
-  }
-  expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
-  expect(json.limit).toBe(params.limit || 50)
-}
-
-describe("internal search", () => {
-  it("default query", async () => {
-    const response = await search.paginatedSearch({}, PARAMS)
-    checkLucene(response, `*:*`)
-  })
-
-  it("test equal query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        equal: {
-          column: "1",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND column:"1"`)
-  })
-
-  it("test notEqual query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        notEqual: {
-          column: "1",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND !column:"1"`)
-  })
-
-  it("test OR query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        allOr: true,
-        equal: {
-          column: "2",
-        },
-        notEqual: {
-          column: "1",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `(column:"2" OR !column:"1")`)
-  })
-
-  it("test AND query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        equal: {
-          column: "2",
-        },
-        notEqual: {
-          column: "1",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
-  })
-
-  it("test pagination query", async () => {
-    const updatedParams = {
-      ...PARAMS,
-      limit: 100,
-      bookmark: "awd",
-      sort: "column",
-    }
-    const response = await search.paginatedSearch(
-      {
-        string: {
-          column: "2",
-        },
-      },
-      updatedParams
-    )
-    checkLucene(response, `*:* AND column:2*`, updatedParams)
-  })
-
-  it("test range query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        range: {
-          column: { low: 1, high: 2 },
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
-  })
-
-  it("test empty query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        empty: {
-          column: "",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
-  })
-
-  it("test notEmpty query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        notEmpty: {
-          column: "",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
-  })
-
-  it("test oneOf query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        oneOf: {
-          column: ["a", "b"],
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
-  })
-
-  it("test contains query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        contains: {
-          column: ["a"],
-          colArr: [1, 2, 3],
-        },
-      },
-      PARAMS
-    )
-    checkLucene(
-      response,
-      `(*:* AND column:(a) AND colArr:(1 AND 2 AND 3))`,
-      PARAMS
-    )
-  })
-
-  it("test multiple of same column", async () => {
-    const response = await search.paginatedSearch(
-      {
-        allOr: true,
-        equal: {
-          "1:column": "a",
-          "2:column": "b",
-          "3:column": "c",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
-  })
-
-  it("check a weird case for lucene building", async () => {
-    const response = await search.paginatedSearch(
-      {
-        equal: {
-          "1:1:column": "a",
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
-  })
-
-  it("test containsAny query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        containsAny: {
-          column: ["a", "b", "c"],
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
-  })
-
-  it("test notContains query", async () => {
-    const response = await search.paginatedSearch(
-      {
-        notContains: {
-          column: ["a", "b", "c"],
-        },
-      },
-      PARAMS
-    )
-    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
-  })
-
-  it("test equal without version query", async () => {
-    PARAMS.version = undefined
-    const response = await search.paginatedSearch(
-      {
-        equal: {
-          column: "1",
-        },
-      },
-      PARAMS
-    )
-
-    const query = response.rows[0].query
-    const json = JSON.parse(query)
-    if (PARAMS.sort) {
-      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-    }
-    if (PARAMS.bookmark) {
-      expect(json.bookmark).toBe(PARAMS.bookmark)
-    }
-    expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
-  })
-})

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -0,0 +1,131 @@
+import { Datasource, FieldType, Row, Table } from "@budibase/types"
+
+import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
+import { search } from "../../../../../sdk/app/rows/search"
+import { generator } from "@budibase/backend-core/tests"
+import {
+  DatabaseName,
+  getDatasource,
+} from "../../../../../integrations/tests/utils"
+import { tableForDatasource } from "../../../../../tests/utilities/structures"
+
+// These test cases are only for things that cannot be tested through the API
+// (e.g. limiting searches to returning specific fields). If it's possible to
+// test through the API, it should be done there instead.
+describe.each([
+  ["lucene", undefined],
+  ["sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+])("search sdk (%s)", (name, dsProvider) => {
+  const isSqs = name === "sqs"
+  const isLucene = name === "lucene"
+  const isInternal = isLucene || isSqs
+  const config = new TestConfiguration()
+
+  let envCleanup: (() => void) | undefined
+  let datasource: Datasource | undefined
+  let table: Table
+  let rows: Row[]
+
+  beforeAll(async () => {
+    if (isSqs) {
+      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+    }
+    await config.init()
+
+    if (dsProvider) {
+      datasource = await config.createDatasource({
+        datasource: await dsProvider,
+      })
+    }
+
+    table = await config.api.table.save(
+      tableForDatasource(datasource, {
+        primary: ["id"],
+        schema: {
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            autocolumn: true,
+          },
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          surname: {
+            name: "surname",
+            type: FieldType.STRING,
+          },
+          age: {
+            name: "age",
+            type: FieldType.NUMBER,
+          },
+          address: {
+            name: "address",
+            type: FieldType.STRING,
+          },
+        },
+      })
+    )
+
+    rows = []
+    for (let i = 0; i < 10; i++) {
+      rows.push(
+        await config.api.row.save(table._id!, {
+          name: generator.first(),
+          surname: generator.last(),
+          age: generator.age(),
+          address: generator.address(),
+        })
+      )
+    }
+  })
+
+  afterAll(async () => {
+    config.end()
+    if (envCleanup) {
+      envCleanup()
+    }
+  })
+
+  it("querying by fields will always return data attribute columns", async () => {
+    await config.doInContext(config.appId, async () => {
+      const { rows } = await search({
+        tableId: table._id!,
+        query: {},
+        fields: ["name", "age"],
+      })
+
+      expect(rows).toHaveLength(10)
+      for (const row of rows) {
+        const keys = Object.keys(row)
+        expect(keys).toContain("name")
+        expect(keys).toContain("age")
+        expect(keys).not.toContain("surname")
+        expect(keys).not.toContain("address")
+      }
+    })
+  })
+
+  !isInternal &&
+    it("will decode _id in oneOf query", async () => {
+      await config.doInContext(config.appId, async () => {
+        const result = await search({
+          tableId: table._id!,
+          query: {
+            oneOf: {
+              _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
+            },
+          },
+        })
+
+        expect(result.rows).toHaveLength(3)
+        expect(result.rows.map(row => row.id)).toEqual(
+          expect.arrayContaining([1, 4, 8])
+        )
+      })
+    })
+})

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -126,16 +126,25 @@ export default class AliasTables {
   }
 
   reverse<T extends Row | Row[]>(rows: T): T {
+    const mapping = new Map()
+
     const process = (row: Row) => {
       const final: Row = {}
-      for (let [key, value] of Object.entries(row)) {
+      for (const key of Object.keys(row)) {
-        if (!key.includes(".")) {
+        let mappedKey = mapping.get(key)
-          final[key] = value
+        if (!mappedKey) {
+          const dotLocation = key.indexOf(".")
+          if (dotLocation === -1) {
+            mappedKey = key
           } else {
-          const [alias, column] = key.split(".")
+            const alias = key.slice(0, dotLocation)
+            const column = key.slice(dotLocation + 1)
             const tableName = this.tableAliases[alias] || alias
-          final[`${tableName}.${column}`] = value
+            mappedKey = `${tableName}.${column}`
           }
+          mapping.set(key, mappedKey)
+        }
+        final[mappedKey] = row[key]
       }
       return final
     }

packages/server/src/tests/utilities/api/row.ts

@@ -10,6 +10,7 @@ import {
   RowSearchParams,
   DeleteRows,
   DeleteRow,
+  PaginatedSearchRowResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -133,12 +134,20 @@ export class RowAPI extends TestAPI {
     )
   }
 
-  search = async (
+  search = async <T extends RowSearchParams>(
     sourceId: string,
-    params?: RowSearchParams,
+    params?: T,
     expectations?: Expectations
-  ): Promise<SearchRowResponse> => {
+  ): Promise<
-    return await this._post<SearchRowResponse>(`/api/${sourceId}/search`, {
+    T extends { paginate: true }
+      ? PaginatedSearchRowResponse
+      : SearchRowResponse
+  > => {
+    return await this._post<
+      T extends { paginate: true }
+        ? PaginatedSearchRowResponse
+        : SearchRowResponse
+    >(`/api/${sourceId}/search`, {
       body: params,
       expectations,
     })

packages/server/src/tests/utilities/structures.ts

@@ -37,7 +37,7 @@ export function tableForDatasource(
 ): Table {
   return merge(
     {
-      name: generator.guid(),
+      name: generator.guid().substring(0, 10),
       type: "table",
       sourceType: datasource
         ? TableSourceType.EXTERNAL

packages/server/src/utilities/rowProcessor/utils.ts

@@ -105,6 +105,9 @@ export function processDates<T extends Row | Row[]>(
     if (schema.type !== FieldType.DATETIME) {
       continue
     }
+    if (schema.dateOnly) {
+      continue
+    }
     if (!schema.timeOnly && !schema.ignoreTimezones) {
       datesWithTZ.push(column)
     }

packages/server/src/utilities/schema.ts

@@ -129,11 +129,16 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
         return
       }
 
-      const { type: columnType } = schema[columnName]
+      const columnSchema = schema[columnName]
+      const { type: columnType } = columnSchema
       if (columnType === FieldType.NUMBER) {
         // If provided must be a valid number
         parsedRow[columnName] = columnData ? Number(columnData) : columnData
-      } else if (columnType === FieldType.DATETIME) {
+      } else if (
+        columnType === FieldType.DATETIME &&
+        !columnSchema.timeOnly &&
+        !columnSchema.dateOnly
+      ) {
         // If provided must be a valid date
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()