Merge branch 'master' into fix/automation-query-rows-table-with-spaces

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.38",
+  "version": "3.2.39",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/middleware/errorHandling.ts

@@ -32,8 +32,12 @@ export async function errorHandling(ctx: any, next: any) {
     }
 
     if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) {
+      let rootErr = err
+      while (rootErr.cause) {
+        rootErr = rootErr.cause
+      }
       // @ts-ignore
-      error.stack = err.stack
+      error.stack = rootErr.stack
     }
 
     ctx.body = error

packages/backend-core/src/sql/sql.ts

@@ -816,14 +816,29 @@ class InternalBuilder {
         filters.oneOf,
         ArrayOperator.ONE_OF,
         (q, key: string, array) => {
+          const schema = this.getFieldSchema(key)
+          const values = Array.isArray(array) ? array : [array]
           if (shouldOr) {
             q = q.or
           }
           if (this.client === SqlClient.ORACLE) {
             // @ts-ignore
             key = this.convertClobs(key)
+          } else if (
+            this.client === SqlClient.SQL_LITE &&
+            schema?.type === FieldType.DATETIME &&
+            schema.dateOnly
+          ) {
+            for (const value of values) {
+              if (value != null) {
+                q = q.or.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              } else {
+                q = q.or.whereNull(key)
+              }
+            }
+            return q
           }
-          return q.whereIn(key, Array.isArray(array) ? array : [array])
+          return q.whereIn(key, values)
         },
         (q, key: string[], array) => {
           if (shouldOr) {
@@ -882,6 +897,19 @@ class InternalBuilder {
         let high = value.high
         let low = value.low
 
+        if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (high != null) {
+            high = `${high.toISOString().slice(0, 10)}T23:59:59.999Z`
+          }
+          if (low != null) {
+            low = low.toISOString().slice(0, 10)
+          }
+        }
+
         if (this.client === SqlClient.ORACLE) {
           rawKey = this.convertClobs(key)
         } else if (
@@ -914,6 +942,7 @@ class InternalBuilder {
     }
     if (filters.equal) {
       iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -928,6 +957,16 @@ class InternalBuilder {
             // @ts-expect-error knex types are wrong, raw is fine here
             subq.whereNotNull(identifier).andWhere(identifier, value)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+          } else {
+            return q.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? = ?, FALSE)`, [
             this.rawQuotedIdentifier(key),
@@ -938,6 +977,7 @@ class InternalBuilder {
     }
     if (filters.notEqual) {
       iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -959,6 +999,18 @@ class InternalBuilder {
               // @ts-expect-error knex types are wrong, raw is fine here
               .or.whereNull(identifier)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.not
+              .whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              .or.whereNull(key)
+          } else {
+            return q.not.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? != ?, TRUE)`, [
             this.rawQuotedIdentifier(key),

packages/backend-core/src/sql/utils.ts

@@ -14,7 +14,7 @@ import environment from "../environment"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}.\d{3}Z)?$/
 const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
 
 export function isExternalTableID(tableId: string) {
@@ -149,15 +149,7 @@ export function isInvalidISODateString(str: string) {
 }
 
 export function isValidISODateString(str: string) {
-  const trimmedValue = str.trim()
+  return ISO_DATE_REGEX.test(str.trim())
-  if (!ISO_DATE_REGEX.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  if (isNaN(d.getTime())) {
-    return false
-  }
-  return d.toISOString() === trimmedValue
 }
 
 export function isValidFilter(value: any) {

packages/builder/src/stores/builder/queries.js

@@ -1,130 +0,0 @@
-import { writable, get, derived } from "svelte/store"
-import { datasources } from "./datasources"
-import { integrations } from "./integrations"
-import { API } from "@/api"
-import { duplicateName } from "@/helpers/duplicate"
-
-const sortQueries = queryList => {
-  queryList.sort((q1, q2) => {
-    return q1.name.localeCompare(q2.name)
-  })
-}
-
-export function createQueriesStore() {
-  const store = writable({
-    list: [],
-    selectedQueryId: null,
-  })
-  const derivedStore = derived(store, $store => ({
-    ...$store,
-    selected: $store.list?.find(q => q._id === $store.selectedQueryId),
-  }))
-
-  const fetch = async () => {
-    const queries = await API.getQueries()
-    sortQueries(queries)
-    store.update(state => ({
-      ...state,
-      list: queries,
-    }))
-  }
-
-  const save = async (datasourceId, query) => {
-    const _integrations = get(integrations)
-    const dataSource = get(datasources).list.filter(
-      ds => ds._id === datasourceId
-    )
-    // Check if readable attribute is found
-    if (dataSource.length !== 0) {
-      const integration = _integrations[dataSource[0].source]
-      const readable = integration.query[query.queryVerb].readable
-      if (readable) {
-        query.readable = readable
-      }
-    }
-    query.datasourceId = datasourceId
-    const savedQuery = await API.saveQuery(query)
-    store.update(state => {
-      const idx = state.list.findIndex(query => query._id === savedQuery._id)
-      const queries = state.list
-      if (idx >= 0) {
-        queries.splice(idx, 1, savedQuery)
-      } else {
-        queries.push(savedQuery)
-      }
-      sortQueries(queries)
-      return {
-        list: queries,
-        selectedQueryId: savedQuery._id,
-      }
-    })
-    return savedQuery
-  }
-
-  const importQueries = async ({ data, datasourceId }) => {
-    return await API.importQueries(datasourceId, data)
-  }
-
-  const select = id => {
-    store.update(state => ({
-      ...state,
-      selectedQueryId: id,
-    }))
-  }
-
-  const preview = async query => {
-    const result = await API.previewQuery(query)
-    // Assume all the fields are strings and create a basic schema from the
-    // unique fields returned by the server
-    const schema = {}
-    for (let [field, metadata] of Object.entries(result.schema)) {
-      schema[field] = metadata || { type: "string" }
-    }
-    return { ...result, schema, rows: result.rows || [] }
-  }
-
-  const deleteQuery = async query => {
-    await API.deleteQuery(query._id, query._rev)
-    store.update(state => {
-      state.list = state.list.filter(existing => existing._id !== query._id)
-      return state
-    })
-  }
-
-  const duplicate = async query => {
-    let list = get(store).list
-    const newQuery = { ...query }
-    const datasourceId = query.datasourceId
-
-    delete newQuery._id
-    delete newQuery._rev
-    newQuery.name = duplicateName(
-      query.name,
-      list.map(q => q.name)
-    )
-
-    return await save(datasourceId, newQuery)
-  }
-
-  const removeDatasourceQueries = datasourceId => {
-    store.update(state => ({
-      ...state,
-      list: state.list.filter(table => table.datasourceId !== datasourceId),
-    }))
-  }
-
-  return {
-    subscribe: derivedStore.subscribe,
-    fetch,
-    init: fetch,
-    select,
-    save,
-    import: importQueries,
-    delete: deleteQuery,
-    preview,
-    duplicate,
-    removeDatasourceQueries,
-  }
-}
-
-export const queries = createQueriesStore()

packages/builder/src/stores/builder/queries.ts

@@ -0,0 +1,156 @@
+import { derived, get, Writable } from "svelte/store"
+import { datasources } from "./datasources"
+import { integrations } from "./integrations"
+import { API } from "@/api"
+import { duplicateName } from "@/helpers/duplicate"
+import { DerivedBudiStore } from "@/stores/BudiStore"
+import {
+  Query,
+  QueryPreview,
+  PreviewQueryResponse,
+  SaveQueryRequest,
+  ImportRestQueryRequest,
+  QuerySchema,
+} from "@budibase/types"
+
+const sortQueries = (queryList: Query[]) => {
+  queryList.sort((q1, q2) => {
+    return q1.name.localeCompare(q2.name)
+  })
+}
+
+interface BuilderQueryStore {
+  list: Query[]
+  selectedQueryId: string | null
+}
+
+interface DerivedQueryStore extends BuilderQueryStore {
+  selected?: Query
+}
+
+export class QueryStore extends DerivedBudiStore<
+  BuilderQueryStore,
+  DerivedQueryStore
+> {
+  constructor() {
+    const makeDerivedStore = (store: Writable<BuilderQueryStore>) => {
+      return derived(store, ($store): DerivedQueryStore => {
+        return {
+          list: $store.list,
+          selectedQueryId: $store.selectedQueryId,
+          selected: $store.list?.find(q => q._id === $store.selectedQueryId),
+        }
+      })
+    }
+
+    super(
+      {
+        list: [],
+        selectedQueryId: null,
+      },
+      makeDerivedStore
+    )
+
+    this.select = this.select.bind(this)
+  }
+
+  async fetch() {
+    const queries = await API.getQueries()
+    sortQueries(queries)
+    this.store.update(state => ({
+      ...state,
+      list: queries,
+    }))
+  }
+
+  async save(datasourceId: string, query: SaveQueryRequest) {
+    const _integrations = get(integrations)
+    const dataSource = get(datasources).list.filter(
+      ds => ds._id === datasourceId
+    )
+    // Check if readable attribute is found
+    if (dataSource.length !== 0) {
+      const integration = _integrations[dataSource[0].source]
+      const readable = integration.query[query.queryVerb].readable
+      if (readable) {
+        query.readable = readable
+      }
+    }
+    query.datasourceId = datasourceId
+    const savedQuery = await API.saveQuery(query)
+    this.store.update(state => {
+      const idx = state.list.findIndex(query => query._id === savedQuery._id)
+      const queries = state.list
+      if (idx >= 0) {
+        queries.splice(idx, 1, savedQuery)
+      } else {
+        queries.push(savedQuery)
+      }
+      sortQueries(queries)
+      return {
+        list: queries,
+        selectedQueryId: savedQuery._id || null,
+      }
+    })
+    return savedQuery
+  }
+
+  async importQueries(data: ImportRestQueryRequest) {
+    return await API.importQueries(data)
+  }
+
+  select(id: string | null) {
+    this.store.update(state => ({
+      ...state,
+      selectedQueryId: id,
+    }))
+  }
+
+  async preview(query: QueryPreview): Promise<PreviewQueryResponse> {
+    const result = await API.previewQuery(query)
+    // Assume all the fields are strings and create a basic schema from the
+    // unique fields returned by the server
+    const schema: Record<string, QuerySchema> = {}
+    for (let [field, metadata] of Object.entries(result.schema)) {
+      schema[field] = (metadata as QuerySchema) || { type: "string" }
+    }
+    return { ...result, schema, rows: result.rows || [] }
+  }
+
+  async delete(query: Query) {
+    if (!query._id || !query._rev) {
+      throw new Error("Query ID or Revision is missing")
+    }
+    await API.deleteQuery(query._id, query._rev)
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(existing => existing._id !== query._id),
+    }))
+  }
+
+  async duplicate(query: Query) {
+    let list = get(this.store).list
+    const newQuery = { ...query }
+    const datasourceId = query.datasourceId
+
+    delete newQuery._id
+    delete newQuery._rev
+    newQuery.name = duplicateName(
+      query.name,
+      list.map(q => q.name)
+    )
+
+    return await this.save(datasourceId, newQuery)
+  }
+
+  removeDatasourceQueries(datasourceId: string) {
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(table => table.datasourceId !== datasourceId),
+    }))
+  }
+
+  init = this.fetch
+}
+
+export const queries = new QueryStore()

packages/frontend-core/src/fetch/UserFetch.ts

@@ -2,11 +2,7 @@ import { get } from "svelte/store"
 import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 import { utils } from "@budibase/shared-core"
-import {
+import { SearchFilters, SearchUsersRequest } from "@budibase/types"
-  BasicOperator,
-  SearchFilters,
-  SearchUsersRequest,
-} from "@budibase/types"
 
 interface UserFetchQuery {
   appId: string
@@ -56,7 +52,7 @@ export default class UserFetch extends DataFetch<
 
     const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
       ? rest
-      : { [BasicOperator.EMPTY]: { email: null } }
+      : {}
 
     try {
       const opts: SearchUsersRequest = {

packages/server/src/api/routes/tests/row.spec.ts

@@ -2341,7 +2341,7 @@ if (descriptions.length) {
               [FieldType.ARRAY]: ["options 2", "options 4"],
               [FieldType.NUMBER]: generator.natural(),
               [FieldType.BOOLEAN]: generator.bool(),
-              [FieldType.DATETIME]: generator.date().toISOString(),
+              [FieldType.DATETIME]: generator.date().toISOString().slice(0, 10),
               [FieldType.ATTACHMENTS]: [setup.structures.basicAttachment()],
               [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
               [FieldType.FORMULA]: undefined, // generated field

packages/server/src/api/routes/tests/search.spec.ts

@@ -1683,6 +1683,151 @@ if (descriptions.length) {
                 })
               })
 
+            describe("datetime - date only", () => {
+              describe.each([true, false])(
+                "saved with timestamp: %s",
+                saveWithTimestamp => {
+                  describe.each([true, false])(
+                    "search with timestamp: %s",
+                    searchWithTimestamp => {
+                      const SAVE_SUFFIX = saveWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
+                      const SEARCH_SUFFIX = searchWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
+
+                      const JAN_1ST = `2020-01-01`
+                      const JAN_10TH = `2020-01-10`
+                      const JAN_30TH = `2020-01-30`
+                      const UNEXISTING_DATE = `2020-01-03`
+                      const NULL_DATE__ID = `null_date__id`
+
+                      beforeAll(async () => {
+                        tableOrViewId = await createTableOrView({
+                          dateid: { name: "dateid", type: FieldType.STRING },
+                          date: {
+                            name: "date",
+                            type: FieldType.DATETIME,
+                            dateOnly: true,
+                          },
+                        })
+
+                        await createRows([
+                          { dateid: NULL_DATE__ID, date: null },
+                          { date: `${JAN_1ST}${SAVE_SUFFIX}` },
+                          { date: `${JAN_10TH}${SAVE_SUFFIX}` },
+                        ])
+                      })
+
+                      describe("equal", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("successfully finds an ISO8601 row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_10TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_10TH }])
+                        })
+
+                        it("finds a row with ISO8601 timestamp", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            equal: {
+                              date: `${UNEXISTING_DATE}${SEARCH_SUFFIX}`,
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+
+                      describe("notEqual", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_30TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
+                      })
+
+                      describe("oneOf", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            oneOf: { date: [`${JAN_1ST}${SEARCH_SUFFIX}`] },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            oneOf: {
+                              date: [`${UNEXISTING_DATE}${SEARCH_SUFFIX}`],
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+
+                      describe("range", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("successfully finds multiple rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_10TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                          ])
+                        })
+
+                        it("successfully finds no rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                                high: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+                    }
+                  )
+                }
+              )
+            })
+
             isInternal &&
               !isInMemory &&
               describe("AI Column", () => {

packages/server/src/utilities/rowProcessor/index.ts

@@ -411,6 +411,15 @@ export async function coreOutputProcessing(
           row[property] = `${hours}:${minutes}:${seconds}`
         }
       }
+    } else if (column.type === FieldType.DATETIME && column.dateOnly) {
+      for (const row of rows) {
+        if (typeof row[property] === "string") {
+          row[property] = new Date(row[property])
+        }
+        if (row[property] instanceof Date) {
+          row[property] = row[property].toISOString().slice(0, 10)
+        }
+      }
     } else if (column.type === FieldType.LINK) {
       for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time

packages/shared-core/src/filters.ts

@@ -699,7 +699,27 @@ export function runQuery<T extends Record<string, any>>(
       return docValue._id === testValue
     }
 
-    return docValue === testValue
+    if (docValue === testValue) {
+      return true
+    }
+
+    if (docValue == null && testValue != null) {
+      return false
+    }
+
+    if (docValue != null && testValue == null) {
+      return false
+    }
+
+    const leftDate = dayjs(docValue)
+    if (leftDate.isValid()) {
+      const rightDate = dayjs(testValue)
+      if (rightDate.isValid()) {
+        return leftDate.isSame(rightDate)
+      }
+    }
+
+    return false
   }
 
   const not =

packages/types/src/sdk/featureFlag.ts

@@ -1,9 +1,15 @@
 export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
+
+  // Account-portal
+  DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL",
 }
 
 export const FeatureFlagDefaults = {
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
+
+  // Account-portal
+  [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,
 }
 
 export type FeatureFlags = typeof FeatureFlagDefaults