Merge pull request #6206 from Budibase/ignore-timezones-master

Dates without timezones (master)

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -15,6 +15,7 @@
   export let placeholder = null
   export let appendTo = undefined
   export let timeOnly = false
+  export let ignoreTimezones = false
 
   const dispatch = createEventDispatcher()
   const flatpickrId = `${uuid()}-wrapper`
@@ -50,19 +51,35 @@
 
   const handleChange = event => {
     const [dates] = event.detail
+    const noTimezone = enableTime && !timeOnly && ignoreTimezones
     let newValue = dates[0]
     if (newValue) {
       newValue = newValue.toISOString()
     }
-    // if time only set date component to 2000-01-01
+
+    // If time only set date component to 2000-01-01
     if (timeOnly) {
       newValue = `2000-01-01T${newValue.split("T")[1]}`
     }
-    // date only, offset for timezone so always right date
+
+    // For date-only fields, construct a manual timestamp string without a time
+    // or time zone
     else if (!enableTime) {
-      const offset = dates[0].getTimezoneOffset() * 60000
+      const year = dates[0].getFullYear()
-      newValue = new Date(dates[0].getTime() - offset).toISOString()
+      const month = `${dates[0].getMonth() + 1}`.padStart(2, "0")
+      const day = `${dates[0].getDate()}`.padStart(2, "0")
+      newValue = `${year}-${month}-${day}T00:00:00.000`
     }
+
+    // For non-timezone-aware fields, create an ISO 8601 timestamp of the exact
+    // time picked, without timezone
+    else if (noTimezone) {
+      const offset = dates[0].getTimezoneOffset() * 60000
+      newValue = new Date(dates[0].getTime() - offset)
+        .toISOString()
+        .slice(0, -1)
+    }
+
     dispatch("change", newValue)
   }
 
@@ -112,10 +129,12 @@
       // Treat as numerical timestamp
       date = new Date(parseInt(val))
     }
+
     time = date.getTime()
     if (isNaN(time)) {
       return null
     }
+
     // By rounding to the nearest second we avoid locking up in an endless
     // loop in the builder, caused by potentially enriching {{ now }} to every
     // millisecond.

packages/bbui/src/Form/DatePicker.svelte

@@ -12,6 +12,7 @@
   export let timeOnly = false
   export let placeholder = null
   export let appendTo = undefined
+  export let ignoreTimezones = false
 
   const dispatch = createEventDispatcher()
 
@@ -30,6 +31,7 @@
     {enableTime}
     {timeOnly}
     {appendTo}
+    {ignoreTimezones}
     on:change={onChange}
   />
 </Field>

packages/bbui/src/Tooltip/TooltipWrapper.svelte

@@ -39,7 +39,6 @@
     position: relative;
     display: flex;
     justify-content: center;
-    margin-top: 1px;
     margin-left: 5px;
     margin-right: 5px;
   }

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -170,28 +170,29 @@ export function makeDatasourceFormComponents(datasource) {
           optionsType: "select",
           optionsSource: "schema",
         })
-      }
+      } else if (fieldType === "longform") {
-      if (fieldType === "longform") {
         component.customProps({
           format: "auto",
         })
-      }
+      } else if (fieldType === "array") {
-      if (fieldType === "array") {
         component.customProps({
           placeholder: "Choose an option",
           optionsSource: "schema",
         })
-      }
+      } else if (fieldType === "link") {
-
-      if (fieldType === "link") {
         let placeholder =
           fieldSchema.relationshipType === "one-to-many"
             ? "Choose an option"
             : "Choose some options"
         component.customProps({ placeholder })
-      }
+      } else if (fieldType === "boolean") {
-      if (fieldType === "boolean") {
         component.customProps({ text: field, label: "" })
+      } else if (fieldType === "datetime") {
+        component.customProps({
+          enableTime: !fieldSchema?.dateOnly,
+          timeOnly: fieldSchema?.timeOnly,
+          ignoreTimezones: fieldSchema.ignoreTimezones,
+        })
       }
       components.push(component)
     }

packages/builder/src/components/backend/DataTable/RowFieldControl.svelte

@@ -53,6 +53,7 @@
     {label}
     timeOnly={isTimeStamp}
     enableTime={!meta?.dateOnly}
+    ignoreTimezones={meta?.ignoreTimezones}
     bind:value
   />
 {:else if type === "attachment"}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -14,7 +14,7 @@
   } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
-  import { tables } from "stores/backend"
+  import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -63,6 +63,7 @@
   let primaryDisplay =
     $tables.selected.primaryDisplay == null ||
     $tables.selected.primaryDisplay === field.name
+  let isCreating = originalName == null
 
   let table = $tables.selected
   let indexes = [...($tables.selected.indexes || [])]
@@ -81,6 +82,9 @@
     (field.type === LINK_TYPE && !field.tableId) ||
     Object.keys(errors).length !== 0
   $: errors = checkErrors(field)
+  $: datasource = $datasources.list.find(
+    source => source._id === table?.sourceId
+  )
 
   // used to select what different options can be displayed for column type
   $: canBeSearched =
@@ -430,6 +434,18 @@
       bind:value={field.constraints.datetime.earliest}
     />
     <DatePicker label="Latest" bind:value={field.constraints.datetime.latest} />
+    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
+      <div>
+        <Label
+          tooltip={isCreating
+            ? null
+            : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+        >
+          Time zones
+        </Label>
+        <Toggle bind:value={field.ignoreTimezones} text="Ignore time zones" />
+      </div>
+    {/if}
   {:else if field.type === "number"}
     <Input
       type="number"

packages/client/manifest.json

@@ -2620,16 +2620,22 @@
       },
       {
         "type": "boolean",
-        "label": "Show Time",
+        "label": "Show time",
         "key": "enableTime",
         "defaultValue": true
       },
       {
         "type": "boolean",
-        "label": "Time Only",
+        "label": "Time only",
         "key": "timeOnly",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Ignore time zones",
+        "key": "ignoreTimezones",
+        "defaultValue": false
+      },
       {
         "type": "text",
         "label": "Default value",

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -8,6 +8,7 @@
   export let disabled = false
   export let enableTime = false
   export let timeOnly = false
+  export let ignoreTimezones = false
   export let validation
   export let defaultValue
   export let onChange
@@ -43,6 +44,7 @@
       appendTo={document.getElementById("flatpickr-root")}
       {enableTime}
       {timeOnly}
+      {ignoreTimezones}
       {placeholder}
     />
   {/if}

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -29,7 +29,10 @@ import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 // @ts-ignore
 import { cloneDeep } from "lodash/fp"
-import { processFormulas } from "../../../utilities/rowProcessor/utils"
+import {
+  processFormulas,
+  processDates,
+} from "../../../utilities/rowProcessor/utils"
 // @ts-ignore
 import { getAppDB } from "@budibase/backend-core/context"
 
@@ -434,7 +437,13 @@ module External {
           relationships
         )
       }
-      return processFormulas(table, Object.values(finalRows)).map((row: Row) =>
+
+      // Process some additional data types
+      let finalRowArray = Object.values(finalRows)
+      finalRowArray = processDates(table, finalRowArray)
+      finalRowArray = processFormulas(table, finalRowArray)
+
+      return finalRowArray.map((row: Row) =>
         this.squashRelationshipColumns(table, row, relationships)
       )
     }

packages/server/src/definitions/common.ts

@@ -25,6 +25,7 @@ export interface FieldSchema {
   formula?: string
   formulaType?: string
   main?: boolean
+  ignoreTimezones?: boolean
   meta?: {
     toTable: string
     toKey: string

packages/server/src/integrations/base/sqlTable.ts

@@ -61,7 +61,9 @@ function generateSchema(
         schema.boolean(key)
         break
       case FieldTypes.DATETIME:
-        schema.datetime(key)
+        schema.datetime(key, {
+          useTz: !column.ignoreTimezones,
+        })
         break
       case FieldTypes.ARRAY:
         schema.json(key)

packages/server/src/integrations/mysql.ts

@@ -15,7 +15,6 @@ import {
 } from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 import dayjs from "dayjs"
-import { FieldTypes } from "../constants"
 const { NUMBER_REGEX } = require("../utilities")
 
 module MySQLModule {
@@ -30,6 +29,7 @@ module MySQLModule {
     database: string
     ssl?: { [key: string]: any }
     rejectUnauthorized: boolean
+    typeCast: Function
   }
 
   const SCHEMA: Integration = {
@@ -89,6 +89,8 @@ module MySQLModule {
     },
   }
 
+  const TimezoneAwareDateTypes = ["timestamp"]
+
   function bindingTypeCoerce(bindings: any[]) {
     for (let i = 0; i < bindings.length; i++) {
       const binding = bindings[i]
@@ -131,7 +133,19 @@ module MySQLModule {
       }
       // @ts-ignore
       delete config.rejectUnauthorized
-      this.config = config
+      this.config = {
+        ...config,
+        typeCast: function (field: any, next: any) {
+          if (
+            field.type == "DATETIME" ||
+            field.type === "DATE" ||
+            field.type === "TIMESTAMP"
+          ) {
+            return field.string()
+          }
+          return next()
+        },
+      }
     }
 
     getBindingIdentifier(): string {

packages/server/src/integrations/postgres.ts

@@ -16,10 +16,19 @@ import {
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module PostgresModule {
-  const { Client } = require("pg")
+  const { Client, types } = require("pg")
   const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
+  // Return "date" and "timestamp" types as plain strings.
+  // This lets us reference the original stored timezone.
+  // types is undefined when running in a test env for some reason.
+  if (types) {
+    types.setTypeParser(1114, (val: any) => val) // timestamp
+    types.setTypeParser(1082, (val: any) => val) // date
+    types.setTypeParser(1184, (val: any) => val) // timestampz
+  }
+
   const JSON_REGEX = /'{.*}'::json/s
 
   interface PostgresConfig {

packages/server/src/utilities/rowProcessor/utils.js

@@ -65,3 +65,28 @@ exports.processFormulas = (
   }
   return single ? rows[0] : rows
 }
+
+/**
+ * Processes any date columns and ensures that those without the ignoreTimezones
+ * flag set are parsed as UTC rather than local time.
+ */
+exports.processDates = (table, rows) => {
+  let datesWithTZ = []
+  for (let [column, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.DATETIME) {
+      continue
+    }
+    if (!schema.ignoreTimezones) {
+      datesWithTZ.push(column)
+    }
+  }
+
+  for (let row of rows) {
+    for (let col of datesWithTZ) {
+      if (row[col] && typeof row[col] === "string" && !row[col].endsWith("Z")) {
+        row[col] = new Date(row[col]).toISOString()
+      }
+    }
+  }
+  return rows
+}

packages/server/yarn.lock

@@ -1014,10 +1014,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.194":
+"@budibase/backend-core@1.0.195":
-  version "1.0.194"
+  version "1.0.195"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.194.tgz#08b2b1aec3c88efbc7cfb14145ce6f199475c538"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.195.tgz#ee40c690ae4a54febab8b140c9bbb7d04221f3b9"
-  integrity sha512-BbnJFtAioJeD9tQfSwc2uftFK8SagREgSfUYv06dfnf/NsmkrONzZiTon1oA57S7ifcSiu+WZf87lNX0k8pwfQ==
+  integrity sha512-6diWgRV9t4DU3kXteJJAhCxyta9m1wvzN7vNbflhY4kYJeg7BC+7jcvc2r8zl6s1vVeSW4ic5/ueSLRaTDySuw==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1092,12 +1092,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.194":
+"@budibase/pro@1.0.195":
-  version "1.0.194"
+  version "1.0.195"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.194.tgz#fbf977b292b9a6dbf7b072b2e0379dcf4379943a"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.195.tgz#368652398d1da95f160fc0192b77144b11147ff5"
-  integrity sha512-LSqVwlhKWwFNnC3acvLnCzbeBoze1Ma6GELE/b5ZxS65owsigu0KC6T/UuujEbU9xqbFJ3R6uV+4Fz4NUibLIw==
+  integrity sha512-zf1f1exHop4m6vda5hObUnTZa2PIBRnc4e0r9iqFbzvGBMfBLGUhGzu23JEwNYaS2xhWHj2FNv4/IVzIyLG4eA==
   dependencies:
-    "@budibase/backend-core" "1.0.194"
+    "@budibase/backend-core" "1.0.195"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":

packages/worker/yarn.lock

@@ -293,10 +293,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.194":
+"@budibase/backend-core@1.0.195":
-  version "1.0.194"
+  version "1.0.195"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.194.tgz#08b2b1aec3c88efbc7cfb14145ce6f199475c538"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.195.tgz#ee40c690ae4a54febab8b140c9bbb7d04221f3b9"
-  integrity sha512-BbnJFtAioJeD9tQfSwc2uftFK8SagREgSfUYv06dfnf/NsmkrONzZiTon1oA57S7ifcSiu+WZf87lNX0k8pwfQ==
+  integrity sha512-6diWgRV9t4DU3kXteJJAhCxyta9m1wvzN7vNbflhY4kYJeg7BC+7jcvc2r8zl6s1vVeSW4ic5/ueSLRaTDySuw==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -322,12 +322,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.194":
+"@budibase/pro@1.0.195":
-  version "1.0.194"
+  version "1.0.195"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.194.tgz#fbf977b292b9a6dbf7b072b2e0379dcf4379943a"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.195.tgz#368652398d1da95f160fc0192b77144b11147ff5"
-  integrity sha512-LSqVwlhKWwFNnC3acvLnCzbeBoze1Ma6GELE/b5ZxS65owsigu0KC6T/UuujEbU9xqbFJ3R6uV+4Fz4NUibLIw==
+  integrity sha512-zf1f1exHop4m6vda5hObUnTZa2PIBRnc4e0r9iqFbzvGBMfBLGUhGzu23JEwNYaS2xhWHj2FNv4/IVzIyLG4eA==
   dependencies:
-    "@budibase/backend-core" "1.0.194"
+    "@budibase/backend-core" "1.0.195"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":