Merge branch 'master' into fix/sqs-internal-columns

.github/workflows/budibase_ci.yml

@@ -179,13 +179,6 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Test server
-        env:
-          DD_CIVISIBILITY_AGENTLESS_ENABLED: true
-          DD_API_KEY: "${{ secrets.DATADOG_API_KEY }}"
-          DD_SITE: "datadoghq.eu"
-          NODE_OPTIONS: "-r dd-trace/ci/init"
-          DD_ENV: "ci"
-          DD_SERVICE: "budibase/packages/server"
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
             yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}

packages/backend-core/src/db/constants.ts

@@ -1,14 +1,5 @@
-export const CONSTANT_INTERNAL_ROW_COLS = [
+export {
-  "_id",
+  CONSTANT_INTERNAL_ROW_COLS,
-  "_rev",
+  CONSTANT_EXTERNAL_ROW_COLS,
-  "type",
+  isInternalColumnName,
-  "createdAt",
+} from "@budibase/shared-core"
-  "updatedAt",
-  "tableId",
-] as const
-
-export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
-
-export function isInternalColumnName(name: string): boolean {
-  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
-}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -17,6 +17,8 @@
     SWITCHABLE_TYPES,
     ValidColumnNameRegex,
     helpers,
+    CONSTANT_INTERNAL_ROW_COLS,
+    CONSTANT_EXTERNAL_ROW_COLS,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -52,7 +54,6 @@
   const DATE_TYPE = FieldType.DATETIME
 
   const dispatch = createEventDispatcher()
-  const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
   const { dispatch: gridDispatch, rows } = getContext("grid")
 
   export let field
@@ -487,20 +488,27 @@
       })
     }
     const newError = {}
+    const prohibited = externalTable
+      ? CONSTANT_EXTERNAL_ROW_COLS
+      : CONSTANT_INTERNAL_ROW_COLS
     if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
-    } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
+    } else if (
-      newError.name = `${PROHIBITED_COLUMN_NAMES.join(
+      prohibited.some(
+        name => fieldInfo?.name?.toLowerCase() === name.toLowerCase()
+      )
+    ) {
+      newError.name = `${prohibited.join(
         ", "
-      )} are not allowed as column names`
+      )} are not allowed as column names - case insensitive.`
     } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
 
     if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) {
-      newError.subtype = `Auto Column requires a type`
+      newError.subtype = `Auto Column requires a type.`
     }
 
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte

@@ -3,6 +3,7 @@
   import { Button } from "@budibase/bbui"
   import GridCell from "../cells/GridCell.svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
+  import { BlankRowID } from "../lib/constants"
 
   const {
     renderedRows,
@@ -17,6 +18,7 @@
     isDragging,
     buttonColumnWidth,
     showVScrollbar,
+    dispatch,
   } = getContext("grid")
 
   let container
@@ -89,6 +91,17 @@
           </GridCell>
         </div>
       {/each}
+      <div
+        class="row blank"
+        on:mouseenter={$isDragging ? null : () => ($hoveredRowId = BlankRowID)}
+        on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+      >
+        <GridCell
+          width={$buttonColumnWidth}
+          highlighted={$hoveredRowId === BlankRowID}
+          on:click={() => dispatch("add-row-inline")}
+        />
+      </div>
     </GridScrollWrapper>
   </div>
 </div>
@@ -129,8 +142,11 @@
     align-items: center;
     gap: 4px;
   }
+  .blank :global(.cell:hover) {
+    cursor: pointer;
+  }
 
-  /* Add left cell border */
+  /* Add left cell border to all cells */
   .button-column :global(.cell) {
     border-left: var(--cell-border);
   }

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -26,7 +26,7 @@
     MaxCellRenderOverflow,
     GutterWidth,
     DefaultRowHeight,
-    Padding,
+    VPadding,
     SmallRowHeight,
     ControlsHeight,
     ScrollBarSize,
@@ -119,7 +119,7 @@
   // Derive min height and make available in context
   const minHeight = derived(rowHeight, $height => {
     const heightForControls = showControls ? ControlsHeight : 0
-    return Padding + SmallRowHeight + $height + heightForControls
+    return VPadding + SmallRowHeight + $height + heightForControls
   })
   context = { ...context, minHeight }
 
@@ -354,8 +354,13 @@
     transition: none;
   }
 
-  /* Overrides */
+  /* Overrides for quiet */
-  .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)) {
+  .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)),
+  .grid.quiet :global(.sticky-column .row > .cell),
+  .grid.quiet :global(.new-row .row > .cell:not(:last-child)) {
     border-right: none;
   }
+  .grid.quiet :global(.sticky-column:before) {
+    display: none;
+  }
 </style>

packages/frontend-core/src/components/grid/layout/GridBody.svelte

@@ -2,6 +2,7 @@
   import { getContext, onMount } from "svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import GridRow from "./GridRow.svelte"
+  import GridCell from "../cells/GridCell.svelte"
   import { BlankRowID } from "../lib/constants"
   import ButtonColumn from "./ButtonColumn.svelte"
 
@@ -46,7 +47,6 @@
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
-<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div bind:this={body} class="grid-body">
   <GridScrollWrapper scrollHorizontally scrollVertically attachHandlers>
     {#each $renderedRows as row, idx}
@@ -54,13 +54,16 @@
     {/each}
     {#if $config.canAddRows}
       <div
-        class="blank"
+        class="row blank"
-        class:highlighted={$hoveredRowId === BlankRowID}
-        style="width:{columnsWidth}px"
         on:mouseenter={$isDragging ? null : () => ($hoveredRowId = BlankRowID)}
         on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-        on:click={() => dispatch("add-row-inline")}
+      >
-      />
+        <GridCell
+          width={columnsWidth}
+          highlighted={$hoveredRowId === BlankRowID}
+          on:click={() => dispatch("add-row-inline")}
+        />
+      </div>
     {/if}
   </GridScrollWrapper>
   {#if $props.buttons?.length}
@@ -76,15 +79,13 @@
     overflow: hidden;
     flex: 1 1 auto;
   }
-  .blank {
+  .row {
-    height: var(--row-height);
+    display: flex;
-    background: var(--cell-background);
+    flex-direction: row;
-    border-bottom: var(--cell-border);
+    justify-content: flex-start;
-    border-right: var(--cell-border);
+    align-items: stretch;
-    position: absolute;
   }
-  .blank.highlighted {
+  .blank :global(.cell:hover) {
-    background: var(--cell-background-hover);
     cursor: pointer;
   }
 </style>

packages/frontend-core/src/components/grid/layout/NewRow.svelte

@@ -31,6 +31,7 @@
     filter,
     inlineFilters,
     columnRenderMap,
+    scrollTop,
   } = getContext("grid")
 
   let visible = false
@@ -43,6 +44,21 @@
   $: $datasource, (visible = false)
   $: selectedRowCount = Object.values($selectedRows).length
   $: hasNoRows = !$rows.length
+  $: renderedRowCount = $renderedRows.length
+  $: offset = getOffset($hasNextPage, renderedRowCount, $rowHeight, $scrollTop)
+
+  const getOffset = (hasNextPage, rowCount, rowHeight, scrollTop) => {
+    // If we have a next page of data then we aren't truly at the bottom, so we
+    // render the add row component at the top
+    if (hasNextPage) {
+      return 0
+    }
+    offset = rowCount * rowHeight - (scrollTop % rowHeight)
+    if (rowCount !== 0) {
+      offset -= 1
+    }
+    return offset
+  }
 
   const addRow = async () => {
     // Blur the active cell and tick to let final value updates propagate
@@ -85,12 +101,6 @@
       return
     }
 
-    // If we have a next page of data then we aren't truly at the bottom, so we
-    // render the add row component at the top
-    if ($hasNextPage) {
-      offset = 0
-    }
-
     // If we don't have a next page then we're at the bottom and can scroll to
     // the max available offset
     else {
@@ -98,10 +108,6 @@
         ...state,
         top: $maxScrollTop,
       }))
-      offset = $renderedRows.length * $rowHeight - ($maxScrollTop % $rowHeight)
-      if ($renderedRows.length !== 0) {
-        offset -= 1
-      }
     }
 
     // Update state and select initial cell
@@ -171,39 +177,41 @@
 <!-- Only show new row functionality if we have any columns -->
 {#if visible}
   <div
-    class="container"
+    class="new-row"
     class:floating={offset > 0}
     style="--offset:{offset}px; --sticky-width:{width}px;"
   >
     <div class="underlay sticky" transition:fade|local={{ duration: 130 }} />
     <div class="underlay" transition:fade|local={{ duration: 130 }} />
     <div class="sticky-column" transition:fade|local={{ duration: 130 }}>
-      <GutterCell expandable on:expand={addViaModal} rowHovered>
+      <div class="row">
-        <Icon name="Add" color="var(--spectrum-global-color-gray-500)" />
+        <GutterCell expandable on:expand={addViaModal} rowHovered>
-        {#if isAdding}
+          <Icon name="Add" color="var(--spectrum-global-color-gray-500)" />
-          <div in:fade={{ duration: 130 }} class="loading-overlay" />
-        {/if}
-      </GutterCell>
-      {#if $stickyColumn}
-        {@const cellId = getCellID(NewRowID, $stickyColumn.name)}
-        <DataCell
-          {cellId}
-          rowFocused
-          column={$stickyColumn}
-          row={newRow}
-          focused={$focusedCellId === cellId}
-          width={$stickyColumn.width}
-          {updateValue}
-          topRow={offset === 0}
-        >
-          {#if $stickyColumn?.schema?.autocolumn}
-            <div class="readonly-overlay">Can't edit auto column</div>
-          {/if}
           {#if isAdding}
             <div in:fade={{ duration: 130 }} class="loading-overlay" />
           {/if}
-        </DataCell>
+        </GutterCell>
-      {/if}
+        {#if $stickyColumn}
+          {@const cellId = getCellID(NewRowID, $stickyColumn.name)}
+          <DataCell
+            {cellId}
+            rowFocused
+            column={$stickyColumn}
+            row={newRow}
+            focused={$focusedCellId === cellId}
+            width={$stickyColumn.width}
+            {updateValue}
+            topRow={offset === 0}
+          >
+            {#if $stickyColumn?.schema?.autocolumn}
+              <div class="readonly-overlay">Can't edit auto column</div>
+            {/if}
+            {#if isAdding}
+              <div in:fade={{ duration: 130 }} class="loading-overlay" />
+            {/if}
+          </DataCell>
+        {/if}
+      </div>
     </div>
     <div class="normal-columns" transition:fade|local={{ duration: 130 }}>
       <GridScrollWrapper scrollHorizontally attachHandlers>
@@ -270,7 +278,7 @@
     margin-left: -6px;
   }
 
-  .container {
+  .new-row {
     position: absolute;
     top: var(--default-row-height);
     left: 0;
@@ -280,10 +288,10 @@
     flex-direction: row;
     align-items: stretch;
   }
-  .container :global(.cell) {
+  .new-row :global(.cell) {
     --cell-background: var(--spectrum-global-color-gray-75) !important;
   }
-  .container.floating :global(.cell) {
+  .new-row.floating :global(.cell) {
     height: calc(var(--row-height) + 1px);
     border-top: var(--cell-border);
   }
@@ -312,8 +320,10 @@
     pointer-events: all;
     z-index: 3;
     position: absolute;
-    top: calc(var(--row-height) + var(--offset) + 24px);
+    top: calc(
-    left: 18px;
+      var(--row-height) + var(--offset) + var(--default-row-height) / 2
+    );
+    left: calc(var(--default-row-height) / 2);
   }
   .button-with-keys {
     display: flex;

packages/frontend-core/src/components/grid/layout/StickyColumn.svelte

@@ -66,62 +66,58 @@
 
   <!-- svelte-ignore a11y-no-static-element-interactions -->
   <!-- svelte-ignore a11y-click-events-have-key-events -->
-  <div class="content">
+  <GridScrollWrapper scrollVertically attachHandlers>
-    <GridScrollWrapper scrollVertically attachHandlers>
+    {#each $renderedRows as row, idx}
-      {#each $renderedRows as row, idx}
+      {@const rowSelected = !!$selectedRows[row._id]}
-        {@const rowSelected = !!$selectedRows[row._id]}
+      {@const rowHovered = $hoveredRowId === row._id}
-        {@const rowHovered = $hoveredRowId === row._id}
+      {@const rowFocused = $focusedRow?._id === row._id}
-        {@const rowFocused = $focusedRow?._id === row._id}
+      {@const cellId = getCellID(row._id, $stickyColumn?.name)}
-        {@const cellId = getCellID(row._id, $stickyColumn?.name)}
+      <div
-        <div
+        class="row"
-          class="row"
+        on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
-          on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
+        on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-          on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+        on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
-          on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
+      >
-        >
+        <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
-          <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
+        {#if $stickyColumn}
-          {#if $stickyColumn}
+          <DataCell
-            <DataCell
+            {row}
-              {row}
+            {cellId}
-              {cellId}
+            {rowFocused}
-              {rowFocused}
+            selected={rowSelected}
-              selected={rowSelected}
+            highlighted={rowHovered || rowFocused}
-              highlighted={rowHovered || rowFocused}
+            rowIdx={row.__idx}
-              rowIdx={row.__idx}
+            topRow={idx === 0}
-              topRow={idx === 0}
+            focused={$focusedCellId === cellId}
-              focused={$focusedCellId === cellId}
+            selectedUser={$selectedCellMap[cellId]}
-              selectedUser={$selectedCellMap[cellId]}
+            width={$stickyColumn.width}
-              width={$stickyColumn.width}
+            column={$stickyColumn}
-              column={$stickyColumn}
+            contentLines={$contentLines}
-              contentLines={$contentLines}
+          />
-            />
+        {/if}
-          {/if}
+      </div>
-        </div>
+    {/each}
-      {/each}
+    {#if $config.canAddRows}
-      {#if $config.canAddRows}
+      <div
-        <div
+        class="row blank"
-          class="row new"
+        on:mouseenter={$isDragging ? null : () => ($hoveredRowId = BlankRowID)}
-          on:mouseenter={$isDragging
+        on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-            ? null
+        on:click={() => dispatch("add-row-inline")}
-            : () => ($hoveredRowId = BlankRowID)}
+      >
-          on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+        <GutterCell rowHovered={$hoveredRowId === BlankRowID}>
-          on:click={() => dispatch("add-row-inline")}
+          <Icon name="Add" color="var(--spectrum-global-color-gray-500)" />
-        >
+        </GutterCell>
-          <GutterCell rowHovered={$hoveredRowId === BlankRowID}>
+        {#if $stickyColumn}
-            <Icon name="Add" color="var(--spectrum-global-color-gray-500)" />
+          <GridCell
-          </GutterCell>
+            width={$stickyColumn.width}
-          {#if $stickyColumn}
+            highlighted={$hoveredRowId === BlankRowID}
-            <GridCell
+          >
-              width={$stickyColumn.width}
+            <KeyboardShortcut padded keybind="Ctrl+Enter" />
-              highlighted={$hoveredRowId === BlankRowID}
+          </GridCell>
-            >
+        {/if}
-              <KeyboardShortcut padded keybind="Ctrl+Enter" />
+      </div>
-            </GridCell>
+    {/if}
-          {/if}
+  </GridScrollWrapper>
-        </div>
-      {/if}
-    </GridScrollWrapper>
-  </div>
 </div>
 
 <style>
@@ -174,11 +170,7 @@
     justify-content: flex-start;
     align-items: stretch;
   }
-  .content {
+  .blank :global(.cell:hover) {
-    position: relative;
-    flex: 1 1 auto;
-  }
-  .row.new :global(*:hover) {
     cursor: pointer;
   }
 </style>

packages/frontend-core/src/components/grid/lib/constants.js

@@ -1,12 +1,13 @@
-export const Padding = 100
-export const ScrollBarSize = 8
-export const GutterWidth = 72
-export const DefaultColumnWidth = 200
-export const MinColumnWidth = 80
 export const SmallRowHeight = 36
 export const MediumRowHeight = 64
 export const LargeRowHeight = 92
 export const DefaultRowHeight = SmallRowHeight
+export const VPadding = SmallRowHeight * 2
+export const HPadding = 40
+export const ScrollBarSize = 8
+export const GutterWidth = 72
+export const DefaultColumnWidth = 200
+export const MinColumnWidth = 80
 export const NewRowID = "new"
 export const BlankRowID = "blank"
 export const RowPageSize = 100

packages/frontend-core/src/components/grid/stores/scroll.js

@@ -1,6 +1,12 @@
 import { writable, derived, get } from "svelte/store"
 import { tick } from "svelte"
-import { Padding, GutterWidth, FocusedCellMinOffset } from "../lib/constants"
+import {
+  GutterWidth,
+  FocusedCellMinOffset,
+  ScrollBarSize,
+  HPadding,
+  VPadding,
+} from "../lib/constants"
 import { parseCellID } from "../lib/utils"
 
 export const createStores = () => {
@@ -34,28 +40,15 @@ export const deriveStores = context => {
   // Memoize store primitives
   const stickyColumnWidth = derived(stickyColumn, $col => $col?.width || 0, 0)
 
-  // Derive vertical limits
-  const contentHeight = derived(
-    [rows, rowHeight],
-    ([$rows, $rowHeight]) => ($rows.length + 1) * $rowHeight + Padding,
-    0
-  )
-  const maxScrollTop = derived(
-    [height, contentHeight],
-    ([$height, $contentHeight]) => Math.max($contentHeight - $height, 0),
-    0
-  )
-
   // Derive horizontal limits
   const contentWidth = derived(
     [visibleColumns, stickyColumnWidth, buttonColumnWidth],
     ([$visibleColumns, $stickyColumnWidth, $buttonColumnWidth]) => {
-      const space = Math.max(Padding, $buttonColumnWidth - 1)
+      let width = GutterWidth + $buttonColumnWidth + $stickyColumnWidth
-      let width = GutterWidth + space + $stickyColumnWidth
       $visibleColumns.forEach(col => {
         width += col.width
       })
-      return width
+      return width + HPadding
     },
     0
   )
@@ -71,14 +64,6 @@ export const deriveStores = context => {
     },
     0
   )
-
-  // Derive whether to show scrollbars or not
-  const showVScrollbar = derived(
-    [contentHeight, height],
-    ([$contentHeight, $height]) => {
-      return $contentHeight > $height
-    }
-  )
   const showHScrollbar = derived(
     [contentWidth, screenWidth],
     ([$contentWidth, $screenWidth]) => {
@@ -86,6 +71,30 @@ export const deriveStores = context => {
     }
   )
 
+  // Derive vertical limits
+  const contentHeight = derived(
+    [rows, rowHeight, showHScrollbar],
+    ([$rows, $rowHeight, $showHScrollbar]) => {
+      let height = ($rows.length + 1) * $rowHeight + VPadding
+      if ($showHScrollbar) {
+        height += ScrollBarSize * 2
+      }
+      return height
+    },
+    0
+  )
+  const maxScrollTop = derived(
+    [height, contentHeight],
+    ([$height, $contentHeight]) => Math.max($contentHeight - $height, 0),
+    0
+  )
+  const showVScrollbar = derived(
+    [contentHeight, height],
+    ([$contentHeight, $height]) => {
+      return $contentHeight > $height
+    }
+  )
+
   return {
     contentHeight,
     contentWidth,

packages/server/package.json

@@ -99,7 +99,7 @@
     "mysql2": "3.9.8",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
-    "openai": "^3.2.1",
+    "openai": "^4.52.1",
     "openapi-types": "9.3.1",
     "pg": "8.10.0",
     "pouchdb": "7.3.0",

packages/server/src/api/routes/tests/table.spec.ts

@@ -276,6 +276,31 @@ describe.each([
         })
     })
 
+    isInternal &&
+      it("shouldn't allow duplicate column names", async () => {
+        const saveTableRequest: SaveTableRequest = {
+          ...basicTable(),
+        }
+        saveTableRequest.schema["Type"] = {
+          type: FieldType.STRING,
+          name: "Type",
+        }
+        // allow the "Type" column - internal columns aren't case sensitive
+        await config.api.table.save(saveTableRequest, {
+          status: 200,
+        })
+        saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" }
+        saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" }
+
+        await config.api.table.save(saveTableRequest, {
+          status: 400,
+          body: {
+            message:
+              'Column(s) "foo" are duplicated - check for other columns with these name (case in-sensitive)',
+          },
+        })
+      })
+
     it("should add a new column for an internal DB table", async () => {
       const saveTableRequest: SaveTableRequest = {
         ...basicTable(),

packages/server/src/automations/steps/openai.ts

@@ -1,4 +1,5 @@
-import { Configuration, OpenAIApi } from "openai"
+import { OpenAI } from "openai"
+
 import {
   AutomationActionStepId,
   AutomationStepSchema,
@@ -75,13 +76,11 @@ export async function run({ inputs }: AutomationStepInput) {
   }
 
   try {
-    const configuration = new Configuration({
+    const openai = new OpenAI({
       apiKey: environment.OPENAI_API_KEY,
     })
 
-    const openai = new OpenAIApi(configuration)
+    const completion = await openai.chat.completions.create({
-
-    const completion = await openai.createChatCompletion({
       model: inputs.model,
       messages: [
         {
@@ -90,8 +89,7 @@ export async function run({ inputs }: AutomationStepInput) {
         },
       ],
     })
-
+    const response = completion?.choices[0]?.message?.content
-    const response = completion?.data?.choices[0]?.message?.content
 
     return {
       response,

packages/server/src/automations/tests/openai.spec.ts

@@ -1,15 +1,13 @@
 const setup = require("./utilities")
 
 import environment from "../../environment"
-import openai from "openai"
+import { OpenAI } from "openai"
 
-jest.mock(
+jest.mock("openai", () => ({
-  "openai",
+  OpenAI: jest.fn().mockImplementation(() => ({
-  jest.fn(() => ({
+    chat: {
-    Configuration: jest.fn(),
+      completions: {
-    OpenAIApi: jest.fn(() => ({
+        create: jest.fn(() => ({
-      createChatCompletion: jest.fn(() => ({
-        data: {
           choices: [
             {
               message: {
@@ -17,15 +15,13 @@ jest.mock(
               },
             },
           ],
-        },
+        })),
-      })),
+      },
-    })),
+    },
-  }))
+  })),
-)
+}))
 
-const mockedOpenAIApi = openai.OpenAIApi as jest.MockedClass<
+const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
-  typeof openai.OpenAIApi
->
 
 const OPENAI_PROMPT = "What is the meaning of life?"
 
@@ -73,14 +69,18 @@ describe("test the openai action", () => {
   })
 
   it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => {
-    mockedOpenAIApi.mockImplementation(
+    mockedOpenAI.mockImplementation(
       () =>
         ({
-          createChatCompletion: jest.fn(() => {
+          chat: {
-            throw new Error(
+            completions: {
-              "An error occurred while calling createChatCompletion"
+              create: jest.fn(() => {
-            )
+                throw new Error(
-          }),
+                  "An error occurred while calling createChatCompletion"
+                )
+              }),
+            },
+          },
         } as any)
     )
 

packages/server/src/sdk/app/tables/internal/index.ts

@@ -17,6 +17,7 @@ import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
 import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
 import { context } from "@budibase/backend-core"
+import { findDuplicateInternalColumns } from "@budibase/shared-core"
 import { getTable } from "../getters"
 import { checkAutoColumns } from "./utils"
 import * as viewsSdk from "../../views"
@@ -44,6 +45,17 @@ export async function save(
   if (hasTypeChanged(table, oldTable)) {
     throw new Error("A column type has changed.")
   }
+
+  // check for case sensitivity - we don't want to allow duplicated columns
+  const duplicateColumn = findDuplicateInternalColumns(table)
+  if (duplicateColumn.length) {
+    throw new Error(
+      `Column(s) "${duplicateColumn.join(
+        ", "
+      )}" are duplicated - check for other columns with these name (case in-sensitive)`
+    )
+  }
+
   // check that subtypes have been maintained
   table = checkAutoColumns(table, oldTable)
 

packages/shared-core/src/constants/index.ts

@@ -1,5 +1,6 @@
 export * from "./api"
 export * from "./fields"
+export * from "./rows"
 
 export const OperatorOptions = {
   Equals: {

packages/shared-core/src/constants/rows.ts

@@ -0,0 +1,14 @@
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
+
+export function isInternalColumnName(name: string): boolean {
+  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
+}

packages/shared-core/src/table.ts

@@ -1,4 +1,5 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, Table } from "@budibase/types"
+import { CONSTANT_INTERNAL_ROW_COLS } from "./constants"
 
 const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
@@ -51,3 +52,27 @@ export function canBeDisplayColumn(type: FieldType): boolean {
 export function canBeSortColumn(type: FieldType): boolean {
   return !!allowSortColumnByType[type]
 }
+
+export function findDuplicateInternalColumns(table: Table): string[] {
+  // maintains the case of keys
+  const casedKeys = Object.keys(table.schema)
+  // get the column names
+  const uncasedKeys = casedKeys.map(colName => colName.toLowerCase())
+  // there are duplicates
+  const set = new Set(uncasedKeys)
+  let duplicates: string[] = []
+  if (set.size !== uncasedKeys.length) {
+    for (let key of set.keys()) {
+      const count = uncasedKeys.filter(name => name === key).length
+      if (count > 1) {
+        duplicates.push(key)
+      }
+    }
+  }
+  for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+    if (casedKeys.find(key => key === internalColumn)) {
+      duplicates.push(internalColumn)
+    }
+  }
+  return duplicates
+}

yarn.lock

@@ -5911,6 +5911,14 @@
     "@types/node" "*"
     form-data "^3.0.0"
 
+"@types/node-fetch@^2.6.4":
+  version "2.6.11"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
+  integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
+  dependencies:
+    "@types/node" "*"
+    form-data "^4.0.0"
+
 "@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@^20.4.5":
   version "20.12.10"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.10.tgz#8f0c3f12b0f075eee1fe20c1afb417e9765bef76"
@@ -7476,7 +7484,7 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0:
+axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11480,6 +11488,11 @@ forever-agent@~0.6.1:
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
   integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==
 
+form-data-encoder@1.7.2:
+  version "1.7.2"
+  resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040"
+  integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==
+
 form-data@4.0.0, form-data@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
@@ -11516,6 +11529,14 @@ form-data@~2.3.2:
     combined-stream "^1.0.6"
     mime-types "^2.1.12"
 
+formdata-node@^4.3.2:
+  version "4.4.1"
+  resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2"
+  integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==
+  dependencies:
+    node-domexception "1.0.0"
+    web-streams-polyfill "4.0.0-beta.3"
+
 formidable@^1.1.1:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
@@ -16262,6 +16283,11 @@ node-addon-api@^6.1.0:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76"
   integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==
 
+node-domexception@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
+  integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
+
 node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
@@ -16891,13 +16917,19 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
-openai@^3.2.1:
+openai@4.52.1:
-  version "3.2.1"
+  version "4.52.1"
-  resolved "https://registry.yarnpkg.com/openai/-/openai-3.2.1.tgz#1fa35bdf979cbde8453b43f2dd3a7d401ee40866"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
-  integrity sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==
+  integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
   dependencies:
-    axios "^0.26.0"
+    "@types/node" "^18.11.18"
-    form-data "^4.0.0"
+    "@types/node-fetch" "^2.6.4"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+    web-streams-polyfill "^3.2.1"
 
 openapi-response-validator@^9.2.0:
   version "9.3.1"
@@ -22334,6 +22366,16 @@ wcwidth@^1.0.0, wcwidth@^1.0.1:
   dependencies:
     defaults "^1.0.3"
 
+web-streams-polyfill@4.0.0-beta.3:
+  version "4.0.0-beta.3"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
+  integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
+
+web-streams-polyfill@^3.2.1:
+  version "3.3.3"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b"
+  integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==
+
 webfinger@^0.4.2:
   version "0.4.2"
   resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d"