From fe951babf56fabaddc983d6c1565c3c4d584cb27 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 26 Mar 2025 13:20:25 +0100
Subject: [PATCH 001/101] Initial table generation prompt

---
 .../_components/AITableCreationModal.svelte   | 34 +++++++++++++++++++
 .../builder/app/[application]/data/new.svelte | 25 +++++++++++---
 packages/frontend-core/src/api/ai.ts          | 17 +++++++++-
 packages/pro                                  |  2 +-
 packages/types/src/api/web/ai.ts              |  8 +++++
 5 files changed, 80 insertions(+), 6 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
new file mode 100644
index 0000000000..a25dfdbbb8
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -0,0 +1,34 @@
+<script lang="ts">
+  import { API } from "@/api"
+  import { keepOpen, Modal, ModalContent, TextArea } from "@budibase/bbui"
+
+  let modal: Modal
+  let prompt: string =
+    "I need a ticketing system where employees can submit tickets. A ticket has a title, description, priority, status, and an assigned technician"
+  let result: string
+
+  export function show() {
+    modal.show()
+  }
+
+  export async function generate() {
+    result = ""
+
+    const { response } = await API.generateTables({ prompt })
+    result = response ?? ""
+    return keepOpen
+  }
+</script>
+
+<Modal bind:this={modal}>
+  <ModalContent
+    title="AI generation"
+    size="XL"
+    confirmText="Generate"
+    onConfirm={generate}
+    showCancelButton={false}
+  >
+    <TextArea label="Table Name" bind:value={prompt} />
+    <TextArea label="Result" value={result} disabled minHeight={350} />
+  </ModalContent>
+</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
index 2ba23686ab..1518d64110 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { API } from "@/api"
   import {
     tables,
@@ -15,9 +15,12 @@
   import IntegrationIcon from "@/components/backend/DatasourceNavigator/IntegrationIcon.svelte"
   import CreationPage from "@/components/common/CreationPage.svelte"
   import ICONS from "@/components/backend/DatasourceNavigator/icons/index.js"
+  import AiTableCreationModal from "./_components/AITableCreationModal.svelte"
+  import { onMount } from "svelte"
 
-  let internalTableModal
-  let externalDatasourceModal
+  let internalTableModal: CreateInternalTableModal
+  let externalDatasourceModal: CreateExternalDatasourceModal
+  let aiTablesModal: AiTableCreationModal
 
   let sampleDataLoading = false
   let externalDatasourceLoading = false
@@ -37,6 +40,10 @@
       notifications.error("Error creating datasource")
     }
   }
+
+  onMount(() => {
+    aiTablesModal.show()
+  })
 </script>
 
 <CreateInternalTableModal bind:this={internalTableModal} />
@@ -46,6 +53,8 @@
   bind:this={externalDatasourceModal}
 />
 
+<AiTableCreationModal bind:this={aiTablesModal} />
+
 <CreationPage
   showClose={hasData($datasources, $tables)}
   onClose={() => $goto("./table")}
@@ -60,7 +69,7 @@
 
   <div class="options">
     <DatasourceOption
-      on:click={internalTableModal.show}
+      on:click={() => internalTableModal.show()}
       title="Create new table"
       description="Non-relational"
       {disabled}
@@ -83,6 +92,14 @@
     >
       <svelte:component this={ICONS.BUDIBASE} height="20" width="20" />
     </DatasourceOption>
+    <DatasourceOption
+      on:click={() => aiTablesModal.show()}
+      title="Generate it using AI"
+      description="Non-relational"
+      {disabled}
+    >
+      <Icon name="MagicWand" size="M" />
+    </DatasourceOption>
   </div>
 
   <div class="subHeading">
diff --git a/packages/frontend-core/src/api/ai.ts b/packages/frontend-core/src/api/ai.ts
index da4c23a23c..926def93ca 100644
--- a/packages/frontend-core/src/api/ai.ts
+++ b/packages/frontend-core/src/api/ai.ts
@@ -1,9 +1,17 @@
-import { GenerateJsRequest, GenerateJsResponse } from "@budibase/types"
+import {
+  GenerateJsRequest,
+  GenerateJsResponse,
+  GenerateTablesRequest,
+  GenerateTablesResponse,
+} from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
 export interface AIEndpoints {
   generateCronExpression: (prompt: string) => Promise<{ message: string }>
   generateJs: (req: GenerateJsRequest) => Promise<GenerateJsResponse>
+  generateTables: (
+    req: GenerateTablesRequest
+  ) => Promise<GenerateTablesResponse>
 }
 
 export const buildAIEndpoints = (API: BaseAPIClient): AIEndpoints => ({
@@ -23,4 +31,11 @@ export const buildAIEndpoints = (API: BaseAPIClient): AIEndpoints => ({
       body: req,
     })
   },
+
+  generateTables: async req => {
+    return await API.post({
+      url: "/api/ai/tables",
+      body: req,
+    })
+  },
 })
diff --git a/packages/pro b/packages/pro
index ae33c80d3a..29b8850d7a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ae33c80d3a7f289535951669e48f8d8a437467cc
+Subproject commit 29b8850d7af58c7e33be5c15fe900840bc69093e
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 3962422b77..036858cb15 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -8,3 +8,11 @@ export interface GenerateJsRequest {
 export interface GenerateJsResponse {
   code: string
 }
+
+export interface GenerateTablesRequest {
+  prompt: string
+}
+
+export interface GenerateTablesResponse {
+  response?: string
+}

From 323e2fefcd6cebd3b57a003b7c44b42a9f77e950 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 26 Mar 2025 14:13:05 +0100
Subject: [PATCH 002/101] Move code to server for easy dev

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 17 +++++++++++++++++
 packages/server/src/api/routes/ai.ts      |  9 +++++++++
 packages/server/src/api/routes/index.ts   |  2 ++
 4 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/api/controllers/ai.ts
 create mode 100644 packages/server/src/api/routes/ai.ts

diff --git a/packages/pro b/packages/pro
index 29b8850d7a..0dd8e42568 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 29b8850d7af58c7e33be5c15fe900840bc69093e
+Subproject commit 0dd8e42568bcc7f1d3ca4143221ae72257df8bf8
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
new file mode 100644
index 0000000000..db49923b90
--- /dev/null
+++ b/packages/server/src/api/controllers/ai.ts
@@ -0,0 +1,17 @@
+import {
+  Ctx,
+  GenerateTablesRequest,
+  GenerateTablesResponse,
+} from "@budibase/types"
+import { getLLM } from "packages/pro/src/ai"
+
+export async function generateTables(
+  ctx: Ctx<GenerateTablesRequest, GenerateTablesResponse>
+) {
+  const llm = await getLLM()
+  const response = await llm?.generateTables(ctx.request.body.prompt)
+
+  ctx.body = {
+    response: response?.message ?? undefined,
+  }
+}
diff --git a/packages/server/src/api/routes/ai.ts b/packages/server/src/api/routes/ai.ts
new file mode 100644
index 0000000000..14dbf6443e
--- /dev/null
+++ b/packages/server/src/api/routes/ai.ts
@@ -0,0 +1,9 @@
+import * as ai from "../controllers/ai"
+import Router from "@koa/router"
+import { auth } from "@budibase/backend-core"
+
+const router: Router = new Router()
+
+router.post("/api/ai/tables", auth.builderOrAdmin, ai.generateTables)
+
+export default router
diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts
index 253b0f5a82..8c363ede72 100644
--- a/packages/server/src/api/routes/index.ts
+++ b/packages/server/src/api/routes/index.ts
@@ -30,6 +30,7 @@ import Router from "@koa/router"
 import { api as pro } from "@budibase/pro"
 import rowActionRoutes from "./rowAction"
 import oauth2Routes from "./oauth2"
+import aiRoutes from "./ai"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
@@ -74,4 +75,5 @@ export const mainRoutes: Router[] = [
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
+  aiRoutes,
 ]

From c47e7571cd828a9dfa88f115fa8d2bf267e27db4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 12:06:48 +0100
Subject: [PATCH 003/101] Handle errors

---
 .../data/_components/AITableCreationModal.svelte         | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index a25dfdbbb8..7bae3aea59 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -13,9 +13,12 @@
 
   export async function generate() {
     result = ""
-
-    const { response } = await API.generateTables({ prompt })
-    result = response ?? ""
+    try {
+      const { response } = await API.generateTables({ prompt })
+      result = response ?? ""
+    } catch {
+      // TODO
+    }
     return keepOpen
   }
 </script>

From a4d6c263d14574eef11c09ce7f4168848ca46cbb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 15:31:02 +0100
Subject: [PATCH 004/101] Display error

---
 .../data/_components/AITableCreationModal.svelte     | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index 7bae3aea59..9c64d698e0 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -1,6 +1,12 @@
 <script lang="ts">
   import { API } from "@/api"
-  import { keepOpen, Modal, ModalContent, TextArea } from "@budibase/bbui"
+  import {
+    keepOpen,
+    Modal,
+    ModalContent,
+    notifications,
+    TextArea,
+  } from "@budibase/bbui"
 
   let modal: Modal
   let prompt: string =
@@ -16,8 +22,8 @@
     try {
       const { response } = await API.generateTables({ prompt })
       result = response ?? ""
-    } catch {
-      // TODO
+    } catch (e: any) {
+      notifications.error(e.message)
     }
     return keepOpen
   }

From e6413948a56880f2bbd01c6fd57230fb07bd9871 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 15:34:36 +0100
Subject: [PATCH 005/101] Initial generation

---
 .gitignore                                |  1 +
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 40 +++++++++++++++++++++--
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 21637edfbe..f548a62da6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,3 +112,4 @@ budibase-datasource
 
 *.iml
 .nx
+llm-output/*
diff --git a/packages/pro b/packages/pro
index 0dd8e42568..0e1d1f449d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 0dd8e42568bcc7f1d3ca4143221ae72257df8bf8
+Subproject commit 0e1d1f449dbdd96aed851ed3a1c5b0dced7bc383
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index db49923b90..92a3deb770 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -1,16 +1,52 @@
 import {
-  Ctx,
   GenerateTablesRequest,
   GenerateTablesResponse,
+  SourceName,
+  UserCtx,
 } from "@budibase/types"
 import { getLLM } from "packages/pro/src/ai"
+import * as tableController from "./public/tables"
+import { context } from "@budibase/backend-core"
+import sdk from "../../sdk"
+import { generateDatasourceID } from "../../db/utils"
+import fs from "fs"
+import path from "path"
 
 export async function generateTables(
-  ctx: Ctx<GenerateTablesRequest, GenerateTablesResponse>
+  ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
 ) {
   const llm = await getLLM()
+  llm!.maxTokens = 1200
   const response = await llm?.generateTables(ctx.request.body.prompt)
 
+  const count = (await sdk.datasources.fetch()).length
+  const { id: dsId } = await context.getAppDB().put({
+    _id: generateDatasourceID(),
+    name: `Test ${count}`,
+    type: "budibase",
+
+    source: SourceName.BUDIBASE,
+    config: {},
+  })
+
+  console.warn(response?.message)
+
+  fs.writeFileSync(
+    path.join(process.env.PWD!, `../../llm-output/response_${Date.now()}.json`),
+    response?.message || ""
+  )
+
+  const json = JSON.parse(response!.message || "")
+  for (const table of json) {
+    await tableController.create(
+      {
+        request: { body: { ...table, sourceId: dsId } },
+        user: ctx.user,
+      } as any,
+      async () => {}
+    )
+  }
+
   ctx.body = {
     response: response?.message ?? undefined,
   }

From 2fd8e744fff1cced58cbee5a58b1b6caf57a5382 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 16:43:22 +0100
Subject: [PATCH 006/101] Redirects

---
 .../_components/AITableCreationModal.svelte   | 22 +++++++++++--------
 .../builder/app/[application]/data/new.svelte |  5 -----
 packages/server/src/api/controllers/ai.ts     | 15 +++++++------
 packages/types/src/api/web/ai.ts              |  1 +
 4 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index 9c64d698e0..c76d468eec 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -1,12 +1,8 @@
 <script lang="ts">
   import { API } from "@/api"
-  import {
-    keepOpen,
-    Modal,
-    ModalContent,
-    notifications,
-    TextArea,
-  } from "@budibase/bbui"
+  import { datasources, tables } from "@/stores/builder"
+  import { Modal, ModalContent, notifications, TextArea } from "@budibase/bbui"
+  import { goto } from "@roxi/routify"
 
   let modal: Modal
   let prompt: string =
@@ -20,12 +16,20 @@
   export async function generate() {
     result = ""
     try {
-      const { response } = await API.generateTables({ prompt })
+      const { response, createdTables } = await API.generateTables({ prompt })
       result = response ?? ""
+
+      const [tableToRedirect] = createdTables.sort((a, b) =>
+        a.name.localeCompare(b.name)
+      )
+
+      notifications.success(`Tables created successfully.`)
+      await datasources.fetch()
+      await tables.fetch()
+      $goto(`./table/${tableToRedirect.id}`)
     } catch (e: any) {
       notifications.error(e.message)
     }
-    return keepOpen
   }
 </script>
 
diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
index 1518d64110..ba49df801e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
@@ -16,7 +16,6 @@
   import CreationPage from "@/components/common/CreationPage.svelte"
   import ICONS from "@/components/backend/DatasourceNavigator/icons/index.js"
   import AiTableCreationModal from "./_components/AITableCreationModal.svelte"
-  import { onMount } from "svelte"
 
   let internalTableModal: CreateInternalTableModal
   let externalDatasourceModal: CreateExternalDatasourceModal
@@ -40,10 +39,6 @@
       notifications.error("Error creating datasource")
     }
   }
-
-  onMount(() => {
-    aiTablesModal.show()
-  })
 </script>
 
 <CreateInternalTableModal bind:this={internalTableModal} />
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 92a3deb770..67e4639418 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -37,17 +37,18 @@ export async function generateTables(
   )
 
   const json = JSON.parse(response!.message || "")
+  const createdTables: GenerateTablesResponse["createdTables"] = []
   for (const table of json) {
-    await tableController.create(
-      {
-        request: { body: { ...table, sourceId: dsId } },
-        user: ctx.user,
-      } as any,
-      async () => {}
-    )
+    const saveCtx = {
+      request: { body: { ...table, sourceId: dsId } },
+      user: ctx.user,
+    } as any
+    await tableController.create(saveCtx, async () => {})
+    createdTables.push({ id: saveCtx.body._id, name: saveCtx.body.name })
   }
 
   ctx.body = {
     response: response?.message ?? undefined,
+    createdTables,
   }
 }
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 036858cb15..b2ad74687f 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -15,4 +15,5 @@ export interface GenerateTablesRequest {
 
 export interface GenerateTablesResponse {
   response?: string
+  createdTables: { id: string; name: string }[]
 }

From b25b2e2b74cf54f44eec9af9d83598cfdb4e90c5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 17:32:46 +0100
Subject: [PATCH 007/101] Add cache mechanism for development

---
 .../_components/AITableCreationModal.svelte   | 28 +++++++++++++------
 packages/pro                                  |  2 +-
 packages/server/src/api/controllers/ai.ts     | 18 ++++++++----
 packages/types/src/api/web/ai.ts              |  2 ++
 4 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index c76d468eec..64a08cccf3 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -1,23 +1,32 @@
 <script lang="ts">
   import { API } from "@/api"
   import { datasources, tables } from "@/stores/builder"
-  import { Modal, ModalContent, notifications, TextArea } from "@budibase/bbui"
+  import {
+    Modal,
+    ModalContent,
+    notifications,
+    TextArea,
+    Toggle,
+  } from "@budibase/bbui"
+  import type { GenerateTablesRequest } from "@budibase/types"
   import { goto } from "@roxi/routify"
 
   let modal: Modal
-  let prompt: string =
-    "I need a ticketing system where employees can submit tickets. A ticket has a title, description, priority, status, and an assigned technician"
-  let result: string
+
+  let modalData: GenerateTablesRequest = {
+    prompt:
+      "I need a ticketing system where employees can submit tickets. A ticket has a title, description, priority, status, and an assigned technician",
+    addData: true,
+    useCached: true,
+  }
 
   export function show() {
     modal.show()
   }
 
   export async function generate() {
-    result = ""
     try {
-      const { response, createdTables } = await API.generateTables({ prompt })
-      result = response ?? ""
+      const { createdTables } = await API.generateTables(modalData)
 
       const [tableToRedirect] = createdTables.sort((a, b) =>
         a.name.localeCompare(b.name)
@@ -41,7 +50,8 @@
     onConfirm={generate}
     showCancelButton={false}
   >
-    <TextArea label="Table Name" bind:value={prompt} />
-    <TextArea label="Result" value={result} disabled minHeight={350} />
+    <TextArea label="Table Name" bind:value={modalData.prompt} />
+    <Toggle label="Add data" bind:value={modalData.addData} />
+    <Toggle label="Use cached" bind:value={modalData.useCached} />
   </ModalContent>
 </Modal>
diff --git a/packages/pro b/packages/pro
index 0e1d1f449d..b39116a002 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 0e1d1f449dbdd96aed851ed3a1c5b0dced7bc383
+Subproject commit b39116a0021f57ea5f64cd182644c6dfb37af236
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 67e4639418..3804542be1 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -11,13 +11,19 @@ import sdk from "../../sdk"
 import { generateDatasourceID } from "../../db/utils"
 import fs from "fs"
 import path from "path"
+import { createHash } from "crypto"
 
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
 ) {
+  const { prompt, useCached } = ctx.request.body
+
   const llm = await getLLM()
   llm!.maxTokens = 1200
-  const response = await llm?.generateTables(ctx.request.body.prompt)
+
+  const cacheKey = createHash("md5").update(prompt).digest("hex")
+
+  const response = await llm?.generateTables(prompt, useCached ? cacheKey : "")
 
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
@@ -31,10 +37,12 @@ export async function generateTables(
 
   console.warn(response?.message)
 
-  fs.writeFileSync(
-    path.join(process.env.PWD!, `../../llm-output/response_${Date.now()}.json`),
-    response?.message || ""
-  )
+  if (!useCached) {
+    fs.writeFileSync(
+      path.join(process.env.PWD!, `../../llm-output/${cacheKey}.json`),
+      response?.message || ""
+    )
+  }
 
   const json = JSON.parse(response!.message || "")
   const createdTables: GenerateTablesResponse["createdTables"] = []
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index b2ad74687f..b2e483b572 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -11,6 +11,8 @@ export interface GenerateJsResponse {
 
 export interface GenerateTablesRequest {
   prompt: string
+  addData: boolean
+  useCached: boolean
 }
 
 export interface GenerateTablesResponse {

From ffc9426b9df752837199471faf0e03692bcbcd7b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 17:41:49 +0100
Subject: [PATCH 008/101] Allow extending the response

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index b39116a002..9a2597344d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit b39116a0021f57ea5f64cd182644c6dfb37af236
+Subproject commit 9a2597344dd365cc9ad4020b65e477b2224a3142
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 3804542be1..f0498ee453 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -21,7 +21,9 @@ export async function generateTables(
   const llm = await getLLM()
   llm!.maxTokens = 1200
 
-  const cacheKey = createHash("md5").update(prompt).digest("hex")
+  const cacheKey = `${createHash("md5").update(prompt).digest("hex")}_${
+    useCached ? 1 : 0
+  }`
 
   const response = await llm?.generateTables(prompt, useCached ? cacheKey : "")
 
@@ -46,7 +48,7 @@ export async function generateTables(
 
   const json = JSON.parse(response!.message || "")
   const createdTables: GenerateTablesResponse["createdTables"] = []
-  for (const table of json) {
+  for (const table of json.structure) {
     const saveCtx = {
       request: { body: { ...table, sourceId: dsId } },
       user: ctx.user,

From f2a37196019efc7baad7ab77c079242247d049c1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 27 Mar 2025 17:57:18 +0100
Subject: [PATCH 009/101] Generate data on the response

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/pro b/packages/pro
index 9a2597344d..54a25598b6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 9a2597344dd365cc9ad4020b65e477b2224a3142
+Subproject commit 54a25598b6badd15238fd2df2fe59e779bbaac97
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index f0498ee453..d522ddab96 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -16,16 +16,20 @@ import { createHash } from "crypto"
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
 ) {
-  const { prompt, useCached } = ctx.request.body
+  const { prompt, useCached, addData } = ctx.request.body
 
   const llm = await getLLM()
   llm!.maxTokens = 1200
 
   const cacheKey = `${createHash("md5").update(prompt).digest("hex")}_${
-    useCached ? 1 : 0
+    addData ? 1 : 0
   }`
 
-  const response = await llm?.generateTables(prompt, useCached ? cacheKey : "")
+  const response = await llm?.generateTables(
+    prompt,
+    useCached ? cacheKey : "",
+    addData
+  )
 
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({

From f36c162685de7abb01989343d5f2309cfc17795e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 28 Mar 2025 10:43:48 +0100
Subject: [PATCH 010/101] Generate data

---
 packages/server/src/api/controllers/ai.ts | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index d522ddab96..2f8bb9b457 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -53,12 +53,24 @@ export async function generateTables(
   const json = JSON.parse(response!.message || "")
   const createdTables: GenerateTablesResponse["createdTables"] = []
   for (const table of json.structure) {
-    const saveCtx = {
+    const createTableCtx = {
       request: { body: { ...table, sourceId: dsId } },
       user: ctx.user,
     } as any
-    await tableController.create(saveCtx, async () => {})
-    createdTables.push({ id: saveCtx.body._id, name: saveCtx.body.name })
+    await tableController.create(createTableCtx, async () => {})
+    const createdTable = createTableCtx.body
+    createdTables.push({ id: createdTable._id, name: createdTable.name })
+
+    const dataToAdd = json.data[table.name]
+    for (const entry of dataToAdd || []) {
+      await sdk.rows.save(
+        createdTable._id,
+        {
+          ...entry,
+        },
+        ctx.user._id
+      )
+    }
   }
 
   ctx.body = {

From 121dcd51aa56f587a4e411934fd8d0a6b7c70736 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 28 Mar 2025 10:59:31 +0100
Subject: [PATCH 011/101] Keep history

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index e946c119f8..251788a2bf 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit e946c119f840be74038438f162274520d9a2839f
+Subproject commit 251788a2bf43f988db8e5bad39f83dff078b7aed
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 2f8bb9b457..1c1e8ed387 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -27,7 +27,7 @@ export async function generateTables(
 
   const response = await llm?.generateTables(
     prompt,
-    useCached ? cacheKey : "",
+    useCached ? `${cacheKey}/latest` : "",
     addData
   )
 
@@ -44,8 +44,13 @@ export async function generateTables(
   console.warn(response?.message)
 
   if (!useCached) {
+    const dir = path.join(process.env.PWD!, `../../llm-output/${cacheKey}`)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir)
+    }
+    fs.writeFileSync(path.join(dir, "latest.json"), response?.message || "")
     fs.writeFileSync(
-      path.join(process.env.PWD!, `../../llm-output/${cacheKey}.json`),
+      path.join(dir, `${Date.now()}.json`),
       response?.message || ""
     )
   }

From ec9fc6540ecfdef5745520b40c3e5c7cf7c9370d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 28 Mar 2025 11:42:49 +0100
Subject: [PATCH 012/101] Save prompt

---
 packages/server/src/api/controllers/ai.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 1c1e8ed387..ae9c5b00cd 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -48,6 +48,7 @@ export async function generateTables(
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir)
     }
+    fs.writeFileSync(path.join(dir, "prompt.txt"), prompt)
     fs.writeFileSync(path.join(dir, "latest.json"), response?.message || "")
     fs.writeFileSync(
       path.join(dir, `${Date.now()}.json`),
@@ -66,7 +67,7 @@ export async function generateTables(
     const createdTable = createTableCtx.body
     createdTables.push({ id: createdTable._id, name: createdTable.name })
 
-    const dataToAdd = json.data[table.name]
+    const dataToAdd = json.data?.[table.name]
     for (const entry of dataToAdd || []) {
       await sdk.rows.save(
         createdTable._id,

From 26bc239ae46b266f131a3b25e2aa732fcb22ec75 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 28 Mar 2025 14:20:25 +0100
Subject: [PATCH 013/101] Fix editing AI created tables

---
 packages/backend-core/src/sql/utils.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 746a949ef3..33f51b23ac 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -47,6 +47,9 @@ export function isExternalTable(table: Table) {
   if (
     table?.sourceId &&
     table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    !table.sourceId.startsWith(
+      DocumentType.DATASOURCE + SEPARATOR + "bb_internal_"
+    ) &&
     table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
   ) {
     return true

From 79ac7040af44e38674b6b8ea534d6e9526250ba5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 28 Mar 2025 14:20:49 +0100
Subject: [PATCH 014/101] Fix editing AI created tables

---
 packages/server/src/api/controllers/ai.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index ae9c5b00cd..f9d68d955d 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -1,4 +1,5 @@
 import {
+  DocumentType,
   GenerateTablesRequest,
   GenerateTablesResponse,
   SourceName,
@@ -6,9 +7,8 @@ import {
 } from "@budibase/types"
 import { getLLM } from "packages/pro/src/ai"
 import * as tableController from "./public/tables"
-import { context } from "@budibase/backend-core"
+import { context, utils } from "@budibase/backend-core"
 import sdk from "../../sdk"
-import { generateDatasourceID } from "../../db/utils"
 import fs from "fs"
 import path from "path"
 import { createHash } from "crypto"
@@ -33,7 +33,7 @@ export async function generateTables(
 
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
-    _id: generateDatasourceID(),
+    _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
     name: `Test ${count}`,
     type: "budibase",
 

From d59796f31b9c09208f92f28ad8f1bf177b153374 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 11:03:27 +0200
Subject: [PATCH 015/101] Update dev prompt used

---
 .../[application]/data/_components/AITableCreationModal.svelte  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index 64a08cccf3..d4634c373d 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -15,7 +15,7 @@
 
   let modalData: GenerateTablesRequest = {
     prompt:
-      "I need a ticketing system where employees can submit tickets. A ticket has a title, description, priority, status, and an assigned technician",
+      "I need a ticketing system where users can submit tickets. A ticket has a title, description, priority, status, and an assigned technician",
     addData: true,
     useCached: true,
   }

From 462a0fd9bf525274b4e4942a3b428fa947a668cd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 12:22:11 +0200
Subject: [PATCH 016/101] Handle relationships

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 80 ++++++++++++++++++++---
 2 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/packages/pro b/packages/pro
index 251788a2bf..7d83635fbf 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 251788a2bf43f988db8e5bad39f83dff078b7aed
+Subproject commit 7d83635fbf2ae970a2dc07145951597b5f06d8b0
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index f9d68d955d..1373535913 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -1,8 +1,11 @@
 import {
   DocumentType,
+  FieldType,
   GenerateTablesRequest,
   GenerateTablesResponse,
+  RelationshipFieldMetadata,
   SourceName,
+  Table,
   UserCtx,
 } from "@budibase/types"
 import { getLLM } from "packages/pro/src/ai"
@@ -56,26 +59,83 @@ export async function generateTables(
     )
   }
 
-  const json = JSON.parse(response!.message || "")
+  const json = JSON.parse(response!.message || "") as {
+    structure: Table[]
+    data: Record<string, any[]>
+  }
   const createdTables: GenerateTablesResponse["createdTables"] = []
+
   for (const table of json.structure) {
     const createTableCtx = {
-      request: { body: { ...table, sourceId: dsId } },
+      request: {
+        body: {
+          ...table,
+          sourceId: dsId,
+          schema: {},
+          primaryDisplay: undefined,
+        },
+      },
       user: ctx.user,
+      throw: ctx.throw,
     } as any
     await tableController.create(createTableCtx, async () => {})
     const createdTable = createTableCtx.body
     createdTables.push({ id: createdTable._id, name: createdTable.name })
+    table._id = createdTable._id
+  }
 
-    const dataToAdd = json.data?.[table.name]
-    for (const entry of dataToAdd || []) {
-      await sdk.rows.save(
-        createdTable._id,
-        {
-          ...entry,
+  for (const table of json.structure) {
+    for (const fieldKey of Object.keys(table.schema).filter(
+      f => table.schema[f].type === FieldType.LINK
+    )) {
+      const field = table.schema[fieldKey] as RelationshipFieldMetadata
+      const linkedTable = createdTables.find(t => t.name === field.tableId)
+      if (!linkedTable) {
+        throw `Table ${field.tableId} not found in the json response.`
+      }
+      field.tableId = linkedTable.id
+    }
+  }
+
+  for (const table of json.structure) {
+    const readTableCtx = {
+      params: { tableId: table._id },
+      user: ctx.user,
+      throw: ctx.throw,
+    } as any
+
+    await tableController.read(readTableCtx, async () => {})
+
+    const updateTableCtx = {
+      request: {
+        body: {
+          ...readTableCtx.body,
+          schema: {
+            ...readTableCtx.body.schema,
+            ...table.schema,
+          },
+          displayName: table.primaryDisplay,
         },
-        ctx.user._id
-      )
+      },
+      params: { tableId: table._id },
+      user: ctx.user,
+      throw: ctx.throw,
+    } as any
+    await tableController.update(updateTableCtx, async () => {})
+  }
+
+  if (addData) {
+    for (const table of json.structure) {
+      const dataToAdd = json.data?.[table.name]
+      for (const entry of dataToAdd || []) {
+        await sdk.rows.save(
+          table._id!,
+          {
+            ...entry,
+          },
+          ctx.user._id
+        )
+      }
     }
   }
 

From ead0833fc70f08ec23313610d6d20920d249f584 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 12:24:51 +0200
Subject: [PATCH 017/101] Fix display name

---
 packages/server/src/api/controllers/ai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 1373535913..16df0923db 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -114,7 +114,7 @@ export async function generateTables(
             ...readTableCtx.body.schema,
             ...table.schema,
           },
-          displayName: table.primaryDisplay,
+          primaryDisplay: table.primaryDisplay,
         },
       },
       params: { tableId: table._id },

From d23b88019e6d03b9fd0c2713cb1e88a937604cfa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 12:47:31 +0200
Subject: [PATCH 018/101] Create data (without links)

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 7d83635fbf..162b5412b4 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 7d83635fbf2ae970a2dc07145951597b5f06d8b0
+Subproject commit 162b5412b44428841b43496e5fed8c545ae1b9f5
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 16df0923db..b3d8ce064e 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -125,13 +125,25 @@ export async function generateTables(
   }
 
   if (addData) {
+    const createdData: Record<string, Record<string, string>> = {}
     for (const table of json.structure) {
+      createdData[table.name] = {}
       const dataToAdd = json.data?.[table.name]
+
+      const linksOverride: Record<string, null> = {}
+      for (const fieldKey of Object.keys(table.schema).filter(
+        f => table.schema[f].type === FieldType.LINK
+      )) {
+        linksOverride[fieldKey] = null
+      }
+
       for (const entry of dataToAdd || []) {
         await sdk.rows.save(
           table._id!,
           {
             ...entry,
+            ...linksOverride,
+            _id: undefined,
           },
           ctx.user._id
         )

From bd9ea2e951a3d409576951b772c376f68da03f1f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 13:19:23 +0200
Subject: [PATCH 019/101] Add data to links

---
 packages/server/src/api/controllers/ai.ts | 49 +++++++++++++++++++++--
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index b3d8ce064e..5b7bf71c50 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -125,9 +125,13 @@ export async function generateTables(
   }
 
   if (addData) {
-    const createdData: Record<string, Record<string, string>> = {}
+    const createdData: Record<string, string> = {}
+    const toUpdateLinks: {
+      tableId: string
+      rowId: string
+      data: Record<string, string>
+    }[] = []
     for (const table of json.structure) {
-      createdData[table.name] = {}
       const dataToAdd = json.data?.[table.name]
 
       const linksOverride: Record<string, null> = {}
@@ -138,7 +142,7 @@ export async function generateTables(
       }
 
       for (const entry of dataToAdd || []) {
-        await sdk.rows.save(
+        const createdRow = await sdk.rows.save(
           table._id!,
           {
             ...entry,
@@ -147,8 +151,47 @@ export async function generateTables(
           },
           ctx.user._id
         )
+
+        createdData[entry._id] = createdRow.row._id!
+
+        const overridenLinks = Object.keys(linksOverride).reduce<
+          Record<string, string>
+        >((acc, l) => {
+          if (entry[l]) {
+            acc[l] = entry[l]
+          }
+          return acc
+        }, {})
+
+        if (Object.keys(overridenLinks)) {
+          toUpdateLinks.push({
+            tableId: createdRow.table._id!,
+            rowId: createdRow.row._id!,
+            data: overridenLinks,
+          })
+        }
       }
     }
+
+    for (const data of toUpdateLinks) {
+      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+
+      const updatedLinks = Object.keys(data.data).reduce<
+        Record<string, string>
+      >((acc, d) => {
+        acc[d] = createdData[data.data[d]]
+        return acc
+      }, {})
+
+      await sdk.rows.save(
+        data.tableId,
+        {
+          ...persistedRow,
+          ...updatedLinks,
+        },
+        ctx.user._id
+      )
+    }
   }
 
   ctx.body = {

From b16cafe8fa367ec7bccdbc9a3395059db7974c22 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 14:51:51 +0200
Subject: [PATCH 020/101] Improvements

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/pro b/packages/pro
index 162b5412b4..703247c76a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 162b5412b44428841b43496e5fed8c545ae1b9f5
+Subproject commit 703247c76a6d008ce425318b9c038a1509c71df4
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 5b7bf71c50..de1c631bd6 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -60,12 +60,12 @@ export async function generateTables(
   }
 
   const json = JSON.parse(response!.message || "") as {
-    structure: Table[]
+    tables: Table[]
     data: Record<string, any[]>
   }
   const createdTables: GenerateTablesResponse["createdTables"] = []
 
-  for (const table of json.structure) {
+  for (const table of Object.values(json.tables)) {
     const createTableCtx = {
       request: {
         body: {
@@ -84,7 +84,7 @@ export async function generateTables(
     table._id = createdTable._id
   }
 
-  for (const table of json.structure) {
+  for (const table of Object.values(json.tables)) {
     for (const fieldKey of Object.keys(table.schema).filter(
       f => table.schema[f].type === FieldType.LINK
     )) {
@@ -97,7 +97,7 @@ export async function generateTables(
     }
   }
 
-  for (const table of json.structure) {
+  for (const table of Object.values(json.tables)) {
     const readTableCtx = {
       params: { tableId: table._id },
       user: ctx.user,
@@ -131,7 +131,7 @@ export async function generateTables(
       rowId: string
       data: Record<string, string>
     }[] = []
-    for (const table of json.structure) {
+    for (const table of Object.values(json.tables)) {
       const dataToAdd = json.data?.[table.name]
 
       const linksOverride: Record<string, null> = {}

From 4817849615d12a6fb96a15f7e498679419d87245 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 15:00:21 +0200
Subject: [PATCH 021/101] Clean

---
 packages/server/src/api/controllers/ai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index de1c631bd6..ec651a0095 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -173,7 +173,7 @@ export async function generateTables(
       }
     }
 
-    for (const data of toUpdateLinks) {
+    for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
       const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
       const updatedLinks = Object.keys(data.data).reduce<

From b52a5c3082a329ca267887d3d4b1e4e805279f16 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 31 Mar 2025 16:33:17 +0200
Subject: [PATCH 022/101] Fix relationships

---
 packages/server/src/api/controllers/ai.ts | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index ec651a0095..0254c5738f 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -125,11 +125,11 @@ export async function generateTables(
   }
 
   if (addData) {
-    const createdData: Record<string, string> = {}
+    const createdData: Record<string, Record<string, string>> = {}
     const toUpdateLinks: {
       tableId: string
       rowId: string
-      data: Record<string, string>
+      data: Record<string, { rowId: string; tableId: string }>
     }[] = []
     for (const table of Object.values(json.tables)) {
       const dataToAdd = json.data?.[table.name]
@@ -152,13 +152,17 @@ export async function generateTables(
           ctx.user._id
         )
 
-        createdData[entry._id] = createdRow.row._id!
+        createdData[table._id!] ??= {}
+        createdData[table._id!][entry._id] = createdRow.row._id!
 
         const overridenLinks = Object.keys(linksOverride).reduce<
-          Record<string, string>
+          Record<string, { rowId: string; tableId: string }>
         >((acc, l) => {
           if (entry[l]) {
-            acc[l] = entry[l]
+            acc[l] = {
+              tableId: (table.schema[l] as RelationshipFieldMetadata).tableId,
+              rowId: entry[l],
+            }
           }
           return acc
         }, {})
@@ -179,7 +183,7 @@ export async function generateTables(
       const updatedLinks = Object.keys(data.data).reduce<
         Record<string, string>
       >((acc, d) => {
-        acc[d] = createdData[data.data[d]]
+        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
         return acc
       }, {})
 

From 595ddbaea599c9ca4b704a56491858603111062e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 10:46:48 +0200
Subject: [PATCH 023/101] Clean

---
 packages/server/src/api/controllers/ai.ts | 1 -
 packages/types/src/api/web/ai.ts          | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 0254c5738f..86218392dd 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -199,7 +199,6 @@ export async function generateTables(
   }
 
   ctx.body = {
-    response: response?.message ?? undefined,
     createdTables,
   }
 }
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index b2e483b572..d22221dc0d 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -16,6 +16,5 @@ export interface GenerateTablesRequest {
 }
 
 export interface GenerateTablesResponse {
-  response?: string
   createdTables: { id: string; name: string }[]
 }

From ad0b9f8b7a686809ff1bce7df1cb7a716a9c075d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 11:16:42 +0200
Subject: [PATCH 024/101] Initial zod usage

---
 packages/pro                              |   2 +-
 packages/server/src/api/controllers/ai.ts | 230 ++++++++++------------
 2 files changed, 110 insertions(+), 122 deletions(-)

diff --git a/packages/pro b/packages/pro
index 703247c76a..0e940ad427 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 703247c76a6d008ce425318b9c038a1509c71df4
+Subproject commit 0e940ad427ef13ff48bc0911b18743734c34ccce
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 86218392dd..1dff73ef91 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -3,13 +3,11 @@ import {
   FieldType,
   GenerateTablesRequest,
   GenerateTablesResponse,
-  RelationshipFieldMetadata,
   SourceName,
-  Table,
+  TableSourceType,
   UserCtx,
 } from "@budibase/types"
 import { getLLM } from "packages/pro/src/ai"
-import * as tableController from "./public/tables"
 import { context, utils } from "@budibase/backend-core"
 import sdk from "../../sdk"
 import fs from "fs"
@@ -44,51 +42,41 @@ export async function generateTables(
     config: {},
   })
 
-  console.warn(response?.message)
-
   if (!useCached) {
     const dir = path.join(process.env.PWD!, `../../llm-output/${cacheKey}`)
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir)
     }
     fs.writeFileSync(path.join(dir, "prompt.txt"), prompt)
-    fs.writeFileSync(path.join(dir, "latest.json"), response?.message || "")
+    fs.writeFileSync(path.join(dir, "latest.json"), JSON.stringify(response))
     fs.writeFileSync(
       path.join(dir, `${Date.now()}.json`),
-      response?.message || ""
+      JSON.stringify(response)
     )
   }
 
-  const json = JSON.parse(response!.message || "") as {
-    tables: Table[]
-    data: Record<string, any[]>
-  }
   const createdTables: GenerateTablesResponse["createdTables"] = []
 
-  for (const table of Object.values(json.tables)) {
-    const createTableCtx = {
-      request: {
-        body: {
-          ...table,
-          sourceId: dsId,
-          schema: {},
-          primaryDisplay: undefined,
-        },
-      },
-      user: ctx.user,
-      throw: ctx.throw,
-    } as any
-    await tableController.create(createTableCtx, async () => {})
-    const createdTable = createTableCtx.body
-    createdTables.push({ id: createdTable._id, name: createdTable.name })
-    table._id = createdTable._id
+  for (const table of response!.tables) {
+    const { _id, ...structure } = table.structure
+    const createdTable = await sdk.tables.create({
+      ...structure,
+      sourceId: dsId,
+      schema: {},
+      primaryDisplay: undefined,
+      sourceType: TableSourceType.INTERNAL,
+      type: "table",
+    })
+
+    createdTables.push({ id: createdTable._id!, name: createdTable.name })
+    table.structure._id = createdTable._id!
   }
 
-  for (const table of Object.values(json.tables)) {
-    for (const fieldKey of Object.keys(table.schema).filter(
-      f => table.schema[f].type === FieldType.LINK
+  for (const table of Object.values(response!.tables)) {
+    for (const field of table.structure.schema.filter(
+      f => f.type === FieldType.LINK
     )) {
-      const field = table.schema[fieldKey] as RelationshipFieldMetadata
+      // const field =  table.schema[fieldKey] as RelationshipFieldMetadata
       const linkedTable = createdTables.find(t => t.name === field.tableId)
       if (!linkedTable) {
         throw `Table ${field.tableId} not found in the json response.`
@@ -97,106 +85,106 @@ export async function generateTables(
     }
   }
 
-  for (const table of Object.values(json.tables)) {
-    const readTableCtx = {
-      params: { tableId: table._id },
-      user: ctx.user,
-      throw: ctx.throw,
-    } as any
+  // for (const table of Object.values(json.tables)) {
+  //   const readTableCtx = {
+  //     params: { tableId: table._id },
+  //     user: ctx.user,
+  //     throw: ctx.throw,
+  //   } as any
 
-    await tableController.read(readTableCtx, async () => {})
+  //   await tableController.read(readTableCtx, async () => {})
 
-    const updateTableCtx = {
-      request: {
-        body: {
-          ...readTableCtx.body,
-          schema: {
-            ...readTableCtx.body.schema,
-            ...table.schema,
-          },
-          primaryDisplay: table.primaryDisplay,
-        },
-      },
-      params: { tableId: table._id },
-      user: ctx.user,
-      throw: ctx.throw,
-    } as any
-    await tableController.update(updateTableCtx, async () => {})
-  }
+  //   const updateTableCtx = {
+  //     request: {
+  //       body: {
+  //         ...readTableCtx.body,
+  //         schema: {
+  //           ...readTableCtx.body.schema,
+  //           ...table.schema,
+  //         },
+  //         primaryDisplay: table.primaryDisplay,
+  //       },
+  //     },
+  //     params: { tableId: table._id },
+  //     user: ctx.user,
+  //     throw: ctx.throw,
+  //   } as any
+  //   await tableController.update(updateTableCtx, async () => {})
+  // }
 
-  if (addData) {
-    const createdData: Record<string, Record<string, string>> = {}
-    const toUpdateLinks: {
-      tableId: string
-      rowId: string
-      data: Record<string, { rowId: string; tableId: string }>
-    }[] = []
-    for (const table of Object.values(json.tables)) {
-      const dataToAdd = json.data?.[table.name]
+  // if (addData) {
+  //   const createdData: Record<string, Record<string, string>> = {}
+  //   const toUpdateLinks: {
+  //     tableId: string
+  //     rowId: string
+  //     data: Record<string, { rowId: string; tableId: string }>
+  //   }[] = []
+  //   for (const table of Object.values(json.tables)) {
+  //     const dataToAdd = json.data?.[table.name]
 
-      const linksOverride: Record<string, null> = {}
-      for (const fieldKey of Object.keys(table.schema).filter(
-        f => table.schema[f].type === FieldType.LINK
-      )) {
-        linksOverride[fieldKey] = null
-      }
+  //     const linksOverride: Record<string, null> = {}
+  //     for (const fieldKey of Object.keys(table.schema).filter(
+  //       f => table.schema[f].type === FieldType.LINK
+  //     )) {
+  //       linksOverride[fieldKey] = null
+  //     }
 
-      for (const entry of dataToAdd || []) {
-        const createdRow = await sdk.rows.save(
-          table._id!,
-          {
-            ...entry,
-            ...linksOverride,
-            _id: undefined,
-          },
-          ctx.user._id
-        )
+  //     for (const entry of dataToAdd || []) {
+  //       const createdRow = await sdk.rows.save(
+  //         table._id!,
+  //         {
+  //           ...entry,
+  //           ...linksOverride,
+  //           _id: undefined,
+  //         },
+  //         ctx.user._id
+  //       )
 
-        createdData[table._id!] ??= {}
-        createdData[table._id!][entry._id] = createdRow.row._id!
+  //       createdData[table._id!] ??= {}
+  //       createdData[table._id!][entry._id] = createdRow.row._id!
 
-        const overridenLinks = Object.keys(linksOverride).reduce<
-          Record<string, { rowId: string; tableId: string }>
-        >((acc, l) => {
-          if (entry[l]) {
-            acc[l] = {
-              tableId: (table.schema[l] as RelationshipFieldMetadata).tableId,
-              rowId: entry[l],
-            }
-          }
-          return acc
-        }, {})
+  //       const overridenLinks = Object.keys(linksOverride).reduce<
+  //         Record<string, { rowId: string; tableId: string }>
+  //       >((acc, l) => {
+  //         if (entry[l]) {
+  //           acc[l] = {
+  //             tableId: (table.schema[l] as RelationshipFieldMetadata).tableId,
+  //             rowId: entry[l],
+  //           }
+  //         }
+  //         return acc
+  //       }, {})
 
-        if (Object.keys(overridenLinks)) {
-          toUpdateLinks.push({
-            tableId: createdRow.table._id!,
-            rowId: createdRow.row._id!,
-            data: overridenLinks,
-          })
-        }
-      }
-    }
+  //       if (Object.keys(overridenLinks)) {
+  //         toUpdateLinks.push({
+  //           tableId: createdRow.table._id!,
+  //           rowId: createdRow.row._id!,
+  //           data: overridenLinks,
+  //         })
+  //       }
+  //     }
+  //   }
 
-    for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
-      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+  //   for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
+  //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-      const updatedLinks = Object.keys(data.data).reduce<
-        Record<string, string>
-      >((acc, d) => {
-        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-        return acc
-      }, {})
+  //     const updatedLinks = Object.keys(data.data).reduce<
+  //       Record<string, string>
+  //     >((acc, d) => {
+  //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+  //       return acc
+  //     }, {})
 
-      await sdk.rows.save(
-        data.tableId,
-        {
-          ...persistedRow,
-          ...updatedLinks,
-        },
-        ctx.user._id
-      )
-    }
-  }
+  //     await sdk.rows.save(
+  //       data.tableId,
+  //       {
+  //         ...persistedRow,
+  //         ...updatedLinks,
+  //       },
+  //       ctx.user._id
+  //     )
+  //   }
+  // }
 
   ctx.body = {
     createdTables,

From 6d176ccb747f041e470e0b9ff54a78f52659390e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 11:36:20 +0200
Subject: [PATCH 025/101] Generate schema

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 68 ++++++++++++++---------
 2 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/packages/pro b/packages/pro
index 0e940ad427..ca0b9af3cd 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 0e940ad427ef13ff48bc0911b18743734c34ccce
+Subproject commit ca0b9af3cdd2efd010dd5c466f85d030c408b776
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 1dff73ef91..04ea368863 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -4,6 +4,7 @@ import {
   GenerateTablesRequest,
   GenerateTablesResponse,
   SourceName,
+  TableSchema,
   TableSourceType,
   UserCtx,
 } from "@budibase/types"
@@ -26,7 +27,7 @@ export async function generateTables(
     addData ? 1 : 0
   }`
 
-  const response = await llm?.generateTables(
+  const response = await llm!.generateTables(
     prompt,
     useCached ? `${cacheKey}/latest` : "",
     addData
@@ -57,7 +58,7 @@ export async function generateTables(
 
   const createdTables: GenerateTablesResponse["createdTables"] = []
 
-  for (const table of response!.tables) {
+  for (const table of response.tables) {
     const { _id, ...structure } = table.structure
     const createdTable = await sdk.tables.create({
       ...structure,
@@ -72,7 +73,7 @@ export async function generateTables(
     table.structure._id = createdTable._id!
   }
 
-  for (const table of Object.values(response!.tables)) {
+  for (const table of Object.values(response.tables)) {
     for (const field of table.structure.schema.filter(
       f => f.type === FieldType.LINK
     )) {
@@ -85,32 +86,45 @@ export async function generateTables(
     }
   }
 
-  // for (const table of Object.values(json.tables)) {
-  //   const readTableCtx = {
-  //     params: { tableId: table._id },
-  //     user: ctx.user,
-  //     throw: ctx.throw,
-  //   } as any
+  for (const { structure: table } of Object.values(response.tables)) {
+    const storedTable = await sdk.tables.getTable(table._id)
 
-  //   await tableController.read(readTableCtx, async () => {})
+    await sdk.tables.update({
+      ...storedTable,
+      schema: {
+        ...storedTable.schema,
+        ...table.schema.reduce<TableSchema>((acc, field) => {
+          acc[field.name] = field
+          return acc
+        }, {}),
+      },
+      primaryDisplay: table.primaryDisplay,
+    })
+    // const readTableCtx = {
+    //   params: { tableId: table._id },
+    //   user: ctx.user,
+    //   throw: ctx.throw,
+    // } as any
 
-  //   const updateTableCtx = {
-  //     request: {
-  //       body: {
-  //         ...readTableCtx.body,
-  //         schema: {
-  //           ...readTableCtx.body.schema,
-  //           ...table.schema,
-  //         },
-  //         primaryDisplay: table.primaryDisplay,
-  //       },
-  //     },
-  //     params: { tableId: table._id },
-  //     user: ctx.user,
-  //     throw: ctx.throw,
-  //   } as any
-  //   await tableController.update(updateTableCtx, async () => {})
-  // }
+    // await tableController.read(readTableCtx, async () => {})
+
+    // const updateTableCtx = {
+    //   request: {
+    //     body: {
+    //       ...readTableCtx.body,
+    //       schema: {
+    //         ...readTableCtx.body.schema,
+    //         ...table.schema,
+    //       },
+    //       primaryDisplay: table.primaryDisplay,
+    //     },
+    //   },
+    //   params: { tableId: table._id },
+    //   user: ctx.user,
+    //   throw: ctx.throw,
+    // } as any
+    // await tableController.update(updateTableCtx, async () => {})
+  }
 
   // if (addData) {
   //   const createdData: Record<string, Record<string, string>> = {}

From 35a2003eddc7267eec9fa774141eda03be86c294 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 14:06:17 +0200
Subject: [PATCH 026/101] Add data (without links)

---
 packages/pro                              |   2 +-
 packages/server/src/api/controllers/ai.ts | 161 ++++++++++------------
 2 files changed, 70 insertions(+), 93 deletions(-)

diff --git a/packages/pro b/packages/pro
index ca0b9af3cd..1dfb5a9337 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ca0b9af3cdd2efd010dd5c466f85d030c408b776
+Subproject commit 1dfb5a9337fefa29861026fea86551132491e0bf
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 04ea368863..00ef80dd97 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -59,9 +59,8 @@ export async function generateTables(
   const createdTables: GenerateTablesResponse["createdTables"] = []
 
   for (const table of response.tables) {
-    const { _id, ...structure } = table.structure
     const createdTable = await sdk.tables.create({
-      ...structure,
+      ...table.structure,
       sourceId: dsId,
       schema: {},
       primaryDisplay: undefined,
@@ -100,105 +99,83 @@ export async function generateTables(
       },
       primaryDisplay: table.primaryDisplay,
     })
-    // const readTableCtx = {
-    //   params: { tableId: table._id },
-    //   user: ctx.user,
-    //   throw: ctx.throw,
-    // } as any
-
-    // await tableController.read(readTableCtx, async () => {})
-
-    // const updateTableCtx = {
-    //   request: {
-    //     body: {
-    //       ...readTableCtx.body,
-    //       schema: {
-    //         ...readTableCtx.body.schema,
-    //         ...table.schema,
-    //       },
-    //       primaryDisplay: table.primaryDisplay,
-    //     },
-    //   },
-    //   params: { tableId: table._id },
-    //   user: ctx.user,
-    //   throw: ctx.throw,
-    // } as any
-    // await tableController.update(updateTableCtx, async () => {})
   }
 
-  // if (addData) {
-  //   const createdData: Record<string, Record<string, string>> = {}
-  //   const toUpdateLinks: {
-  //     tableId: string
-  //     rowId: string
-  //     data: Record<string, { rowId: string; tableId: string }>
-  //   }[] = []
-  //   for (const table of Object.values(json.tables)) {
-  //     const dataToAdd = json.data?.[table.name]
+  if (addData) {
+    const createdData: Record<string, Record<string, string>> = {}
+    const toUpdateLinks: {
+      tableId: string
+      rowId: string
+      data: Record<string, { rowId: string; tableId: string }>
+    }[] = []
+    for (const table of Object.values(response.tables)) {
+      const linksOverride: Record<string, null> = {}
+      for (const field of table.structure.schema.filter(
+        f => f.type === FieldType.LINK
+      )) {
+        linksOverride[field.name] = null
+      }
 
-  //     const linksOverride: Record<string, null> = {}
-  //     for (const fieldKey of Object.keys(table.schema).filter(
-  //       f => table.schema[f].type === FieldType.LINK
-  //     )) {
-  //       linksOverride[fieldKey] = null
-  //     }
+      for (const entry of table.data || []) {
+        const createdRow = await sdk.rows.save(
+          table.structure._id,
+          {
+            ...entry.values.reduce<Record<string, any>>((acc, v) => {
+              acc[v.key] = v.value
+              return acc
+            }, {}),
+            ...linksOverride,
+            _id: undefined,
+          },
+          ctx.user._id
+        )
 
-  //     for (const entry of dataToAdd || []) {
-  //       const createdRow = await sdk.rows.save(
-  //         table._id!,
-  //         {
-  //           ...entry,
-  //           ...linksOverride,
-  //           _id: undefined,
-  //         },
-  //         ctx.user._id
-  //       )
+        createdData[table.structure._id] ??= {}
+        createdData[table.structure._id][entry.id] = createdRow.row._id!
 
-  //       createdData[table._id!] ??= {}
-  //       createdData[table._id!][entry._id] = createdRow.row._id!
+        const overridenLinks = Object.keys(linksOverride).reduce<
+          Record<string, { rowId: string; tableId: string }>
+        >((acc, l) => {
+          if (entry.values.find(f => f.key === l)) {
+            acc[l] = {
+              tableId: (table.structure.schema.find(f => f.name === l) as any)
+                .tableId,
+              rowId: entry.id,
+            }
+          }
+          return acc
+        }, {})
 
-  //       const overridenLinks = Object.keys(linksOverride).reduce<
-  //         Record<string, { rowId: string; tableId: string }>
-  //       >((acc, l) => {
-  //         if (entry[l]) {
-  //           acc[l] = {
-  //             tableId: (table.schema[l] as RelationshipFieldMetadata).tableId,
-  //             rowId: entry[l],
-  //           }
-  //         }
-  //         return acc
-  //       }, {})
+        if (Object.keys(overridenLinks)) {
+          toUpdateLinks.push({
+            tableId: createdRow.table._id!,
+            rowId: createdRow.row._id!,
+            data: overridenLinks,
+          })
+        }
+      }
+    }
 
-  //       if (Object.keys(overridenLinks)) {
-  //         toUpdateLinks.push({
-  //           tableId: createdRow.table._id!,
-  //           rowId: createdRow.row._id!,
-  //           data: overridenLinks,
-  //         })
-  //       }
-  //     }
-  //   }
+    //   for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
+    //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-  //   for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
-  //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+    //     const updatedLinks = Object.keys(data.data).reduce<
+    //       Record<string, string>
+    //     >((acc, d) => {
+    //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+    //       return acc
+    //     }, {})
 
-  //     const updatedLinks = Object.keys(data.data).reduce<
-  //       Record<string, string>
-  //     >((acc, d) => {
-  //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-  //       return acc
-  //     }, {})
-
-  //     await sdk.rows.save(
-  //       data.tableId,
-  //       {
-  //         ...persistedRow,
-  //         ...updatedLinks,
-  //       },
-  //       ctx.user._id
-  //     )
-  //   }
-  // }
+    //     await sdk.rows.save(
+    //       data.tableId,
+    //       {
+    //         ...persistedRow,
+    //         ...updatedLinks,
+    //       },
+    //       ctx.user._id
+    //     )
+    //   }
+  }
 
   ctx.body = {
     createdTables,

From 160a9030c7ee76072e0f23fb56030e0c07c4dde9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 14:21:28 +0200
Subject: [PATCH 027/101] Populate links

---
 packages/server/src/api/controllers/ai.ts | 38 +++++++++++------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 00ef80dd97..040331a99a 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -140,13 +140,13 @@ export async function generateTables(
             acc[l] = {
               tableId: (table.structure.schema.find(f => f.name === l) as any)
                 .tableId,
-              rowId: entry.id,
+              rowId: entry.values.find(f => f.key === l)!.value as any,
             }
           }
           return acc
         }, {})
 
-        if (Object.keys(overridenLinks)) {
+        if (Object.keys(overridenLinks).length) {
           toUpdateLinks.push({
             tableId: createdRow.table._id!,
             rowId: createdRow.row._id!,
@@ -156,25 +156,25 @@ export async function generateTables(
       }
     }
 
-    //   for (const data of toUpdateLinks.filter(d => Object.keys(d.data).length)) {
-    //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+    for (const data of toUpdateLinks) {
+      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-    //     const updatedLinks = Object.keys(data.data).reduce<
-    //       Record<string, string>
-    //     >((acc, d) => {
-    //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-    //       return acc
-    //     }, {})
+      const updatedLinks = Object.keys(data.data).reduce<
+        Record<string, string>
+      >((acc, d) => {
+        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+        return acc
+      }, {})
 
-    //     await sdk.rows.save(
-    //       data.tableId,
-    //       {
-    //         ...persistedRow,
-    //         ...updatedLinks,
-    //       },
-    //       ctx.user._id
-    //     )
-    //   }
+      await sdk.rows.save(
+        data.tableId,
+        {
+          ...persistedRow,
+          ...updatedLinks,
+        },
+        ctx.user._id
+      )
+    }
   }
 
   ctx.body = {

From 51826ea502f41ada01e544640a370c122e9e2c0d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 15:09:53 +0200
Subject: [PATCH 028/101] Use gpt-4o

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 1dfb5a9337..50ca69df2a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1dfb5a9337fefa29861026fea86551132491e0bf
+Subproject commit 50ca69df2a225937c9095086829328ce7134a2f4
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 040331a99a..68fea78f82 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -20,7 +20,7 @@ export async function generateTables(
 ) {
   const { prompt, useCached, addData } = ctx.request.body
 
-  const llm = await getLLM()
+  const llm = await getLLM("gpt-4o")
   llm!.maxTokens = 1200
 
   const cacheKey = `${createHash("md5").update(prompt).digest("hex")}_${

From a312ff0e39aba295d157add4c744e4bae2829024 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 16:07:38 +0200
Subject: [PATCH 029/101] Fix types

---
 packages/server/src/api/controllers/ai.ts | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 68fea78f82..d4f49fd746 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -57,6 +57,7 @@ export async function generateTables(
   }
 
   const createdTables: GenerateTablesResponse["createdTables"] = []
+  const tableIds: Record<string, string> = {}
 
   for (const table of response.tables) {
     const createdTable = await sdk.tables.create({
@@ -69,7 +70,7 @@ export async function generateTables(
     })
 
     createdTables.push({ id: createdTable._id!, name: createdTable.name })
-    table.structure._id = createdTable._id!
+    tableIds[table.structure.name] = createdTable._id!
   }
 
   for (const table of Object.values(response.tables)) {
@@ -86,7 +87,7 @@ export async function generateTables(
   }
 
   for (const { structure: table } of Object.values(response.tables)) {
-    const storedTable = await sdk.tables.getTable(table._id)
+    const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
     await sdk.tables.update({
       ...storedTable,
@@ -117,8 +118,9 @@ export async function generateTables(
       }
 
       for (const entry of table.data || []) {
+        const tableId = tableIds[table.structure.name]
         const createdRow = await sdk.rows.save(
-          table.structure._id,
+          tableId,
           {
             ...entry.values.reduce<Record<string, any>>((acc, v) => {
               acc[v.key] = v.value
@@ -130,8 +132,8 @@ export async function generateTables(
           ctx.user._id
         )
 
-        createdData[table.structure._id] ??= {}
-        createdData[table.structure._id][entry.id] = createdRow.row._id!
+        createdData[tableId] ??= {}
+        createdData[tableId][entry.id] = createdRow.row._id!
 
         const overridenLinks = Object.keys(linksOverride).reduce<
           Record<string, { rowId: string; tableId: string }>

From b6ec0c3856d6d5e68235bb8b68594a627e96ec71 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 16:17:42 +0200
Subject: [PATCH 030/101] Typo

---
 .../[application]/data/_components/AITableCreationModal.svelte  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index d4634c373d..3912c36611 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -50,7 +50,7 @@
     onConfirm={generate}
     showCancelButton={false}
   >
-    <TextArea label="Table Name" bind:value={modalData.prompt} />
+    <TextArea label="Prompt" bind:value={modalData.prompt} />
     <Toggle label="Add data" bind:value={modalData.addData} />
     <Toggle label="Use cached" bind:value={modalData.useCached} />
   </ModalContent>

From 2259e9d1325575f007db253eacaa71ffeed2dd69 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 17:07:17 +0200
Subject: [PATCH 031/101] Use formula correctly

---
 packages/server/src/api/controllers/ai.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index d4f49fd746..3f6346d4d0 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -84,6 +84,12 @@ export async function generateTables(
       }
       field.tableId = linkedTable.id
     }
+
+    for (const field of table.structure.schema.filter(
+      f => f.type === FieldType.FORMULA
+    )) {
+      field.formula = `{{ js "${btoa(field.formula)}" }}`
+    }
   }
 
   for (const { structure: table } of Object.values(response.tables)) {

From aa10eb49fcd29ea32dd0e0cd3ea32721708b0099 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 1 Apr 2025 17:07:25 +0200
Subject: [PATCH 032/101] Update formula

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 50ca69df2a..d2ce83fe77 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 50ca69df2a225937c9095086829328ce7134a2f4
+Subproject commit d2ce83fe773f7da2dd9d8eb8bd2512d5bbb07a9d

From 68a39193c8f3f5c94eb3bfcaa534fca777da52d7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 2 Apr 2025 13:32:56 +0200
Subject: [PATCH 033/101] Process uploads

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 62 +++++++++++++++++++++--
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/packages/pro b/packages/pro
index d2ce83fe77..ee9eb351c9 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d2ce83fe773f7da2dd9d8eb8bd2512d5bbb07a9d
+Subproject commit ee9eb351c94340808858bbc7e881a48983f80f34
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 3f6346d4d0..efa2ba795e 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -1,3 +1,4 @@
+import * as uuid from "uuid"
 import {
   DocumentType,
   FieldType,
@@ -6,14 +7,19 @@ import {
   SourceName,
   TableSchema,
   TableSourceType,
+  Upload,
   UserCtx,
 } from "@budibase/types"
 import { getLLM } from "packages/pro/src/ai"
-import { context, utils } from "@budibase/backend-core"
+import { context, objectStore, utils } from "@budibase/backend-core"
 import sdk from "../../sdk"
-import fs from "fs"
-import path from "path"
+import fs, { mkdirSync } from "fs"
+import path, { join } from "path"
 import { createHash } from "crypto"
+import { pipeline } from "stream"
+import { promisify } from "util"
+import fetch from "node-fetch"
+import { ObjectStoreBuckets } from "../../constants"
 
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
@@ -123,8 +129,21 @@ export async function generateTables(
         linksOverride[field.name] = null
       }
 
+      const attachmentColumns = table.structure.schema.filter(
+        f => f.type === FieldType.ATTACHMENT_SINGLE
+      )
+
       for (const entry of table.data || []) {
         const tableId = tableIds[table.structure.name]
+
+        const attachmentData: Record<string, any> = {}
+        for (const column of attachmentColumns) {
+          const attachment = await downloadFile(
+            entry.values.find(f => f.key === column.name)!.value as string
+          )
+          attachmentData[column.name] = attachment
+        }
+
         const createdRow = await sdk.rows.save(
           tableId,
           {
@@ -133,6 +152,7 @@ export async function generateTables(
               return acc
             }, {}),
             ...linksOverride,
+            ...attachmentData,
             _id: undefined,
           },
           ctx.user._id
@@ -189,3 +209,39 @@ export async function generateTables(
     createdTables,
   }
 }
+
+async function downloadFile(url: string): Promise<Upload> {
+  const res = await fetch(url)
+
+  const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
+
+  if (!fs.existsSync(tmpPath)) {
+    mkdirSync(tmpPath)
+  }
+
+  const extension = [...res.url.split(".")].pop()!.split("?")[0]
+
+  const destination = path.resolve(tmpPath, `${uuid.v4()}${extension}`)
+  const fileStream = fs.createWriteStream(destination, { flags: "wx" })
+
+  await promisify(pipeline)(res.body, fileStream)
+
+  const processedFileName = path.basename(destination)
+
+  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+  const response = await objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: s3Key,
+    path: destination,
+    type: "image/jpeg",
+  })
+
+  return {
+    size: fileStream.bytesWritten,
+    name: processedFileName,
+    url: await objectStore.getAppFileUrl(s3Key),
+    extension,
+    key: response.Key!,
+  }
+}

From 02a2eaf78bb0b1d2318b29eecc74bf64d7a35d68 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 2 Apr 2025 13:33:14 +0200
Subject: [PATCH 034/101] Hide cache option is not local

---
 .../data/_components/AITableCreationModal.svelte           | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index 3912c36611..42f8e715db 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -1,6 +1,7 @@
 <script lang="ts">
   import { API } from "@/api"
   import { datasources, tables } from "@/stores/builder"
+  import { admin } from "@/stores/portal"
   import {
     Modal,
     ModalContent,
@@ -17,7 +18,7 @@
     prompt:
       "I need a ticketing system where users can submit tickets. A ticket has a title, description, priority, status, and an assigned technician",
     addData: true,
-    useCached: true,
+    useCached: $admin.isDev,
   }
 
   export function show() {
@@ -52,6 +53,8 @@
   >
     <TextArea label="Prompt" bind:value={modalData.prompt} />
     <Toggle label="Add data" bind:value={modalData.addData} />
-    <Toggle label="Use cached" bind:value={modalData.useCached} />
+    {#if $admin.isDev}
+      <Toggle label="Use cached" bind:value={modalData.useCached} />
+    {/if}
   </ModalContent>
 </Modal>

From ffd3e2b2c016c0dd9d7eabf12618d7b5953fc2ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 2 Apr 2025 15:31:21 +0200
Subject: [PATCH 035/101] Move cached data to tmp

---
 .gitignore                                | 1 -
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 6 +++++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index f548a62da6..21637edfbe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,4 +112,3 @@ budibase-datasource
 
 *.iml
 .nx
-llm-output/*
diff --git a/packages/pro b/packages/pro
index 8e3db23290..8e49f29207 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 8e3db232903de9dc30706fc5ab712a14116a5935
+Subproject commit 8e49f29207555b9fa43eb33063041bebc0e0b5b6
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index efa2ba795e..d6d7f983e2 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -50,7 +50,11 @@ export async function generateTables(
   })
 
   if (!useCached) {
-    const dir = path.join(process.env.PWD!, `../../llm-output/${cacheKey}`)
+    const baseDir = path.join(objectStore.budibaseTempDir(), "llm-output")
+    if (!fs.existsSync(baseDir)) {
+      fs.mkdirSync(baseDir)
+    }
+    const dir = path.join(baseDir, cacheKey)
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir)
     }

From 1f3b7753722619f566645b8580824fc20ec8e2f7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 2 Apr 2025 15:32:45 +0200
Subject: [PATCH 036/101] Handle error

---
 packages/server/src/api/controllers/ai.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index d6d7f983e2..c7921505fb 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -27,7 +27,10 @@ export async function generateTables(
   const { prompt, useCached, addData } = ctx.request.body
 
   const llm = await getLLM("gpt-4o")
-  llm!.maxTokens = 1200
+  if (!llm) {
+    ctx.throw("LLM not available")
+  }
+  llm.maxTokens = 1200
 
   const cacheKey = `${createHash("md5").update(prompt).digest("hex")}_${
     addData ? 1 : 0

From 1f6495c186cb8b0658a74a13f3d1488ef58dc723 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 13:38:14 +0200
Subject: [PATCH 037/101] Use tooling

---
 packages/pro                              |   2 +-
 packages/server/src/api/controllers/ai.ts | 228 ++++++++++------------
 2 files changed, 107 insertions(+), 123 deletions(-)

diff --git a/packages/pro b/packages/pro
index 8e49f29207..5ee3fced42 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 8e49f29207555b9fa43eb33063041bebc0e0b5b6
+Subproject commit 5ee3fced420c9fb1bda554fa881f4c86c344a91c
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index c7921505fb..d8f7ef03b6 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -10,38 +10,18 @@ import {
   Upload,
   UserCtx,
 } from "@budibase/types"
-import { getLLM } from "packages/pro/src/ai"
+import { ai } from "@budibase/pro"
 import { context, objectStore, utils } from "@budibase/backend-core"
 import sdk from "../../sdk"
 import fs, { mkdirSync } from "fs"
 import path, { join } from "path"
-import { createHash } from "crypto"
+
 import { pipeline } from "stream"
 import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
 
-export async function generateTables(
-  ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
-) {
-  const { prompt, useCached, addData } = ctx.request.body
-
-  const llm = await getLLM("gpt-4o")
-  if (!llm) {
-    ctx.throw("LLM not available")
-  }
-  llm.maxTokens = 1200
-
-  const cacheKey = `${createHash("md5").update(prompt).digest("hex")}_${
-    addData ? 1 : 0
-  }`
-
-  const response = await llm!.generateTables(
-    prompt,
-    useCached ? `${cacheKey}/latest` : "",
-    addData
-  )
-
+async function generateTablesDelegate(data: ai.GenerationStructure) {
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
     _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
@@ -52,27 +32,10 @@ export async function generateTables(
     config: {},
   })
 
-  if (!useCached) {
-    const baseDir = path.join(objectStore.budibaseTempDir(), "llm-output")
-    if (!fs.existsSync(baseDir)) {
-      fs.mkdirSync(baseDir)
-    }
-    const dir = path.join(baseDir, cacheKey)
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir)
-    }
-    fs.writeFileSync(path.join(dir, "prompt.txt"), prompt)
-    fs.writeFileSync(path.join(dir, "latest.json"), JSON.stringify(response))
-    fs.writeFileSync(
-      path.join(dir, `${Date.now()}.json`),
-      JSON.stringify(response)
-    )
-  }
-
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
-  for (const table of response.tables) {
+  for (const table of data.tables) {
     const createdTable = await sdk.tables.create({
       ...table.structure,
       sourceId: dsId,
@@ -86,7 +49,7 @@ export async function generateTables(
     tableIds[table.structure.name] = createdTable._id!
   }
 
-  for (const table of Object.values(response.tables)) {
+  for (const table of Object.values(data.tables)) {
     for (const field of table.structure.schema.filter(
       f => f.type === FieldType.LINK
     )) {
@@ -105,7 +68,7 @@ export async function generateTables(
     }
   }
 
-  for (const { structure: table } of Object.values(response.tables)) {
+  for (const { structure: table } of Object.values(data.tables)) {
     const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
     await sdk.tables.update({
@@ -121,96 +84,117 @@ export async function generateTables(
     })
   }
 
-  if (addData) {
-    const createdData: Record<string, Record<string, string>> = {}
-    const toUpdateLinks: {
-      tableId: string
-      rowId: string
-      data: Record<string, { rowId: string; tableId: string }>
-    }[] = []
-    for (const table of Object.values(response.tables)) {
-      const linksOverride: Record<string, null> = {}
-      for (const field of table.structure.schema.filter(
-        f => f.type === FieldType.LINK
-      )) {
-        linksOverride[field.name] = null
-      }
+  // if (addData) {
+  //   const createdData: Record<string, Record<string, string>> = {}
+  //   const toUpdateLinks: {
+  //     tableId: string
+  //     rowId: string
+  //     data: Record<string, { rowId: string; tableId: string }>
+  //   }[] = []
+  //   for (const table of Object.values(response.tables)) {
+  //     const linksOverride: Record<string, null> = {}
+  //     for (const field of table.structure.schema.filter(
+  //       f => f.type === FieldType.LINK
+  //     )) {
+  //       linksOverride[field.name] = null
+  //     }
 
-      const attachmentColumns = table.structure.schema.filter(
-        f => f.type === FieldType.ATTACHMENT_SINGLE
-      )
+  //     const attachmentColumns = table.structure.schema.filter(
+  //       f => f.type === FieldType.ATTACHMENT_SINGLE
+  //     )
 
-      for (const entry of table.data || []) {
-        const tableId = tableIds[table.structure.name]
+  //     for (const entry of table.data || []) {
+  //       const tableId = tableIds[table.structure.name]
 
-        const attachmentData: Record<string, any> = {}
-        for (const column of attachmentColumns) {
-          const attachment = await downloadFile(
-            entry.values.find(f => f.key === column.name)!.value as string
-          )
-          attachmentData[column.name] = attachment
-        }
+  //       const attachmentData: Record<string, any> = {}
+  //       for (const column of attachmentColumns) {
+  //         const attachment = await downloadFile(
+  //           entry.values.find(f => f.key === column.name)!.value as string
+  //         )
+  //         attachmentData[column.name] = attachment
+  //       }
 
-        const createdRow = await sdk.rows.save(
-          tableId,
-          {
-            ...entry.values.reduce<Record<string, any>>((acc, v) => {
-              acc[v.key] = v.value
-              return acc
-            }, {}),
-            ...linksOverride,
-            ...attachmentData,
-            _id: undefined,
-          },
-          ctx.user._id
-        )
+  //       const createdRow = await sdk.rows.save(
+  //         tableId,
+  //         {
+  //           ...entry.values.reduce<Record<string, any>>((acc, v) => {
+  //             acc[v.key] = v.value
+  //             return acc
+  //           }, {}),
+  //           ...linksOverride,
+  //           ...attachmentData,
+  //           _id: undefined,
+  //         },
+  //         ctx.user._id
+  //       )
 
-        createdData[tableId] ??= {}
-        createdData[tableId][entry.id] = createdRow.row._id!
+  //       createdData[tableId] ??= {}
+  //       createdData[tableId][entry.id] = createdRow.row._id!
 
-        const overridenLinks = Object.keys(linksOverride).reduce<
-          Record<string, { rowId: string; tableId: string }>
-        >((acc, l) => {
-          if (entry.values.find(f => f.key === l)) {
-            acc[l] = {
-              tableId: (table.structure.schema.find(f => f.name === l) as any)
-                .tableId,
-              rowId: entry.values.find(f => f.key === l)!.value as any,
-            }
-          }
-          return acc
-        }, {})
+  //       const overridenLinks = Object.keys(linksOverride).reduce<
+  //         Record<string, { rowId: string; tableId: string }>
+  //       >((acc, l) => {
+  //         if (entry.values.find(f => f.key === l)) {
+  //           acc[l] = {
+  //             tableId: (table.structure.schema.find(f => f.name === l) as any)
+  //               .tableId,
+  //             rowId: entry.values.find(f => f.key === l)!.value as any,
+  //           }
+  //         }
+  //         return acc
+  //       }, {})
 
-        if (Object.keys(overridenLinks).length) {
-          toUpdateLinks.push({
-            tableId: createdRow.table._id!,
-            rowId: createdRow.row._id!,
-            data: overridenLinks,
-          })
-        }
-      }
-    }
+  //       if (Object.keys(overridenLinks).length) {
+  //         toUpdateLinks.push({
+  //           tableId: createdRow.table._id!,
+  //           rowId: createdRow.row._id!,
+  //           data: overridenLinks,
+  //         })
+  //       }
+  //     }
+  //   }
 
-    for (const data of toUpdateLinks) {
-      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+  //   for (const data of toUpdateLinks) {
+  //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-      const updatedLinks = Object.keys(data.data).reduce<
-        Record<string, string>
-      >((acc, d) => {
-        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-        return acc
-      }, {})
+  //     const updatedLinks = Object.keys(data.data).reduce<
+  //       Record<string, string>
+  //     >((acc, d) => {
+  //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+  //       return acc
+  //     }, {})
 
-      await sdk.rows.save(
-        data.tableId,
-        {
-          ...persistedRow,
-          ...updatedLinks,
-        },
-        ctx.user._id
-      )
-    }
+  //     await sdk.rows.save(
+  //       data.tableId,
+  //       {
+  //         ...persistedRow,
+  //         ...updatedLinks,
+  //       },
+  //       ctx.user._id
+  //     )
+  //   }
+  // }
+
+  return createdTables
+}
+
+export async function generateTables(
+  ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
+) {
+  const { prompt, useCached, addData } = ctx.request.body
+
+  const llm = await ai.getLLM("gpt-4o")
+  if (!llm) {
+    ctx.throw("LLM not available")
   }
+  llm.maxTokens = 5000
+
+  const createdTables = await llm!.generateTables(
+    prompt,
+    useCached,
+    addData,
+    generateTablesDelegate
+  )
 
   ctx.body = {
     createdTables,

From 5e3f01d6ec96ec868ef02e5248422bdd637022c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 13:40:51 +0200
Subject: [PATCH 038/101] Add data

---
 packages/pro                              |   2 +-
 packages/server/src/api/controllers/ai.ts | 167 +++++++++++-----------
 2 files changed, 87 insertions(+), 82 deletions(-)

diff --git a/packages/pro b/packages/pro
index 5ee3fced42..252bb04fac 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 5ee3fced420c9fb1bda554fa881f4c86c344a91c
+Subproject commit 252bb04faceead0452d1190e432bd085ffe9980f
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index d8f7ef03b6..a8883345a4 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -21,7 +21,11 @@ import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
 
-async function generateTablesDelegate(data: ai.GenerationStructure) {
+async function generateTablesDelegate(
+  data: ai.GenerationStructure,
+  addData: boolean,
+  userId: string
+) {
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
     _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
@@ -84,96 +88,96 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
     })
   }
 
-  // if (addData) {
-  //   const createdData: Record<string, Record<string, string>> = {}
-  //   const toUpdateLinks: {
-  //     tableId: string
-  //     rowId: string
-  //     data: Record<string, { rowId: string; tableId: string }>
-  //   }[] = []
-  //   for (const table of Object.values(response.tables)) {
-  //     const linksOverride: Record<string, null> = {}
-  //     for (const field of table.structure.schema.filter(
-  //       f => f.type === FieldType.LINK
-  //     )) {
-  //       linksOverride[field.name] = null
-  //     }
+  if (addData) {
+    const createdData: Record<string, Record<string, string>> = {}
+    const toUpdateLinks: {
+      tableId: string
+      rowId: string
+      data: Record<string, { rowId: string; tableId: string }>
+    }[] = []
+    for (const table of Object.values(data.tables)) {
+      const linksOverride: Record<string, null> = {}
+      for (const field of table.structure.schema.filter(
+        f => f.type === FieldType.LINK
+      )) {
+        linksOverride[field.name] = null
+      }
 
-  //     const attachmentColumns = table.structure.schema.filter(
-  //       f => f.type === FieldType.ATTACHMENT_SINGLE
-  //     )
+      const attachmentColumns = table.structure.schema.filter(
+        f => f.type === FieldType.ATTACHMENT_SINGLE
+      )
 
-  //     for (const entry of table.data || []) {
-  //       const tableId = tableIds[table.structure.name]
+      for (const entry of table.data || []) {
+        const tableId = tableIds[table.structure.name]
 
-  //       const attachmentData: Record<string, any> = {}
-  //       for (const column of attachmentColumns) {
-  //         const attachment = await downloadFile(
-  //           entry.values.find(f => f.key === column.name)!.value as string
-  //         )
-  //         attachmentData[column.name] = attachment
-  //       }
+        const attachmentData: Record<string, any> = {}
+        for (const column of attachmentColumns) {
+          const attachment = await downloadFile(
+            entry.values.find(f => f.key === column.name)!.value as string
+          )
+          attachmentData[column.name] = attachment
+        }
 
-  //       const createdRow = await sdk.rows.save(
-  //         tableId,
-  //         {
-  //           ...entry.values.reduce<Record<string, any>>((acc, v) => {
-  //             acc[v.key] = v.value
-  //             return acc
-  //           }, {}),
-  //           ...linksOverride,
-  //           ...attachmentData,
-  //           _id: undefined,
-  //         },
-  //         ctx.user._id
-  //       )
+        const createdRow = await sdk.rows.save(
+          tableId,
+          {
+            ...entry.values.reduce<Record<string, any>>((acc, v) => {
+              acc[v.key] = v.value
+              return acc
+            }, {}),
+            ...linksOverride,
+            ...attachmentData,
+            _id: undefined,
+          },
+          userId
+        )
 
-  //       createdData[tableId] ??= {}
-  //       createdData[tableId][entry.id] = createdRow.row._id!
+        createdData[tableId] ??= {}
+        createdData[tableId][entry.id] = createdRow.row._id!
 
-  //       const overridenLinks = Object.keys(linksOverride).reduce<
-  //         Record<string, { rowId: string; tableId: string }>
-  //       >((acc, l) => {
-  //         if (entry.values.find(f => f.key === l)) {
-  //           acc[l] = {
-  //             tableId: (table.structure.schema.find(f => f.name === l) as any)
-  //               .tableId,
-  //             rowId: entry.values.find(f => f.key === l)!.value as any,
-  //           }
-  //         }
-  //         return acc
-  //       }, {})
+        const overridenLinks = Object.keys(linksOverride).reduce<
+          Record<string, { rowId: string; tableId: string }>
+        >((acc, l) => {
+          if (entry.values.find(f => f.key === l)) {
+            acc[l] = {
+              tableId: (table.structure.schema.find(f => f.name === l) as any)
+                .tableId,
+              rowId: entry.values.find(f => f.key === l)!.value as any,
+            }
+          }
+          return acc
+        }, {})
 
-  //       if (Object.keys(overridenLinks).length) {
-  //         toUpdateLinks.push({
-  //           tableId: createdRow.table._id!,
-  //           rowId: createdRow.row._id!,
-  //           data: overridenLinks,
-  //         })
-  //       }
-  //     }
-  //   }
+        if (Object.keys(overridenLinks).length) {
+          toUpdateLinks.push({
+            tableId: createdRow.table._id!,
+            rowId: createdRow.row._id!,
+            data: overridenLinks,
+          })
+        }
+      }
+    }
 
-  //   for (const data of toUpdateLinks) {
-  //     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+    for (const data of toUpdateLinks) {
+      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-  //     const updatedLinks = Object.keys(data.data).reduce<
-  //       Record<string, string>
-  //     >((acc, d) => {
-  //       acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-  //       return acc
-  //     }, {})
+      const updatedLinks = Object.keys(data.data).reduce<
+        Record<string, string>
+      >((acc, d) => {
+        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+        return acc
+      }, {})
 
-  //     await sdk.rows.save(
-  //       data.tableId,
-  //       {
-  //         ...persistedRow,
-  //         ...updatedLinks,
-  //       },
-  //       ctx.user._id
-  //     )
-  //   }
-  // }
+      await sdk.rows.save(
+        data.tableId,
+        {
+          ...persistedRow,
+          ...updatedLinks,
+        },
+        userId
+      )
+    }
+  }
 
   return createdTables
 }
@@ -193,6 +197,7 @@ export async function generateTables(
     prompt,
     useCached,
     addData,
+    ctx.userId,
     generateTablesDelegate
   )
 

From 6f5bd875aea10f9b99c013c55baf52355ceadc04 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 14:49:21 +0200
Subject: [PATCH 039/101] Split add data

---
 packages/pro                              |   2 +-
 packages/server/src/api/controllers/ai.ts | 167 +++++++++++-----------
 2 files changed, 82 insertions(+), 87 deletions(-)

diff --git a/packages/pro b/packages/pro
index 252bb04fac..3a5c1fd77d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 252bb04faceead0452d1190e432bd085ffe9980f
+Subproject commit 3a5c1fd77d49715a0a345b2662db2d6232078c47
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index a8883345a4..526d0c8e78 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -21,11 +21,7 @@ import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
 
-async function generateTablesDelegate(
-  data: ai.GenerationStructure,
-  addData: boolean,
-  userId: string
-) {
+async function generateTablesDelegate(data: ai.GenerationStructure) {
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
     _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
@@ -88,98 +84,95 @@ async function generateTablesDelegate(
     })
   }
 
-  if (addData) {
-    const createdData: Record<string, Record<string, string>> = {}
-    const toUpdateLinks: {
-      tableId: string
-      rowId: string
-      data: Record<string, { rowId: string; tableId: string }>
-    }[] = []
-    for (const table of Object.values(data.tables)) {
-      const linksOverride: Record<string, null> = {}
-      for (const field of table.structure.schema.filter(
-        f => f.type === FieldType.LINK
-      )) {
-        linksOverride[field.name] = null
-      }
+  return createdTables
+}
 
-      const attachmentColumns = table.structure.schema.filter(
-        f => f.type === FieldType.ATTACHMENT_SINGLE
-      )
-
-      for (const entry of table.data || []) {
-        const tableId = tableIds[table.structure.name]
-
-        const attachmentData: Record<string, any> = {}
-        for (const column of attachmentColumns) {
-          const attachment = await downloadFile(
-            entry.values.find(f => f.key === column.name)!.value as string
-          )
-          attachmentData[column.name] = attachment
-        }
-
-        const createdRow = await sdk.rows.save(
-          tableId,
-          {
-            ...entry.values.reduce<Record<string, any>>((acc, v) => {
-              acc[v.key] = v.value
-              return acc
-            }, {}),
-            ...linksOverride,
-            ...attachmentData,
-            _id: undefined,
-          },
-          userId
-        )
-
-        createdData[tableId] ??= {}
-        createdData[tableId][entry.id] = createdRow.row._id!
-
-        const overridenLinks = Object.keys(linksOverride).reduce<
-          Record<string, { rowId: string; tableId: string }>
-        >((acc, l) => {
-          if (entry.values.find(f => f.key === l)) {
-            acc[l] = {
-              tableId: (table.structure.schema.find(f => f.name === l) as any)
-                .tableId,
-              rowId: entry.values.find(f => f.key === l)!.value as any,
-            }
-          }
-          return acc
-        }, {})
-
-        if (Object.keys(overridenLinks).length) {
-          toUpdateLinks.push({
-            tableId: createdRow.table._id!,
-            rowId: createdRow.row._id!,
-            data: overridenLinks,
-          })
-        }
-      }
+async function generateDataDelegate(
+  data: Record<string, Record<string, any>>,
+  userId: string
+) {
+  const createdData: Record<string, Record<string, string>> = {}
+  const toUpdateLinks: {
+    tableId: string
+    rowId: string
+    data: Record<string, { rowId: string; tableId: string }>
+  }[] = []
+  for (const tableId of Object.keys(data)) {
+    const table = await sdk.tables.getTable(tableId)
+    const linksOverride: Record<string, null> = {}
+    for (const field of Object.values(table.schema).filter(
+      f => f.type === FieldType.LINK
+    )) {
+      linksOverride[field.name] = null
     }
 
-    for (const data of toUpdateLinks) {
-      const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+    const attachmentColumns = Object.values(table.schema)
+      .filter(f => f.type === FieldType.ATTACHMENT_SINGLE)
+      .map(c => c.name)
 
-      const updatedLinks = Object.keys(data.data).reduce<
-        Record<string, string>
-      >((acc, d) => {
-        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
-        return acc
-      }, {})
+    for (const entry of [data[tableId]]) {
+      const attachmentData: Record<string, any> = {}
+      for (const column of attachmentColumns) {
+        const attachment = await downloadFile(entry[column])
+        attachmentData[column] = attachment
+      }
 
-      await sdk.rows.save(
-        data.tableId,
+      const createdRow = await sdk.rows.save(
+        tableId,
         {
-          ...persistedRow,
-          ...updatedLinks,
+          ...entry,
+          ...linksOverride,
+          ...attachmentData,
+          _id: undefined,
         },
         userId
       )
+
+      createdData[tableId] ??= {}
+      createdData[tableId][entry.id] = createdRow.row._id!
+
+      const overridenLinks = Object.keys(linksOverride).reduce<
+        Record<string, { rowId: string; tableId: string }>
+      >((acc, l) => {
+        if (entry[l]) {
+          acc[l] = {
+            tableId: (table.schema[l] as any).tableId,
+            rowId: entry[l],
+          }
+        }
+        return acc
+      }, {})
+
+      if (Object.keys(overridenLinks).length) {
+        toUpdateLinks.push({
+          tableId: createdRow.table._id!,
+          rowId: createdRow.row._id!,
+          data: overridenLinks,
+        })
+      }
     }
   }
 
-  return createdTables
+  for (const data of toUpdateLinks) {
+    const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+
+    const updatedLinks = Object.keys(data.data).reduce<Record<string, string>>(
+      (acc, d) => {
+        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+        return acc
+      },
+      {}
+    )
+
+    await sdk.rows.save(
+      data.tableId,
+      {
+        ...persistedRow,
+        ...updatedLinks,
+      },
+      userId
+    )
+  }
 }
 
 export async function generateTables(
@@ -198,7 +191,9 @@ export async function generateTables(
     useCached,
     addData,
     ctx.userId,
-    generateTablesDelegate
+    generateTablesDelegate,
+    sdk.tables.getTables,
+    generateDataDelegate
   )
 
   ctx.body = {

From 0e8d14e33e6174817c4d89d61e8a7cbb001d1c64 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 15:07:51 +0200
Subject: [PATCH 040/101] Clean data from initial structure

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 12 +++++-------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/packages/pro b/packages/pro
index 3a5c1fd77d..8195b4aa72 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3a5c1fd77d49715a0a345b2662db2d6232078c47
+Subproject commit 8195b4aa72efe0898f99fb87414b06b4d09380c8
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 526d0c8e78..9d00b103fe 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -37,7 +37,7 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
 
   for (const table of data.tables) {
     const createdTable = await sdk.tables.create({
-      ...table.structure,
+      ...table,
       sourceId: dsId,
       schema: {},
       primaryDisplay: undefined,
@@ -46,13 +46,11 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
     })
 
     createdTables.push({ id: createdTable._id!, name: createdTable.name })
-    tableIds[table.structure.name] = createdTable._id!
+    tableIds[table.name] = createdTable._id!
   }
 
   for (const table of Object.values(data.tables)) {
-    for (const field of table.structure.schema.filter(
-      f => f.type === FieldType.LINK
-    )) {
+    for (const field of table.schema.filter(f => f.type === FieldType.LINK)) {
       // const field =  table.schema[fieldKey] as RelationshipFieldMetadata
       const linkedTable = createdTables.find(t => t.name === field.tableId)
       if (!linkedTable) {
@@ -61,14 +59,14 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
       field.tableId = linkedTable.id
     }
 
-    for (const field of table.structure.schema.filter(
+    for (const field of table.schema.filter(
       f => f.type === FieldType.FORMULA
     )) {
       field.formula = `{{ js "${btoa(field.formula)}" }}`
     }
   }
 
-  for (const { structure: table } of Object.values(data.tables)) {
+  for (const table of Object.values(data.tables)) {
     const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
     await sdk.tables.update({

From b1a148a7d9b35f419d3a22183b7219b0b60c959d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 17:55:13 +0200
Subject: [PATCH 041/101] Define single attachment

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/packages/pro b/packages/pro
index 8195b4aa72..3d2a7a13e4 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 8195b4aa72efe0898f99fb87414b06b4d09380c8
+Subproject commit 3d2a7a13e46df52a4e6f391a51712f4fc5c39494
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 9d00b103fe..b9af20b7de 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -5,6 +5,7 @@ import {
   GenerateTablesRequest,
   GenerateTablesResponse,
   SourceName,
+  Table,
   TableSchema,
   TableSourceType,
   Upload,
@@ -87,7 +88,8 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
 
 async function generateDataDelegate(
   data: Record<string, Record<string, any>>,
-  userId: string
+  userId: string,
+  tables: Record<string, Table>
 ) {
   const createdData: Record<string, Record<string, string>> = {}
   const toUpdateLinks: {
@@ -95,8 +97,8 @@ async function generateDataDelegate(
     rowId: string
     data: Record<string, { rowId: string; tableId: string }>
   }[] = []
-  for (const tableId of Object.keys(data)) {
-    const table = await sdk.tables.getTable(tableId)
+  for (const tableName of Object.keys(data)) {
+    const table = tables[tableName]
     const linksOverride: Record<string, null> = {}
     for (const field of Object.values(table.schema).filter(
       f => f.type === FieldType.LINK
@@ -108,13 +110,14 @@ async function generateDataDelegate(
       .filter(f => f.type === FieldType.ATTACHMENT_SINGLE)
       .map(c => c.name)
 
-    for (const entry of [data[tableId]]) {
+    for (const entry of [data[tableName]]) {
       const attachmentData: Record<string, any> = {}
       for (const column of attachmentColumns) {
         const attachment = await downloadFile(entry[column])
         attachmentData[column] = attachment
       }
 
+      const tableId = tables[tableName]._id!
       const createdRow = await sdk.rows.save(
         tableId,
         {

From 28a48d989989f9dc0a0b8d70ab6b0fe5fe763c34 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 18:04:40 +0200
Subject: [PATCH 042/101] Create many rows

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index 3d2a7a13e4..a93d47c72c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3d2a7a13e46df52a4e6f391a51712f4fc5c39494
+Subproject commit a93d47c72cadebb343a82a5ed655ba65be3288c5
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index b9af20b7de..84e79693ad 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -87,7 +87,7 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
 }
 
 async function generateDataDelegate(
-  data: Record<string, Record<string, any>>,
+  data: Record<string, Record<string, any>[]>,
   userId: string,
   tables: Record<string, Table>
 ) {
@@ -110,7 +110,7 @@ async function generateDataDelegate(
       .filter(f => f.type === FieldType.ATTACHMENT_SINGLE)
       .map(c => c.name)
 
-    for (const entry of [data[tableName]]) {
+    for (const entry of data[tableName]) {
       const attachmentData: Record<string, any> = {}
       for (const column of attachmentColumns) {
         const attachment = await downloadFile(entry[column])

From 6e89eae1bf2065c974e4483f815a93e0172ae4c3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Apr 2025 18:21:01 +0200
Subject: [PATCH 043/101] Populate links

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index a93d47c72c..ed7bdd261c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit a93d47c72cadebb343a82a5ed655ba65be3288c5
+Subproject commit ed7bdd261c87b8dc8af3497998de6918b33f8e57
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 84e79693ad..c44edf1fc3 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -130,7 +130,7 @@ async function generateDataDelegate(
       )
 
       createdData[tableId] ??= {}
-      createdData[tableId][entry.id] = createdRow.row._id!
+      createdData[tableId][entry._id] = createdRow.row._id!
 
       const overridenLinks = Object.keys(linksOverride).reduce<
         Record<string, { rowId: string; tableId: string }>

From 294d5e815dc356c6df7f20e3eba9ced49bf76fab Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 10:58:55 +0200
Subject: [PATCH 044/101] Attachments

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/packages/pro b/packages/pro
index ed7bdd261c..f5db0da3ff 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ed7bdd261c87b8dc8af3497998de6918b33f8e57
+Subproject commit f5db0da3ffb887f450ef1c888bb1a2d40deab55e
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index c44edf1fc3..48975cec53 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -106,15 +106,25 @@ async function generateDataDelegate(
       linksOverride[field.name] = null
     }
 
-    const attachmentColumns = Object.values(table.schema)
-      .filter(f => f.type === FieldType.ATTACHMENT_SINGLE)
-      .map(c => c.name)
+    const attachmentColumns = Object.values(table.schema).filter(f =>
+      [FieldType.ATTACHMENTS, FieldType.ATTACHMENT_SINGLE].includes(f.type)
+    )
 
     for (const entry of data[tableName]) {
       const attachmentData: Record<string, any> = {}
       for (const column of attachmentColumns) {
-        const attachment = await downloadFile(entry[column])
-        attachmentData[column] = attachment
+        attachmentData[column.name] = []
+        if (!Array.isArray(entry[column.name])) {
+          entry[column.name] = [entry[column.name]]
+        }
+        for (const attachmentValue of entry[column.name]) {
+          const attachment = await downloadFile(attachmentValue)
+          attachmentData[column.name].push(attachment)
+        }
+
+        if (column.type === FieldType.ATTACHMENT_SINGLE) {
+          attachmentData[column.name] = attachmentData[column.name][0]
+        }
       }
 
       const tableId = tables[tableName]._id!

From dd4de6655f875d088db79352cda149fd192423f8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 12:16:33 +0200
Subject: [PATCH 045/101] Support files

---
 packages/server/src/api/controllers/ai.ts | 44 +++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 48975cec53..32c729657e 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -212,8 +212,14 @@ export async function generateTables(
   }
 }
 
-async function downloadFile(url: string): Promise<Upload> {
-  const res = await fetch(url)
+async function downloadFile(
+  file: string | { fileName: string; extension: string; content: string }
+): Promise<Upload> {
+  if (typeof file === "object") {
+    return createFile(file)
+  }
+
+  const res = await fetch(file)
 
   const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
 
@@ -247,3 +253,37 @@ async function downloadFile(url: string): Promise<Upload> {
     key: response.Key!,
   }
 }
+
+async function createFile(file: {
+  fileName: string
+  extension: string
+  content: string
+}): Promise<Upload> {
+  const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
+
+  if (!fs.existsSync(tmpPath)) {
+    mkdirSync(tmpPath)
+  }
+
+  const destination = path.resolve(tmpPath, `${file.fileName}${file.extension}`)
+
+  fs.writeFileSync(destination, file.content)
+
+  const processedFileName = path.basename(destination)
+  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+  const response = await objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: s3Key,
+    path: destination,
+    type: "text/plain",
+  })
+
+  return {
+    size: fs.readFileSync(destination).byteLength,
+    name: processedFileName,
+    url: await objectStore.getAppFileUrl(s3Key),
+    extension: file.extension,
+    key: response.Key!,
+  }
+}

From b0d319e8210d3ea7256e434fbc195562234c8d37 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 12:17:34 +0200
Subject: [PATCH 046/101] Support files

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index f5db0da3ff..180d19a611 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit f5db0da3ffb887f450ef1c888bb1a2d40deab55e
+Subproject commit 180d19a611147d2c227c2f885ea723020d6c4775

From 24bee1be4aa3a13b4c5cc5e73f071169bda8ee52 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 12:53:55 +0200
Subject: [PATCH 047/101] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 180d19a611..0fff3b7f1f 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 180d19a611147d2c227c2f885ea723020d6c4775
+Subproject commit 0fff3b7f1fd00148d4391b9cd39271c2c8784f98

From a0f7f4142a55582a005610755e838e39340afe08 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 13:16:02 +0200
Subject: [PATCH 048/101] Max tokens

---
 packages/server/src/api/controllers/ai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 32c729657e..ab4a132c63 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -195,7 +195,7 @@ export async function generateTables(
   if (!llm) {
     ctx.throw("LLM not available")
   }
-  llm.maxTokens = 5000
+  llm.maxTokens = 2000
 
   const createdTables = await llm!.generateTables(
     prompt,

From 9e48a78108e7cb8d30bb3efc203d87b9cbefbdba Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 13:16:46 +0200
Subject: [PATCH 049/101] Pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 0fff3b7f1f..83817787fe 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 0fff3b7f1fd00148d4391b9cd39271c2c8784f98
+Subproject commit 83817787fe3e695527779a30842a22b77a68f1aa

From 487c0d7c7f76cebfc76cce47a7daed69c5c385c9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 13:48:55 +0200
Subject: [PATCH 050/101] Rebasing pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 83817787fe..ca58ddcdc7 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 83817787fe3e695527779a30842a22b77a68f1aa
+Subproject commit ca58ddcdc77478ec9de4defe6821be2fddfc38d4

From 540c3d94cf255844d60de0d14bb213d3eeacb95b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Apr 2025 13:54:30 +0200
Subject: [PATCH 051/101] Use AI translate and categorise

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ca58ddcdc7..a3f4eda49d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ca58ddcdc77478ec9de4defe6821be2fddfc38d4
+Subproject commit a3f4eda49d52d81534b0dc69b3e589a99522a795

From 1bd3dfcd3575c9921531bd59bed0f17e7e1c536f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Apr 2025 11:33:46 +0200
Subject: [PATCH 052/101] Use default model

---
 packages/server/src/api/controllers/ai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index ab4a132c63..92b69668b9 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -191,7 +191,7 @@ export async function generateTables(
 ) {
   const { prompt, useCached, addData } = ctx.request.body
 
-  const llm = await ai.getLLM("gpt-4o")
+  const llm = await ai.getLLM()
   if (!llm) {
     ctx.throw("LLM not available")
   }

From c97f654a81b5fa76174e34fc86aa51f38c81afc2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Apr 2025 12:50:08 +0200
Subject: [PATCH 053/101] Use gpt-4o again

---
 packages/server/src/api/controllers/ai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 92b69668b9..ab4a132c63 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -191,7 +191,7 @@ export async function generateTables(
 ) {
   const { prompt, useCached, addData } = ctx.request.body
 
-  const llm = await ai.getLLM()
+  const llm = await ai.getLLM("gpt-4o")
   if (!llm) {
     ctx.throw("LLM not available")
   }

From 5d784656a4b0a42cc04d056883f73f814745f52a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Apr 2025 14:57:45 +0200
Subject: [PATCH 054/101] Handle wrong attachment links

---
 packages/server/src/api/controllers/ai.ts | 57 +++++++++++++----------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index ab4a132c63..390e60d3fb 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -119,7 +119,9 @@ async function generateDataDelegate(
         }
         for (const attachmentValue of entry[column.name]) {
           const attachment = await downloadFile(attachmentValue)
-          attachmentData[column.name].push(attachment)
+          if (attachment) {
+            attachmentData[column.name].push(attachment)
+          }
         }
 
         if (column.type === FieldType.ATTACHMENT_SINGLE) {
@@ -214,43 +216,48 @@ export async function generateTables(
 
 async function downloadFile(
   file: string | { fileName: string; extension: string; content: string }
-): Promise<Upload> {
+): Promise<Upload | undefined> {
   if (typeof file === "object") {
     return createFile(file)
   }
 
-  const res = await fetch(file)
+  try {
+    const res = await fetch(file)
 
-  const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
+    const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
 
-  if (!fs.existsSync(tmpPath)) {
-    mkdirSync(tmpPath)
-  }
+    if (!fs.existsSync(tmpPath)) {
+      mkdirSync(tmpPath)
+    }
 
-  const extension = [...res.url.split(".")].pop()!.split("?")[0]
+    const extension = [...res.url.split(".")].pop()!.split("?")[0]
 
-  const destination = path.resolve(tmpPath, `${uuid.v4()}${extension}`)
-  const fileStream = fs.createWriteStream(destination, { flags: "wx" })
+    const destination = path.resolve(tmpPath, `${uuid.v4()}${extension}`)
+    const fileStream = fs.createWriteStream(destination, { flags: "wx" })
 
-  await promisify(pipeline)(res.body, fileStream)
+    await promisify(pipeline)(res.body, fileStream)
 
-  const processedFileName = path.basename(destination)
+    const processedFileName = path.basename(destination)
 
-  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+    const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
 
-  const response = await objectStore.upload({
-    bucket: ObjectStoreBuckets.APPS,
-    filename: s3Key,
-    path: destination,
-    type: "image/jpeg",
-  })
+    const response = await objectStore.upload({
+      bucket: ObjectStoreBuckets.APPS,
+      filename: s3Key,
+      path: destination,
+      type: "image/jpeg",
+    })
 
-  return {
-    size: fileStream.bytesWritten,
-    name: processedFileName,
-    url: await objectStore.getAppFileUrl(s3Key),
-    extension,
-    key: response.Key!,
+    return {
+      size: fileStream.bytesWritten,
+      name: processedFileName,
+      url: await objectStore.getAppFileUrl(s3Key),
+      extension,
+      key: response.Key!,
+    }
+  } catch (e) {
+    console.error("Error downloading file", e)
+    return
   }
 }
 

From eaca3919105c3a1268c2e99257079c0bc23d7321 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Apr 2025 15:02:31 +0200
Subject: [PATCH 055/101] Remove default settings

---
 .../data/_components/AITableCreationModal.svelte             | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index 42f8e715db..d5bf7fed7f 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -15,10 +15,9 @@
   let modal: Modal
 
   let modalData: GenerateTablesRequest = {
-    prompt:
-      "I need a ticketing system where users can submit tickets. A ticket has a title, description, priority, status, and an assigned technician",
+    prompt: "",
     addData: true,
-    useCached: $admin.isDev,
+    useCached: $admin.isDev && false,
   }
 
   export function show() {

From dfb5c1d7175b7bcdf46db2e0a031fcc6437eb954 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 10:14:39 +0200
Subject: [PATCH 056/101] Clean

---
 packages/server/src/api/controllers/ai.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 390e60d3fb..d805171c2a 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -52,7 +52,6 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
 
   for (const table of Object.values(data.tables)) {
     for (const field of table.schema.filter(f => f.type === FieldType.LINK)) {
-      // const field =  table.schema[fieldKey] as RelationshipFieldMetadata
       const linkedTable = createdTables.find(t => t.name === field.tableId)
       if (!linkedTable) {
         throw `Table ${field.tableId} not found in the json response.`

From 4e8e725b4b8228bbf2b89a7729dcbb2d0c04065e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 11:01:44 +0200
Subject: [PATCH 057/101] Fix relationship conflicts

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 42ad279dd7..3d037429b0 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 42ad279dd750ac977f5ab9778487e722fea973b4
+Subproject commit 3d037429b09281ed499d91a39fcacb0dd4e27c74
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index d805171c2a..dd6624e422 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -74,7 +74,12 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
       schema: {
         ...storedTable.schema,
         ...table.schema.reduce<TableSchema>((acc, field) => {
-          acc[field.name] = field
+          if (field.type === FieldType.LINK) {
+            const { reverseFieldName, relationshipId, ...rest } = field
+            acc[field.name] = { ...rest, fieldName: reverseFieldName }
+          } else {
+            acc[field.name] = field
+          }
           return acc
         }, {}),
       },

From c0e41686599b7f2959adad2d6ca26df8d9df1cf7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 12:11:57 +0200
Subject: [PATCH 058/101] Avoid duplicates

---
 packages/server/src/api/controllers/ai.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index dd6624e422..4d59365d24 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -66,6 +66,8 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
     }
   }
 
+  const processedRelationships: string[] = []
+
   for (const table of Object.values(data.tables)) {
     const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
@@ -75,8 +77,12 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
         ...storedTable.schema,
         ...table.schema.reduce<TableSchema>((acc, field) => {
           if (field.type === FieldType.LINK) {
-            const { reverseFieldName, relationshipId, ...rest } = field
-            acc[field.name] = { ...rest, fieldName: reverseFieldName }
+            // Avoid circular references
+            if (!processedRelationships.includes(field.relationshipId)) {
+              const { reverseFieldName, relationshipId, ...rest } = field
+              acc[field.name] = { ...rest, fieldName: reverseFieldName }
+              processedRelationships.push(relationshipId)
+            }
           } else {
             acc[field.name] = field
           }

From 80130ba61cef48d6ba21da645453852160cc82ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 12:28:43 +0200
Subject: [PATCH 059/101] Fix wrong generated order

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 13 ++++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 3d037429b0..fd496476d0 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3d037429b09281ed499d91a39fcacb0dd4e27c74
+Subproject commit fd496476d025cccdbbeb2f8e470c461458c0d6ad
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 4d59365d24..6392781e92 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -4,6 +4,7 @@ import {
   FieldType,
   GenerateTablesRequest,
   GenerateTablesResponse,
+  RelationshipType,
   SourceName,
   Table,
   TableSchema,
@@ -79,8 +80,18 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
           if (field.type === FieldType.LINK) {
             // Avoid circular references
             if (!processedRelationships.includes(field.relationshipId)) {
+              // Reversing relationship type, as the name is confusing for the AI gets it in the wrong order
+              const map = {
+                [RelationshipType.MANY_TO_ONE]: RelationshipType.ONE_TO_MANY,
+                [RelationshipType.ONE_TO_MANY]: RelationshipType.MANY_TO_ONE,
+                [RelationshipType.MANY_TO_MANY]: RelationshipType.MANY_TO_MANY,
+              }
               const { reverseFieldName, relationshipId, ...rest } = field
-              acc[field.name] = { ...rest, fieldName: reverseFieldName }
+              acc[field.name] = {
+                ...rest,
+                fieldName: reverseFieldName,
+                relationshipType: map[field.relationshipType] as any,
+              }
               processedRelationships.push(relationshipId)
             }
           } else {

From b8646963b3b7a64a3ae16b49928860955dc60ed7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 13:00:02 +0200
Subject: [PATCH 060/101] Split attachment and image columns

---
 packages/types/src/documents/app/table/constants.ts |  4 ++++
 packages/types/src/documents/app/table/schema.ts    | 11 ++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index 117cfa0eba..fe2e43587a 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -39,6 +39,10 @@ export enum BBReferenceFieldSubType {
   USERS = "users",
 }
 
+export enum AttachmentSubType {
+  IMAGE = "image",
+}
+
 export type SupportedSqlTypes =
   | FieldType.STRING
   | FieldType.BARCODEQR
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index e3f9d3af3f..c29c8af505 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -2,6 +2,7 @@
 // column size, position and whether it can be viewed
 import { FieldType, FormulaResponseType } from "../row"
 import {
+  AttachmentSubType,
   AutoFieldSubType,
   AutoReason,
   BBReferenceFieldSubType,
@@ -142,8 +143,15 @@ export interface BBReferenceSingleFieldMetadata
   default?: string
 }
 
-export interface AttachmentFieldMetadata extends BaseFieldSchema {
+export interface AttachmentFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.ATTACHMENTS
+  subtype?: AttachmentSubType
+}
+export interface SingleAttachmentFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.ATTACHMENT_SINGLE
+  subtype?: AttachmentSubType
 }
 
 export interface FieldConstraints {
@@ -246,6 +254,7 @@ export type FieldSchema =
   | BBReferenceFieldMetadata
   | JsonFieldMetadata
   | AttachmentFieldMetadata
+  | SingleAttachmentFieldMetadata
   | BBReferenceSingleFieldMetadata
   | ArrayFieldMetadata
   | OptionsFieldMetadata

From 020b58dc55cb0926e7ac4ce8570c2531880e71fd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Apr 2025 13:26:04 +0200
Subject: [PATCH 061/101] Fix data relationship handling

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/packages/pro b/packages/pro
index fd496476d0..2ecb23a6b5 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit fd496476d025cccdbbeb2f8e470c461458c0d6ad
+Subproject commit 2ecb23a6b5f0dee64646a1477fe6fd7ccd9d15a9
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 6392781e92..3964bf6b8d 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -116,7 +116,7 @@ async function generateDataDelegate(
   const toUpdateLinks: {
     tableId: string
     rowId: string
-    data: Record<string, { rowId: string; tableId: string }>
+    data: Record<string, { rowId: string[]; tableId: string }>
   }[] = []
   for (const tableName of Object.keys(data)) {
     const table = tables[tableName]
@@ -166,7 +166,7 @@ async function generateDataDelegate(
       createdData[tableId][entry._id] = createdRow.row._id!
 
       const overridenLinks = Object.keys(linksOverride).reduce<
-        Record<string, { rowId: string; tableId: string }>
+        Record<string, { rowId: string[]; tableId: string }>
       >((acc, l) => {
         if (entry[l]) {
           acc[l] = {
@@ -190,9 +190,14 @@ async function generateDataDelegate(
   for (const data of toUpdateLinks) {
     const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
 
-    const updatedLinks = Object.keys(data.data).reduce<Record<string, string>>(
+    const updatedLinks = Object.keys(data.data).reduce<Record<string, any>>(
       (acc, d) => {
-        acc[d] = createdData[data.data[d].tableId][data.data[d].rowId]
+        acc[d] = [
+          ...(persistedRow[d] || []),
+          ...data.data[d].rowId.map(
+            rid => createdData[data.data[d].tableId][rid]
+          ),
+        ]
         return acc
       },
       {}

From 95c3844410c637b7268447fde573f6cec2f6d8de Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 10:54:19 +0200
Subject: [PATCH 062/101] Move logic outside the controller

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 57 +++++++----------------
 2 files changed, 17 insertions(+), 42 deletions(-)

diff --git a/packages/pro b/packages/pro
index 2ecb23a6b5..8f89c2878f 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 2ecb23a6b5f0dee64646a1477fe6fd7ccd9d15a9
+Subproject commit 8f89c2878f32c09c7e007faeb29396df50acd125
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 3964bf6b8d..fc9e45d308 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -4,7 +4,6 @@ import {
   FieldType,
   GenerateTablesRequest,
   GenerateTablesResponse,
-  RelationshipType,
   SourceName,
   Table,
   TableSchema,
@@ -23,7 +22,9 @@ import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
 
-async function generateTablesDelegate(data: ai.GenerationStructure) {
+async function generateTablesDelegate(
+  tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
+) {
   const count = (await sdk.datasources.fetch()).length
   const { id: dsId } = await context.getAppDB().put({
     _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
@@ -37,7 +38,7 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
-  for (const table of data.tables) {
+  for (const table of tables) {
     const createdTable = await sdk.tables.create({
       ...table,
       sourceId: dsId,
@@ -51,54 +52,28 @@ async function generateTablesDelegate(data: ai.GenerationStructure) {
     tableIds[table.name] = createdTable._id!
   }
 
-  for (const table of Object.values(data.tables)) {
-    for (const field of table.schema.filter(f => f.type === FieldType.LINK)) {
-      const linkedTable = createdTables.find(t => t.name === field.tableId)
-      if (!linkedTable) {
-        throw `Table ${field.tableId} not found in the json response.`
+  for (const table of tables) {
+    for (const field of Object.values(table.schema)) {
+      if (field.type === FieldType.LINK) {
+        const linkedTable = createdTables.find(t => t.name === field.tableId)
+        if (!linkedTable) {
+          throw `Table ${field.tableId} not found in the json response.`
+        }
+        field.tableId = linkedTable.id
+      } else if (field.type === FieldType.FORMULA) {
+        field.formula = `{{ js "${btoa(field.formula)}" }}`
       }
-      field.tableId = linkedTable.id
-    }
-
-    for (const field of table.schema.filter(
-      f => f.type === FieldType.FORMULA
-    )) {
-      field.formula = `{{ js "${btoa(field.formula)}" }}`
     }
   }
 
-  const processedRelationships: string[] = []
-
-  for (const table of Object.values(data.tables)) {
+  for (const table of tables) {
     const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
     await sdk.tables.update({
       ...storedTable,
       schema: {
         ...storedTable.schema,
-        ...table.schema.reduce<TableSchema>((acc, field) => {
-          if (field.type === FieldType.LINK) {
-            // Avoid circular references
-            if (!processedRelationships.includes(field.relationshipId)) {
-              // Reversing relationship type, as the name is confusing for the AI gets it in the wrong order
-              const map = {
-                [RelationshipType.MANY_TO_ONE]: RelationshipType.ONE_TO_MANY,
-                [RelationshipType.ONE_TO_MANY]: RelationshipType.MANY_TO_ONE,
-                [RelationshipType.MANY_TO_MANY]: RelationshipType.MANY_TO_MANY,
-              }
-              const { reverseFieldName, relationshipId, ...rest } = field
-              acc[field.name] = {
-                ...rest,
-                fieldName: reverseFieldName,
-                relationshipType: map[field.relationshipType] as any,
-              }
-              processedRelationships.push(relationshipId)
-            }
-          } else {
-            acc[field.name] = field
-          }
-          return acc
-        }, {}),
+        ...table.schema,
       },
       primaryDisplay: table.primaryDisplay,
     })

From 03be8b64590a9ed7182ee7a4c0ebf1c13a36edc0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 12:41:13 +0200
Subject: [PATCH 063/101] Improve AI

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 8f89c2878f..3a6342a386 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 8f89c2878f32c09c7e007faeb29396df50acd125
+Subproject commit 3a6342a38634e6bdc638a8cff4271701002d5459

From 0c6bf9de0e711df9b662feea8c40813cde97073a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 12:57:57 +0200
Subject: [PATCH 064/101] Remove caching mechanism

---
 .../data/_components/AITableCreationModal.svelte             | 5 -----
 packages/pro                                                 | 2 +-
 packages/server/src/api/controllers/ai.ts                    | 3 +--
 packages/types/src/api/web/ai.ts                             | 1 -
 4 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
index d5bf7fed7f..8a4b8ba280 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
@@ -1,7 +1,6 @@
 <script lang="ts">
   import { API } from "@/api"
   import { datasources, tables } from "@/stores/builder"
-  import { admin } from "@/stores/portal"
   import {
     Modal,
     ModalContent,
@@ -17,7 +16,6 @@
   let modalData: GenerateTablesRequest = {
     prompt: "",
     addData: true,
-    useCached: $admin.isDev && false,
   }
 
   export function show() {
@@ -52,8 +50,5 @@
   >
     <TextArea label="Prompt" bind:value={modalData.prompt} />
     <Toggle label="Add data" bind:value={modalData.addData} />
-    {#if $admin.isDev}
-      <Toggle label="Use cached" bind:value={modalData.useCached} />
-    {/if}
   </ModalContent>
 </Modal>
diff --git a/packages/pro b/packages/pro
index 3a6342a386..329d8c079e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3a6342a38634e6bdc638a8cff4271701002d5459
+Subproject commit 329d8c079ed3cc829475faade2b8ccc0c0ce63b6
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index fc9e45d308..bb114fe90c 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -192,7 +192,7 @@ async function generateDataDelegate(
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
 ) {
-  const { prompt, useCached, addData } = ctx.request.body
+  const { prompt, addData } = ctx.request.body
 
   const llm = await ai.getLLM("gpt-4o")
   if (!llm) {
@@ -202,7 +202,6 @@ export async function generateTables(
 
   const createdTables = await llm!.generateTables(
     prompt,
-    useCached,
     addData,
     ctx.userId,
     generateTablesDelegate,
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 392d849e32..61123f1ecf 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -33,7 +33,6 @@ export interface GenerateCronResponse {
 export interface GenerateTablesRequest {
   prompt: string
   addData: boolean
-  useCached: boolean
 }
 
 export interface GenerateTablesResponse {

From 4b4c2abfdc969ce7f6486f86baf1cffb7072d58d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 13:24:02 +0200
Subject: [PATCH 065/101] Clean

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 329d8c079e..fc5270482d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 329d8c079ed3cc829475faade2b8ccc0c0ce63b6
+Subproject commit fc5270482d60c1ad636ba8d63ee70b56ccf521e2
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index bb114fe90c..e939d191bf 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -200,7 +200,7 @@ export async function generateTables(
   }
   llm.maxTokens = 2000
 
-  const createdTables = await llm!.generateTables(
+  const createdTables = await llm.generateTables(
     prompt,
     addData,
     ctx.userId,

From ac357c1f3908e6babeeed1a9180346432e18a91b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 14:26:37 +0200
Subject: [PATCH 066/101] Move code to generator

---
 packages/pro                              |  2 +-
 packages/server/src/api/controllers/ai.ts | 22 +++++++++-------------
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/packages/pro b/packages/pro
index fc5270482d..1e63fa32ff 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit fc5270482d60c1ad636ba8d63ee70b56ccf521e2
+Subproject commit 1e63fa32ff9635356c1fafd803b6f1e53f282d7d
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index e939d191bf..19e7d993df 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -194,20 +194,16 @@ export async function generateTables(
 ) {
   const { prompt, addData } = ctx.request.body
 
-  const llm = await ai.getLLM("gpt-4o")
-  if (!llm) {
-    ctx.throw("LLM not available")
-  }
-  llm.maxTokens = 2000
-
-  const createdTables = await llm.generateTables(
-    prompt,
-    addData,
-    ctx.userId,
+  const tableGenerator = new ai.TableGeneration({
     generateTablesDelegate,
-    sdk.tables.getTables,
-    generateDataDelegate
-  )
+    getTablesDelegate: sdk.tables.getTables,
+    generateDataDelegate,
+  })
+  const createdTables = await tableGenerator.generate({
+    userPrompt: prompt,
+    addData,
+    userId: ctx.userId,
+  })
 
   ctx.body = {
     createdTables,

From e480ac05c32a6e603ded010baad97f16b1bc1dd9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 16:07:08 +0200
Subject: [PATCH 067/101] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 9108e85411..ea26aaa992 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 9108e85411fd1b6ba190ff178500637d7e47d759
+Subproject commit ea26aaa992c03702bb8001d4188ec89e7b64f59e

From 3f4edd2cafe5a3de6fd86082d3b6c590bcf7ae8a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 16:35:58 +0200
Subject: [PATCH 068/101] Fix build

---
 packages/types/src/documents/app/table/constants.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index fe2e43587a..3eeb852b4e 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -14,6 +14,7 @@ export type FieldSubType =
   | AutoFieldSubType
   | JsonFieldSubType
   | BBReferenceFieldSubType
+  | AttachmentSubType
 
 export enum AutoFieldSubType {
   CREATED_BY = "createdBy",

From cd244c02c3bbf10838a1fd0dd37dcf9b84a12a97 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 16:55:11 +0200
Subject: [PATCH 069/101] Extract

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/packages/pro b/packages/pro
index 35bdf46cf2..7d637cdb56 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 35bdf46cf26a1056b6771b938fb848755312f447
+Subproject commit 7d637cdb569623909df02a56d0b6e308472c8172
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 19e7d993df..184821c9e2 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -199,11 +199,10 @@ export async function generateTables(
     getTablesDelegate: sdk.tables.getTables,
     generateDataDelegate,
   })
-  const createdTables = await tableGenerator.generate({
-    userPrompt: prompt,
-    addData,
-    userId: ctx.userId,
-  })
+  if (addData) {
+    tableGenerator.withData(ctx.user._id || "")
+  }
+  const createdTables = await tableGenerator.generate(prompt)
 
   ctx.body = {
     createdTables,

From c8fc01fa0c3b3d398f1123f46ee3c562d023dd87 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 17:05:19 +0200
Subject: [PATCH 070/101] Extract utils from controller

---
 packages/server/src/api/controllers/ai.ts  | 90 ++--------------------
 packages/server/src/utilities/fileUtils.ts | 85 ++++++++++++++++++++
 packages/server/src/utilities/index.ts     |  1 +
 3 files changed, 94 insertions(+), 82 deletions(-)
 create mode 100644 packages/server/src/utilities/fileUtils.ts

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 184821c9e2..926546a9a5 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -21,6 +21,8 @@ import { pipeline } from "stream"
 import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
+import { uploadUrl } from "../../utilities"
+import { uploadFile } from "../../utilities/fileUtils"
 
 async function generateTablesDelegate(
   tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
@@ -114,7 +116,12 @@ async function generateDataDelegate(
           entry[column.name] = [entry[column.name]]
         }
         for (const attachmentValue of entry[column.name]) {
-          const attachment = await downloadFile(attachmentValue)
+          let attachment
+          if (typeof attachmentValue === "object") {
+            attachment = await uploadUrl(attachmentValue)
+          } else {
+            attachment = await uploadFile(attachmentValue)
+          }
           if (attachment) {
             attachmentData[column.name].push(attachment)
           }
@@ -208,84 +215,3 @@ export async function generateTables(
     createdTables,
   }
 }
-
-async function downloadFile(
-  file: string | { fileName: string; extension: string; content: string }
-): Promise<Upload | undefined> {
-  if (typeof file === "object") {
-    return createFile(file)
-  }
-
-  try {
-    const res = await fetch(file)
-
-    const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
-
-    if (!fs.existsSync(tmpPath)) {
-      mkdirSync(tmpPath)
-    }
-
-    const extension = [...res.url.split(".")].pop()!.split("?")[0]
-
-    const destination = path.resolve(tmpPath, `${uuid.v4()}${extension}`)
-    const fileStream = fs.createWriteStream(destination, { flags: "wx" })
-
-    await promisify(pipeline)(res.body, fileStream)
-
-    const processedFileName = path.basename(destination)
-
-    const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
-
-    const response = await objectStore.upload({
-      bucket: ObjectStoreBuckets.APPS,
-      filename: s3Key,
-      path: destination,
-      type: "image/jpeg",
-    })
-
-    return {
-      size: fileStream.bytesWritten,
-      name: processedFileName,
-      url: await objectStore.getAppFileUrl(s3Key),
-      extension,
-      key: response.Key!,
-    }
-  } catch (e) {
-    console.error("Error downloading file", e)
-    return
-  }
-}
-
-async function createFile(file: {
-  fileName: string
-  extension: string
-  content: string
-}): Promise<Upload> {
-  const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
-
-  if (!fs.existsSync(tmpPath)) {
-    mkdirSync(tmpPath)
-  }
-
-  const destination = path.resolve(tmpPath, `${file.fileName}${file.extension}`)
-
-  fs.writeFileSync(destination, file.content)
-
-  const processedFileName = path.basename(destination)
-  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
-
-  const response = await objectStore.upload({
-    bucket: ObjectStoreBuckets.APPS,
-    filename: s3Key,
-    path: destination,
-    type: "text/plain",
-  })
-
-  return {
-    size: fs.readFileSync(destination).byteLength,
-    name: processedFileName,
-    url: await objectStore.getAppFileUrl(s3Key),
-    extension: file.extension,
-    key: response.Key!,
-  }
-}
diff --git a/packages/server/src/utilities/fileUtils.ts b/packages/server/src/utilities/fileUtils.ts
new file mode 100644
index 0000000000..d64e4db57b
--- /dev/null
+++ b/packages/server/src/utilities/fileUtils.ts
@@ -0,0 +1,85 @@
+import fs from "fs"
+import path from "path"
+import { pipeline } from "stream"
+import { promisify } from "util"
+import * as uuid from "uuid"
+import fetch from "node-fetch"
+
+import { context, objectStore } from "@budibase/backend-core"
+import { Upload } from "@budibase/types"
+import { ObjectStoreBuckets } from "../constants"
+
+function getTmpPath() {
+  const tmpPath = path.join(objectStore.budibaseTempDir(), "ai-downloads")
+  if (!fs.existsSync(tmpPath)) {
+    fs.mkdirSync(tmpPath)
+  }
+
+  return tmpPath
+}
+
+export async function uploadUrl(url: string): Promise<Upload | undefined> {
+  try {
+    const res = await fetch(url)
+
+    const extension = [...res.url.split(".")].pop()!.split("?")[0]
+
+    const destination = path.resolve(getTmpPath(), `${uuid.v4()}${extension}`)
+    const fileStream = fs.createWriteStream(destination, { flags: "wx" })
+
+    await promisify(pipeline)(res.body, fileStream)
+
+    const processedFileName = path.basename(destination)
+
+    const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+    const response = await objectStore.upload({
+      bucket: ObjectStoreBuckets.APPS,
+      filename: s3Key,
+      path: destination,
+      type: "image/jpeg",
+    })
+
+    return {
+      size: fileStream.bytesWritten,
+      name: processedFileName,
+      url: await objectStore.getAppFileUrl(s3Key),
+      extension,
+      key: response.Key!,
+    }
+  } catch (e) {
+    console.error("Error downloading file", e)
+    return
+  }
+}
+
+export async function uploadFile(file: {
+  fileName: string
+  extension: string
+  content: string
+}): Promise<Upload> {
+  const destination = path.resolve(
+    getTmpPath(),
+    `${file.fileName}${file.extension}`
+  )
+
+  fs.writeFileSync(destination, file.content)
+
+  const processedFileName = path.basename(destination)
+  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+  const response = await objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: s3Key,
+    path: destination,
+    type: "text/plain",
+  })
+
+  return {
+    size: fs.readFileSync(destination).byteLength,
+    name: processedFileName,
+    url: await objectStore.getAppFileUrl(s3Key),
+    extension: file.extension,
+    key: response.Key!,
+  }
+}
diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts
index f1b32c81f3..19a44b79ce 100644
--- a/packages/server/src/utilities/index.ts
+++ b/packages/server/src/utilities/index.ts
@@ -1,3 +1,4 @@
+export * from "./fileUtils"
 import env from "../environment"
 import { context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"

From 7f41d085f7bd034cf3bd09e44b41d4633fb77fef Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 17:16:24 +0200
Subject: [PATCH 071/101] Extract core from controller

---
 packages/server/src/api/controllers/ai.ts     | 194 +-----------------
 .../server/src/sdk/app/ai/helpers/index.ts    |   2 +
 .../server/src/sdk/app/ai/helpers/rows.ts     | 115 +++++++++++
 .../server/src/sdk/app/ai/helpers/table.ts    |  70 +++++++
 packages/server/src/sdk/app/ai/index.ts       |   1 +
 packages/server/src/sdk/index.ts              |   2 +
 6 files changed, 192 insertions(+), 192 deletions(-)
 create mode 100644 packages/server/src/sdk/app/ai/helpers/index.ts
 create mode 100644 packages/server/src/sdk/app/ai/helpers/rows.ts
 create mode 100644 packages/server/src/sdk/app/ai/helpers/table.ts
 create mode 100644 packages/server/src/sdk/app/ai/index.ts

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 926546a9a5..173a9e8fc4 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -1,200 +1,10 @@
-import * as uuid from "uuid"
 import {
-  DocumentType,
-  FieldType,
   GenerateTablesRequest,
   GenerateTablesResponse,
-  SourceName,
-  Table,
-  TableSchema,
-  TableSourceType,
-  Upload,
   UserCtx,
 } from "@budibase/types"
 import { ai } from "@budibase/pro"
-import { context, objectStore, utils } from "@budibase/backend-core"
 import sdk from "../../sdk"
-import fs, { mkdirSync } from "fs"
-import path, { join } from "path"
-
-import { pipeline } from "stream"
-import { promisify } from "util"
-import fetch from "node-fetch"
-import { ObjectStoreBuckets } from "../../constants"
-import { uploadUrl } from "../../utilities"
-import { uploadFile } from "../../utilities/fileUtils"
-
-async function generateTablesDelegate(
-  tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
-) {
-  const count = (await sdk.datasources.fetch()).length
-  const { id: dsId } = await context.getAppDB().put({
-    _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
-    name: `Test ${count}`,
-    type: "budibase",
-
-    source: SourceName.BUDIBASE,
-    config: {},
-  })
-
-  const createdTables: GenerateTablesResponse["createdTables"] = []
-  const tableIds: Record<string, string> = {}
-
-  for (const table of tables) {
-    const createdTable = await sdk.tables.create({
-      ...table,
-      sourceId: dsId,
-      schema: {},
-      primaryDisplay: undefined,
-      sourceType: TableSourceType.INTERNAL,
-      type: "table",
-    })
-
-    createdTables.push({ id: createdTable._id!, name: createdTable.name })
-    tableIds[table.name] = createdTable._id!
-  }
-
-  for (const table of tables) {
-    for (const field of Object.values(table.schema)) {
-      if (field.type === FieldType.LINK) {
-        const linkedTable = createdTables.find(t => t.name === field.tableId)
-        if (!linkedTable) {
-          throw `Table ${field.tableId} not found in the json response.`
-        }
-        field.tableId = linkedTable.id
-      } else if (field.type === FieldType.FORMULA) {
-        field.formula = `{{ js "${btoa(field.formula)}" }}`
-      }
-    }
-  }
-
-  for (const table of tables) {
-    const storedTable = await sdk.tables.getTable(tableIds[table.name])
-
-    await sdk.tables.update({
-      ...storedTable,
-      schema: {
-        ...storedTable.schema,
-        ...table.schema,
-      },
-      primaryDisplay: table.primaryDisplay,
-    })
-  }
-
-  return createdTables
-}
-
-async function generateDataDelegate(
-  data: Record<string, Record<string, any>[]>,
-  userId: string,
-  tables: Record<string, Table>
-) {
-  const createdData: Record<string, Record<string, string>> = {}
-  const toUpdateLinks: {
-    tableId: string
-    rowId: string
-    data: Record<string, { rowId: string[]; tableId: string }>
-  }[] = []
-  for (const tableName of Object.keys(data)) {
-    const table = tables[tableName]
-    const linksOverride: Record<string, null> = {}
-    for (const field of Object.values(table.schema).filter(
-      f => f.type === FieldType.LINK
-    )) {
-      linksOverride[field.name] = null
-    }
-
-    const attachmentColumns = Object.values(table.schema).filter(f =>
-      [FieldType.ATTACHMENTS, FieldType.ATTACHMENT_SINGLE].includes(f.type)
-    )
-
-    for (const entry of data[tableName]) {
-      const attachmentData: Record<string, any> = {}
-      for (const column of attachmentColumns) {
-        attachmentData[column.name] = []
-        if (!Array.isArray(entry[column.name])) {
-          entry[column.name] = [entry[column.name]]
-        }
-        for (const attachmentValue of entry[column.name]) {
-          let attachment
-          if (typeof attachmentValue === "object") {
-            attachment = await uploadUrl(attachmentValue)
-          } else {
-            attachment = await uploadFile(attachmentValue)
-          }
-          if (attachment) {
-            attachmentData[column.name].push(attachment)
-          }
-        }
-
-        if (column.type === FieldType.ATTACHMENT_SINGLE) {
-          attachmentData[column.name] = attachmentData[column.name][0]
-        }
-      }
-
-      const tableId = tables[tableName]._id!
-      const createdRow = await sdk.rows.save(
-        tableId,
-        {
-          ...entry,
-          ...linksOverride,
-          ...attachmentData,
-          _id: undefined,
-        },
-        userId
-      )
-
-      createdData[tableId] ??= {}
-      createdData[tableId][entry._id] = createdRow.row._id!
-
-      const overridenLinks = Object.keys(linksOverride).reduce<
-        Record<string, { rowId: string[]; tableId: string }>
-      >((acc, l) => {
-        if (entry[l]) {
-          acc[l] = {
-            tableId: (table.schema[l] as any).tableId,
-            rowId: entry[l],
-          }
-        }
-        return acc
-      }, {})
-
-      if (Object.keys(overridenLinks).length) {
-        toUpdateLinks.push({
-          tableId: createdRow.table._id!,
-          rowId: createdRow.row._id!,
-          data: overridenLinks,
-        })
-      }
-    }
-  }
-
-  for (const data of toUpdateLinks) {
-    const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
-
-    const updatedLinks = Object.keys(data.data).reduce<Record<string, any>>(
-      (acc, d) => {
-        acc[d] = [
-          ...(persistedRow[d] || []),
-          ...data.data[d].rowId.map(
-            rid => createdData[data.data[d].tableId][rid]
-          ),
-        ]
-        return acc
-      },
-      {}
-    )
-
-    await sdk.rows.save(
-      data.tableId,
-      {
-        ...persistedRow,
-        ...updatedLinks,
-      },
-      userId
-    )
-  }
-}
 
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
@@ -202,9 +12,9 @@ export async function generateTables(
   const { prompt, addData } = ctx.request.body
 
   const tableGenerator = new ai.TableGeneration({
-    generateTablesDelegate,
+    generateTablesDelegate: sdk.ai.helpers.generateTables,
     getTablesDelegate: sdk.tables.getTables,
-    generateDataDelegate,
+    generateDataDelegate: sdk.ai.helpers.generateRows,
   })
   if (addData) {
     tableGenerator.withData(ctx.user._id || "")
diff --git a/packages/server/src/sdk/app/ai/helpers/index.ts b/packages/server/src/sdk/app/ai/helpers/index.ts
new file mode 100644
index 0000000000..396c3a24f5
--- /dev/null
+++ b/packages/server/src/sdk/app/ai/helpers/index.ts
@@ -0,0 +1,2 @@
+export * from "./table"
+export * from "./rows"
diff --git a/packages/server/src/sdk/app/ai/helpers/rows.ts b/packages/server/src/sdk/app/ai/helpers/rows.ts
new file mode 100644
index 0000000000..4bc1601e41
--- /dev/null
+++ b/packages/server/src/sdk/app/ai/helpers/rows.ts
@@ -0,0 +1,115 @@
+import { FieldType, Table } from "@budibase/types"
+import sdk from "../../.."
+import { uploadFile, uploadUrl } from "../../../../utilities"
+
+export async function generateRows(
+  data: Record<string, Record<string, any>[]>,
+  userId: string,
+  tables: Record<string, Table>
+) {
+  const createdData: Record<string, Record<string, string>> = {}
+  const toUpdateLinks: {
+    tableId: string
+    rowId: string
+    data: Record<string, { rowId: string[]; tableId: string }>
+  }[] = []
+  for (const tableName of Object.keys(data)) {
+    const table = tables[tableName]
+    const linksOverride: Record<string, null> = {}
+    for (const field of Object.values(table.schema).filter(
+      f => f.type === FieldType.LINK
+    )) {
+      linksOverride[field.name] = null
+    }
+
+    const attachmentColumns = Object.values(table.schema).filter(f =>
+      [FieldType.ATTACHMENTS, FieldType.ATTACHMENT_SINGLE].includes(f.type)
+    )
+
+    for (const entry of data[tableName]) {
+      const attachmentData: Record<string, any> = {}
+      for (const column of attachmentColumns) {
+        attachmentData[column.name] = []
+        if (!Array.isArray(entry[column.name])) {
+          entry[column.name] = [entry[column.name]]
+        }
+        for (const attachmentValue of entry[column.name]) {
+          let attachment
+          if (typeof attachmentValue === "object") {
+            attachment = await uploadUrl(attachmentValue)
+          } else {
+            attachment = await uploadFile(attachmentValue)
+          }
+          if (attachment) {
+            attachmentData[column.name].push(attachment)
+          }
+        }
+
+        if (column.type === FieldType.ATTACHMENT_SINGLE) {
+          attachmentData[column.name] = attachmentData[column.name][0]
+        }
+      }
+
+      const tableId = tables[tableName]._id!
+      const createdRow = await sdk.rows.save(
+        tableId,
+        {
+          ...entry,
+          ...linksOverride,
+          ...attachmentData,
+          _id: undefined,
+        },
+        userId
+      )
+
+      createdData[tableId] ??= {}
+      createdData[tableId][entry._id] = createdRow.row._id!
+
+      const overridenLinks = Object.keys(linksOverride).reduce<
+        Record<string, { rowId: string[]; tableId: string }>
+      >((acc, l) => {
+        if (entry[l]) {
+          acc[l] = {
+            tableId: (table.schema[l] as any).tableId,
+            rowId: entry[l],
+          }
+        }
+        return acc
+      }, {})
+
+      if (Object.keys(overridenLinks).length) {
+        toUpdateLinks.push({
+          tableId: createdRow.table._id!,
+          rowId: createdRow.row._id!,
+          data: overridenLinks,
+        })
+      }
+    }
+  }
+
+  for (const data of toUpdateLinks) {
+    const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
+
+    const updatedLinks = Object.keys(data.data).reduce<Record<string, any>>(
+      (acc, d) => {
+        acc[d] = [
+          ...(persistedRow[d] || []),
+          ...data.data[d].rowId.map(
+            rid => createdData[data.data[d].tableId][rid]
+          ),
+        ]
+        return acc
+      },
+      {}
+    )
+
+    await sdk.rows.save(
+      data.tableId,
+      {
+        ...persistedRow,
+        ...updatedLinks,
+      },
+      userId
+    )
+  }
+}
diff --git a/packages/server/src/sdk/app/ai/helpers/table.ts b/packages/server/src/sdk/app/ai/helpers/table.ts
new file mode 100644
index 0000000000..4e6c5b7c70
--- /dev/null
+++ b/packages/server/src/sdk/app/ai/helpers/table.ts
@@ -0,0 +1,70 @@
+import {
+  DocumentType,
+  FieldType,
+  GenerateTablesResponse,
+  SourceName,
+  TableSchema,
+  TableSourceType,
+} from "@budibase/types"
+import sdk from "../../.."
+import { context, utils } from "@budibase/backend-core"
+
+export async function generateTables(
+  tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
+) {
+  const count = (await sdk.datasources.fetch()).length
+  const { id: dsId } = await context.getAppDB().put({
+    _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
+    name: `Test ${count}`,
+    type: "budibase",
+
+    source: SourceName.BUDIBASE,
+    config: {},
+  })
+
+  const createdTables: GenerateTablesResponse["createdTables"] = []
+  const tableIds: Record<string, string> = {}
+
+  for (const table of tables) {
+    const createdTable = await sdk.tables.create({
+      ...table,
+      sourceId: dsId,
+      schema: {},
+      primaryDisplay: undefined,
+      sourceType: TableSourceType.INTERNAL,
+      type: "table",
+    })
+
+    createdTables.push({ id: createdTable._id!, name: createdTable.name })
+    tableIds[table.name] = createdTable._id!
+  }
+
+  for (const table of tables) {
+    for (const field of Object.values(table.schema)) {
+      if (field.type === FieldType.LINK) {
+        const linkedTable = createdTables.find(t => t.name === field.tableId)
+        if (!linkedTable) {
+          throw `Table ${field.tableId} not found in the json response.`
+        }
+        field.tableId = linkedTable.id
+      } else if (field.type === FieldType.FORMULA) {
+        field.formula = `{{ js "${btoa(field.formula)}" }}`
+      }
+    }
+  }
+
+  for (const table of tables) {
+    const storedTable = await sdk.tables.getTable(tableIds[table.name])
+
+    await sdk.tables.update({
+      ...storedTable,
+      schema: {
+        ...storedTable.schema,
+        ...table.schema,
+      },
+      primaryDisplay: table.primaryDisplay,
+    })
+  }
+
+  return createdTables
+}
diff --git a/packages/server/src/sdk/app/ai/index.ts b/packages/server/src/sdk/app/ai/index.ts
new file mode 100644
index 0000000000..e315c1ceea
--- /dev/null
+++ b/packages/server/src/sdk/app/ai/index.ts
@@ -0,0 +1 @@
+export * as helpers from "./helpers"
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
index 5a4c20ff45..a994e899be 100644
--- a/packages/server/src/sdk/index.ts
+++ b/packages/server/src/sdk/index.ts
@@ -14,6 +14,7 @@ import * as rowActions from "./app/rowActions"
 import * as screens from "./app/screens"
 import * as common from "./app/common"
 import * as oauth2 from "./app/oauth2"
+import * as ai from "./app/ai"
 
 const sdk = {
   backups,
@@ -32,6 +33,7 @@ const sdk = {
   rowActions,
   common,
   oauth2,
+  ai,
 }
 
 // default export for TS

From eaa3c410d1a4fd34c948c05ca32acf66323f8582 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 17:20:02 +0200
Subject: [PATCH 072/101] Add comments

---
 packages/backend-core/src/sql/utils.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 144a2f39ab..cc078ada0e 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -47,6 +47,7 @@ export function isExternalTable(table: Table) {
   if (
     table?.sourceId &&
     table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    // TODO handle properly when the product part is defined
     !table.sourceId.startsWith(
       DocumentType.DATASOURCE + SEPARATOR + "bb_internal_"
     ) &&

From d15881edd36cb606b6e0c0980094b67c88e54625 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 17:31:04 +0200
Subject: [PATCH 073/101] Fix

---
 packages/server/src/sdk/app/ai/helpers/rows.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/ai/helpers/rows.ts b/packages/server/src/sdk/app/ai/helpers/rows.ts
index 4bc1601e41..be7dcef3f0 100644
--- a/packages/server/src/sdk/app/ai/helpers/rows.ts
+++ b/packages/server/src/sdk/app/ai/helpers/rows.ts
@@ -36,9 +36,9 @@ export async function generateRows(
         for (const attachmentValue of entry[column.name]) {
           let attachment
           if (typeof attachmentValue === "object") {
-            attachment = await uploadUrl(attachmentValue)
-          } else {
             attachment = await uploadFile(attachmentValue)
+          } else {
+            attachment = await uploadUrl(attachmentValue)
           }
           if (attachment) {
             attachmentData[column.name].push(attachment)

From c3652cf5f7d3bad7cc110fea23a06503a4454b40 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Apr 2025 17:37:11 +0200
Subject: [PATCH 074/101] Fix undefined reference

---
 .../data/_components/CreateInternalTableModal.svelte            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateInternalTableModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateInternalTableModal.svelte
index 6fee783363..a6080bf685 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateInternalTableModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateInternalTableModal.svelte
@@ -6,7 +6,7 @@
   let modal
   let promptUpload = false
 
-  export function show({ promptUpload: newPromptUpload = false }) {
+  export function show({ promptUpload: newPromptUpload = false } = {}) {
     promptUpload = newPromptUpload
     modal.show()
   }

From 148f429a1978b077d6e90948c0d03a442f90dcd7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 12:40:17 +0200
Subject: [PATCH 075/101] Use init

---
 packages/pro                              | 2 +-
 packages/server/src/api/controllers/ai.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 764069afd1..b6c62d7629 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 764069afd1dbb99cde73c6ad6ba550cd77398d7d
+Subproject commit b6c62d7629af20efe1f91b09eac6abdd123fbb53
diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 173a9e8fc4..617f63c629 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -11,7 +11,7 @@ export async function generateTables(
 ) {
   const { prompt, addData } = ctx.request.body
 
-  const tableGenerator = new ai.TableGeneration({
+  const tableGenerator = await ai.TableGeneration.init({
     generateTablesDelegate: sdk.ai.helpers.generateTables,
     getTablesDelegate: sdk.tables.getTables,
     generateDataDelegate: sdk.ai.helpers.generateRows,

From adcfff90a96107a479eb62b7267168b2c5ca5492 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 12:41:32 +0200
Subject: [PATCH 076/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index b6c62d7629..d83ce2f49a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit b6c62d7629af20efe1f91b09eac6abdd123fbb53
+Subproject commit d83ce2f49a8883e64ca08ffd2c34787804a11fc1

From 79eb796c8f445e7d37f79b4ad0d61df481cfea01 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 12:44:07 +0200
Subject: [PATCH 077/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index d83ce2f49a..2a28e8bad5 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d83ce2f49a8883e64ca08ffd2c34787804a11fc1
+Subproject commit 2a28e8bad597ccc8ca73958e2ef530b751072992

From 7d8d5d604be784c372ea3ff651e1497c7a54cbce Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 13:48:50 +0200
Subject: [PATCH 078/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 2a28e8bad5..ae229305f8 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 2a28e8bad597ccc8ca73958e2ef530b751072992
+Subproject commit ae229305f8a9d6e3ac23bf345e01f47e4baf51d8

From b381d1221e36155402d8e6e3b1af0f82eb30da19 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 14:59:13 +0200
Subject: [PATCH 079/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ae229305f8..a1cbcdc61b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ae229305f8a9d6e3ac23bf345e01f47e4baf51d8
+Subproject commit a1cbcdc61b9831715f7eb991f9154df0274bcdd0

From 0cacae8fdb25567e83afd09f3cdc44b86d34c415 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Apr 2025 17:39:24 +0200
Subject: [PATCH 080/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 61ab80f72c..1f25a2e3b2 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 61ab80f72c5402ada70cf82b8f7a30b86ef38b2e
+Subproject commit 1f25a2e3b2564d0a619ae3260374fdbda86a73ab

From 44bde43f98b71335ed68e0021ccada896ff5dde2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 17 Apr 2025 15:59:29 +0200
Subject: [PATCH 081/101] Implement validations

---
 packages/pro                     | 2 +-
 packages/types/src/api/web/ai.ts | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 3efda0bc39..e54b201e30 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3efda0bc39ded6c43f7eb7492dcb287d92bf1475
+Subproject commit e54b201e30fc6a1af276556b9899ff132553558a
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 5b028d52d7..4d60b335ed 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -5,7 +5,11 @@ export interface Message {
   content: string
 }
 
-export enum StructuredOutput {}
+export enum StructuredOutput {
+  TableGeneration,
+  GenerateAIColumns,
+  GenerateData,
+}
 
 export type ResponseFormat = "text" | "json" | StructuredOutput
 

From f1e305b13c5a9a536b8816bcea3cf064db19080b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 17 Apr 2025 17:08:13 +0200
Subject: [PATCH 082/101] Replace keys for jsonschema

---
 packages/pro                                    |  2 +-
 packages/server/src/api/routes/tests/ai.spec.ts | 15 ++++++++-------
 packages/types/src/api/web/ai.ts                |  9 ++-------
 3 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/packages/pro b/packages/pro
index e54b201e30..68a16c6d09 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit e54b201e30fc6a1af276556b9899ff132553558a
+Subproject commit 68a16c6d0966f0799fd672cb0f3f4f627957628a
diff --git a/packages/server/src/api/routes/tests/ai.spec.ts b/packages/server/src/api/routes/tests/ai.spec.ts
index 0afd0e274f..d348231883 100644
--- a/packages/server/src/api/routes/tests/ai.spec.ts
+++ b/packages/server/src/api/routes/tests/ai.spec.ts
@@ -1,4 +1,5 @@
 import { z } from "zod"
+import { zodResponseFormat } from "openai/helpers/zod"
 import { mockChatGPTResponse } from "../../../tests/utilities/mocks/ai/openai"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import nock from "nock"
@@ -11,11 +12,10 @@ import {
   PlanModel,
   PlanType,
   ProviderConfig,
-  StructuredOutput,
 } from "@budibase/types"
 import { context } from "@budibase/backend-core"
 import { generator, mocks } from "@budibase/backend-core/tests"
-import { ai, quotas } from "@budibase/pro"
+import { quotas } from "@budibase/pro"
 import { MockLLMResponseFn } from "../../../tests/utilities/mocks/ai"
 import { mockAnthropicResponse } from "../../../tests/utilities/mocks/ai/anthropic"
 
@@ -414,11 +414,12 @@ describe("BudibaseAI", () => {
       expect(usage.monthly.current.budibaseAICredits).toBe(0)
 
       const gptResponse = generator.guid()
-      const structuredOutput = generator.word() as unknown as StructuredOutput
-      ai.structuredOutputs[structuredOutput] = {
-        key: generator.word(),
-        validator: z.object({ name: z.string() }),
-      }
+      const structuredOutput = zodResponseFormat(
+        z.object({
+          [generator.word()]: z.string(),
+        }),
+        "key"
+      )
       mockChatGPTResponse(gptResponse, { format: structuredOutput })
       const { message } = await config.api.ai.chat({
         messages: [{ role: "user", content: "Hello!" }],
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 4d60b335ed..88a16e029a 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -1,3 +1,4 @@
+import openai from "openai"
 import { EnrichedBinding } from "../../ui"
 
 export interface Message {
@@ -5,13 +6,7 @@ export interface Message {
   content: string
 }
 
-export enum StructuredOutput {
-  TableGeneration,
-  GenerateAIColumns,
-  GenerateData,
-}
-
-export type ResponseFormat = "text" | "json" | StructuredOutput
+export type ResponseFormat = "text" | "json" | openai.ResponseFormatJSONSchema
 
 export interface ChatCompletionRequest {
   messages: Message[]

From fa901d16ac94cfd1d4864ddb3a137fa2917677d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 17 Apr 2025 17:10:29 +0200
Subject: [PATCH 083/101] Remove unnused modal

---
 .../_components/AITableCreationModal.svelte   | 54 -------------------
 .../data/_components/AITableGeneration.svelte | 21 +++++++-
 2 files changed, 19 insertions(+), 56 deletions(-)
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
deleted file mode 100644
index 8a4b8ba280..0000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableCreationModal.svelte
+++ /dev/null
@@ -1,54 +0,0 @@
-<script lang="ts">
-  import { API } from "@/api"
-  import { datasources, tables } from "@/stores/builder"
-  import {
-    Modal,
-    ModalContent,
-    notifications,
-    TextArea,
-    Toggle,
-  } from "@budibase/bbui"
-  import type { GenerateTablesRequest } from "@budibase/types"
-  import { goto } from "@roxi/routify"
-
-  let modal: Modal
-
-  let modalData: GenerateTablesRequest = {
-    prompt: "",
-    addData: true,
-  }
-
-  export function show() {
-    modal.show()
-  }
-
-  export async function generate() {
-    try {
-      const { createdTables } = await API.generateTables(modalData)
-
-      const [tableToRedirect] = createdTables.sort((a, b) =>
-        a.name.localeCompare(b.name)
-      )
-
-      notifications.success(`Tables created successfully.`)
-      await datasources.fetch()
-      await tables.fetch()
-      $goto(`./table/${tableToRedirect.id}`)
-    } catch (e: any) {
-      notifications.error(e.message)
-    }
-  }
-</script>
-
-<Modal bind:this={modal}>
-  <ModalContent
-    title="AI generation"
-    size="XL"
-    confirmText="Generate"
-    onConfirm={generate}
-    showCancelButton={false}
-  >
-    <TextArea label="Prompt" bind:value={modalData.prompt} />
-    <Toggle label="Add data" bind:value={modalData.addData} />
-  </ModalContent>
-</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
index bc6ea6fa0d..6217ea44a8 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
@@ -1,16 +1,33 @@
 <script lang="ts">
   import { API } from "@/api"
   import AiInput from "@/components/common/ai/AIInput.svelte"
+  import { datasources, tables } from "@/stores/builder"
   import { auth, licensing } from "@/stores/portal"
   import { ActionButton, notifications } from "@budibase/bbui"
+  import { goto } from "@roxi/routify"
 
   let promptText = ""
 
   $: isEnabled = $auth?.user?.llm && !$licensing.aiCreditsExceeded
 
   async function submitPrompt(message: string) {
-    await API.generateTables({ prompt: message, addData: true })
-    notifications.success("Tables created successfully!")
+    try {
+      const { createdTables } = await API.generateTables({
+        prompt: message,
+        addData: true,
+      })
+
+      const [tableToRedirect] = createdTables.sort((a, b) =>
+        a.name.localeCompare(b.name)
+      )
+
+      notifications.success(`Tables created successfully.`)
+      await datasources.fetch()
+      await tables.fetch()
+      $goto(`./table/${tableToRedirect.id}`)
+    } catch (e: any) {
+      notifications.error(e.message)
+    }
   }
 
   const examplePrompts = [

From d6b319ca1f72f8e701395635fcc2e6ae02c75e20 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 17 Apr 2025 17:12:24 +0200
Subject: [PATCH 084/101] Clean

---
 .../pages/builder/app/[application]/data/new.svelte  | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
index 5604e935b0..5b1209046e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte
@@ -18,11 +18,9 @@
   import AiTableGeneration from "./_components/AITableGeneration.svelte"
   import { featureFlag } from "@/helpers"
   import { FeatureFlag } from "@budibase/types"
-  import AiTableCreationModal from "./_components/AITableCreationModal.svelte"
 
   let internalTableModal: CreateInternalTableModal
   let externalDatasourceModal: CreateExternalDatasourceModal
-  let aiTablesModal: AiTableCreationModal
 
   let sampleDataLoading = false
   let externalDatasourceLoading = false
@@ -55,8 +53,6 @@
   bind:this={externalDatasourceModal}
 />
 
-<AiTableCreationModal bind:this={aiTablesModal} />
-
 <CreationPage
   showClose={hasData($datasources, $tables)}
   onClose={() => $goto("./table")}
@@ -99,14 +95,6 @@
     >
       <svelte:component this={ICONS.BUDIBASE} height="20" width="20" />
     </DatasourceOption>
-    <DatasourceOption
-      on:click={() => aiTablesModal.show()}
-      title="Generate it using AI"
-      description="Non-relational"
-      {disabled}
-    >
-      <Icon name="MagicWand" size="M" />
-    </DatasourceOption>
   </div>
 
   <div class="subHeading">

From 6e3e7b04a049b9cf11afc795dd70f3c3a21a6e1b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 17 Apr 2025 17:55:48 +0200
Subject: [PATCH 085/101] Update submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 68a16c6d09..a5f0f62dc2 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 68a16c6d0966f0799fd672cb0f3f4f627957628a
+Subproject commit a5f0f62dc2edca1c95f448212637afa9b7ea1f0b

From a4707f1a30e7600f5faa56d99727be7df0accc96 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 11:30:59 +0200
Subject: [PATCH 086/101] Improve file madness

---
 packages/pro                                  |  2 +-
 .../server/src/sdk/app/ai/helpers/rows.ts     | 52 ++++++++++---------
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/packages/pro b/packages/pro
index a5f0f62dc2..c2e1666b33 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit a5f0f62dc2edca1c95f448212637afa9b7ea1f0b
+Subproject commit c2e1666b33cc6f0c5d88db8e173644e566256a80
diff --git a/packages/server/src/sdk/app/ai/helpers/rows.ts b/packages/server/src/sdk/app/ai/helpers/rows.ts
index be7dcef3f0..c380003761 100644
--- a/packages/server/src/sdk/app/ai/helpers/rows.ts
+++ b/packages/server/src/sdk/app/ai/helpers/rows.ts
@@ -1,4 +1,4 @@
-import { FieldType, Table } from "@budibase/types"
+import { FieldSchema, FieldType, Table } from "@budibase/types"
 import sdk from "../../.."
 import { uploadFile, uploadUrl } from "../../../../utilities"
 
@@ -27,36 +27,14 @@ export async function generateRows(
     )
 
     for (const entry of data[tableName]) {
-      const attachmentData: Record<string, any> = {}
-      for (const column of attachmentColumns) {
-        attachmentData[column.name] = []
-        if (!Array.isArray(entry[column.name])) {
-          entry[column.name] = [entry[column.name]]
-        }
-        for (const attachmentValue of entry[column.name]) {
-          let attachment
-          if (typeof attachmentValue === "object") {
-            attachment = await uploadFile(attachmentValue)
-          } else {
-            attachment = await uploadUrl(attachmentValue)
-          }
-          if (attachment) {
-            attachmentData[column.name].push(attachment)
-          }
-        }
+      await processAttachments(entry, attachmentColumns)
 
-        if (column.type === FieldType.ATTACHMENT_SINGLE) {
-          attachmentData[column.name] = attachmentData[column.name][0]
-        }
-      }
-
-      const tableId = tables[tableName]._id!
+      const tableId = table._id!
       const createdRow = await sdk.rows.save(
         tableId,
         {
           ...entry,
           ...linksOverride,
-          ...attachmentData,
           _id: undefined,
         },
         userId
@@ -113,3 +91,27 @@ export async function generateRows(
     )
   }
 }
+async function processAttachments(
+  entry: Record<string, any>,
+  attachmentColumns: FieldSchema[]
+) {
+  function processAttachment(value: any) {
+    if (typeof value === "object") {
+      return uploadFile(value)
+    }
+
+    return uploadUrl(value)
+  }
+
+  for (const column of attachmentColumns) {
+    if (!Array.isArray(entry[column.name])) {
+      entry[column.name] = await processAttachment(entry[column.name])
+    } else {
+      entry[column.name] = await Promise.all(
+        entry[column.name].map((attachment: any) =>
+          processAttachment(attachment)
+        )
+      )
+    }
+  }
+}

From 586591aa6a1e146f3a32767537aaf00c8305404c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 11:40:35 +0200
Subject: [PATCH 087/101] Store new tables as internal tables

---
 packages/backend-core/src/sql/utils.ts           |  4 ----
 .../DatasourceNavigator/datasourceUtils.js       |  3 ++-
 packages/server/src/sdk/app/ai/helpers/table.ts  | 16 ++--------------
 .../src/sdk/app/datasources/datasources.ts       |  3 ++-
 4 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index cc078ada0e..5bd9e4d12a 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -47,10 +47,6 @@ export function isExternalTable(table: Table) {
   if (
     table?.sourceId &&
     table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
-    // TODO handle properly when the product part is defined
-    !table.sourceId.startsWith(
-      DocumentType.DATASOURCE + SEPARATOR + "bb_internal_"
-    ) &&
     table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
   ) {
     return true
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/datasourceUtils.js b/packages/builder/src/components/backend/DatasourceNavigator/datasourceUtils.js
index 5e38cc9bfe..f23866c267 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/datasourceUtils.js
+++ b/packages/builder/src/components/backend/DatasourceNavigator/datasourceUtils.js
@@ -1,4 +1,5 @@
 import { TableNames } from "@/constants"
+import { INTERNAL_TABLE_SOURCE_ID } from "@budibase/types"
 
 const showDatasourceOpen = ({
   selected,
@@ -41,7 +42,7 @@ const containsActiveEntity = (
   // Check for hardcoded datasource edge cases
   if (
     isActive("./datasource/bb_internal") &&
-    datasource._id === "bb_internal"
+    datasource._id === INTERNAL_TABLE_SOURCE_ID
   ) {
     return true
   }
diff --git a/packages/server/src/sdk/app/ai/helpers/table.ts b/packages/server/src/sdk/app/ai/helpers/table.ts
index 4e6c5b7c70..a51847d1b4 100644
--- a/packages/server/src/sdk/app/ai/helpers/table.ts
+++ b/packages/server/src/sdk/app/ai/helpers/table.ts
@@ -1,37 +1,25 @@
 import {
-  DocumentType,
   FieldType,
   GenerateTablesResponse,
-  SourceName,
+  INTERNAL_TABLE_SOURCE_ID,
   TableSchema,
   TableSourceType,
 } from "@budibase/types"
 import sdk from "../../.."
-import { context, utils } from "@budibase/backend-core"
 
 export async function generateTables(
   tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
 ) {
-  const count = (await sdk.datasources.fetch()).length
-  const { id: dsId } = await context.getAppDB().put({
-    _id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
-    name: `Test ${count}`,
-    type: "budibase",
-
-    source: SourceName.BUDIBASE,
-    config: {},
-  })
-
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
   for (const table of tables) {
     const createdTable = await sdk.tables.create({
       ...table,
-      sourceId: dsId,
       schema: {},
       primaryDisplay: undefined,
       sourceType: TableSourceType.INTERNAL,
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       type: "table",
     })
 
diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts
index 962ed4c74e..409657440d 100644
--- a/packages/server/src/sdk/app/datasources/datasources.ts
+++ b/packages/server/src/sdk/app/datasources/datasources.ts
@@ -11,6 +11,7 @@ import {
   Row,
   RestConfig,
   SourceName,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
@@ -51,7 +52,7 @@ export async function fetch(opts?: {
   )
 
   const internal = internalTables.rows.reduce((acc: any, row: Row) => {
-    const sourceId = row.doc.sourceId || "bb_internal"
+    const sourceId = row.doc.sourceId || INTERNAL_TABLE_SOURCE_ID
     acc[sourceId] = acc[sourceId] || []
     acc[sourceId].push(row.doc)
     return acc

From 3be9655bcfe1a33f664ea339ecf8d0630b82ce6b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 12:27:05 +0200
Subject: [PATCH 088/101] Move getSequentialName to shared-core

---
 packages/builder/src/helpers/duplicate.ts     | 116 +-----------------
 packages/shared-core/src/helpers/duplicate.ts | 113 +++++++++++++++++
 packages/shared-core/src/helpers/index.ts     |   1 +
 .../src/helpers/tests/duplicate.test.ts       |   0
 4 files changed, 118 insertions(+), 112 deletions(-)
 create mode 100644 packages/shared-core/src/helpers/duplicate.ts
 rename packages/{builder => shared-core}/src/helpers/tests/duplicate.test.ts (100%)

diff --git a/packages/builder/src/helpers/duplicate.ts b/packages/builder/src/helpers/duplicate.ts
index eb705c6525..e0a281658f 100644
--- a/packages/builder/src/helpers/duplicate.ts
+++ b/packages/builder/src/helpers/duplicate.ts
@@ -1,113 +1,5 @@
-/**
- * Duplicates a name with respect to a collection of existing names
- * e.g.
- *    name           all names          result
- *   ------        -----------         --------
- * ("foo")       ["foo"]              "foo 1"
- * ("foo")       ["foo", "foo 1"]     "foo 2"
- * ("foo 1")     ["foo", "foo 1"]     "foo 2"
- * ("foo")       ["foo", "foo 2"]     "foo 1"
- *
- * Repl
- */
-export const duplicateName = (name: string, allNames: string[]) => {
-  const duplicatePattern = new RegExp(`\\s(\\d+)$`)
-  const baseName = name.split(duplicatePattern)[0]
-  const isDuplicate = new RegExp(`${baseName}\\s(\\d+)$`)
+// export * from
+import { helpers } from "@budibase/shared-core"
 
-  // get the sequence from matched names
-  const sequence: number[] = []
-  allNames.filter(n => {
-    if (n === baseName) {
-      return true
-    }
-    const match = n.match(isDuplicate)
-    if (match) {
-      sequence.push(parseInt(match[1]))
-      return true
-    }
-    return false
-  })
-  sequence.sort((a, b) => a - b)
-  // get the next number in the sequence
-  let number
-  if (sequence.length === 0) {
-    number = 1
-  } else {
-    // get the next number in the sequence
-    for (let i = 0; i < sequence.length; i++) {
-      if (sequence[i] !== i + 1) {
-        number = i + 1
-        break
-      }
-    }
-    if (!number) {
-      number = sequence.length + 1
-    }
-  }
-
-  return `${baseName} ${number}`
-}
-
-/**
- * More flexible alternative to the above function, which handles getting the
- * next sequential name from an array of existing items while accounting for
- * any type of prefix, and being able to deeply retrieve that name from the
- * existing item array.
- *
- * Examples with a prefix of "foo":
- * [] => "foo"
- * ["foo"] => "foo2"
- * ["foo", "foo6"] => "foo7"
- *
- * Examples with a prefix of "foo " (space at the end):
- * [] => "foo"
- * ["foo"] => "foo 2"
- * ["foo", "foo 6"] => "foo 7"
- *
- * @param items the array of existing items
- * @param prefix the string prefix of each name, including any spaces desired
- * @param getName optional function to extract the name for an item, if not a
- *  flat array of strings
- */
-export const getSequentialName = <T extends any>(
-  items: T[] | null,
-  prefix: string | null,
-  {
-    getName,
-    numberFirstItem,
-    separator = "",
-  }: {
-    getName?: (item: T) => string
-    numberFirstItem?: boolean
-    separator?: string
-  } = {}
-) => {
-  if (!prefix?.length) {
-    return ""
-  }
-  const trimmedPrefix = prefix.trim()
-  const firstName = numberFirstItem ? `${prefix}1` : trimmedPrefix
-  if (!items?.length) {
-    return firstName
-  }
-  let max = 0
-  items.forEach(item => {
-    const name = getName?.(item) ?? item
-    if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) {
-      return
-    }
-    const split = name.split(trimmedPrefix)
-    if (split.length !== 2) {
-      return
-    }
-    if (split[1].trim() === "") {
-      split[1] = "1"
-    }
-    const num = parseInt(split[1])
-    if (num > max) {
-      max = num
-    }
-  })
-  return max === 0 ? firstName : `${prefix}${separator}${max + 1}`
-}
+export const duplicateName = helpers.duplicateName
+export const getSequentialName = helpers.getSequentialName
diff --git a/packages/shared-core/src/helpers/duplicate.ts b/packages/shared-core/src/helpers/duplicate.ts
new file mode 100644
index 0000000000..eb705c6525
--- /dev/null
+++ b/packages/shared-core/src/helpers/duplicate.ts
@@ -0,0 +1,113 @@
+/**
+ * Duplicates a name with respect to a collection of existing names
+ * e.g.
+ *    name           all names          result
+ *   ------        -----------         --------
+ * ("foo")       ["foo"]              "foo 1"
+ * ("foo")       ["foo", "foo 1"]     "foo 2"
+ * ("foo 1")     ["foo", "foo 1"]     "foo 2"
+ * ("foo")       ["foo", "foo 2"]     "foo 1"
+ *
+ * Repl
+ */
+export const duplicateName = (name: string, allNames: string[]) => {
+  const duplicatePattern = new RegExp(`\\s(\\d+)$`)
+  const baseName = name.split(duplicatePattern)[0]
+  const isDuplicate = new RegExp(`${baseName}\\s(\\d+)$`)
+
+  // get the sequence from matched names
+  const sequence: number[] = []
+  allNames.filter(n => {
+    if (n === baseName) {
+      return true
+    }
+    const match = n.match(isDuplicate)
+    if (match) {
+      sequence.push(parseInt(match[1]))
+      return true
+    }
+    return false
+  })
+  sequence.sort((a, b) => a - b)
+  // get the next number in the sequence
+  let number
+  if (sequence.length === 0) {
+    number = 1
+  } else {
+    // get the next number in the sequence
+    for (let i = 0; i < sequence.length; i++) {
+      if (sequence[i] !== i + 1) {
+        number = i + 1
+        break
+      }
+    }
+    if (!number) {
+      number = sequence.length + 1
+    }
+  }
+
+  return `${baseName} ${number}`
+}
+
+/**
+ * More flexible alternative to the above function, which handles getting the
+ * next sequential name from an array of existing items while accounting for
+ * any type of prefix, and being able to deeply retrieve that name from the
+ * existing item array.
+ *
+ * Examples with a prefix of "foo":
+ * [] => "foo"
+ * ["foo"] => "foo2"
+ * ["foo", "foo6"] => "foo7"
+ *
+ * Examples with a prefix of "foo " (space at the end):
+ * [] => "foo"
+ * ["foo"] => "foo 2"
+ * ["foo", "foo 6"] => "foo 7"
+ *
+ * @param items the array of existing items
+ * @param prefix the string prefix of each name, including any spaces desired
+ * @param getName optional function to extract the name for an item, if not a
+ *  flat array of strings
+ */
+export const getSequentialName = <T extends any>(
+  items: T[] | null,
+  prefix: string | null,
+  {
+    getName,
+    numberFirstItem,
+    separator = "",
+  }: {
+    getName?: (item: T) => string
+    numberFirstItem?: boolean
+    separator?: string
+  } = {}
+) => {
+  if (!prefix?.length) {
+    return ""
+  }
+  const trimmedPrefix = prefix.trim()
+  const firstName = numberFirstItem ? `${prefix}1` : trimmedPrefix
+  if (!items?.length) {
+    return firstName
+  }
+  let max = 0
+  items.forEach(item => {
+    const name = getName?.(item) ?? item
+    if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) {
+      return
+    }
+    const split = name.split(trimmedPrefix)
+    if (split.length !== 2) {
+      return
+    }
+    if (split[1].trim() === "") {
+      split[1] = "1"
+    }
+    const num = parseInt(split[1])
+    if (num > max) {
+      max = num
+    }
+  })
+  return max === 0 ? firstName : `${prefix}${separator}${max + 1}`
+}
diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts
index b924617a0f..6eecfea304 100644
--- a/packages/shared-core/src/helpers/index.ts
+++ b/packages/shared-core/src/helpers/index.ts
@@ -7,3 +7,4 @@ export * as schema from "./schema"
 export * as views from "./views"
 export * as roles from "./roles"
 export * as lists from "./lists"
+export * from "./duplicate"
diff --git a/packages/builder/src/helpers/tests/duplicate.test.ts b/packages/shared-core/src/helpers/tests/duplicate.test.ts
similarity index 100%
rename from packages/builder/src/helpers/tests/duplicate.test.ts
rename to packages/shared-core/src/helpers/tests/duplicate.test.ts

From 08b8dc6f8a1ec1f0908d2ac17061e99cca89b87e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 12:33:32 +0200
Subject: [PATCH 089/101] Avoid duplicate

---
 packages/server/src/sdk/app/ai/helpers/table.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/ai/helpers/table.ts b/packages/server/src/sdk/app/ai/helpers/table.ts
index a51847d1b4..4d11e43a32 100644
--- a/packages/server/src/sdk/app/ai/helpers/table.ts
+++ b/packages/server/src/sdk/app/ai/helpers/table.ts
@@ -6,6 +6,7 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import sdk from "../../.."
+import { helpers } from "@budibase/shared-core"
 
 export async function generateTables(
   tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
@@ -13,9 +14,17 @@ export async function generateTables(
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
+  const existingTableNames = (await sdk.tables.getAllInternalTables()).map(
+    t => t.name
+  )
+
   for (const table of tables) {
+    const name = helpers.getSequentialName(existingTableNames, table.name, {
+      separator: " ",
+    })
     const createdTable = await sdk.tables.create({
       ...table,
+      name,
       schema: {},
       primaryDisplay: undefined,
       sourceType: TableSourceType.INTERNAL,
@@ -23,7 +32,7 @@ export async function generateTables(
       type: "table",
     })
 
-    createdTables.push({ id: createdTable._id!, name: createdTable.name })
+    createdTables.push({ id: createdTable._id!, name: table.name })
     tableIds[table.name] = createdTable._id!
   }
 

From 92746023865a31750ec66fdeefb116b1ae246379 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 12:46:09 +0200
Subject: [PATCH 090/101] Validate before creation

---
 packages/server/src/sdk/app/ai/helpers/table.ts | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/sdk/app/ai/helpers/table.ts b/packages/server/src/sdk/app/ai/helpers/table.ts
index 4d11e43a32..c090ab11b1 100644
--- a/packages/server/src/sdk/app/ai/helpers/table.ts
+++ b/packages/server/src/sdk/app/ai/helpers/table.ts
@@ -14,6 +14,16 @@ export async function generateTables(
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
+  for (const table of tables) {
+    for (const linkField of Object.values(table.schema).filter(
+      f => f.type === FieldType.LINK
+    )) {
+      if (!tables.find(t => t.name === linkField.tableId)) {
+        throw `Table ${linkField.tableId} not found in the json response.`
+      }
+    }
+  }
+
   const existingTableNames = (await sdk.tables.getAllInternalTables()).map(
     t => t.name
   )
@@ -39,11 +49,7 @@ export async function generateTables(
   for (const table of tables) {
     for (const field of Object.values(table.schema)) {
       if (field.type === FieldType.LINK) {
-        const linkedTable = createdTables.find(t => t.name === field.tableId)
-        if (!linkedTable) {
-          throw `Table ${field.tableId} not found in the json response.`
-        }
-        field.tableId = linkedTable.id
+        field.tableId = tableIds[field.tableId]
       } else if (field.type === FieldType.FORMULA) {
         field.formula = `{{ js "${btoa(field.formula)}" }}`
       }

From 98cbb40ba1482f64edad387ecb4dfd984c103203 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 12:51:29 +0200
Subject: [PATCH 091/101] Destroy tables on creation error

---
 .../server/src/sdk/app/ai/helpers/table.ts    | 93 ++++++++++---------
 1 file changed, 50 insertions(+), 43 deletions(-)

diff --git a/packages/server/src/sdk/app/ai/helpers/table.ts b/packages/server/src/sdk/app/ai/helpers/table.ts
index c090ab11b1..987df6781c 100644
--- a/packages/server/src/sdk/app/ai/helpers/table.ts
+++ b/packages/server/src/sdk/app/ai/helpers/table.ts
@@ -14,59 +14,66 @@ export async function generateTables(
   const createdTables: GenerateTablesResponse["createdTables"] = []
   const tableIds: Record<string, string> = {}
 
-  for (const table of tables) {
-    for (const linkField of Object.values(table.schema).filter(
-      f => f.type === FieldType.LINK
-    )) {
-      if (!tables.find(t => t.name === linkField.tableId)) {
-        throw `Table ${linkField.tableId} not found in the json response.`
+  try {
+    for (const table of tables) {
+      for (const linkField of Object.values(table.schema).filter(
+        f => f.type === FieldType.LINK
+      )) {
+        if (!tables.find(t => t.name === linkField.tableId)) {
+          throw `Table ${linkField.tableId} not found in the json response.`
+        }
       }
     }
-  }
 
-  const existingTableNames = (await sdk.tables.getAllInternalTables()).map(
-    t => t.name
-  )
+    const existingTableNames = (await sdk.tables.getAllInternalTables()).map(
+      t => t.name
+    )
 
-  for (const table of tables) {
-    const name = helpers.getSequentialName(existingTableNames, table.name, {
-      separator: " ",
-    })
-    const createdTable = await sdk.tables.create({
-      ...table,
-      name,
-      schema: {},
-      primaryDisplay: undefined,
-      sourceType: TableSourceType.INTERNAL,
-      sourceId: INTERNAL_TABLE_SOURCE_ID,
-      type: "table",
-    })
+    for (const table of tables) {
+      const name = helpers.getSequentialName(existingTableNames, table.name, {
+        separator: " ",
+      })
+      const createdTable = await sdk.tables.create({
+        ...table,
+        name,
+        schema: {},
+        primaryDisplay: undefined,
+        sourceType: TableSourceType.INTERNAL,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        type: "table",
+      })
 
-    createdTables.push({ id: createdTable._id!, name: table.name })
-    tableIds[table.name] = createdTable._id!
-  }
+      createdTables.push({ id: createdTable._id!, name: table.name })
+      tableIds[table.name] = createdTable._id!
+    }
 
-  for (const table of tables) {
-    for (const field of Object.values(table.schema)) {
-      if (field.type === FieldType.LINK) {
-        field.tableId = tableIds[field.tableId]
-      } else if (field.type === FieldType.FORMULA) {
-        field.formula = `{{ js "${btoa(field.formula)}" }}`
+    for (const table of tables) {
+      for (const field of Object.values(table.schema)) {
+        if (field.type === FieldType.LINK) {
+          field.tableId = tableIds[field.tableId]
+        } else if (field.type === FieldType.FORMULA) {
+          field.formula = `{{ js "${btoa(field.formula)}" }}`
+        }
       }
     }
-  }
 
-  for (const table of tables) {
-    const storedTable = await sdk.tables.getTable(tableIds[table.name])
+    for (const table of tables) {
+      const storedTable = await sdk.tables.getTable(tableIds[table.name])
 
-    await sdk.tables.update({
-      ...storedTable,
-      schema: {
-        ...storedTable.schema,
-        ...table.schema,
-      },
-      primaryDisplay: table.primaryDisplay,
-    })
+      await sdk.tables.update({
+        ...storedTable,
+        schema: {
+          ...storedTable.schema,
+          ...table.schema,
+        },
+        primaryDisplay: table.primaryDisplay,
+      })
+    }
+  } catch (e) {
+    const tables = await sdk.tables.getTables(createdTables.map(t => t.id))
+    await Promise.all(tables.map(sdk.tables.internal.destroy))
+
+    throw e
   }
 
   return createdTables

From 63302a421dc50bcf201db1300f2321a5b123e41a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 13:43:26 +0200
Subject: [PATCH 092/101] Fix test

---
 packages/shared-core/src/helpers/tests/duplicate.test.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/shared-core/src/helpers/tests/duplicate.test.ts b/packages/shared-core/src/helpers/tests/duplicate.test.ts
index 5e956d7b1c..465f11abea 100644
--- a/packages/shared-core/src/helpers/tests/duplicate.test.ts
+++ b/packages/shared-core/src/helpers/tests/duplicate.test.ts
@@ -1,4 +1,3 @@
-import { expect, describe, it } from "vitest"
 import { duplicateName, getSequentialName } from "../duplicate"
 
 describe("duplicate", () => {

From 2b99685bbc8043672c0964ca87d1bf52d1e471f3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 17:37:08 +0200
Subject: [PATCH 093/101] Remove add data flag

---
 packages/server/src/api/controllers/ai.ts | 6 ++----
 packages/types/src/api/web/ai.ts          | 1 -
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/ai.ts b/packages/server/src/api/controllers/ai.ts
index 617f63c629..1f28954a4f 100644
--- a/packages/server/src/api/controllers/ai.ts
+++ b/packages/server/src/api/controllers/ai.ts
@@ -9,16 +9,14 @@ import sdk from "../../sdk"
 export async function generateTables(
   ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
 ) {
-  const { prompt, addData } = ctx.request.body
+  const { prompt } = ctx.request.body
 
   const tableGenerator = await ai.TableGeneration.init({
     generateTablesDelegate: sdk.ai.helpers.generateTables,
     getTablesDelegate: sdk.tables.getTables,
     generateDataDelegate: sdk.ai.helpers.generateRows,
   })
-  if (addData) {
-    tableGenerator.withData(ctx.user._id || "")
-  }
+  tableGenerator.withData(ctx.user._id || "")
   const createdTables = await tableGenerator.generate(prompt)
 
   ctx.body = {
diff --git a/packages/types/src/api/web/ai.ts b/packages/types/src/api/web/ai.ts
index 88a16e029a..a2ec508d76 100644
--- a/packages/types/src/api/web/ai.ts
+++ b/packages/types/src/api/web/ai.ts
@@ -37,7 +37,6 @@ export interface GenerateCronResponse {
 
 export interface GenerateTablesRequest {
   prompt: string
-  addData: boolean
 }
 
 export interface GenerateTablesResponse {

From a4a647061059c6f2c19375958a26449f977e876d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 18:37:45 +0200
Subject: [PATCH 094/101] Basic generateTable tests

---
 packages/pro                                  |   2 +-
 .../server/src/api/routes/tests/ai.spec.ts    | 316 +++++++++++++++++-
 packages/server/src/tests/utilities/api/ai.ts |  14 +
 3 files changed, 330 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index c2e1666b33..a35af6c175 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit c2e1666b33cc6f0c5d88db8e173644e566256a80
+Subproject commit a35af6c175819daa7c3201258ab7c9eab89c45a6
diff --git a/packages/server/src/api/routes/tests/ai.spec.ts b/packages/server/src/api/routes/tests/ai.spec.ts
index d348231883..2374f5fc61 100644
--- a/packages/server/src/api/routes/tests/ai.spec.ts
+++ b/packages/server/src/api/routes/tests/ai.spec.ts
@@ -6,16 +6,21 @@ import nock from "nock"
 import { configs, env, features, setEnv } from "@budibase/backend-core"
 import {
   AIInnerConfig,
+  AIOperationEnum,
+  AttachmentSubType,
+  AutoFieldSubType,
   ConfigType,
   Feature,
+  FieldType,
   License,
   PlanModel,
   PlanType,
   ProviderConfig,
+  RelationshipType,
 } from "@budibase/types"
 import { context } from "@budibase/backend-core"
 import { generator, mocks } from "@budibase/backend-core/tests"
-import { quotas } from "@budibase/pro"
+import { quotas, ai } from "@budibase/pro"
 import { MockLLMResponseFn } from "../../../tests/utilities/mocks/ai"
 import { mockAnthropicResponse } from "../../../tests/utilities/mocks/ai/anthropic"
 
@@ -432,4 +437,313 @@ describe("BudibaseAI", () => {
       expect(usage.monthly.current.budibaseAICredits).toBeGreaterThan(0)
     })
   })
+
+  describe("POST /api/ai/tables", () => {
+    let licenseKey = "test-key"
+    let internalApiKey = "api-key"
+
+    let envCleanup: () => void
+    let featureCleanup: () => void
+    beforeAll(() => {
+      envCleanup = setEnv({
+        SELF_HOSTED: false,
+        INTERNAL_API_KEY: internalApiKey,
+      })
+      featureCleanup = features.testutils.setFeatureFlags("*", {
+        AI_TABLE_GENERATION: true,
+      })
+    })
+
+    afterAll(() => {
+      featureCleanup()
+      envCleanup()
+    })
+
+    beforeEach(async () => {
+      await config.newTenant()
+      nock.cleanAll()
+      const license: License = {
+        plan: {
+          type: PlanType.FREE,
+          model: PlanModel.PER_USER,
+          usesInvoicing: false,
+        },
+        features: [Feature.BUDIBASE_AI],
+        quotas: {} as any,
+        tenantId: config.tenantId,
+      }
+      nock(env.ACCOUNT_PORTAL_URL)
+        .get(`/api/license/${licenseKey}`)
+        .reply(200, license)
+    })
+
+    it("handles correct chat response", async () => {
+      const prompt = "Create me a table for managing IT tickets"
+      const generationStructure: ai.GenerationStructure = {
+        tables: [
+          {
+            name: "Tickets",
+            primaryDisplay: "Title",
+            schema: [
+              {
+                name: "Title",
+                type: FieldType.STRING,
+                constraints: {
+                  presence: true,
+                },
+              },
+              {
+                name: "Description",
+                type: FieldType.LONGFORM,
+                constraints: {
+                  presence: true,
+                },
+              },
+              {
+                name: "Priority",
+                type: FieldType.OPTIONS,
+                constraints: {
+                  inclusion: ["Low", "Medium", "High"],
+                  presence: true,
+                },
+              },
+              {
+                name: "Status",
+                type: FieldType.OPTIONS,
+                constraints: {
+                  inclusion: ["Open", "In Progress", "Closed"],
+                  presence: true,
+                },
+              },
+              {
+                name: "Assignee",
+                type: FieldType.LINK,
+                tableId: "Employees",
+                relationshipType: RelationshipType.MANY_TO_ONE,
+                reverseFieldName: "AssignedTickets",
+                relationshipId: "TicketUser",
+              },
+              {
+                name: "Created Date",
+                type: FieldType.DATETIME,
+                ignoreTimezones: false,
+                dateOnly: true,
+              },
+              {
+                name: "Resolution Time (Days)",
+                type: FieldType.FORMULA,
+                formula:
+                  'return (new Date() - new Date($("Created Date"))) / (1000 * 60 * 60 * 24);',
+                responseType: FieldType.NUMBER,
+              },
+              {
+                name: "Attachment",
+                type: FieldType.ATTACHMENT_SINGLE,
+              },
+            ],
+          },
+          {
+            name: "Employees",
+            primaryDisplay: "First Name",
+            schema: [
+              {
+                name: "First Name",
+                type: FieldType.STRING,
+                constraints: {
+                  presence: true,
+                },
+              },
+              {
+                name: "Last Name",
+                type: FieldType.STRING,
+                constraints: {
+                  presence: true,
+                },
+              },
+              {
+                name: "Position",
+                type: FieldType.STRING,
+                constraints: {
+                  presence: true,
+                },
+              },
+              {
+                name: "Photo",
+                type: FieldType.ATTACHMENT_SINGLE,
+                subtype: AttachmentSubType.IMAGE,
+              },
+              {
+                name: "Documents",
+                type: FieldType.ATTACHMENTS,
+              },
+
+              {
+                name: "AssignedTickets",
+                type: FieldType.LINK,
+                tableId: "Tickets",
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                reverseFieldName: "Assignee",
+                relationshipId: "TicketUser",
+              },
+            ],
+          },
+        ],
+      }
+      mockChatGPTResponse(JSON.stringify(generationStructure), {
+        format: zodResponseFormat(ai.generationStructure, "key"),
+      })
+
+      const aiColumnGeneration: ai.AIColumnSchemas = {
+        Tickets: [
+          {
+            name: "Ticket Summary",
+            type: FieldType.AI,
+            operation: AIOperationEnum.SUMMARISE_TEXT,
+            columns: ["Title", "Description"],
+          },
+          {
+            name: "Translated Description",
+            type: FieldType.AI,
+            operation: AIOperationEnum.TRANSLATE,
+            column: "Description",
+            language: "es",
+          },
+        ],
+        Employees: [
+          {
+            name: "Role Category",
+            type: FieldType.AI,
+            operation: AIOperationEnum.CATEGORISE_TEXT,
+            columns: ["Position"],
+            categories: "Manager,Staff,Intern,Contractor",
+          },
+        ],
+      }
+      mockChatGPTResponse(JSON.stringify(aiColumnGeneration), {
+        format: zodResponseFormat(
+          ai.aiColumnSchemas(generationStructure),
+          "key"
+        ),
+      })
+
+      nock("http://photourl.com").get("/any").reply(200)
+
+      const dataGeneration: Record<string, Record<string, any>[]> = {
+        Tickets: [],
+        "Employees 2": [
+          {
+            "First Name": "Joshua",
+            "Last Name": "Lee",
+            Position: "Application Developer",
+            Photo: "http://photourl.com/any",
+            Documents: [
+              {
+                name: "development_guidelines.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+              {
+                name: "project_documents.txt",
+                extension: ".txt",
+                content: "any content",
+              },
+            ],
+            "Role Category": "Staff",
+          },
+          {
+            "First Name": "Emily",
+            "Last Name": "Davis",
+            Position: "Software Deployment Technician",
+            Photo: "http://photourl.com/any",
+            Documents: [
+              {
+                name: "software_license_list.txt",
+                extension: ".txt",
+                content: "any content",
+              },
+              {
+                name: "deployment_guide.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+              {
+                name: "installation_logs.txt",
+                extension: ".txt",
+                content: "any content",
+              },
+            ],
+            "Role Category": "Staff",
+          },
+          {
+            "First Name": "James",
+            "Last Name": "Smith",
+            Position: "IT Support Specialist",
+            Photo: "http://photourl.com/any",
+            Documents: [
+              {
+                name: "certificates.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+              {
+                name: "employment_contract.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+            ],
+            "Role Category": "Staff",
+          },
+          {
+            "First Name": "Jessica",
+            "Last Name": "Taylor",
+            Position: "Cybersecurity Analyst",
+            Photo: "http://photourl.com/any",
+            Documents: [
+              {
+                name: "security_audit_report.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+              {
+                name: "incident_response_plan.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+            ],
+            "Role Category": "Staff",
+          },
+          {
+            "First Name": "Ashley",
+            "Last Name": "Harris",
+            Position: "Database Administrator",
+            Photo: "http://photourl.com/any",
+            Documents: [
+              {
+                name: "database_backup.txt",
+                extension: ".txt",
+                content: "any content",
+              },
+              {
+                name: "permission_settings.pdf",
+                extension: ".pdf",
+                content: "any content",
+              },
+            ],
+            "Role Category": "Staff",
+          },
+        ],
+      }
+      mockChatGPTResponse(JSON.stringify(dataGeneration), {
+        format: zodResponseFormat(ai.tableDataStructuredOutput([]), "key"),
+      })
+
+      mockChatGPTResponse("Mock LLM Response")
+
+      const { createdTables } = await config.api.ai.generateTables({ prompt })
+      expect(createdTables).toEqual([
+        { id: expect.stringMatching(/ta_\w+/), name: "Tickets" },
+        { id: expect.stringMatching(/ta_\w+/), name: "Employees" },
+      ])
+    })
+  })
 })
diff --git a/packages/server/src/tests/utilities/api/ai.ts b/packages/server/src/tests/utilities/api/ai.ts
index efaa321f09..4e590ff808 100644
--- a/packages/server/src/tests/utilities/api/ai.ts
+++ b/packages/server/src/tests/utilities/api/ai.ts
@@ -5,6 +5,8 @@ import {
   GenerateCronResponse,
   GenerateJsRequest,
   GenerateJsResponse,
+  GenerateTablesRequest,
+  GenerateTablesResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 import { constants } from "@budibase/backend-core"
@@ -44,4 +46,16 @@ export class AIAPI extends TestAPI {
       expectations,
     })
   }
+
+  generateTables = async (
+    req: GenerateTablesRequest,
+    expectations?: Expectations
+  ): Promise<GenerateTablesResponse> => {
+    const headers: Record<string, string> = {}
+    return await this._post<GenerateTablesResponse>(`/api/ai/tables`, {
+      body: req,
+      headers,
+      expectations,
+    })
+  }
 }

From ddd809b568c62ef6ec03fc24a645d2112d9bcee9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 18:50:10 +0200
Subject: [PATCH 095/101] Cleanups

---
 .../server/src/api/routes/tests/ai.spec.ts    | 80 +++++++++----------
 1 file changed, 38 insertions(+), 42 deletions(-)

diff --git a/packages/server/src/api/routes/tests/ai.spec.ts b/packages/server/src/api/routes/tests/ai.spec.ts
index 2374f5fc61..89d910a22d 100644
--- a/packages/server/src/api/routes/tests/ai.spec.ts
+++ b/packages/server/src/api/routes/tests/ai.spec.ts
@@ -8,7 +8,6 @@ import {
   AIInnerConfig,
   AIOperationEnum,
   AttachmentSubType,
-  AutoFieldSubType,
   ConfigType,
   Feature,
   FieldType,
@@ -439,16 +438,8 @@ describe("BudibaseAI", () => {
   })
 
   describe("POST /api/ai/tables", () => {
-    let licenseKey = "test-key"
-    let internalApiKey = "api-key"
-
-    let envCleanup: () => void
     let featureCleanup: () => void
     beforeAll(() => {
-      envCleanup = setEnv({
-        SELF_HOSTED: false,
-        INTERNAL_API_KEY: internalApiKey,
-      })
       featureCleanup = features.testutils.setFeatureFlags("*", {
         AI_TABLE_GENERATION: true,
       })
@@ -456,27 +447,41 @@ describe("BudibaseAI", () => {
 
     afterAll(() => {
       featureCleanup()
-      envCleanup()
     })
 
     beforeEach(async () => {
       await config.newTenant()
       nock.cleanAll()
-      const license: License = {
-        plan: {
-          type: PlanType.FREE,
-          model: PlanModel.PER_USER,
-          usesInvoicing: false,
-        },
-        features: [Feature.BUDIBASE_AI],
-        quotas: {} as any,
-        tenantId: config.tenantId,
-      }
-      nock(env.ACCOUNT_PORTAL_URL)
-        .get(`/api/license/${licenseKey}`)
-        .reply(200, license)
     })
 
+    const mockAIGenerationStructure = (
+      generationStructure: ai.GenerationStructure
+    ) =>
+      mockChatGPTResponse(JSON.stringify(generationStructure), {
+        format: zodResponseFormat(ai.generationStructure, "key"),
+      })
+
+    const mockAIColumnGeneration = (
+      generationStructure: ai.GenerationStructure,
+      aiColumnGeneration: ai.AIColumnSchemas
+    ) =>
+      mockChatGPTResponse(JSON.stringify(aiColumnGeneration), {
+        format: zodResponseFormat(
+          ai.aiColumnSchemas(generationStructure),
+          "key"
+        ),
+      })
+
+    const mockDataGeneration = (
+      dataGeneration: Record<string, Record<string, any>[]>
+    ) =>
+      mockChatGPTResponse(JSON.stringify(dataGeneration), {
+        format: zodResponseFormat(ai.tableDataStructuredOutput([]), "key"),
+      })
+
+    const mockProcessAIColumn = (response: string) =>
+      mockChatGPTResponse(response)
+
     it("handles correct chat response", async () => {
       const prompt = "Create me a table for managing IT tickets"
       const generationStructure: ai.GenerationStructure = {
@@ -589,9 +594,7 @@ describe("BudibaseAI", () => {
           },
         ],
       }
-      mockChatGPTResponse(JSON.stringify(generationStructure), {
-        format: zodResponseFormat(ai.generationStructure, "key"),
-      })
+      mockAIGenerationStructure(generationStructure)
 
       const aiColumnGeneration: ai.AIColumnSchemas = {
         Tickets: [
@@ -619,14 +622,9 @@ describe("BudibaseAI", () => {
           },
         ],
       }
-      mockChatGPTResponse(JSON.stringify(aiColumnGeneration), {
-        format: zodResponseFormat(
-          ai.aiColumnSchemas(generationStructure),
-          "key"
-        ),
-      })
+      mockAIColumnGeneration(generationStructure, aiColumnGeneration)
 
-      nock("http://photourl.com").get("/any").reply(200)
+      nock("https://photourl.com").get("/any.png").reply(200).persist()
 
       const dataGeneration: Record<string, Record<string, any>[]> = {
         Tickets: [],
@@ -635,7 +633,7 @@ describe("BudibaseAI", () => {
             "First Name": "Joshua",
             "Last Name": "Lee",
             Position: "Application Developer",
-            Photo: "http://photourl.com/any",
+            Photo: "https://photourl.com/any.png",
             Documents: [
               {
                 name: "development_guidelines.pdf",
@@ -654,7 +652,7 @@ describe("BudibaseAI", () => {
             "First Name": "Emily",
             "Last Name": "Davis",
             Position: "Software Deployment Technician",
-            Photo: "http://photourl.com/any",
+            Photo: "https://photourl.com/any.png",
             Documents: [
               {
                 name: "software_license_list.txt",
@@ -678,7 +676,7 @@ describe("BudibaseAI", () => {
             "First Name": "James",
             "Last Name": "Smith",
             Position: "IT Support Specialist",
-            Photo: "http://photourl.com/any",
+            Photo: "https://photourl.com/any.png",
             Documents: [
               {
                 name: "certificates.pdf",
@@ -697,7 +695,7 @@ describe("BudibaseAI", () => {
             "First Name": "Jessica",
             "Last Name": "Taylor",
             Position: "Cybersecurity Analyst",
-            Photo: "http://photourl.com/any",
+            Photo: "https://photourl.com/any.png",
             Documents: [
               {
                 name: "security_audit_report.pdf",
@@ -716,7 +714,7 @@ describe("BudibaseAI", () => {
             "First Name": "Ashley",
             "Last Name": "Harris",
             Position: "Database Administrator",
-            Photo: "http://photourl.com/any",
+            Photo: "https://photourl.com/any.png",
             Documents: [
               {
                 name: "database_backup.txt",
@@ -733,11 +731,9 @@ describe("BudibaseAI", () => {
           },
         ],
       }
-      mockChatGPTResponse(JSON.stringify(dataGeneration), {
-        format: zodResponseFormat(ai.tableDataStructuredOutput([]), "key"),
-      })
+      mockDataGeneration(dataGeneration)
 
-      mockChatGPTResponse("Mock LLM Response")
+      mockProcessAIColumn("Mock LLM Response")
 
       const { createdTables } = await config.api.ai.generateTables({ prompt })
       expect(createdTables).toEqual([

From 4ccdcf1f2f3f2de5a5efd1880626a2b5685db85a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 19:08:45 +0200
Subject: [PATCH 096/101] Add schema to checks

---
 .../server/src/api/routes/tests/ai.spec.ts    | 128 ++++++++++++++++++
 1 file changed, 128 insertions(+)

diff --git a/packages/server/src/api/routes/tests/ai.spec.ts b/packages/server/src/api/routes/tests/ai.spec.ts
index 89d910a22d..a53e81b234 100644
--- a/packages/server/src/api/routes/tests/ai.spec.ts
+++ b/packages/server/src/api/routes/tests/ai.spec.ts
@@ -740,6 +740,134 @@ describe("BudibaseAI", () => {
         { id: expect.stringMatching(/ta_\w+/), name: "Tickets" },
         { id: expect.stringMatching(/ta_\w+/), name: "Employees" },
       ])
+
+      const tables = [
+        await config.api.table.get(createdTables[0].id),
+        await config.api.table.get(createdTables[1].id),
+      ]
+      expect(tables).toEqual([
+        expect.objectContaining({
+          name: "Tickets",
+          schema: {
+            Title: {
+              name: "Title",
+              type: "string",
+              constraints: {
+                presence: true,
+              },
+            },
+            Description: {
+              name: "Description",
+              type: "longform",
+              constraints: {
+                presence: true,
+              },
+            },
+            Priority: {
+              name: "Priority",
+              type: "options",
+              constraints: {
+                inclusion: ["Low", "Medium", "High"],
+                presence: true,
+              },
+            },
+            Status: {
+              name: "Status",
+              type: "options",
+              constraints: {
+                inclusion: ["Open", "In Progress", "Closed"],
+                presence: true,
+              },
+            },
+            Assignee: {
+              name: "Assignee",
+              type: "link",
+              tableId: createdTables[1].id,
+              fieldName: "AssignedTickets",
+              relationshipType: "one-to-many",
+            },
+            "Created Date": {
+              name: "Created Date",
+              type: "datetime",
+              ignoreTimezones: false,
+              dateOnly: true,
+            },
+            "Resolution Time (Days)": {
+              name: "Resolution Time (Days)",
+              type: "formula",
+              formula:
+                '{{ js "cmV0dXJuIChuZXcgRGF0ZSgpIC0gbmV3IERhdGUoJCgiQ3JlYXRlZCBEYXRlIikpKSAvICgxMDAwICogNjAgKiA2MCAqIDI0KTs=" }}',
+              responseType: "number",
+            },
+            Attachment: {
+              name: "Attachment",
+              type: "attachment_single",
+            },
+            "Ticket Summary": {
+              name: "Ticket Summary",
+              type: "ai",
+              operation: "SUMMARISE_TEXT",
+              columns: ["Title", "Description"],
+            },
+            "Translated Description": {
+              name: "Translated Description",
+              type: "ai",
+              operation: "TRANSLATE",
+              column: "Description",
+              language: "es",
+            },
+          },
+        }),
+        expect.objectContaining({
+          name: "Employees 2",
+          schema: {
+            "First Name": {
+              constraints: {
+                presence: true,
+              },
+              name: "First Name",
+              type: "string",
+            },
+            "Last Name": {
+              constraints: {
+                presence: true,
+              },
+              name: "Last Name",
+              type: "string",
+            },
+            Photo: {
+              name: "Photo",
+              subtype: "image",
+              type: "attachment_single",
+            },
+            Position: {
+              constraints: {
+                presence: true,
+              },
+              name: "Position",
+              type: "string",
+            },
+            AssignedTickets: {
+              fieldName: "Assignee",
+              name: "AssignedTickets",
+              relationshipType: "many-to-one",
+              tableId: createdTables[0].id,
+              type: "link",
+            },
+            Documents: {
+              name: "Documents",
+              type: "attachment",
+            },
+            "Role Category": {
+              categories: "Manager,Staff,Intern,Contractor",
+              columns: ["Position"],
+              name: "Role Category",
+              operation: "CATEGORISE_TEXT",
+              type: "ai",
+            },
+          },
+        }),
+      ])
     })
   })
 })

From 862fd7cf88240b6df0c164f0bf6a44409552001c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 20:03:54 +0200
Subject: [PATCH 097/101] Improve tests

---
 .../server/src/api/routes/tests/ai.spec.ts    | 66 +++++++++++++++++--
 1 file changed, 60 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/ai.spec.ts b/packages/server/src/api/routes/tests/ai.spec.ts
index a53e81b234..9fea336360 100644
--- a/packages/server/src/api/routes/tests/ai.spec.ts
+++ b/packages/server/src/api/routes/tests/ai.spec.ts
@@ -627,7 +627,60 @@ describe("BudibaseAI", () => {
       nock("https://photourl.com").get("/any.png").reply(200).persist()
 
       const dataGeneration: Record<string, Record<string, any>[]> = {
-        Tickets: [],
+        Tickets: [
+          {
+            Title: "System slow performance",
+            Description:
+              "User reports significant slowdowns when using multiple applications simultaneously on their PC.",
+            Priority: "Medium",
+            Status: "Closed",
+            "Created Date": "2025-04-17",
+            Attachment: {
+              name: "performance_logs.txt",
+              extension: ".txt",
+              content: "performance logs",
+            },
+          },
+          {
+            Title: "Email delivery failure",
+            Description:
+              "Emails sent to external clients are bouncing back. Bounce back message: '550: Recipient address rejected'.",
+            Priority: "Medium",
+            Status: "In Progress",
+            "Created Date": "2025-04-19",
+            Attachment: {
+              name: "email_bounce_back.txt",
+              extension: ".txt",
+              content: "Email delivery failure",
+            },
+          },
+          {
+            Title: "Software installation request",
+            Description:
+              "Request to install Adobe Photoshop on user’s workstation for design work.",
+            Priority: "Low",
+            Status: "In Progress",
+            "Created Date": "2025-04-18",
+            Attachment: {
+              name: "software_request_form.pdf",
+              extension: ".pdf",
+              content: "Software installation request",
+            },
+          },
+          {
+            Title: "Unable to connect to VPN",
+            Description:
+              "User is experiencing issues when trying to connect to the VPN. Error message: 'VPN connection failed due to incorrect credentials'.",
+            Priority: "High",
+            Status: "Open",
+            "Created Date": "2025-04-20",
+            Attachment: {
+              name: "vpn_error_screenshot.pdf",
+              extension: ".pdf",
+              content: "vpn error",
+            },
+          },
+        ],
         "Employees 2": [
           {
             "First Name": "Joshua",
@@ -646,7 +699,6 @@ describe("BudibaseAI", () => {
                 content: "any content",
               },
             ],
-            "Role Category": "Staff",
           },
           {
             "First Name": "Emily",
@@ -670,7 +722,6 @@ describe("BudibaseAI", () => {
                 content: "any content",
               },
             ],
-            "Role Category": "Staff",
           },
           {
             "First Name": "James",
@@ -689,7 +740,6 @@ describe("BudibaseAI", () => {
                 content: "any content",
               },
             ],
-            "Role Category": "Staff",
           },
           {
             "First Name": "Jessica",
@@ -708,7 +758,6 @@ describe("BudibaseAI", () => {
                 content: "any content",
               },
             ],
-            "Role Category": "Staff",
           },
           {
             "First Name": "Ashley",
@@ -727,7 +776,6 @@ describe("BudibaseAI", () => {
                 content: "any content",
               },
             ],
-            "Role Category": "Staff",
           },
         ],
       }
@@ -868,6 +916,12 @@ describe("BudibaseAI", () => {
           },
         }),
       ])
+
+      const tickets = await config.api.row.fetch(createdTables[0].id)
+      expect(tickets).toHaveLength(4)
+
+      const employees = await config.api.row.fetch(createdTables[1].id)
+      expect(employees).toHaveLength(5)
     })
   })
 })

From fc952979da423c6cf3fc90dc4fbc02bec9b4448f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Apr 2025 21:08:26 +0200
Subject: [PATCH 098/101] Fix types

---
 .../app/[application]/data/_components/AITableGeneration.svelte  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
index 6217ea44a8..3a17356489 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/_components/AITableGeneration.svelte
@@ -14,7 +14,6 @@
     try {
       const { createdTables } = await API.generateTables({
         prompt: message,
-        addData: true,
       })
 
       const [tableToRedirect] = createdTables.sort((a, b) =>

From 5f88010a2e59d7653005f212b9892fadf2307c55 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Apr 2025 10:13:49 +0200
Subject: [PATCH 099/101] Add margin-left

---
 packages/builder/src/components/common/ai/AIInput.svelte | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/components/common/ai/AIInput.svelte b/packages/builder/src/components/common/ai/AIInput.svelte
index 5a7992e2ec..2ef2231796 100644
--- a/packages/builder/src/components/common/ai/AIInput.svelte
+++ b/packages/builder/src/components/common/ai/AIInput.svelte
@@ -265,6 +265,7 @@
   .ai-icon {
     width: 18px;
     height: 18px;
+    margin-left: 4px;
     margin-right: 8px;
     flex-shrink: 0;
     cursor: var(--ai-icon-cursor, pointer);

From d81640ef470d3237025b5b9129d70e1638377a66 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Apr 2025 10:21:48 +0200
Subject: [PATCH 100/101] Handle submit

---
 packages/builder/src/components/common/ai/AIInput.svelte | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/ai/AIInput.svelte b/packages/builder/src/components/common/ai/AIInput.svelte
index 2ef2231796..8e64fea7f3 100644
--- a/packages/builder/src/components/common/ai/AIInput.svelte
+++ b/packages/builder/src/components/common/ai/AIInput.svelte
@@ -31,6 +31,8 @@
   $: disabled = !aiEnabled || creditsExceeded || readonly || promptLoading
   $: animateBorder = !disabled && expanded
 
+  $: canSubmit = !readonly && !!value
+
   function collapse() {
     dispatch("collapse")
     expanded = expandedOnly
@@ -62,7 +64,7 @@
   }
 
   async function onPromptSubmit() {
-    if (readonly) {
+    if (!canSubmit) {
       return
     }
     promptLoading = true
@@ -155,6 +157,7 @@
             ? "#6E56FF"
             : "var(--spectrum-global-color-gray-600)"}
           size="S"
+          disabled={!canSubmit}
           hoverable={!readonly}
           hoverColor="#6E56FF"
           name={promptLoading ? "StopCircle" : "PlayCircle"}

From 45d95ae0be2979b171fe52618a58e45346105ed5 Mon Sep 17 00:00:00 2001
From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com>
Date: Thu, 24 Apr 2025 11:06:07 +0100
Subject: [PATCH 101/101] 15974 add missing plan to creator check (#16005)

* Change creator functionality to premium

* Remove unused isEnterprisePlan function

* Add ENTERPRISE_BASIC constant

* Renamed isPremiumPlan to isPremiumOrAbove based on feedback.

* Added premium_plus plan to creators.

* Added premium_plus to PlanType list.
---
 packages/builder/src/components/common/RoleSelect.svelte | 1 +
 packages/frontend-core/src/constants.ts                  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 15d4bd815d..4316e339dc 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -28,6 +28,7 @@
 
   $: isPremiumOrAbove = [
     Constants.PlanType.PREMIUM,
+    Constants.PlanType.PREMIUM_PLUS,
     Constants.PlanType.ENTERPRISE,
     Constants.PlanType.ENTERPRISE_BASIC_TRIAL,
     Constants.PlanType.ENTERPRISE_BASIC,
diff --git a/packages/frontend-core/src/constants.ts b/packages/frontend-core/src/constants.ts
index 178614dd19..954d77856c 100644
--- a/packages/frontend-core/src/constants.ts
+++ b/packages/frontend-core/src/constants.ts
@@ -86,6 +86,7 @@ export const PlanType = {
   PRO: "pro",
   BUSINESS: "business",
   PREMIUM: "premium",
+  PREMIUM_PLUS: "premium_plus",
   ENTERPRISE: "enterprise",
   ENTERPRISE_BASIC_TRIAL: "enterprise_basic_trial",
 }