Merge branch 'master' into fix/csv-importing-file-refresh

packages/server/src/api/controllers/table/index.ts

@@ -14,16 +14,20 @@ import { events, HTTPError } from "@budibase/backend-core"
 import {
   BulkImportRequest,
   BulkImportResponse,
+  CsvToJsonRequest,
+  CsvToJsonResponse,
   FetchTablesResponse,
   MigrateRequest,
   MigrateResponse,
-  Row,
   SaveTableRequest,
   SaveTableResponse,
   Table,
   TableResponse,
   TableSourceType,
   UserCtx,
+  ValidateNewTableImportRequest,
+  ValidateTableImportRequest,
+  ValidateTableImportResponse,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -144,7 +148,9 @@ export async function bulkImport(
   ctx.body = { message: `Bulk rows created.` }
 }
 
-export async function csvToJson(ctx: UserCtx) {
+export async function csvToJson(
+  ctx: UserCtx<CsvToJsonRequest, CsvToJsonResponse>
+) {
   const { csvString } = ctx.request.body
 
   const result = await jsonFromCsvString(csvString)
@@ -153,8 +159,10 @@ export async function csvToJson(ctx: UserCtx) {
   ctx.body = result
 }
 
-export async function validateNewTableImport(ctx: UserCtx) {
+export async function validateNewTableImport(
-  const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
+  ctx: UserCtx<ValidateNewTableImportRequest, ValidateTableImportResponse>
+) {
+  const { rows, schema } = ctx.request.body
 
   if (isRows(rows) && isSchema(schema)) {
     ctx.status = 200
@@ -164,8 +172,10 @@ export async function validateNewTableImport(ctx: UserCtx) {
   }
 }
 
-export async function validateExistingTableImport(ctx: UserCtx) {
+export async function validateExistingTableImport(
-  const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body
+  ctx: UserCtx<ValidateTableImportRequest, ValidateTableImportResponse>
+) {
+  const { rows, tableId } = ctx.request.body
 
   let schema = null
   if (tableId) {

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -15,6 +15,7 @@ import {
   Table,
   TableSchema,
   SupportedSqlTypes,
+  JsonFieldSubType,
 } from "@budibase/types"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import { tableForDatasource } from "../../../tests/utilities/structures"
@@ -288,7 +289,10 @@ describe("/datasources", () => {
             name: "options",
             type: FieldType.OPTIONS,
             constraints: {
-              presence: { allowEmpty: false },
+              presence: {
+                allowEmpty: false,
+              },
+              inclusion: [],
             },
           },
           [FieldType.NUMBER]: {
@@ -302,6 +306,10 @@ describe("/datasources", () => {
           [FieldType.ARRAY]: {
             name: "array",
             type: FieldType.ARRAY,
+            constraints: {
+              type: JsonFieldSubType.ARRAY,
+              inclusion: [],
+            },
           },
           [FieldType.DATETIME]: {
             name: "datetime",

packages/server/src/api/routes/tests/row.spec.ts

@@ -32,6 +32,7 @@ import {
   TableSourceType,
   UpdatedRowEventEmitter,
   TableSchema,
+  JsonFieldSubType,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import _, { merge } from "lodash"
@@ -102,7 +103,7 @@ describe.each([
   ): SaveTableRequest {
     const defaultSchema: TableSchema = {
       id: {
-        type: FieldType.AUTO,
+        type: FieldType.NUMBER,
         name: "id",
         autocolumn: true,
         constraints: {
@@ -384,7 +385,7 @@ describe.each([
         const arrayField: FieldSchema = {
           type: FieldType.ARRAY,
           constraints: {
-            type: "array",
+            type: JsonFieldSubType.ARRAY,
             presence: false,
             inclusion: ["One", "Two", "Three"],
           },

packages/server/src/api/routes/tests/search.spec.ts

@@ -20,6 +20,7 @@ import {
   Datasource,
   EmptyFilterOption,
   FieldType,
+  JsonFieldSubType,
   RelationshipType,
   Row,
   RowSearchParams,
@@ -1494,7 +1495,10 @@ describe.each([
         numbers: {
           name: "numbers",
           type: FieldType.ARRAY,
-          constraints: { inclusion: ["one", "two", "three"] },
+          constraints: {
+            type: JsonFieldSubType.ARRAY,
+            inclusion: ["one", "two", "three"],
+          },
         },
       })
       await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }])

packages/server/src/api/routes/tests/table.spec.ts

@@ -398,6 +398,7 @@ describe.each([
                   name: "auto",
                   autocolumn: true,
                   type: FieldType.AUTO,
+                  subtype: AutoFieldSubType.AUTO_ID,
                 },
               },
             },

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -56,7 +56,7 @@ describe.each([
       primary: ["id"],
       schema: {
         id: {
-          type: FieldType.AUTO,
+          type: FieldType.NUMBER,
           name: "id",
           autocolumn: true,
           constraints: {
@@ -241,7 +241,7 @@ describe.each([
         schema: {
           id: {
             name: "id",
-            type: FieldType.AUTO,
+            type: FieldType.NUMBER,
             autocolumn: true,
             visible: true,
           },
@@ -1555,7 +1555,7 @@ describe.each([
             schema: {
               id: {
                 name: "id",
-                type: FieldType.AUTO,
+                type: FieldType.NUMBER,
                 autocolumn: true,
               },
               name: {

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -17,6 +17,7 @@ import {
   AutoFieldSubType,
   Datasource,
   FieldType,
+  JsonFieldSubType,
   RelationshipType,
   Row,
   SourceName,
@@ -131,7 +132,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
     "Item Tags": {
       type: FieldType.ARRAY,
       constraints: {
-        type: FieldType.ARRAY,
+        type: JsonFieldSubType.ARRAY,
         presence: {
           allowEmpty: false,
         },
@@ -153,7 +154,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
     Status: {
       type: FieldType.ARRAY,
       constraints: {
-        type: FieldType.ARRAY,
+        type: JsonFieldSubType.ARRAY,
         presence: {
           allowEmpty: false,
         },
@@ -291,7 +292,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
     "Employee Level": {
       type: FieldType.ARRAY,
       constraints: {
-        type: FieldType.ARRAY,
+        type: JsonFieldSubType.ARRAY,
         presence: false,
         inclusion: ["Manager", "Junior", "Senior", "Apprentice", "Contractor"],
       },
@@ -535,7 +536,7 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
     "Expense Tags": {
       type: FieldType.ARRAY,
       constraints: {
-        type: FieldType.ARRAY,
+        type: JsonFieldSubType.ARRAY,
         presence: {
           allowEmpty: false,
         },

packages/server/src/integrations/utils/utils.ts

@@ -150,22 +150,28 @@ export function generateColumnDefinition(config: {
     }).internal
   }
 
-  const constraints: {
+  let schema: FieldSchema
-    presence: boolean
-    inclusion?: string[]
-  } = {
-    presence,
-  }
   if (foundType === FieldType.OPTIONS) {
-    constraints.inclusion = options
+    schema = {
-  }
+      type: foundType,
-
+      externalType,
-  const schema: FieldSchema = {
+      autocolumn,
-    type: foundType,
+      name,
-    externalType,
+      constraints: {
-    autocolumn,
+        presence,
-    name,
+        inclusion: options!,
-    constraints,
+      },
+    }
+  } else {
+    schema = {
+      type: foundType,
+      externalType,
+      autocolumn,
+      name,
+      constraints: {
+        presence,
+      },
+    }
   }
   if (schema.type === FieldType.DATETIME) {
     schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -55,7 +55,7 @@ describe.each([
         schema: {
           id: {
             name: "id",
-            type: FieldType.AUTO,
+            type: FieldType.NUMBER,
             autocolumn: true,
           },
           name: {

packages/types/src/api/web/app/rows.ts

@@ -37,7 +37,7 @@ export interface PaginatedSearchRowResponse
     PaginationResponse {}
 
 export interface ExportRowsRequest {
-  rows: string[]
+  rows?: string[]
   columns?: string[]
   query?: SearchFilters
   sort?: string

packages/types/src/api/web/app/table.ts

@@ -1,4 +1,4 @@
-import { Row, Table, TableRequest, View } from "../../../documents"
+import { Row, Table, TableRequest, TableSchema, View } from "../../../documents"
 import { ViewV2Enriched } from "../../../sdk"
 
 export type TableViewsResponse = { [key: string]: View | ViewV2Enriched }
@@ -32,3 +32,28 @@ export interface MigrateRequest {
 export interface MigrateResponse {
   message: string
 }
+
+export interface ValidateNewTableImportRequest {
+  rows: Row[]
+  schema: TableSchema
+}
+
+export interface ValidateTableImportRequest {
+  tableId?: string
+  rows: Row[]
+}
+
+export interface ValidateTableImportResponse {
+  schemaValidation: {
+    [field: string]: boolean
+  }
+  allValid: boolean
+  invalidColumns: Array<string>
+  errors: Record<string, string>
+}
+
+export interface CsvToJsonRequest {
+  csvString: string
+}
+
+export type CsvToJsonResponse = any[]

packages/types/src/documents/app/table/schema.ts

@@ -64,7 +64,7 @@ export interface AutoColumnFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.AUTO
   autocolumn: true
-  subtype?: AutoFieldSubType
+  subtype: AutoFieldSubType
   lastID?: number
   // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc)
   autoReason?: AutoReason
@@ -157,6 +157,21 @@ export interface FieldConstraints {
   }
 }
 
+export interface OptionsFieldMetadata extends BaseFieldSchema {
+  type: FieldType.OPTIONS
+  constraints: FieldConstraints & {
+    inclusion: string[]
+  }
+}
+
+export interface ArrayFieldMetadata extends BaseFieldSchema {
+  type: FieldType.ARRAY
+  constraints: FieldConstraints & {
+    type: JsonFieldSubType.ARRAY
+    inclusion: string[]
+  }
+}
+
 interface BaseFieldSchema extends UIFieldMetadata {
   type: FieldType
   name: string
@@ -182,6 +197,8 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.BB_REFERENCE_SINGLE
     | FieldType.ATTACHMENTS
     | FieldType.STRING
+    | FieldType.ARRAY
+    | FieldType.OPTIONS
   >
 }
 
@@ -198,6 +215,8 @@ export type FieldSchema =
   | JsonFieldMetadata
   | AttachmentFieldMetadata
   | BBReferenceSingleFieldMetadata
+  | ArrayFieldMetadata
+  | OptionsFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema