MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge master.

This commit is contained in:
Sam Rose 2024-08-12 09:43:49 +01:00
parent 083b595d50 4d7f459529
commit bcefa398a4
No known key found for this signature in database
13 changed files with 136 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/backend-core/src/sql/sqlTable.ts
View File

@ -28,16 +28,25 @@ function generateSchema(
oldTable: null | Table = null,
renamed?: RenameColumn
) {
let primaryKey = table && table.primary ? table.primary[0] : null
let primaryKeys = table && table.primary ? table.primary : []
const columns = Object.values(table.schema)
// all columns in a junction table will be meta
let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta)
let isJunction = metaCols.length === columns.length
let columnTypeSet: string[] = []
// can't change primary once its set for now
if (primaryKey && !oldTable && !isJunction) {
schema.increments(primaryKey).primary()
} else if (!oldTable && isJunction) {
schema.primary(metaCols.map(col => col.name))
if (!oldTable) {
// junction tables are special - we have an expected format
if (isJunction) {
schema.primary(metaCols.map(col => col.name))
} else if (primaryKeys.length === 1) {
schema.increments(primaryKeys[0]).primary()
// note that we've set its type
columnTypeSet.push(primaryKeys[0])
} else {
schema.primary(primaryKeys)
}
}
// check if any columns need added
@ -49,7 +58,7 @@ function generateSchema(
const oldColumn = oldTable ? oldTable.schema[key] : null
if (
(oldColumn && oldColumn.type) ||
(primaryKey === key && !isJunction) ||
columnTypeSet.includes(key) ||
renamed?.updated === key
) {
continue
@ -61,7 +70,12 @@ function generateSchema(
case FieldType.LONGFORM:
case FieldType.BARCODEQR:
case FieldType.BB_REFERENCE_SINGLE:
schema.text(key)
// primary key strings have to have a length in some DBs
if (primaryKeys.includes(key)) {
schema.string(key, 255)
} else {
schema.text(key)
}
break
case FieldType.NUMBER:
// if meta is specified then this is a junction table entry

1
packages/server/package.json
View File

@ -144,6 +144,7 @@
"copyfiles": "2.4.1",
"docker-compose": "0.23.17",
"jest": "29.7.0",
"jest-extended": "^4.0.2",
"jest-openapi": "0.14.2",
"nock": "13.5.4",
"nodemon": "2.0.15",

5
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -195,12 +195,13 @@ export class ExternalRequest<T extends Operation> {
if (filters) {
// need to map over the filters and make sure the _id field isn't present
let prefix = 1
for (const operator of Object.values(filters)) {
for (const [operatorType, operator] of Object.entries(filters)) {
const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
for (const field of Object.keys(operator || {})) {
if (dbCore.removeKeyNumbering(field) === "_id") {
if (primary) {
const parts = breakRowIdField(operator[field])
if (primary.length > 1) {
if (primary.length > 1 && isArrayOp) {
operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
id: primary,
values: parts[0],

3
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -71,8 +71,7 @@ export function basicProcessing({
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
for (let field of Object.values(table.schema)) {
const fieldName = field.name
for (let fieldName of Object.keys(table.schema)) {
let value = extractFieldValue({
row,
tableName: table.name,

13
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -1,5 +1,5 @@
import * as setup from "./utilities"
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import { checkBuilderEndpoint, allowUndefined } from "./utilities/TestFunctions"
import { getCachedVariable } from "../../../threads/utils"
import { context, events } from "@budibase/backend-core"
import sdk from "../../../sdk"
@ -380,21 +380,24 @@ describe("/datasources", () => {
persisted?.entities &&
Object.entries(persisted.entities).reduce<Record<string, Table>>(
(acc, [tableName, table]) => {
acc[tableName] = {
acc[tableName] = expect.objectContaining({
...table,
primaryDisplay: expect.not.stringMatching(
new RegExp(`^${table.primaryDisplay || ""}$`)
),
schema: Object.entries(table.schema).reduce<TableSchema>(
(acc, [fieldName, field]) => {
acc[fieldName] = expect.objectContaining({
acc[fieldName] = {
...field,
})
externalType: allowUndefined(expect.any(String)),
constraints: allowUndefined(expect.any(Object)),
autocolumn: allowUndefined(expect.any(Boolean)),
}
return acc
},
{}
),
}
})
return acc
},
{}

66
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -41,6 +41,7 @@ import { dataFilters } from "@budibase/shared-core"
import { Knex } from "knex"
import { structures } from "@budibase/backend-core/tests"
import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
import { generateRowIdField } from "../../../integrations/utils"
describe.each([
["in-memory", undefined],
@ -2355,6 +2356,35 @@ describe.each([
})
})
describe("Invalid column definitions", () => {
beforeAll(async () => {
// need to create an invalid table - means ignoring typescript
table = await createTable({
// @ts-ignore
invalid: {
type: FieldType.STRING,
},
name: {
name: "name",
type: FieldType.STRING,
},
})
await createRows([
{ name: "foo", invalid: "id1" },
{ name: "bar", invalid: "id2" },
])
})
it("can get rows with all table data", async () => {
await expectSearch({
query: {},
}).toContain([
{ name: "foo", invalid: "id1" },
{ name: "bar", invalid: "id2" },
])
})
})
describe.each(["data_name_test", "name_data_test", "name_test_data_"])(
"special (%s) case",
column => {
@ -2621,6 +2651,42 @@ describe.each([
})
})
!isInternal &&
describe("search by composite key", () => {
beforeAll(async () => {
table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
idColumn1: {
name: "idColumn1",
type: FieldType.NUMBER,
},
idColumn2: {
name: "idColumn2",
type: FieldType.NUMBER,
},
},
primary: ["idColumn1", "idColumn2"],
})
)
await createRows([{ idColumn1: 1, idColumn2: 2 }])
})
it("can filter by the row ID with limit 1", async () => {
await expectSearch({
query: {
equal: { _id: generateRowIdField([1, 2]) },
},
limit: 1,
}).toContain([
{
idColumn1: 1,
idColumn2: 2,
},
])
})
})
isSql &&
describe("pagination edge case with relationships", () => {
let mainRows: Row[] = []

4
packages/server/src/api/routes/tests/utilities/TestFunctions.ts
View File

@ -184,3 +184,7 @@ export const runInProd = async (func: any) => {
env._set("NODE_ENV", nodeEnv)
env._set("JEST_WORKER_ID", workerId)
}
export function allowUndefined(expectation: jest.Expect) {
return expect.toBeOneOf([expectation, undefined, null])
}

11
packages/server/src/integrations/utils/utils.ts
View File

@ -15,6 +15,7 @@ import { helpers, utils } from "@budibase/shared-core"
import { pipeline } from "stream/promises"
import tmp from "tmp"
import fs from "fs"
import { merge, cloneDeep } from "lodash"
type PrimitiveTypes =
| FieldType.STRING
@ -291,10 +292,16 @@ function copyExistingPropsOver(
const fetchedColumnDefinition: FieldSchema | undefined =
table.schema[key]
table.schema[key] = {
...existingTableSchema[key],
// merge the properties - anything missing will be filled in, old definition preferred
// have to clone due to the way merge works
...merge(
cloneDeep(fetchedColumnDefinition),
existingTableSchema[key]
),
// always take externalType and autocolumn from the fetched definition
externalType:
existingTableSchema[key].externalType ||
table.schema[key]?.externalType,
fetchedColumnDefinition?.externalType,
autocolumn: fetchedColumnDefinition?.autocolumn,
} as FieldSchema
// check constraints which can be fetched from the DB (they could be updated)

5
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -73,13 +73,14 @@ function buildInternalFieldList(
fieldList = fieldList.concat(
PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`)
)
for (let col of Object.values(table.schema)) {
for (let key of Object.keys(table.schema)) {
const col = table.schema[key]
const isRelationship = col.type === FieldType.LINK
if (!opts?.relationships && isRelationship) {
continue
}
if (!isRelationship) {
fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
} else {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(table => table._id === linkCol.tableId)

6
packages/server/src/sdk/app/rows/utils.ts
View File

@ -12,6 +12,7 @@ import {
Table,
TableSchema,
SqlClient,
ArrayOperator,
} from "@budibase/types"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
@ -311,3 +312,8 @@ function validateTimeOnlyField(
return res
}
// type-guard check
export function isArrayFilter(operator: any): operator is ArrayOperator {
return Object.values(ArrayOperator).includes(operator)
}

2
packages/server/src/tests/jestSetup.ts
View File

@ -1,8 +1,10 @@
import env from "../environment"
import * as matchers from "jest-extended"
import { env as coreEnv, timers } from "@budibase/backend-core"
import { testContainerUtils } from "@budibase/backend-core/tests"
import nock from "nock"
expect.extend(matchers)
if (!process.env.CI) {
// set a longer timeout in dev for debugging 100 seconds
jest.setTimeout(100 * 1000)

3
packages/server/src/utilities/schema.ts
View File

@ -146,7 +146,8 @@ export function parse(rows: Rows, table: Table): Rows {
return rows.map(row => {
const parsedRow: Row = {}
Object.entries(row).forEach(([columnName, columnData]) => {
Object.keys(row).forEach(columnName => {
const columnData = row[columnName]
const schema = table.schema
if (!(columnName in schema)) {
// Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case

12
yarn.lock
View File

@ -13614,7 +13614,7 @@ jest-config@^29.7.0:
slash "^3.0.0"
strip-json-comments "^3.1.1"
"jest-diff@>=29.4.3 < 30", jest-diff@^29.4.1, jest-diff@^29.7.0:
"jest-diff@>=29.4.3 < 30", jest-diff@^29.0.0, jest-diff@^29.4.1, jest-diff@^29.7.0:
version "29.7.0"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a"
integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==
@ -13673,12 +13673,20 @@ jest-environment-node@^29.7.0:
jest-mock "^29.7.0"
jest-util "^29.7.0"
jest-extended@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/jest-extended/-/jest-extended-4.0.2.tgz#d23b52e687cedf66694e6b2d77f65e211e99e021"
integrity sha512-FH7aaPgtGYHc9mRjriS0ZEHYM5/W69tLrFTIdzm+yJgeoCmmrSB/luSfMSqWP9O29QWHPEmJ4qmU6EwsZideog==
dependencies:
jest-diff "^29.0.0"
jest-get-type "^29.0.0"
jest-get-type@^26.3.0:
version "26.3.0"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0"
integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==
jest-get-type@^29.6.3:
jest-get-type@^29.0.0, jest-get-type@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==