MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/signature-field-and-component

This commit is contained in:
Dean 2024-04-17 09:07:17 +01:00
parent eed8deca35 9292ac7fae
commit 0e4cda458d
37 changed files with 318 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hosting/.env
View File

@ -17,6 +17,7 @@ APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
COUCH_DB_SQS_PORT=4006
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
PLUGINS_DIR=
ROLLING_LOG_MAX_SIZE=
ROLLING_LOG_MAX_SIZE=

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.23.5",
"version": "2.23.6",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344
Subproject commit 328c84234d11d97d840f0eb2c72665b04ba9e4f8

2
packages/backend-core/src/environment.ts
View File

@ -107,7 +107,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

9
packages/builder/src/components/common/Dropzone.svelte
View File

@ -27,14 +27,6 @@
return []
}
}
async function deleteAttachments(fileList) {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
</script>
<Dropzone
@ -42,6 +34,5 @@
{label}
{...$$restProps}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>

5
packages/client/src/components/app/NavItem.svelte
View File

@ -157,6 +157,11 @@
width: 100%;
}
/* Use normal theme colors for links when using a top nav */
.dropdown:not(.left) .sublinks a {
color: var(--spectrum-alias-text-color);
}
/* Left dropdowns */
.dropdown.left .sublinks-wrapper {
display: none;

12
packages/client/src/components/app/forms/AttachmentField.svelte
View File

@ -58,17 +58,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteAttachments({
keys: fileList,
tableId: formContext?.dataSource?.tableId,
})
} catch (error) {
return []
}
}
const handleChange = e => {
const value = fieldApiMapper.set(e.detail)
const changed = fieldApi.setValue(value)
@ -98,7 +87,6 @@
error={fieldState.error}
on:change={handleChange}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
{handleTooManyFiles}
{maximum}

28
packages/frontend-core/src/api/attachments.js
View File

@ -61,34 +61,6 @@ export const buildAttachmentEndpoints = API => {
})
return { publicUrl }
},
/**
* Deletes attachments from the bucket.
* @param keys the attachments to delete
* @param tableId the associated table ID
*/
deleteAttachments: async ({ keys, tableId }) => {
return await API.post({
url: `/api/attachments/${tableId}/delete`,
body: {
keys,
},
})
},
/**
* Deletes attachments from the builder bucket.
* @param keys the attachments to delete
*/
deleteBuilderAttachments: async keys => {
return await API.post({
url: `/api/attachments/delete`,
body: {
keys,
},
})
},
/**
* Download an attachment from a row given its column name.
* @param datasourceId the ID of the datasource to download from

9
packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
View File

@ -61,14 +61,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
onMount(() => {
api = {
focus: () => open(),
@ -101,7 +93,6 @@
on:change={e => onChange(e.detail)}
maximum={maximum || schema.constraints?.length?.maximum}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>
</div>

2
packages/pro

@ -1 +1 @@
Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a
Subproject commit c68183402b8fb17248572006531d5293ffc8a9ac

2
packages/server/package.json
View File

@ -125,7 +125,7 @@
"@babel/preset-env": "7.16.11",
"@swc/core": "1.3.71",
"@swc/jest": "0.2.27",
"@types/archiver": "^6.0.2",
"@types/archiver": "6.0.2",
"@types/global-agent": "2.1.1",
"@types/google-spreadsheet": "3.1.5",
"@types/jest": "29.5.5",

19
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { db as dbCore } from "@budibase/backend-core"
import AliasTables from "./alias"
import sdk from "../../../sdk"
import env from "../../../environment"
@ -120,6 +119,9 @@ async function removeManyToManyRelationships(
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
},
})
} else {
return []
@ -134,6 +136,9 @@ async function removeOneToManyRelationships(rowId: string, table: Table) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
filters,
meta: {
table,
},
})
} else {
return []
@ -249,6 +254,9 @@ export class ExternalRequest<T extends Operation> {
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(table._id!, Operation.READ),
filters: buildFilters(rowId, {}, table),
meta: {
table,
},
})
if (Array.isArray(response) && response.length > 0) {
return response[0]
@ -396,6 +404,9 @@ export class ExternalRequest<T extends Operation> {
[fieldName]: row[lookupField],
},
},
meta: {
table,
},
})
// this is the response from knex if no rows found
const rows: Row[] =
@ -426,6 +437,7 @@ export class ExternalRequest<T extends Operation> {
// if we're creating (in a through table) need to wipe the existing ones first
const promises = []
const related = await this.lookupRelations(mainTableId, row)
const table = this.getTable(mainTableId)!
for (let relationship of relationships) {
const { key, tableId, isUpdate, id, ...rest } = relationship
const body: { [key: string]: any } = processObjectSync(rest, row, {})
@ -471,6 +483,9 @@ export class ExternalRequest<T extends Operation> {
// if we're doing many relationships then we're writing, only one response
body,
filters: buildFilters(id, {}, linkTable),
meta: {
table,
},
})
)
} else {
@ -618,7 +633,7 @@ export class ExternalRequest<T extends Operation> {
if (env.SQL_ALIASING_DISABLE) {
response = await getDatasourceAndQuery(json)
} else {
const aliasing = new AliasTables(Object.keys(this.tables))
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
response = await aliasing.queryWithAliasing(json)
}

9
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -62,12 +62,12 @@ export function basicProcessing({
row,
table,
isLinked,
internal,
sqs,
}: {
row: Row
table: Table
isLinked: boolean
internal?: boolean
sqs?: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
@ -84,12 +84,13 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
if (!internal) {
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
} else {
for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
const columns = Object.keys(table.schema)
for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,

4
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
continue
}
let linked = await basicProcessing({
let linked = basicProcessing({
row,
table: linkedTable,
isLinked: true,
internal: opts?.sqs,
sqs: opts?.sqs,
})
if (!linked._id) {
continue

6
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
let rowId = row._id
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
row._id = rowId
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
row,
table,
isLinked: false,
internal: opts?.sqs,
sqs: opts?.sqs,
}),
table
)
@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
tables,
row,
finalRows,
relationships
relationships,
opts
)
}

7
packages/server/src/api/controllers/static/index.ts
View File

@ -127,13 +127,6 @@ export const uploadFile = async function (
)
}
export const deleteObjects = async function (ctx: Ctx) {
ctx.body = await objectStore.deleteFiles(
ObjectStoreBuckets.APPS,
ctx.request.body.keys
)
}
const requiresMigration = async (ctx: Ctx) => {
const appId = context.getAppId()
if (!appId) {

1
packages/server/src/api/controllers/table/ExternalRequest.ts
View File

@ -22,6 +22,7 @@ export async function makeTableRequest(
operation,
},
meta: {
table,
tables,
},
table,

11
packages/server/src/api/routes/static.ts
View File

@ -32,11 +32,6 @@ router
.get("/builder/:file*", controller.serveBuilder)
.get("/api/assets/client", controller.serveClientLibrary)
.post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
.post(
"/api/attachments/delete",
authorized(BUILDER),
controller.deleteObjects
)
.post("/api/beta/:feature", controller.toggleBetaUiFeature)
.post(
"/api/attachments/:tableId/upload",
@ -44,12 +39,6 @@ router
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.uploadFile
)
.post(
"/api/attachments/:tableId/delete",
paramResource("tableId"),
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.deleteObjects
)
.get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
.get("/app/:appUrl/:path*", controller.serveApp)
.get("/:appId/:path*", controller.serveApp)

1
packages/server/src/db/utils.ts
View File

@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
export const SQS_DATASOURCE_INTERNAL = "internal"
export const ViewName = dbCore.ViewName
export const InternalTables = dbCore.InternalTable
export const UNICODE_MAX = dbCore.UNICODE_MAX

2
packages/server/src/environment.ts
View File

@ -28,6 +28,7 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@ -39,6 +40,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,

28
packages/server/src/integrations/base/sql.ts
View File

@ -22,6 +22,8 @@ import {
SortDirection,
SqlQueryBinding,
Table,
TableSourceType,
INTERNAL_TABLE_SOURCE_ID,
} from "@budibase/types"
import environment from "../../environment"
@ -135,6 +137,18 @@ function generateSelectStatement(
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
return table?._id
} else {
return table?.name
}
}
class InternalBuilder {
private readonly client: string
@ -149,7 +163,7 @@ class InternalBuilder {
tableName: string,
opts: { aliases?: Record<string, string>; relationship?: boolean }
): Knex.QueryBuilder {
function getTableName(name: string) {
function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
return alias || name
}
@ -161,11 +175,11 @@ class InternalBuilder {
const updatedKey = dbCore.removeKeyNumbering(key)
const isRelationshipField = updatedKey.includes(".")
if (!opts.relationship && !isRelationshipField) {
fn(`${getTableName(tableName)}.${updatedKey}`, value)
fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
}
if (opts.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
fn(`${getTableName(filterTableName)}.${property}`, value)
fn(`${getTableAlias(filterTableName)}.${property}`, value)
}
}
}
@ -346,9 +360,10 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
@ -729,12 +744,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
results: Record<string, any>[],
aliases?: Record<string, string>
): Record<string, any>[] {
const tableName = getTableName(table)
for (const [name, field] of Object.entries(table.schema)) {
if (!this._isJsonColumn(field)) {
continue
}
const tableName = aliases?.[table.name] || table.name
const fullName = `${tableName}.${name}`
const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
const fullName = `${aliasedTableName}.${name}`
for (let row of results) {
if (typeof row[fullName] === "string") {
row[fullName] = JSON.parse(row[fullName])

23
packages/server/src/integrations/base/sqlTable.ts
View File

@ -1,19 +1,20 @@
import { Knex, knex } from "knex"
import {
RelationshipType,
FieldSubtype,
FieldType,
NumberFieldMetadata,
Operation,
QueryJson,
RelationshipType,
RenameColumn,
Table,
FieldType,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
import { utils } from "@budibase/shared-core"
import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
import { utils } from "@budibase/shared-core"
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
@ -105,13 +106,13 @@ function generateSchema(
column.relationshipType !== RelationshipType.MANY_TO_MANY
) {
if (!column.foreignKey || !column.tableId) {
throw "Invalid relationship schema"
throw new Error("Invalid relationship schema")
}
const { tableName } = breakExternalTableId(column.tableId)
// @ts-ignore
const relatedTable = tables[tableName]
if (!relatedTable) {
throw "Referenced table doesn't exist"
throw new Error("Referenced table doesn't exist")
}
const relatedPrimary = relatedTable.primary[0]
const externalType = relatedTable.schema[relatedPrimary].externalType
@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
let query: Knex.SchemaBuilder
if (!json.table || !json.meta || !json.meta.tables) {
throw "Cannot execute without table being specified"
throw new Error("Cannot execute without table being specified")
}
if (json.table.sourceType === TableSourceType.INTERNAL) {
throw new Error("Cannot perform table actions for SQS.")
}
switch (this._operation(json)) {
case Operation.CREATE_TABLE:
query = buildCreateTable(client, json.table, json.meta.tables)
break
case Operation.UPDATE_TABLE:
if (!json.meta || !json.meta.table) {
throw "Must specify old table for update"
throw new Error("Must specify old table for update")
}
// renameColumn does not work for MySQL, so return a raw query
if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
query = buildDeleteTable(client, json.table)
break
default:
throw "Table operation is of unknown type"
throw new Error("Table operation is of unknown type")
}
return getNativeSql(query)
}

38
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -9,6 +9,14 @@ import {
} from "@budibase/types"
const TABLE_NAME = "test"
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: TABLE_NAME,
primary: ["id"],
}
function endpoint(table: any, operation: any) {
return {
@ -25,6 +33,10 @@ function generateReadJson({
sort,
paginate,
}: any = {}): QueryJson {
const tableObj = { ...TABLE }
if (table) {
tableObj.name = table
}
return {
endpoint: endpoint(table || TABLE_NAME, "READ"),
resource: {
@ -34,14 +46,7 @@ function generateReadJson({
sort: sort || {},
paginate: paginate || {},
meta: {
table: {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: table || TABLE_NAME,
primary: ["id"],
} as any,
table: tableObj,
},
}
}
@ -49,6 +54,9 @@ function generateReadJson({
function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
return {
endpoint: endpoint(table, "CREATE"),
meta: {
table: TABLE,
},
body,
}
}
@ -58,7 +66,15 @@ function generateUpdateJson({
body = {},
filters = {},
meta = {},
}: {
table: string
body?: any
filters?: any
meta?: any
}): QueryJson {
if (!meta.table) {
meta.table = table
}
return {
endpoint: endpoint(table, "UPDATE"),
filters,
@ -70,6 +86,9 @@ function generateUpdateJson({
function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
return {
endpoint: endpoint(table, "DELETE"),
meta: {
table: TABLE,
},
filters,
}
}
@ -102,6 +121,9 @@ function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}

19
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -4,12 +4,26 @@ import {
QueryJson,
SourceName,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
import sdk from "../../sdk"
// this doesn't exist strictly
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: "tableName",
primary: ["id"],
}
const AliasTables = sdk.rows.AliasTables
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
@ -220,6 +234,9 @@ describe("Captures of real examples", () => {
resource: {
fields,
},
meta: {
table: TABLE,
},
}
}

2
packages/server/src/sdk/app/rows/index.ts
View File

@ -3,6 +3,7 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
import AliasTables from "./sqlAlias"
export default {
...attachments,
@ -10,4 +11,5 @@ export default {
...search,
utils,
external,
AliasTables,
}

11
packages/server/src/sdk/app/rows/search.ts
View File

@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
export { isValidFilter } from "../../../integrations/utils"
@ -76,12 +78,15 @@ export async function search(
}
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (isExternalTable) {
return external.search(options)
return external.search(options, table)
} else if (env.SQS_SEARCH_ENABLE) {
return sqs.search(options)
return sqs.search(options, table)
} else {
return internal.search(options)
return internal.search(options, table)
}
}

13
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -8,6 +8,7 @@ import {
SearchFilters,
RowSearchParams,
SearchResponse,
Table,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import { handleRequest } from "../../../../api/controllers/row/external"
@ -18,13 +19,13 @@ import {
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { HTTPError, db } from "@budibase/backend-core"
import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
@ -68,8 +69,6 @@ export async function search(
}
try {
const table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
let rows = await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
@ -150,11 +149,15 @@ export async function exportRows(
}
const datasource = await sdk.datasources.get(datasourceId!)
const table = await sdk.tables.getTable(tableId)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query: requestQuery, sort, sortOrder })
let result = await search(
{ tableId, query: requestQuery, sort, sortOrder },
table
)
let rows: Row[] = []
let headers

22
packages/server/src/sdk/app/rows/search/internal.ts
View File

@ -1,6 +1,6 @@
import { context, db, HTTPError } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
import { fullSearch, paginatedSearch } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
import {
Database,
@ -33,7 +33,8 @@ import pick from "lodash/pick"
import { breakRowIdField } from "../../../../integrations/utils"
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
@ -51,8 +52,6 @@ export async function search(
query: {},
}
let table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
@ -122,12 +121,15 @@ export async function exportRows(
result = await outputProcessing<Row[]>(table, response)
} else if (query) {
let searchResponse = await search({
tableId,
query,
sort,
sortOrder,
})
let searchResponse = await search(
{
tableId,
query,
sort,
sortOrder,
},
table
)
result = searchResponse.rows
}

99
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -20,7 +20,12 @@ import {
} from "../../../../api/controllers/row/utils"
import sdk from "../../../index"
import { context } from "@budibase/backend-core"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
import {
CONSTANT_INTERNAL_ROW_COLS,
SQS_DATASOURCE_INTERNAL,
} from "../../../../db/utils"
import AliasTables from "../sqlAlias"
import { outputProcessing } from "../../../../utilities/rowProcessor"
function buildInternalFieldList(
table: Table,
@ -31,19 +36,19 @@ function buildInternalFieldList(
fieldList = fieldList.concat(
CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
)
if (opts.relationships) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.LINK) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(
table => table._id === linkCol.tableId
)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
} else {
fieldList.push(`${table._id}.${col.name}`)
}
for (let col of Object.values(table.schema)) {
const isRelationship = col.type === FieldType.LINK
if (!opts.relationships && isRelationship) {
continue
}
if (isRelationship) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(table => table._id === linkCol.tableId)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
} else {
fieldList.push(`${table._id}.${col.name}`)
}
}
return fieldList
@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
}
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId, paginate, query, ...params } = options
const { paginate, query, ...params } = options
const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
const allTables = await sdk.tables.getAllInternalTables()
const allTablesMap = buildTableMap(allTables)
const table = allTables.find(table => table._id === tableId)
if (!table) {
throw new Error("Unable to find table")
}
@ -111,7 +116,7 @@ export async function search(
const request: QueryJson = {
endpoint: {
// not important, we query ourselves
datasourceId: "internal",
datasourceId: SQS_DATASOURCE_INTERNAL,
entityId: table._id!,
operation: Operation.READ,
},
@ -154,34 +159,44 @@ export async function search(
}
}
try {
const query = builder._query(request, {
disableReturning: true,
const alias = new AliasTables(allTables.map(table => table.name))
const rows = await alias.queryWithAliasing(request, async json => {
const query = builder._query(json, {
disableReturning: true,
})
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql,
bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const db = context.getAppDB()
return await db.sql<Row>(sql, bindings)
})
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql,
bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const db = context.getAppDB()
const rows = await db.sql<Row>(sql, bindings)
// process from the format of tableId.column to expected format
const processed = await sqlOutputProcessing(
rows,
table!,
allTablesMap,
relationships,
{
sqs: true,
}
)
return {
rows: await sqlOutputProcessing(
rows,
table!,
allTablesMap,
relationships,
{
sqs: true,
}
),
// final row processing for response
rows: await outputProcessing<Row[]>(table, processed, {
preserveLinks: true,
squash: true,
}),
}
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message

6
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -112,7 +112,7 @@ describe("external search", () => {
tableId,
query: {},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -130,7 +130,7 @@ describe("external search", () => {
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -157,7 +157,7 @@ describe("external search", () => {
},
},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])

4
packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
View File

@ -81,7 +81,7 @@ describe("internal", () => {
tableId,
query: {},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -99,7 +99,7 @@ describe("internal", () => {
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(

28
packages/server/src/api/controllers/row/alias.ts → packages/server/src/sdk/app/rows/sqlAlias.ts
View File

@ -6,11 +6,12 @@ import {
Row,
SearchFilters,
} from "@budibase/types"
import { getSQLClient } from "../../../sdk/app/rows/utils"
import { getSQLClient } from "./utils"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import datasources from "../datasources"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
@ -156,12 +157,19 @@ export default class AliasTables {
}
async queryWithAliasing(
json: QueryJson
json: QueryJson,
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
let aliasingEnabled: boolean, datasource: Datasource | undefined
if (isSqs) {
aliasingEnabled = true
} else {
datasource = await datasources.get(datasourceId)
aliasingEnabled = this.isAliasingEnabled(json, datasource)
}
const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@ -207,7 +215,15 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
const response = await makeExternalQuery(datasource, json)
let response: DatasourcePlusQueryResponse
if (datasource && !isSqs) {
response = await makeExternalQuery(datasource, json)
} else if (queryFn) {
response = await queryFn(json)
} else {
throw new Error("No supplied method to perform aliased query")
}
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {

64
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -1,8 +1,19 @@
import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
import {
FieldType,
RelationshipFieldMetadata,
SQLiteDefinition,
SQLiteTable,
SQLiteTables,
SQLiteType,
Table,
} from "@budibase/types"
import { cloneDeep } from "lodash"
import tablesSdk from "../"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
import {
CONSTANT_INTERNAL_ROW_COLS,
generateJunctionTableID,
} from "../../../../db/utils"
const BASIC_SQLITE_DOC: SQLiteDefinition = {
_id: SQLITE_DESIGN_DOC_ID,
@ -37,9 +48,42 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BB_REFERENCE]: SQLiteType.TEXT,
}
function mapTable(table: Table): { [key: string]: SQLiteType } {
function buildRelationshipDefinitions(
table: Table,
relationshipColumn: RelationshipFieldMetadata
): {
tableId: string
definition: SQLiteTable
} {
const tableId = table._id!,
relatedTableId = relationshipColumn.tableId
return {
tableId: generateJunctionTableID(tableId, relatedTableId),
definition: {
["doc1.rowId"]: SQLiteType.TEXT,
["doc1.tableId"]: SQLiteType.TEXT,
["doc1.fieldName"]: SQLiteType.TEXT,
["doc2.rowId"]: SQLiteType.TEXT,
["doc2.tableId"]: SQLiteType.TEXT,
["doc2.fieldName"]: SQLiteType.TEXT,
tableId: SQLiteType.TEXT,
},
}
}
// this can generate relationship tables as part of the mapping
function mapTable(table: Table): SQLiteTables {
const tables: SQLiteTables = {}
const fields: Record<string, SQLiteType> = {}
for (let [key, column] of Object.entries(table.schema)) {
// relationships should be handled differently
if (column.type === FieldType.LINK) {
const { tableId, definition } = buildRelationshipDefinitions(
table,
column
)
tables[tableId] = { fields: definition }
}
if (!FieldTypeMap[column.type]) {
throw new Error(`Unable to map type "${column.type}" to SQLite type`)
}
@ -50,10 +94,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
constantMap[col] = SQLiteType.TEXT
})
return {
const thisTable: SQLiteTable = {
...constantMap,
...fields,
}
tables[table._id!] = { fields: thisTable }
return tables
}
// nothing exists, need to iterate though existing tables
@ -61,8 +107,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
const tables = await tablesSdk.getAllInternalTables()
const definition = cloneDeep(BASIC_SQLITE_DOC)
for (let table of tables) {
definition.sql.tables[table._id!] = {
fields: mapTable(table),
definition.sql.tables = {
...definition.sql.tables,
...mapTable(table),
}
}
return definition
@ -76,8 +123,9 @@ export async function addTableToSqlite(table: Table) {
} catch (err) {
definition = await buildBaseDefinition()
}
definition.sql.tables[table._id!] = {
fields: mapTable(table),
definition.sql.tables = {
...definition.sql.tables,
...mapTable(table),
}
await db.put(definition)
}

5
packages/server/src/tests/utilities/api/datasource.ts
View File

@ -60,7 +60,10 @@ export class DatasourceAPI extends TestAPI {
})
}
query = async (query: QueryJson, expectations?: Expectations) => {
query = async (
query: Omit<QueryJson, "meta">,
expectations?: Expectations
) => {
return await this._post<any>(`/api/datasources/query`, {
body: query,
expectations,

20
packages/types/src/documents/app/sqlite.ts
View File

@ -6,17 +6,23 @@ export enum SQLiteType {
NUMERIC = "NUMERIC",
}
export type SQLiteTable = Record<
string,
SQLiteType | { field: string; type: SQLiteType }
>
export type SQLiteTables = Record<
string,
{
fields: SQLiteTable
}
>
export interface SQLiteDefinition {
_id: string
language: string
sql: {
tables: {
[tableName: string]: {
fields: {
[key: string]: SQLiteType | { field: string; type: SQLiteType }
}
}
}
tables: SQLiteTables
options: {
table_name: string
}

4
packages/types/src/sdk/search.ts
View File

@ -90,8 +90,8 @@ export interface QueryJson {
paginate?: PaginationJson
body?: Row | Row[]
table?: Table
meta?: {
table?: Table
meta: {
table: Table
tables?: Record<string, Table>
renamed?: RenameColumn
}

2
yarn.lock
View File

@ -5174,7 +5174,7 @@
dependencies:
"@types/node" "*"
"@types/archiver@^6.0.2":
"@types/archiver@6.0.2":
version "6.0.2"
resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2"
integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==