{table.name}
- Point Helm at this repo to see charts.
- - \ No newline at end of file diff --git a/docs/index.yaml b/docs/index.yaml deleted file mode 100644 index 04543c147e..0000000000 --- a/docs/index.yaml +++ /dev/null @@ -1,132 +0,0 @@ -apiVersion: v1 -entries: - budibase: - - apiVersion: v2 - appVersion: 0.9.169 - created: "2021-10-20T14:27:23.521358+01:00" - dependencies: - - condition: services.couchdb.enabled - name: couchdb - repository: https://apache.github.io/couchdb-helm - version: 3.3.4 - - condition: ingress.nginx - name: ingress-nginx - repository: https://github.com/kubernetes/ingress-nginx - version: 3.35.0 - description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes. - digest: 57f365d799fcaace4658883cb8ec961a7905383a68acf065af4f6e57f9878ff8 - keywords: - - low-code - - database - - cluster - name: budibase - sources: - - https://github.com/Budibase/budibase - - https://budibase.com - type: application - urls: - - https://budibase.github.io/budibase/budibase-0.2.2.tgz - version: 0.2.2 - - apiVersion: v2 - appVersion: 0.9.163 - created: "2021-10-20T14:27:23.5153+01:00" - dependencies: - - condition: services.couchdb.enabled - name: couchdb - repository: https://apache.github.io/couchdb-helm - version: 3.3.4 - - condition: ingress.nginx - name: ingress-nginx - repository: https://github.com/kubernetes/ingress-nginx - version: 3.35.0 - description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes. - digest: ebac6d8631cc38b266c3689508b5123f5afc395f23bdb02738be26c7cae0b0b5 - keywords: - - low-code - - database - - cluster - name: budibase - sources: - - https://github.com/Budibase/budibase - - https://budibase.com - type: application - urls: - - https://budibase.github.io/budibase/budibase-0.2.1.tgz - version: 0.2.1 - - apiVersion: v2 - appVersion: 0.9.163 - created: "2021-10-20T14:27:23.510041+01:00" - dependencies: - - condition: services.couchdb.enabled - name: couchdb - repository: https://apache.github.io/couchdb-helm - version: 3.3.4 - - condition: ingress.nginx - name: ingress-nginx - repository: https://github.com/kubernetes/ingress-nginx - version: 3.35.0 - description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes. - digest: f369536c0eac1f6959d51e8ce6d74a87a7a9df29ae84fb9cbed0a273ab77429b - keywords: - - low-code - - database - - cluster - name: budibase - sources: - - https://github.com/Budibase/budibase - - https://budibase.com - type: application - urls: - - https://budibase.github.io/budibase/budibase-0.2.0.tgz - version: 0.2.0 - - apiVersion: v2 - appVersion: 0.9.56 - created: "2021-10-20T14:27:23.504543+01:00" - dependencies: - - condition: services.couchdb.enabled - name: couchdb - repository: https://apache.github.io/couchdb-helm - version: 3.3.4 - - name: ingress-nginx - repository: https://github.com/kubernetes/ingress-nginx - version: 3.35.0 - description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes. - digest: 8dc4f2ed4d98cad5adf25936aefea680042d3e4e17832f846b961fd8708ad192 - keywords: - - low-code - - database - - cluster - name: budibase - sources: - - https://github.com/Budibase/budibase - - https://budibase.com - type: application - urls: - - https://budibase.github.io/budibase/budibase-0.1.1.tgz - version: 0.1.1 - - apiVersion: v2 - appVersion: 0.9.56 - created: "2021-10-20T14:27:23.496847+01:00" - dependencies: - - condition: services.couchdb.enabled - name: couchdb - repository: https://apache.github.io/couchdb-helm - version: 3.3.4 - - name: ingress-nginx - repository: https://github.com/kubernetes/ingress-nginx - version: 3.35.0 - description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes. - digest: 08031b0803cce0eff64472e569d454d9176119c8207aa9873a9c95ee66cc7d3f - keywords: - - low-code - - database - - cluster - name: budibase - sources: - - https://github.com/Budibase/budibase - - https://budibase.com - type: application - urls: - - https://budibase.github.io/budibase/budibase-0.1.0.tgz - version: 0.1.0 -generated: "2021-10-20T14:27:23.491132+01:00" diff --git a/hosting/digitalocean/README.md b/hosting/digitalocean/README.md new file mode 100644 index 0000000000..72c1950d17 --- /dev/null +++ b/hosting/digitalocean/README.md @@ -0,0 +1,19 @@ +# Budibase DigitalOcean One Click +You will find in this directory configuration for packaging and creating a snapshot for the Budibase 1 click Digitalocean build. We use this configuration to have an immutable and reproducible build package for Digitalocean, that rarely needs updated. + +## Prerequisites +You must install Hashicorps `packer` to build the snapshot for digitalocean. Follow the instructions to install packer [here](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli) + +You must have the `DIGITALOCEAN_TOKEN` environment variable set, so that packer can reach out to the digitalocean API for build information. + +## Building +Just run the following command: +``` +yarn build:digitalocean +``` + +## Uploading to Marketplace +You can upload the snapshot to the Digitalocean vendor portal at the following link (Requires vendor account): + +https://marketplace.digitalocean.com/vendorportal + diff --git a/hosting/digitalocean/build.sh b/hosting/digitalocean/build.sh new file mode 100755 index 0000000000..743629ca12 --- /dev/null +++ b/hosting/digitalocean/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +packer build template.json diff --git a/hosting/digitalocean/files/etc/update-motd.d/99-one-click b/hosting/digitalocean/files/etc/update-motd.d/99-one-click new file mode 100644 index 0000000000..0f087a26ee --- /dev/null +++ b/hosting/digitalocean/files/etc/update-motd.d/99-one-click @@ -0,0 +1,19 @@ +#!/bin/sh +# +# Configured as part of the DigitalOcean 1-Click Image build process + +myip=$(hostname -I | awk '{print$1}') +cat <Creating your Budibase app from your selected template...
+tags from text +function removeTags(text) { + return text ? text.replace(tagsRegex, "") : text +} + +function addInfo(projectJson) { + let info = {} + info["title"] = projectJson.title || projectJson.name + info["version"] = projectJson.version + info["description"] = projectJson.description + return info +} + +/** + * Extracts paths provided in json format + * post, patch, put request parameters are extracted in body + * get and delete are extracted to path parameters + * @param apidocJson + * @returns {{}} + */ +function extractPaths(apidocJson) { + let apiPaths = groupByUrl(apidocJson) + let paths = {} + for (let i = 0; i < apiPaths.length; i++) { + let verbs = apiPaths[i].verbs + let url = verbs[0].url + let pattern = pathToRegexp(url, null) + let matches = pattern.exec(url) + + // Surrounds URL parameters with curly brackets -> :email with {email} + let pathKeys = [] + for (let j = 1; j < matches.length; j++) { + let key = matches[j].substr(1) + url = url.replace(matches[j], "{" + key + "}") + pathKeys.push(key) + } + + for (let j = 0; j < verbs.length; j++) { + let verb = verbs[j] + let type = verb.type + + let obj = (paths[url] = paths[url] || {}) + + if (type === "post" || type === "patch" || type === "put") { + _.extend( + obj, + createPostPushPutOutput(verb, swagger.definitions, pathKeys) + ) + } else { + _.extend(obj, createGetDeleteOutput(verb, swagger.definitions)) + } + } + } + return paths +} + +function createPostPushPutOutput(verbs, definitions, pathKeys) { + let pathItemObject = {} + let verbDefinitionResult = createVerbDefinitions(verbs, definitions) + + let params = [] + let pathParams = createPathParameters(verbs, pathKeys) + pathParams = _.filter(pathParams, function (param) { + let hasKey = pathKeys.indexOf(param.name) !== -1 + return !(param.in === "path" && !hasKey) + }) + + params = params.concat(pathParams) + let required = + verbs.parameter && + verbs.parameter.fields && + verbs.parameter.fields.Parameter && + verbs.parameter.fields.Parameter.length > 0 + + params.push({ + in: "body", + name: "body", + description: removeTags(verbs.description), + required: required, + schema: { + $ref: "#/definitions/" + verbDefinitionResult.topLevelParametersRef, + }, + }) + + pathItemObject[verbs.type] = { + tags: [verbs.group], + summary: removeTags(verbs.description), + consumes: ["application/json"], + produces: ["application/json"], + parameters: params, + } + + if (verbDefinitionResult.topLevelSuccessRef) { + pathItemObject[verbs.type].responses = { + 200: { + description: "successful operation", + schema: { + type: verbDefinitionResult.topLevelSuccessRefType, + items: { + $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef, + }, + }, + }, + } + } + + return pathItemObject +} + +function createVerbDefinitions(verbs, definitions) { + let result = { + topLevelParametersRef: null, + topLevelSuccessRef: null, + topLevelSuccessRefType: null, + } + let defaultObjectName = verbs.name + + let fieldArrayResult = {} + if (verbs && verbs.parameter && verbs.parameter.fields) { + fieldArrayResult = createFieldArrayDefinitions( + verbs.parameter.fields.Parameter, + definitions, + verbs.name, + defaultObjectName + ) + result.topLevelParametersRef = fieldArrayResult.topLevelRef + } + + if (verbs && verbs.success && verbs.success.fields) { + fieldArrayResult = createFieldArrayDefinitions( + verbs.success.fields["Success 200"], + definitions, + verbs.name, + defaultObjectName + ) + result.topLevelSuccessRef = fieldArrayResult.topLevelRef + result.topLevelSuccessRefType = fieldArrayResult.topLevelRefType + } + + return result +} + +function createFieldArrayDefinitions( + fieldArray, + definitions, + topLevelRef, + defaultObjectName +) { + let result = { + topLevelRef: topLevelRef, + topLevelRefType: null, + } + + if (!fieldArray) { + return result + } + + for (let i = 0; i < fieldArray.length; i++) { + let parameter = fieldArray[i] + + let nestedName = createNestedName(parameter.field) + let objectName = nestedName.objectName + if (!objectName) { + objectName = defaultObjectName + } + let type = parameter.type + if (i === 0) { + result.topLevelRefType = type + if (parameter.type === "Object") { + objectName = nestedName.propertyName + nestedName.propertyName = null + } else if (parameter.type === "Array") { + objectName = nestedName.propertyName + nestedName.propertyName = null + result.topLevelRefType = "array" + } + result.topLevelRef = objectName + } + + definitions[objectName] = definitions[objectName] || { + properties: {}, + required: [], + } + + if (nestedName.propertyName) { + let prop = { + type: (parameter.type || "").toLowerCase(), + description: removeTags(parameter.description), + } + if (parameter.type === "Object") { + prop.$ref = "#/definitions/" + parameter.field + } + + let typeIndex = type.indexOf("[]") + if (typeIndex !== -1 && typeIndex === type.length - 2) { + prop.type = "array" + prop.items = { + type: type.slice(0, type.length - 2), + } + } + + definitions[objectName]["properties"][nestedName.propertyName] = prop + if (!parameter.optional) { + let arr = definitions[objectName]["required"] + if (arr.indexOf(nestedName.propertyName) === -1) { + arr.push(nestedName.propertyName) + } + } + } + } + + return result +} + +function createNestedName(field) { + let propertyName = field + let objectName + let propertyNames = field.split(".") + if (propertyNames && propertyNames.length > 1) { + propertyName = propertyNames[propertyNames.length - 1] + propertyNames.pop() + objectName = propertyNames.join(".") + } + + return { + propertyName: propertyName, + objectName: objectName, + } +} + +/** + * Generate get, delete method output + * @param verbs + * @param definitions + * @returns {{}} + */ +function createGetDeleteOutput(verbs, definitions) { + let pathItemObject = {} + verbs.type = verbs.type === "del" ? "delete" : verbs.type + + let verbDefinitionResult = createVerbDefinitions(verbs, definitions) + pathItemObject[verbs.type] = { + tags: [verbs.group], + summary: removeTags(verbs.description), + consumes: ["application/json"], + produces: ["application/json"], + parameters: createPathParameters(verbs), + } + if (verbDefinitionResult.topLevelSuccessRef) { + pathItemObject[verbs.type].responses = { + 200: { + description: "successful operation", + schema: { + type: verbDefinitionResult.topLevelSuccessRefType, + items: { + $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef, + }, + }, + }, + } + } + return pathItemObject +} + +/** + * Iterate through all method parameters and create array of parameter objects which are stored as path parameters + * @param verbs + * @returns {Array} + */ +function createPathParameters(verbs) { + let pathItemObject = [] + if (verbs.parameter && verbs.parameter.fields.Parameter) { + for (let i = 0; i < verbs.parameter.fields.Parameter.length; i++) { + let param = verbs.parameter.fields.Parameter[i] + let field = param.field + let type = param.type + pathItemObject.push({ + name: field, + in: type === "file" ? "formData" : "path", + required: !param.optional, + type: param.type.toLowerCase(), + description: removeTags(param.description), + }) + } + } + return pathItemObject +} + +function groupByUrl(apidocJson) { + return _.chain(apidocJson) + .groupBy("url") + .toPairs() + .map(function (element) { + return _.zipObject(["url", "verbs"], element) + }) + .value() +} + +module.exports = toSwagger diff --git a/packages/server/scripts/integrations/oracle/docker-compose.yml b/packages/server/scripts/integrations/oracle/docker-compose.yml index 5cd5e02f81..c54cd0a40b 100644 --- a/packages/server/scripts/integrations/oracle/docker-compose.yml +++ b/packages/server/scripts/integrations/oracle/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.8" services: db: - container_name: oracle-xe + restart: always platform: linux/x86_64 image: container-registry.oracle.com/database/express:18.4.0-xe environment: diff --git a/packages/server/scripts/integrations/oracle/oracle.md b/packages/server/scripts/integrations/oracle/oracle.md index 912de08b65..6c2d7a9252 100644 --- a/packages/server/scripts/integrations/oracle/oracle.md +++ b/packages/server/scripts/integrations/oracle/oracle.md @@ -84,7 +84,7 @@ The `HR` schema is populated with dummy data by default in oracle for testing pu To connect to the HR schema first update the user password and unlock the account by performing ```sql ALTER USER hr ACCOUNT UNLOCK; -ALTER USER hr IDENTIFIED BY hr +ALTER USER hr IDENTIFIED BY hr; ``` You should now be able to connect to the hr schema using the credentials hr/hr diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index ef2913e812..6f55834a49 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -15,14 +15,12 @@ const { generateAppID, getLayoutParams, getScreenParams, - generateScreenID, generateDevAppID, DocumentTypes, AppStatus, } = require("../../db/utils") const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles") const { BASE_LAYOUTS } = require("../../constants/layouts") -const { createHomeScreen } = require("../../constants/screens") const { cloneDeep } = require("lodash/fp") const { processObject } = require("@budibase/string-templates") const { @@ -409,9 +407,5 @@ const createEmptyAppPackage = async (ctx, app) => { screensAndLayouts.push(await processObject(cloned, app)) } - const homeScreen = createHomeScreen(app) - homeScreen._id = generateScreenID() - screensAndLayouts.push(homeScreen) - await db.bulkDocs(screensAndLayouts) } diff --git a/packages/server/src/api/controllers/query.js b/packages/server/src/api/controllers/query.js index 502ef5e67b..cf6f03f00f 100644 --- a/packages/server/src/api/controllers/query.js +++ b/packages/server/src/api/controllers/query.js @@ -1,8 +1,11 @@ const { processString } = require("@budibase/string-templates") const CouchDB = require("../../db") -const { generateQueryID, getQueryParams } = require("../../db/utils") +const { + generateQueryID, + getQueryParams, + isProdAppID, +} = require("../../db/utils") const { BaseQueryVerbs } = require("../../constants") -const env = require("../../environment") const { Thread, ThreadType } = require("../../threads") const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 }) @@ -90,10 +93,9 @@ exports.find = async function (ctx) { const db = new CouchDB(ctx.appId) const query = enrichQueries(await db.get(ctx.params.queryId)) // remove properties that could be dangerous in real app - if (env.isProd()) { + if (isProdAppID(ctx.appId)) { delete query.fields delete query.parameters - delete query.schema } ctx.body = query } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 2226dc99be..af199561dc 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,4 +1,5 @@ import { + FilterTypes, IncludeRelationships, Operation, PaginationJson, @@ -118,8 +119,13 @@ module External { } // check the row and filters to make sure they aren't a key of some sort if (config.filters) { - for (let filter of Object.values(config.filters)) { - if (typeof filter !== "object" || Object.keys(filter).length === 0) { + for (let [key, filter] of Object.entries(config.filters)) { + // oneOf is an array, don't iterate it + if ( + typeof filter !== "object" || + Object.keys(filter).length === 0 || + key === FilterTypes.ONE_OF + ) { continue } iterateObject(filter) diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 660618aef0..10a5c9746a 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -44,14 +44,10 @@ exports.save = async function (ctx) { // the column if you want to change the type if (oldTable && oldTable.schema) { for (let propKey of Object.keys(tableToSave.schema)) { - let column = tableToSave.schema[propKey] let oldColumn = oldTable.schema[propKey] if (oldColumn && oldColumn.type === "internal") { oldColumn.type = "auto" } - if (oldColumn && oldColumn.type !== column.type) { - ctx.throw(400, "Cannot change the type of a column") - } } } diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 0623fc64b5..e4086e8071 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -75,6 +75,7 @@ exports.handleDataImport = async (appId, user, table, dataImport) => { if (!dataImport || !dataImport.csvString) { return table } + const db = new CouchDB(appId) // Populate the table with rows imported from CSV in a bulk update const data = await csvParser.transform({ diff --git a/packages/server/src/api/controllers/user.js b/packages/server/src/api/controllers/user.js index 6dfaf9847d..95e9a9969d 100644 --- a/packages/server/src/api/controllers/user.js +++ b/packages/server/src/api/controllers/user.js @@ -81,7 +81,7 @@ exports.syncUser = async function (ctx) { throw err } } - const roles = user.roles + const roles = deleting ? {} : user.roles // remove props which aren't useful to metadata delete user.password delete user.forceResetPassword diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index 3b43ef2408..e3232323bf 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -5,6 +5,7 @@ const exporters = require("./exporters") const { saveView, getView, getViews, deleteView } = require("./utils") const { fetchView } = require("../row") const { getTable } = require("../table/utils") +const { FieldTypes } = require("../../../constants") exports.fetch = async ctx => { const db = new CouchDB(ctx.appId) @@ -77,6 +78,7 @@ exports.exportView = async ctx => { } await fetchView(ctx) + let rows = ctx.body let schema = view && view.meta && view.meta.schema if (!schema) { @@ -85,11 +87,23 @@ exports.exportView = async ctx => { schema = table.schema } + // remove any relationships + const relationships = Object.entries(schema) + .filter(entry => entry[1].type === FieldTypes.LINK) + .map(entry => entry[0]) + // iterate relationship columns and remove from and row and schema + relationships.forEach(column => { + rows.forEach(row => { + delete row[column] + }) + delete schema[column] + }) + // make sure no "undefined" entries appear in the CSV if (format === exporters.ExportFormats.CSV) { const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { - for (let row of ctx.body) { + for (let row of rows) { if (row[key] == null) { row[key] = "" } @@ -103,5 +117,5 @@ exports.exportView = async ctx => { const filename = `${viewName}.${format}` // send down the file ctx.attachment(filename) - ctx.body = apiFileReturn(exporter(headers, ctx.body)) + ctx.body = apiFileReturn(exporter(headers, rows)) } diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.js index 44f71ad545..8251328df3 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.js @@ -14,24 +14,130 @@ const { const router = Router() router + /** + * @api {get} /api/:tableId/:rowId/enrich Get an enriched row + * @apiName Get an enriched row + * @apiGroup rows + * @apiPermission table read access + * @apiDescription This API is only useful when dealing with rows that have relationships. + * Normally when a row is a returned from the API relationships will only have the structure + * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows + * for each relationship instead. + * + * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched. + * + * @apiSuccess {object} row The response body will be the enriched row. + */ .get( "/api/:tableId/:rowId/enrich", paramSubResource("tableId", "rowId"), authorized(PermissionTypes.TABLE, PermissionLevels.READ), rowController.fetchEnrichedRow ) + /** + * @api {get} /api/:tableId/rows Get all rows in a table + * @apiName Get all rows in a table + * @apiGroup rows + * @apiPermission table read access + * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint. + * This endpoint gets all of the rows within the specified table - it is not heavily used + * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then + * will simply stop. + * + * @apiParam {string} tableId The ID of the table to retrieve all rows within. + * + * @apiSuccess {object[]} rows The response body will be an array of all rows found. + */ .get( "/api/:tableId/rows", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.READ), rowController.fetch ) + /** + * @api {get} /api/:tableId/rows/:rowId Retrieve a single row + * @apiName Retrieve a single row + * @apiGroup rows + * @apiPermission table read access + * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve + * a row by anything other than its _id field, use the search endpoint. + * + * @apiParam {string} tableId The ID of the table to retrieve a row from. + * @apiParam {string} rowId The ID of the row to retrieve. + * + * @apiSuccess {object} body The response body will be the row that was found. + */ .get( "/api/:tableId/rows/:rowId", paramSubResource("tableId", "rowId"), authorized(PermissionTypes.TABLE, PermissionLevels.READ), rowController.find ) + /** + * @api {post} /api/:tableId/search Search for rows in a table + * @apiName Search for rows in a table + * @apiGroup rows + * @apiPermission table read access + * @apiDescription This is the primary method of accessing rows in Budibase, the data provider + * and data UI in the builder are built atop this. All filtering, sorting and pagination is + * handled through this, for internal and external (datasource plus, e.g. SQL) tables. + * + * @apiParam {string} tableId The ID of the table to retrieve rows from. + * + * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true, + * defaults to false. + * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none + * specified then the request will be unfiltered. An example with all of the possible query + * options has been supplied below. + * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned, + * this will be implemented at the database level if supported for performance reasons. This + * is useful when paginating to set exactly how many rows per page. + * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned + * with each successful search request, this should be supplied back to get the next page. + * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to + * sort on. + * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or + * "ascending" as required. + * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search + * being used, either "string" or "number". This is only used for internal tables. + * + * @apiParamExample {json} Example: + * { + * "tableId": "ta_70260ff0b85c467ca74364aefc46f26d", + * "query": { + * "string": {}, + * "fuzzy": {}, + * "range": { + * "columnName": { + * "high": 20, + * "low": 10, + * } + * }, + * "equal": { + * "columnName": "someValue" + * }, + * "notEqual": {}, + * "empty": {}, + * "notEmpty": {}, + * "contains": {}, + * "notContains": {} + * "oneOf": { + * "columnName": ["value"] + * } + * }, + * "limit": 10, + * "sort": "name", + * "sortOrder": "descending", + * "sortType": "string", + * "paginate": true + * } + * + * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters. + * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or + * not there is another page after this request. + * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next + * page. + */ .post( "/api/:tableId/search", paramResource("tableId"), @@ -46,6 +152,30 @@ router authorized(PermissionTypes.TABLE, PermissionLevels.READ), rowController.search ) + /** + * @api {post} /api/:tableId/rows Creates a new row + * @apiName Creates a new row + * @apiGroup rows + * @apiPermission table write access + * @apiDescription This API will create a new row based on the supplied body. If the + * body includes an "_id" field then it will update an existing row if the field + * links to one. Please note that "_id", "_rev" and "tableId" are fields that are + * already used by Budibase tables and cannot be used for columns. + * + * @apiParam {string} tableId The ID of the table to save a row to. + * + * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided. + * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision + * must also be provided. + * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself. + * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches + * a column in the specified table. All other fields will be dropped and not stored. + * + * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this + * is the rows new ID. + * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned. + * @apiSuccess {object} body The contents of the row that was saved will be returned as well. + */ .post( "/api/:tableId/rows", paramResource("tableId"), @@ -53,18 +183,66 @@ router usage, rowController.save ) + /** + * @api {patch} /api/:tableId/rows Updates a row + * @apiName Update a row + * @apiGroup rows + * @apiPermission table write access + * @apiDescription This endpoint is identical to the row creation endpoint but instead it will + * error if an _id isn't provided, it will only function for existing rows. + */ .patch( "/api/:tableId/rows", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), rowController.patch ) + /** + * @api {post} /api/:tableId/rows/validate Validate inputs for a row + * @apiName Validate inputs for a row + * @apiGroup rows + * @apiPermission table write access + * @apiDescription When attempting to save a row you may want to check if the row is valid + * given the table schema, this will iterate through all the constraints on the table and + * check if the request body is valid. + * + * @apiParam {string} tableId The ID of the table the row is to be validated for. + * + * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested + * against the table schema and constraints. + * + * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this + * will be true, if it is not then then errors property will be populated. + * @apiSuccess {object} [errors] A key value map of information about fields on the input + * which do not match the table schema. The key name will be the column names that have breached + * the schema. + */ .post( "/api/:tableId/rows/validate", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), rowController.validate ) + /** + * @api {delete} /api/:tableId/rows Delete rows + * @apiName Delete rows + * @apiGroup rows + * @apiPermission table write access + * @apiDescription This endpoint can delete a single row, or delete them in a bulk + * fashion. + * + * @apiParam {string} tableId The ID of the table the row is to be deleted from. + * + * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this + * key of the request body that are to be deleted. + * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field. + * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its + * revision here. + * + * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array + * of the deleted rows, if deleting a single row then the body will contain a "row" property which + * is the deleted row. + */ .delete( "/api/:tableId/rows", paramResource("tableId"), diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index 62146b0bbe..d8ddbe8133 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -27,13 +27,110 @@ function generateSaveValidator() { } router + /** + * @api {get} /api/tables Fetch all tables + * @apiName Fetch all tables + * @apiGroup tables + * @apiPermission table read access + * @apiDescription This endpoint retrieves all of the tables which have been created in + * an app. This includes all of the external and internal tables; to tell the difference + * between these look for the "type" property on each table, either being "internal" or "external". + * + * @apiSuccess {object[]} body The response body will be the list of tables that was found - as + * this does not take any parameters the only error scenario is no access. + */ .get("/api/tables", authorized(BUILDER), tableController.fetch) + /** + * @api {get} /api/tables/:id Fetch a single table + * @apiName Fetch a single table + * @apiGroup tables + * @apiPermission table read access + * @apiDescription Retrieves a single table this could be be internal or external based on + * the provided table ID. + * + * @apiParam {string} id The ID of the table which is to be retrieved. + * + * @apiSuccess {object[]} body The response body will be the table that was found. + */ .get( "/api/tables/:id", paramResource("id"), authorized(PermissionTypes.TABLE, PermissionLevels.READ), tableController.find ) + /** + * @api {post} /api/tables Save a table + * @apiName Save a table + * @apiGroup tables + * @apiPermission builder + * @apiDescription Create or update a table with this endpoint, this will function for both internal + * external tables. + * + * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified. + * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified. + * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type - + * this will default to internal. + * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the data source ID. If + * building an internal table this does not need to be set, although it will be returned as "bb_internal". + * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply + * supply the table structure to this endpoint with the name changed. + * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this + * object. For each column a "type" and "constraints" must be specified, with some types requiring further information. + * More information about the schema structure can be found in the Typescript definitions. + * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows + * from this table as relationships. + * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction + * of lucene indexes. This functionality is only available for internal tables. + * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this + * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field + * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix + * the rows in the table. This functionality is only available for internal tables. + * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the + * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call + * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating + * or for external tables. + * + * @apiParamExample {json} Example: + * { + * "_id": "ta_05541307fa0f4044abee071ca2a82119", + * "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807", + * "type": "internal", + * "views": {}, + * "name": "tableName", + * "schema": { + * "column": { + * "type": "string", + * "constraints": { + * "type": "string", + * "length": { + * "maximum": null + * }, + * "presence": false + * }, + * "name": "column" + * }, + * }, + * "primaryDisplay": "column", + * "indexes": [], + * "sourceId": "bb_internal", + * "_rename": { + * "old": "columnName", + * "updated": "newColumnName", + * }, + * "dataImport": { + * "csvString": "column\nvalue", + * "primaryDisplay": "column", + * "schema": { + * "column": { + * "type": "string" + * } + * } + * } + * } + * + * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and + * saved to the database. + */ .post( "/api/tables", // allows control over updating a table @@ -42,22 +139,68 @@ router generateSaveValidator(), tableController.save ) + /** + * @api {post} /api/tables/csv/validate Validate a CSV for a table + * @apiName Validate a CSV for a table + * @apiGroup tables + * @apiPermission builder + * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and + * converted into a Budibase schema; this endpoint does this. + * + * @apiParam (Body) {string} csvString The CSV which is to be validated as a string. + * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the + * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can + * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g. + * using a number instead of strings. + * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and + * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When + * importing a CSV to an existing table only fields that are present on the table can be imported. + * + * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for + * the CSV - this will be in the same format used for table schema.s + */ .post( "/api/tables/csv/validate", authorized(BUILDER), tableController.validateCSVSchema ) + /** + * @api {post} /api/tables/:tableId/:revId Delete a table + * @apiName Delete a table + * @apiGroup tables + * @apiPermission builder + * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is + * quite dangerous - it will work for internal and external tables. + * + * @apiParam {string} tableId The ID of the table which is to be deleted. + * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for + * external tables this can simply be set to anything, e.g. "external". + * + * @apiSuccess {string} message A message stating that the table was deleted successfully. + */ .delete( "/api/tables/:tableId/:revId", paramResource("tableId"), authorized(BUILDER), tableController.destroy ) - // this is currently builder only, but in the future - // it could be carried out by an end user in app, - // however some thought will need to be had about - // implications for automations (triggers) - // new trigger type, bulk rows created + /** + * @api {post} /api/tables/:tableId/:revId Import CSV to existing table + * @apiName Import CSV to existing table + * @apiGroup tables + * @apiPermission builder + * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination + * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to + * import the data; please note this will only import fields that already exist on the table/match the type. + * + * @apiParam {string} tableId The ID of the table which the data should be imported to. + * + * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with + * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be + * turned into rows. + * + * @apiSuccess {string} message A message stating that the data was imported successfully. + */ .post( "/api/tables/:tableId/import", paramResource("tableId"), diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index 05e0bc231b..ce1cb80435 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -75,7 +75,6 @@ describe("/applications", () => { .expect("Content-Type", /json/) .expect(200) // should have empty packages - expect(res.body.screens.length).toEqual(1) expect(res.body.layouts.length).toEqual(2) }) }) @@ -88,7 +87,6 @@ describe("/applications", () => { .expect("Content-Type", /json/) .expect(200) expect(res.body.application).toBeDefined() - expect(res.body.screens.length).toEqual(1) expect(res.body.layouts.length).toEqual(2) }) }) diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index 6b9be7447f..37c969aba8 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -1,6 +1,13 @@ -// mock out postgres for this +// Mock out postgres for this jest.mock("pg") +// Mock isProdAppID to we can later mock the implementation and pretend we are +// using prod app IDs +const authDb = require("@budibase/auth/db") +const { isProdAppID } = authDb +const mockIsProdAppID = jest.fn(isProdAppID) +authDb.isProdAppID = mockIsProdAppID + const setup = require("./utilities") const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const { basicQuery, basicDatasource } = setup.structures @@ -19,10 +26,12 @@ describe("/queries", () => { }) async function createInvalidIntegration() { - const datasource = await config.createDatasource({datasource: { - ...basicDatasource().datasource, - source: "INVALID_INTEGRATION", - }}) + const datasource = await config.createDatasource({ + datasource: { + ...basicDatasource().datasource, + source: "INVALID_INTEGRATION", + }, + }) const query = await config.createQuery() return { datasource, query } } @@ -96,11 +105,32 @@ describe("/queries", () => { .set(await config.defaultHeaders()) .expect(200) .expect("Content-Type", /json/) - expect(res.body.fields).toBeUndefined() - expect(res.body.parameters).toBeUndefined() - expect(res.body.schema).toBeUndefined() + expect(res.body.fields).toBeDefined() + expect(res.body.parameters).toBeDefined() + expect(res.body.schema).toBeDefined() }) }) + + it("should remove sensitive info for prod apps", async () => { + // Mock isProdAppID to pretend we are using a prod app + mockIsProdAppID.mockClear() + mockIsProdAppID.mockImplementation(() => true) + + const query = await config.createQuery() + const res = await request + .get(`/api/queries/${query._id}`) + .set(await config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body._id).toEqual(query._id) + expect(res.body.fields).toBeUndefined() + expect(res.body.parameters).toBeUndefined() + expect(res.body.schema).toBeDefined() + + // Reset isProdAppID mock + expect(mockIsProdAppID).toHaveBeenCalledTimes(1) + mockIsProdAppID.mockImplementation(isProdAppID) + }) }) describe("destroy", () => { diff --git a/packages/server/src/api/routes/tests/screen.spec.js b/packages/server/src/api/routes/tests/screen.spec.js index 6a47b78853..c2d60ec006 100644 --- a/packages/server/src/api/routes/tests/screen.spec.js +++ b/packages/server/src/api/routes/tests/screen.spec.js @@ -21,7 +21,7 @@ describe("/screens", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - expect(res.body.length).toEqual(2) + expect(res.body.length).toEqual(1) expect(res.body.some(s => s._id === screen._id)).toEqual(true) }) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index af56dc2368..6711e7c391 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -55,6 +55,17 @@ export enum IncludeRelationships { EXCLUDE = 0, } +export enum FilterTypes { + STRING = "string", + FUZZY = "fuzzy", + RANGE = "range", + EQUAL = "equal", + NOT_EQUAL = "notEqual", + EMPTY = "empty", + NOT_EMPTY = "notEmpty", + ONE_OF = "oneOf", +} + export interface QueryDefinition { type: QueryTypes displayName?: string diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 471774db3d..df4f2d511b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -93,7 +93,7 @@ class InternalBuilder { if (filters.oneOf) { iterate(filters.oneOf, (key, array) => { const fnc = allOr ? "orWhereIn" : "whereIn" - query = query[fnc](key, array) + query = query[fnc](key, Array.isArray(array) ? array : [array]) }) } if (filters.string) { @@ -435,8 +435,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { id = results?.[0].id } else if (sqlClient === SqlClients.MY_SQL) { id = results?.insertId - } else if (sqlClient === SqlClients.ORACLE) { - id = response.outBinds[0][0] } row = processFn( await this.getReturningRow(queryFn, this.checkLookupKeys(id, json)) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 98584e04d2..24a55a273d 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -143,7 +143,7 @@ module MySQLModule { const schema: TableSchema = {} const descResp = await internalQuery( this.client, - { sql: `DESCRIBE ${tableName};` }, + { sql: `DESCRIBE \`${tableName}\`;` }, false ) for (let column of descResp) { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 13658399db..afaa902655 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -348,27 +348,7 @@ module OracleModule { this.schemaErrors = final.errors } - /** - * Knex default returning behaviour does not work with oracle - * Manually add the behaviour for the return column - */ - private addReturning( - query: SqlQuery, - bindings: BindParameters, - returnColumn: string - ) { - if (bindings instanceof Array) { - bindings.push({ dir: oracledb.BIND_OUT }) - query.sql = - query.sql + ` returning \"${returnColumn}\" into :${bindings.length}` - } - } - - private async internalQuery