diff --git a/lerna.json b/lerna.json index 671935c34b..bbe4da4264 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.50", + "version": "2.13.51", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7a2be5a0f0..3ac323a8d4 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -18,14 +18,15 @@ export enum TTL { ONE_DAY = 86400, } -function performExport(funcName: string) { - // @ts-ignore - return (...args: any) => GENERIC[funcName](...args) -} - -export const keys = performExport("keys") -export const get = performExport("get") -export const store = performExport("store") -export const destroy = performExport("delete") -export const withCache = performExport("withCache") -export const bustCache = performExport("bustCache") +export const keys = (...args: Parameters) => + GENERIC.keys(...args) +export const get = (...args: Parameters) => + GENERIC.get(...args) +export const store = (...args: Parameters) => + GENERIC.store(...args) +export const destroy = (...args: Parameters) => + GENERIC.delete(...args) +export const withCache = (...args: Parameters) => + GENERIC.withCache(...args) +export const bustCache = (...args: Parameters) => + GENERIC.bustCache(...args) diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts index 7f5a93f149..db32b520f7 100644 --- a/packages/backend-core/src/cache/passwordReset.ts +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -1,6 +1,6 @@ import * as redis from "../redis/init" import * as utils from "../utils" -import { Duration, DurationType } from "../utils" +import { Duration } from "../utils" const TTL_SECONDS = Duration.fromHours(1).toSeconds() @@ -32,7 +32,18 @@ export async function getCode(code: string): Promise { const client = await redis.getPasswordResetClient() const value = (await client.get(code)) as PasswordReset | undefined if (!value) { - throw "Provided information is not valid, cannot reset password - please try again." + throw new Error( + "Provided information is not valid, cannot reset password - please try again." + ) } return value } + +/** + * Given a reset code this will invalidate it. + * @param code The code provided via the email link. + */ +export async function invalidateCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + await client.delete(code) +} diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c2c0b6b21d..3fec573bb9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -17,6 +17,7 @@ import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" +import { DDInstrumentedDatabase } from "../instrumentation" function buildNano(couchInfo: { url: string; cookie: string }) { return Nano({ @@ -35,7 +36,8 @@ export function DatabaseWithConnection( connection: string, opts?: DatabaseOpts ) { - return new DatabaseImpl(dbName, opts, connection) + const db = new DatabaseImpl(dbName, opts, connection) + return new DDInstrumentedDatabase(db) } export class DatabaseImpl implements Database { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 3e69d49f0e..197770298e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,8 +1,9 @@ import { directCouchQuery, DatabaseImpl } from "./couch" import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" +import { DDInstrumentedDatabase } from "./instrumentation" export function getDB(dbName: string, opts?: DatabaseOpts): Database { - return new DatabaseImpl(dbName, opts) + return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts)) } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts new file mode 100644 index 0000000000..ba5febcba6 --- /dev/null +++ b/packages/backend-core/src/db/instrumentation.ts @@ -0,0 +1,156 @@ +import { + DocumentScope, + DocumentDestroyResponse, + DocumentInsertResponse, + DocumentBulkResponse, + OkResponse, +} from "@budibase/nano" +import { + AllDocsResponse, + AnyDocument, + Database, + DatabaseDumpOpts, + DatabasePutOpts, + DatabaseQueryOpts, + Document, +} from "@budibase/types" +import tracer from "dd-trace" +import { Writable } from "stream" + +export class DDInstrumentedDatabase implements Database { + constructor(private readonly db: Database) {} + + get name(): string { + return this.db.name + } + + exists(): Promise { + return tracer.trace("db.exists", span => { + span?.addTags({ db_name: this.name }) + return this.db.exists() + }) + } + + checkSetup(): Promise> { + return tracer.trace("db.checkSetup", span => { + span?.addTags({ db_name: this.name }) + return this.db.checkSetup() + }) + } + + get(id?: string | undefined): Promise { + return tracer.trace("db.get", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.get(id) + }) + } + + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean | undefined } | undefined + ): Promise { + return tracer.trace("db.getMultiple", span => { + span?.addTags({ + db_name: this.name, + num_docs: ids.length, + allow_missing: opts?.allowMissing, + }) + return this.db.getMultiple(ids, opts) + }) + } + + remove( + id: string | Document, + rev?: string | undefined + ): Promise { + return tracer.trace("db.remove", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.remove(id, rev) + }) + } + + put( + document: AnyDocument, + opts?: DatabasePutOpts | undefined + ): Promise { + return tracer.trace("db.put", span => { + span?.addTags({ db_name: this.name, doc_id: document._id }) + return this.db.put(document, opts) + }) + } + + bulkDocs(documents: AnyDocument[]): Promise { + return tracer.trace("db.bulkDocs", span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkDocs(documents) + }) + } + + allDocs( + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.allDocs", span => { + span?.addTags({ db_name: this.name }) + return this.db.allDocs(params) + }) + } + + query( + viewName: string, + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.query", span => { + span?.addTags({ db_name: this.name, view_name: viewName }) + return this.db.query(viewName, params) + }) + } + + destroy(): Promise { + return tracer.trace("db.destroy", span => { + span?.addTags({ db_name: this.name }) + return this.db.destroy() + }) + } + + compact(): Promise { + return tracer.trace("db.compact", span => { + span?.addTags({ db_name: this.name }) + return this.db.compact() + }) + } + + dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { + return tracer.trace("db.dump", span => { + span?.addTags({ db_name: this.name }) + return this.db.dump(stream, opts) + }) + } + + load(...args: any[]): Promise { + return tracer.trace("db.load", span => { + span?.addTags({ db_name: this.name }) + return this.db.load(...args) + }) + } + + createIndex(...args: any[]): Promise { + return tracer.trace("db.createIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.createIndex(...args) + }) + } + + deleteIndex(...args: any[]): Promise { + return tracer.trace("db.deleteIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.deleteIndex(...args) + }) + } + + getIndexes(...args: any[]): Promise { + return tracer.trace("db.getIndexes", span => { + span?.addTags({ db_name: this.name }) + return this.db.getIndexes(...args) + }) + } +} diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index ad68bd300d..7a051e7f12 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,6 +5,7 @@ import { IdentityType } from "@budibase/types" import env from "../../environment" import * as context from "../../context" import * as correlation from "../correlation" +import tracer from "dd-trace" import { formats } from "dd-trace/ext" import { localFileDestination } from "../system" @@ -116,6 +117,11 @@ if (!env.DISABLE_PINO_LOGGER) { correlationId: correlation.getId(), } + const span = tracer.scope().active() + if (span) { + tracer.inject(span.context(), formats.LOG, contextObject) + } + const mergingObject: any = { err: error, pid: process.pid, diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0657437a3b..b95dace5b2 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -47,7 +47,7 @@ export function createQueue( cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS) // fire off an initial cleanup cleanup().catch(err => { - console.error(`Unable to cleanup automation queue initially - ${err}`) + console.error(`Unable to cleanup ${jobQueue} initially - ${err}`) }) } return queue diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 701e262091..d15453ba62 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -18,6 +18,7 @@ import { SelectableDatabase, getRedisConnectionDetails, } from "./utils" +import { logAlert } from "../logging" import * as timers from "../timers" const RETRY_PERIOD_MS = 2000 @@ -39,21 +40,16 @@ function pickClient(selectDb: number): any { return CLIENTS[selectDb] } -function connectionError( - selectDb: number, - timeout: NodeJS.Timeout, - err: Error | string -) { +function connectionError(timeout: NodeJS.Timeout, err: Error | string) { // manually shut down, ignore errors if (CLOSED) { return } - pickClient(selectDb).disconnect() CLOSED = true // always clear this on error clearTimeout(timeout) CONNECTED = false - console.error("Redis connection failed - " + err) + logAlert("Redis connection failed", err) setTimeout(() => { init() }, RETRY_PERIOD_MS) @@ -79,11 +75,7 @@ function init(selectDb = DEFAULT_SELECT_DB) { // start the timer - only allowed 5 seconds to connect timeout = setTimeout(() => { if (!CONNECTED) { - connectionError( - selectDb, - timeout, - "Did not successfully connect in timeout" - ) + connectionError(timeout, "Did not successfully connect in timeout") } }, STARTUP_TIMEOUT_MS) @@ -106,12 +98,13 @@ function init(selectDb = DEFAULT_SELECT_DB) { // allow the process to exit return } - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("error", (err: Error) => { - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("connect", () => { + console.log(`Connected to Redis DB: ${selectDb}`) clearTimeout(timeout) CONNECTED = true }) diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 326bed3cc5..01fa4899d1 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -2,7 +2,7 @@ import env from "../environment" import * as eventHelpers from "./events" import * as accountSdk from "../accounts" import * as cache from "../cache" -import { doInTenant, getGlobalDB, getIdentity, getTenantId } from "../context" +import { getGlobalDB, getIdentity, getTenantId } from "../context" import * as dbUtils from "../db" import { EmailUnavailableError, HTTPError } from "../errors" import * as platform from "../platform" diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js new file mode 100755 index 0000000000..b81aed3d5d --- /dev/null +++ b/packages/server/scripts/load/create-many-relationships.js @@ -0,0 +1,196 @@ +#!/bin/node +const { + createApp, + getTable, + createRow, + createTable, + getApp, + getRows, +} = require("./utils") + +const Chance = require("chance") + +const generator = new Chance() + +const STUDENT_COUNT = 500 +const SUBJECT_COUNT = 10 + +let { apiKey, appId } = require("yargs") + .demandOption(["apiKey"]) + .option("appId").argv + +const start = Date.now() +async function batchCreate(apiKey, appId, table, items, batchSize = 100) { + let i = 0 + let errors = 0 + + async function createSingleRow(item) { + try { + const row = await createRow(apiKey, appId, table, item) + console.log( + `${table.name} - ${++i} of ${items.length} created (${ + (Date.now() - start) / 1000 + }s)` + ) + return row + } catch { + errors++ + } + } + + const rows = [] + const maxConcurrency = Math.min(batchSize, items.length) + const inFlight = {} + + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + const item = items[itemIndex] + const promise = createSingleRow(item) + .then(result => { + rows.push(result) + }) + .finally(() => { + delete inFlight[itemIndex] + }) + + inFlight[itemIndex] = promise + + if (Object.keys(inFlight).length >= maxConcurrency) { + await Promise.race(Object.values(inFlight)) + } + } + + await Promise.all(Object.values(inFlight)) + + if (errors) { + console.error( + `${table.name} - ${errors} creation errored (${ + (Date.now() - start) / 1000 + }s)` + ) + } + + return rows +} + +const useExistingApp = !!appId + +async function upsertTable(appId, tableName, tableData) { + if (useExistingApp) { + return await getTable(apiKey, appId, tableName) + } + + const table = await createTable(apiKey, appId, { + ...tableData, + name: tableName, + }) + return table +} + +async function run() { + if (!appId) { + const app = appId ? await getApp(apiKey, appId) : await createApp(apiKey) + appId = app._id + + console.log(`App created. Url: http://localhost:10000/builder/app/${appId}`) + } else { + console.log( + `App retrieved. Url: http://localhost:10000/builder/app/${appId}` + ) + } + + const studentsTable = await getTable(apiKey, appId, "Students") + + let studentNumber = studentsTable.schema["Auto ID"].lastID + const students = await batchCreate( + apiKey, + appId, + studentsTable, + Array.from({ length: STUDENT_COUNT }).map(() => ({ + "Student Number": (++studentNumber).toString(), + "First Name": generator.first(), + "Last Name": generator.last(), + Gender: generator.pickone(["M", "F"]), + Grade: generator.pickone(["8", "9", "10", "11"]), + "Tardiness (Days)": generator.integer({ min: 1, max: 100 }), + "Home Number": generator.phone(), + "Attendance_(%)": generator.integer({ min: 0, max: 100 }), + })) + ) + + const subjectTable = await upsertTable(appId, "Subjects", { + schema: { + Name: { + name: "Name", + type: "string", + }, + }, + primaryDisplay: "Name", + }) + + const subjects = useExistingApp + ? await getRows(apiKey, appId, subjectTable._id) + : await batchCreate( + apiKey, + appId, + subjectTable, + Array.from({ length: SUBJECT_COUNT }).map(() => ({ + Name: generator.profession(), + })) + ) + + const gradesTable = await upsertTable(appId, "Grades", { + schema: { + Score: { + name: "Score", + type: "number", + }, + Student: { + name: "Student", + tableId: studentsTable._id, + constraints: { + presence: true, + type: "array", + }, + fieldName: "Grades", + relationshipType: "one-to-many", + type: "link", + }, + Subject: { + name: "Subject", + tableId: subjectTable._id, + constraints: { + presence: true, + type: "array", + }, + fieldName: "Grades", + relationshipType: "one-to-many", + type: "link", + }, + }, + }) + + await batchCreate( + apiKey, + appId, + gradesTable, + students.flatMap(student => + subjects.map(subject => ({ + Score: generator.integer({ min: 0, max: 100 }), + Student: [student], + Subject: [subject], + })) + ) + ) + + console.log( + `Access the app here: http://localhost:10000/builder/app/${appId}` + ) +} + +run() + .then(() => { + console.log(`Done in ${(Date.now() - start) / 1000} seconds`) + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/delete-all-apps.js b/packages/server/scripts/load/delete-all-apps.js new file mode 100755 index 0000000000..5c9e974c7d --- /dev/null +++ b/packages/server/scripts/load/delete-all-apps.js @@ -0,0 +1,29 @@ +#!/bin/node +const { searchApps, deleteApp } = require("./utils") + +if (!process.argv[2]) { + console.error("Please specify an API key as script argument.") + process.exit(-1) +} + +async function run() { + const apiKey = process.argv[2] + const apps = await searchApps(apiKey) + console.log(`Deleting ${apps.length} apps`) + + let deletedApps = 0 + await Promise.all( + apps.map(async app => { + await deleteApp(apiKey, app._id) + console.log(`App ${++deletedApps} of ${apps.length} deleted`) + }) + ) +} + +run() + .then(() => { + console.log("Done!") + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js index 97ff8f1e13..1dabdcec9a 100644 --- a/packages/server/scripts/load/utils.js +++ b/packages/server/scripts/load/utils.js @@ -2,7 +2,8 @@ const fetch = require("node-fetch") const uuid = require("uuid/v4") const URL_APP = "http://localhost:10000/api/public/v1/applications" -const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search" +const URL_TABLE = "http://localhost:10000/api/public/v1/tables" +const URL_SEARCH_TABLE = "http://localhost:10000/api/public/v1/tables/search" async function request(apiKey, url, method, body, appId = undefined) { const headers = { @@ -37,30 +38,64 @@ exports.createApp = async apiKey => { return json.data } -exports.getTable = async (apiKey, appId) => { - const res = await request(apiKey, URL_TABLE, "POST", {}, appId) +exports.getApp = async (apiKey, appId) => { + const res = await request(apiKey, `${URL_APP}/${appId}`, "GET") const json = await res.json() - return json.data[0] + return json.data +} +exports.searchApps = async apiKey => { + const res = await request(apiKey, `${URL_APP}/search`, "POST", {}) + const json = await res.json() + return json.data } -exports.createRow = async (apiKey, appId, table) => { - const body = {} - for (let [key, schema] of Object.entries(table.schema)) { - let fake - switch (schema.type) { - default: - case "string": - fake = schema.constraints.inclusion - ? schema.constraints.inclusion[0] - : "a" - break - case "number": - fake = 1 - break +exports.deleteApp = async (apiKey, appId) => { + const res = await request(apiKey, `${URL_APP}/${appId}`, "DELETE") + return res +} + +exports.getTable = async (apiKey, appId, tableName) => { + const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId) + const json = await res.json() + const table = json.data.find(t => t.name === tableName) + if (!table) { + throw `Table '${tableName} not found` + } + return table +} + +exports.createRow = async (apiKey, appId, table, body) => { + if (!body) { + body = {} + for (let [key, schema] of Object.entries(table.schema)) { + let fake + switch (schema.type) { + default: + case "string": + fake = schema.constraints?.inclusion + ? schema.constraints.inclusion[0] + : "a" + break + case "number": + fake = 1 + break + } + body[key] = fake } - body[key] = fake } const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows` const res = await request(apiKey, url, "POST", body, appId) return (await res.json()).data } + +exports.getRows = async (apiKey, appId, tableId) => { + const url = `${URL_TABLE}/${tableId}/rows/search` + const res = await request(apiKey, url, "POST", {}, appId) + return (await res.json()).data +} + +exports.createTable = async (apiKey, appId, config) => { + const res = await request(apiKey, URL_TABLE, "POST", config, appId) + const json = await res.json() + return json.data +} diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 287b2ae6aa..d741247687 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -26,7 +26,7 @@ import { inputProcessing, outputProcessing, } from "../../../utilities/rowProcessor" -import { cloneDeep, isEqual } from "lodash" +import { cloneDeep } from "lodash" export async function handleRequest( operation: T, @@ -86,50 +86,6 @@ export async function patch(ctx: UserCtx) { } } -export async function save(ctx: UserCtx) { - const inputs = ctx.request.body - const tableId = utils.getTableId(ctx) - - const table = await sdk.tables.getTable(tableId) - const { table: updatedTable, row } = await inputProcessing( - ctx.user?._id, - cloneDeep(table), - inputs - ) - - const validateResult = await sdk.rows.utils.validate({ - row, - tableId, - }) - if (!validateResult.valid) { - throw { validation: validateResult.errors } - } - - const response = await handleRequest(Operation.CREATE, tableId, { - row, - }) - - if (!isEqual(table, updatedTable)) { - await sdk.tables.saveTable(updatedTable) - } - - const rowId = response.row._id - if (rowId) { - const row = await sdk.rows.external.getRow(tableId, rowId, { - relationships: true, - }) - return { - ...response, - row: await outputProcessing(table, row, { - preserveLinks: true, - squash: true, - }), - } - } else { - return response - } -} - export async function find(ctx: UserCtx): Promise { const id = ctx.params.rowId const tableId = utils.getTableId(ctx) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 38731a87e1..7ff8d83e71 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -30,7 +30,7 @@ import { Format } from "../view/exporters" export * as views from "./views" -function pickApi(tableId: any) { +function pickApi(tableId: string) { if (isExternalTableID(tableId)) { return external } @@ -84,9 +84,12 @@ export const save = async (ctx: UserCtx) => { return patch(ctx as UserCtx) } const { row, table, squashed } = await quotas.addRow(() => - quotas.addQuery(() => pickApi(tableId).save(ctx), { - datasourceId: tableId, - }) + quotas.addQuery( + () => sdk.rows.save(tableId, ctx.request.body, ctx.user?._id), + { + datasourceId: tableId, + } + ) ) ctx.status = 200 ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a251724b0a..5c714c098a 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,5 +1,5 @@ import * as linkRows from "../../../db/linkedRows" -import { generateRowID, InternalTables } from "../../../db/utils" +import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { AttachmentCleanup, @@ -94,45 +94,6 @@ export async function patch(ctx: UserCtx) { }) } -export async function save(ctx: UserCtx) { - let inputs = ctx.request.body - inputs.tableId = utils.getTableId(ctx) - - if (!inputs._rev && !inputs._id) { - inputs._id = generateRowID(inputs.tableId) - } - - // this returns the table and row incase they have been updated - const dbTable = await sdk.tables.getTable(inputs.tableId) - - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) - - let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs) - - const validateResult = await sdk.rows.utils.validate({ - row, - table, - }) - - if (!validateResult.valid) { - throw { validation: validateResult.errors } - } - - // make sure link rows are up-to-date - row = (await linkRows.updateLinks({ - eventType: linkRows.EventType.ROW_SAVE, - row, - tableId: row.tableId, - table, - })) as Row - - return finaliseRow(table, row, { - oldTable: dbTable, - updateFormula: true, - }) -} - export async function find(ctx: UserCtx): Promise { const tableId = utils.getTableId(ctx), rowId = ctx.params.rowId diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 8d52b6a05c..3cb7348ee3 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -5,8 +5,8 @@ import { processFormulas, } from "../../../utilities/rowProcessor" import { FieldTypes, FormulaTypes } from "../../../constants" -import { context } from "@budibase/backend-core" -import { Table, Row } from "@budibase/types" +import { context, locks } from "@budibase/backend-core" +import { Table, Row, LockType, LockName } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" import sdk from "../../../sdk" import isEqual from "lodash/isEqual" @@ -149,12 +149,22 @@ export async function finaliseRow( await db.put(table) } catch (err: any) { if (err.status === 409) { - const updatedTable = await sdk.tables.getTable(table._id!) - let response = processAutoColumn(null, updatedTable, row, { - reprocessing: true, - }) - await db.put(response.table) - row = response.row + // Some conflicts with the autocolumns occurred, we need to refetch the table and recalculate + await locks.doWithLock( + { + type: LockType.AUTO_EXTEND, + name: LockName.PROCESS_AUTO_COLUMNS, + resource: table._id, + }, + async () => { + const latestTable = await sdk.tables.getTable(table._id!) + let response = processAutoColumn(null, latestTable, row, { + reprocessing: true, + }) + await db.put(response.table) + row = response.row + } + ) } else { throw err } diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index f27f3f8857..36e0f74bee 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -77,7 +77,7 @@ const publicRouter = new Router({ prefix: PREFIX, }) -if (limiter) { +if (limiter && !env.isDev()) { publicRouter.use(limiter) } diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 04fd36e3d1..0c28787f67 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -16,6 +16,7 @@ import { } from "@budibase/types" import sdk from "../sdk" import { automationsEnabled } from "../features" +import tracer from "dd-trace" const REBOOT_CRON = "@reboot" const WH_STEP_ID = definitions.WEBHOOK.stepId @@ -39,27 +40,62 @@ function loggingArgs(job: AutomationJob) { } export async function processEvent(job: AutomationJob) { - const appId = job.data.event.appId! - const automationId = job.data.automation._id! + return tracer.trace( + "processEvent", + { resource: "automation" }, + async span => { + const appId = job.data.event.appId! + const automationId = job.data.automation._id! - const task = async () => { - try { - // need to actually await these so that an error can be captured properly - console.log("automation running", ...loggingArgs(job)) - - const runFn = () => Runner.run(job) - const result = await quotas.addAutomation(runFn, { + span?.addTags({ + appId, automationId, + job: { + id: job.id, + name: job.name, + attemptsMade: job.attemptsMade, + opts: { + attempts: job.opts.attempts, + priority: job.opts.priority, + delay: job.opts.delay, + repeat: job.opts.repeat, + backoff: job.opts.backoff, + lifo: job.opts.lifo, + timeout: job.opts.timeout, + jobId: job.opts.jobId, + removeOnComplete: job.opts.removeOnComplete, + removeOnFail: job.opts.removeOnFail, + stackTraceLimit: job.opts.stackTraceLimit, + preventParsingData: job.opts.preventParsingData, + }, + }, }) - console.log("automation completed", ...loggingArgs(job)) - return result - } catch (err) { - console.error(`automation was unable to run`, err, ...loggingArgs(job)) - return { err } - } - } - return await context.doInAutomationContext({ appId, automationId, task }) + const task = async () => { + try { + // need to actually await these so that an error can be captured properly + console.log("automation running", ...loggingArgs(job)) + + const runFn = () => Runner.run(job) + const result = await quotas.addAutomation(runFn, { + automationId, + }) + console.log("automation completed", ...loggingArgs(job)) + return result + } catch (err) { + span?.addTags({ error: true }) + console.error( + `automation was unable to run`, + err, + ...loggingArgs(job) + ) + return { err } + } + } + + return await context.doInAutomationContext({ appId, automationId, task }) + } + ) } export async function updateTestHistory( diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts index a9301feb60..ab0381a399 100644 --- a/packages/server/src/jsRunner.ts +++ b/packages/server/src/jsRunner.ts @@ -2,35 +2,44 @@ import vm from "vm" import env from "./environment" import { setJSRunner } from "@budibase/string-templates" import { context, timers } from "@budibase/backend-core" +import tracer from "dd-trace" type TrackerFn = (f: () => T) => T export function init() { setJSRunner((js: string, ctx: vm.Context) => { - const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS - let track: TrackerFn = f => f() - if (perRequestLimit) { - const bbCtx = context.getCurrentContext() - if (bbCtx) { - if (!bbCtx.jsExecutionTracker) { - bbCtx.jsExecutionTracker = - timers.ExecutionTimeTracker.withLimit(perRequestLimit) + return tracer.trace("runJS", {}, span => { + const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS + let track: TrackerFn = f => f() + if (perRequestLimit) { + const bbCtx = context.getCurrentContext() + if (bbCtx) { + if (!bbCtx.jsExecutionTracker) { + bbCtx.jsExecutionTracker = + timers.ExecutionTimeTracker.withLimit(perRequestLimit) + } + track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker) + span?.addTags({ + js: { + limitMS: bbCtx.jsExecutionTracker.limitMs, + elapsedMS: bbCtx.jsExecutionTracker.elapsedMS, + }, + }) } - track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker) } - } - ctx = { - ...ctx, - alert: undefined, - setInterval: undefined, - setTimeout: undefined, - } - vm.createContext(ctx) - return track(() => - vm.runInNewContext(js, ctx, { - timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, - }) - ) + ctx = { + ...ctx, + alert: undefined, + setInterval: undefined, + setTimeout: undefined, + } + vm.createContext(ctx) + return track(() => + vm.runInNewContext(js, ctx, { + timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, + }) + ) + }) }) } diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index 984dd8e5e9..ad6f2afa18 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -12,6 +12,7 @@ import { getCachedSelf } from "../utilities/global" import env from "../environment" import { isWebhookEndpoint } from "./utils" import { UserCtx, ContextUser } from "@budibase/types" +import tracer from "dd-trace" export default async (ctx: UserCtx, next: any) => { // try to get the appID from the request @@ -20,6 +21,11 @@ export default async (ctx: UserCtx, next: any) => { return next() } + if (requestAppId) { + const span = tracer.scope().active() + span?.setTag("appId", requestAppId) + } + // deny access to application preview if (!env.isTest()) { if ( @@ -70,6 +76,14 @@ export default async (ctx: UserCtx, next: any) => { return next() } + if (ctx.user) { + const span = tracer.scope().active() + if (ctx.user._id) { + span?.setTag("userId", ctx.user._id) + } + span?.setTag("tenantId", ctx.user.tenantId) + } + const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined // if the user is not in the right tenant then make sure to wipe their cookie diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index beae02e134..7ad5ea37ff 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -1,6 +1,13 @@ -import { IncludeRelationship, Operation } from "@budibase/types" +import { IncludeRelationship, Operation, Row } from "@budibase/types" import { handleRequest } from "../../../api/controllers/row/external" import { breakRowIdField } from "../../../integrations/utils" +import sdk from "../../../sdk" +import { + inputProcessing, + outputProcessing, +} from "../../../utilities/rowProcessor" +import cloneDeep from "lodash/fp/cloneDeep" +import isEqual from "lodash/fp/isEqual" export async function getRow( tableId: string, @@ -15,3 +22,48 @@ export async function getRow( }) return response ? response[0] : response } + +export async function save( + tableId: string, + inputs: Row, + userId: string | undefined +) { + const table = await sdk.tables.getTable(tableId) + const { table: updatedTable, row } = await inputProcessing( + userId, + cloneDeep(table), + inputs + ) + + const validateResult = await sdk.rows.utils.validate({ + row, + tableId, + }) + if (!validateResult.valid) { + throw { validation: validateResult.errors } + } + + const response = await handleRequest(Operation.CREATE, tableId, { + row, + }) + + if (!isEqual(table, updatedTable)) { + await sdk.tables.saveTable(updatedTable) + } + + const rowId = response.row._id + if (rowId) { + const row = await sdk.rows.external.getRow(tableId, rowId, { + relationships: true, + }) + return { + ...response, + row: await outputProcessing(table, row, { + preserveLinks: true, + squash: true, + }), + } + } else { + return response + } +} diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts new file mode 100644 index 0000000000..14e771b36e --- /dev/null +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -0,0 +1,49 @@ +import { db } from "@budibase/backend-core" +import { Row } from "@budibase/types" +import sdk from "../../../sdk" +import cloneDeep from "lodash/fp/cloneDeep" +import { finaliseRow } from "../../../api/controllers/row/staticFormula" +import { inputProcessing } from "../../../utilities/rowProcessor" +import * as linkRows from "../../../db/linkedRows" + +export async function save( + tableId: string, + inputs: Row, + userId: string | undefined +) { + inputs.tableId = tableId + + if (!inputs._rev && !inputs._id) { + inputs._id = db.generateRowID(inputs.tableId) + } + + // this returns the table and row incase they have been updated + const dbTable = await sdk.tables.getTable(inputs.tableId) + + // need to copy the table so it can be differenced on way out + const tableClone = cloneDeep(dbTable) + + let { table, row } = await inputProcessing(userId, tableClone, inputs) + + const validateResult = await sdk.rows.utils.validate({ + row, + table, + }) + + if (!validateResult.valid) { + throw { validation: validateResult.errors } + } + + // make sure link rows are up-to-date + row = (await linkRows.updateLinks({ + eventType: linkRows.EventType.ROW_SAVE, + row, + tableId: row.tableId, + table, + })) as Row + + return finaliseRow(table, row, { + oldTable: dbTable, + updateFormula: true, + }) +} diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index 8709180f0b..bfd84a715c 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,6 +1,9 @@ import { db as dbCore, context } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" import { getRowParams } from "../../../db/utils" +import { isExternalTableID } from "../../../integrations/utils" +import * as internal from "./internal" +import * as external from "./external" export async function getAllInternalRows(appId?: string) { let db: Database @@ -16,3 +19,18 @@ export async function getAllInternalRows(appId?: string) { ) return response.rows.map(row => row.doc) as Row[] } + +function pickApi(tableId: any) { + if (isExternalTableID(tableId)) { + return external + } + return internal +} + +export async function save( + tableId: string, + row: Row, + userId: string | undefined +) { + return pickApi(tableId).save(tableId, row, userId) +} diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts new file mode 100644 index 0000000000..b60d31b226 --- /dev/null +++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts @@ -0,0 +1,220 @@ +import tk from "timekeeper" +import * as internalSdk from "../internal" + +import { generator } from "@budibase/backend-core/tests" +import { + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, + FieldType, + Table, + AutoFieldSubTypes, +} from "@budibase/types" + +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" +import { cache } from "@budibase/backend-core" + +tk.freeze(Date.now()) + +describe("sdk >> rows >> internal", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.init() + }) + + function makeRow() { + return { + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + } + } + + describe("save", () => { + const tableData: Table = { + name: generator.word(), + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + surname: { + name: "surname", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + type: FieldType.NUMBER, + }, + }, + address: { + name: "address", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + }, + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it("save will persist the row properly", async () => { + const table = await config.createTable(tableData) + const row = makeRow() + + await config.doInContext(config.appId, async () => { + const response = await internalSdk.save( + table._id!, + row, + config.user._id + ) + + expect(response).toEqual({ + table, + row: { + ...row, + type: "row", + _rev: expect.stringMatching("1-.*"), + }, + squashed: { + ...row, + type: "row", + _rev: expect.stringMatching("1-.*"), + }, + }) + + const persistedRow = await config.getRow(table._id!, response.row._id!) + expect(persistedRow).toEqual({ + ...row, + type: "row", + _rev: expect.stringMatching("1-.*"), + createdAt: expect.any(String), + updatedAt: expect.any(String), + }) + }) + }) + + it("auto ids will update when creating new rows", async () => { + const table = await config.createTable({ + ...tableData, + schema: { + ...tableData.schema, + id: { + name: "id", + type: FieldType.AUTO, + subtype: AutoFieldSubTypes.AUTO_ID, + autocolumn: true, + lastID: 0, + }, + }, + }) + const row = makeRow() + + await config.doInContext(config.appId, async () => { + const response = await internalSdk.save( + table._id!, + row, + config.user._id + ) + + expect(response).toEqual({ + table: { + ...table, + schema: { + ...table.schema, + id: { + ...table.schema.id, + lastID: 1, + }, + }, + }, + row: { + ...row, + id: 1, + type: "row", + _rev: expect.stringMatching("1-.*"), + }, + squashed: { + ...row, + id: 1, + type: "row", + _rev: expect.stringMatching("1-.*"), + }, + }) + + const persistedRow = await config.getRow(table._id!, response.row._id!) + expect(persistedRow).toEqual({ + ...row, + type: "row", + id: 1, + _rev: expect.stringMatching("1-.*"), + createdAt: expect.any(String), + updatedAt: expect.any(String), + }) + }) + }) + + it("auto ids will update when creating new rows in parallel", async () => { + function makeRows(count: number) { + return Array.from({ length: count }, () => makeRow()) + } + + const table = await config.createTable({ + ...tableData, + schema: { + ...tableData.schema, + id: { + name: "id", + type: FieldType.AUTO, + subtype: AutoFieldSubTypes.AUTO_ID, + autocolumn: true, + lastID: 0, + }, + }, + }) + + await config.doInContext(config.appId, async () => { + for (const row of makeRows(5)) { + await internalSdk.save(table._id!, row, config.user._id) + } + await Promise.all( + makeRows(10).map(row => + internalSdk.save(table._id!, row, config.user._id) + ) + ) + for (const row of makeRows(5)) { + await internalSdk.save(table._id!, row, config.user._id) + } + }) + + const persistedRows = await config.getRows(table._id!) + expect(persistedRows).toHaveLength(20) + expect(persistedRows).toEqual( + expect.arrayContaining( + Array.from({ length: 20 }).map((_, i) => + expect.objectContaining({ id: i + 1 }) + ) + ) + ) + + const persistedTable = await config.getTable(table._id) + expect((table as any).schema.id.lastID).toBe(0) + expect(persistedTable.schema.id.lastID).toBe(20) + }) + }) +}) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9bb1717f3c..4447899f96 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp" import { performance } from "perf_hooks" import * as sdkUtils from "../sdk/utils" import env from "../environment" +import tracer from "dd-trace" threadUtils.threadSetup() const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId @@ -242,281 +243,347 @@ class Orchestrator { } async execute(): Promise { - // this will retrieve from context created at start of thread - this._context.env = await sdkUtils.getEnvironmentVariables() - let automation = this._automation - let stopped = false - let loopStep: AutomationStep | undefined = undefined + return tracer.trace( + "Orchestrator.execute", + { resource: "automation" }, + async span => { + span?.addTags({ + appId: this._appId, + automationId: this._automation._id, + }) - let stepCount = 0 - let loopStepNumber: any = undefined - let loopSteps: LoopStep[] | undefined = [] - let metadata - let timeoutFlag = false - let wasLoopStep = false - let timeout = this._job.data.event.timeout - // check if this is a recurring automation, - if (isProdAppID(this._appId) && isRecurring(automation)) { - metadata = await this.getMetadata() - const shouldStop = await this.checkIfShouldStop(metadata) - if (shouldStop) { - return - } - } - const start = performance.now() - for (let step of automation.definition.steps) { - let input: any, - iterations = 1, - iterationCount = 0 + // this will retrieve from context created at start of thread + this._context.env = await sdkUtils.getEnvironmentVariables() + let automation = this._automation + let stopped = false + let loopStep: AutomationStep | undefined = undefined - if (timeoutFlag) { - break - } - - if (timeout) { - setTimeout(() => { - timeoutFlag = true - }, timeout || 12000) - } - - stepCount++ - if (step.stepId === LOOP_STEP_ID) { - loopStep = step - loopStepNumber = stepCount - continue - } - - if (loopStep) { - input = await processObject(loopStep.inputs, this._context) - iterations = getLoopIterations(loopStep as LoopStep) - } - for (let index = 0; index < iterations; index++) { - let originalStepInput = cloneDeep(step.inputs) - // Handle if the user has set a max iteration count or if it reaches the max limit set by us - if (loopStep && input.binding) { - let tempOutput = { - items: loopSteps, - iterations: iterationCount, - } - try { - loopStep.inputs.binding = automationUtils.typecastForLooping( - loopStep as LoopStep, - loopStep.inputs as LoopInput - ) - } catch (err) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.INCORRECT_TYPE, - success: false, - }) - loopSteps = undefined - loopStep = undefined - break - } - let item = [] - if ( - typeof loopStep.inputs.binding === "string" && - loopStep.inputs.option === "String" - ) { - item = automationUtils.stringSplit(loopStep.inputs.binding) - } else if (Array.isArray(loopStep.inputs.binding)) { - item = loopStep.inputs.binding - } - this._context.steps[loopStepNumber] = { - currentItem: item[index], - } - - // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it - // Pretty hacky because we need to account for the row object - for (let [key, value] of Object.entries(originalStepInput)) { - if (typeof value === "object") { - for (let [innerKey, innerValue] of Object.entries( - originalStepInput[key] - )) { - if (typeof innerValue === "string") { - originalStepInput[key][innerKey] = - automationUtils.substituteLoopStep( - innerValue, - `steps.${loopStepNumber}` - ) - } else if (typeof value === "object") { - for (let [innerObject, innerValue] of Object.entries( - originalStepInput[key][innerKey] - )) { - originalStepInput[key][innerKey][innerObject] = - automationUtils.substituteLoopStep( - innerValue as string, - `steps.${loopStepNumber}` - ) - } - } - } - } else { - if (typeof value === "string") { - originalStepInput[key] = automationUtils.substituteLoopStep( - value, - `steps.${loopStepNumber}` - ) - } - } - } - - if ( - index === env.AUTOMATION_MAX_ITERATIONS || - index === parseInt(loopStep.inputs.iterations) - ) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.MAX_ITERATIONS, - success: true, - }) - loopSteps = undefined - loopStep = undefined - break - } - - let isFailure = false - const currentItem = this._context.steps[loopStepNumber]?.currentItem - if (currentItem && typeof currentItem === "object") { - isFailure = Object.keys(currentItem).some(value => { - return currentItem[value] === loopStep?.inputs.failure - }) - } else { - isFailure = currentItem && currentItem === loopStep.inputs.failure - } - - if (isFailure) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.FAILURE_CONDITION, - success: false, - }) - loopSteps = undefined - loopStep = undefined - break + let stepCount = 0 + let loopStepNumber: any = undefined + let loopSteps: LoopStep[] | undefined = [] + let metadata + let timeoutFlag = false + let wasLoopStep = false + let timeout = this._job.data.event.timeout + // check if this is a recurring automation, + if (isProdAppID(this._appId) && isRecurring(automation)) { + span?.addTags({ recurring: true }) + metadata = await this.getMetadata() + const shouldStop = await this.checkIfShouldStop(metadata) + if (shouldStop) { + span?.addTags({ shouldStop: true }) + return } } - - // execution stopped, record state for that - if (stopped) { - this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS) - continue - } - - // If it's a loop step, we need to manually add the bindings to the context - let stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject(originalStepInput, this._context) - inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) - - try { - // appId is always passed - const outputs = await stepFn({ - inputs: inputs, - appId: this._appId, - emitter: this._emitter, - context: this._context, + const start = performance.now() + for (let step of automation.definition.steps) { + const stepSpan = tracer.startSpan("Orchestrator.execute.step", { + childOf: span, + }) + stepSpan.addTags({ + resource: "automation", + step: { + stepId: step.stepId, + id: step.id, + name: step.name, + type: step.type, + title: step.stepTitle, + internal: step.internal, + deprecated: step.deprecated, + }, }) - this._context.steps[stepCount] = outputs - // if filter causes us to stop execution don't break the loop, set a var - // so that we can finish iterating through the steps and record that it stopped - if (step.stepId === FILTER_STEP_ID && !outputs.result) { - stopped = true - this.updateExecutionOutput(step.id, step.stepId, step.inputs, { - ...outputs, - ...STOPPED_STATUS, - }) - continue - } - if (loopStep && loopSteps) { - loopSteps.push(outputs) - } else { - this.updateExecutionOutput( - step.id, - step.stepId, - step.inputs, - outputs - ) - } - } catch (err) { - console.error(`Automation error - ${step.stepId} - ${err}`) - return err - } + let input: any, + iterations = 1, + iterationCount = 0 - if (loopStep) { - iterationCount++ - if (index === iterations - 1) { + try { + if (timeoutFlag) { + span?.addTags({ timedOut: true }) + break + } + + if (timeout) { + setTimeout(() => { + timeoutFlag = true + }, timeout || 12000) + } + + stepCount++ + if (step.stepId === LOOP_STEP_ID) { + loopStep = step + loopStepNumber = stepCount + continue + } + + if (loopStep) { + input = await processObject(loopStep.inputs, this._context) + iterations = getLoopIterations(loopStep as LoopStep) + stepSpan?.addTags({ step: { iterations } }) + } + for (let index = 0; index < iterations; index++) { + let originalStepInput = cloneDeep(step.inputs) + // Handle if the user has set a max iteration count or if it reaches the max limit set by us + if (loopStep && input.binding) { + let tempOutput = { + items: loopSteps, + iterations: iterationCount, + } + try { + loopStep.inputs.binding = automationUtils.typecastForLooping( + loopStep as LoopStep, + loopStep.inputs as LoopInput + ) + } catch (err) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.INCORRECT_TYPE, + success: false, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + let item = [] + if ( + typeof loopStep.inputs.binding === "string" && + loopStep.inputs.option === "String" + ) { + item = automationUtils.stringSplit(loopStep.inputs.binding) + } else if (Array.isArray(loopStep.inputs.binding)) { + item = loopStep.inputs.binding + } + this._context.steps[loopStepNumber] = { + currentItem: item[index], + } + + // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it + // Pretty hacky because we need to account for the row object + for (let [key, value] of Object.entries(originalStepInput)) { + if (typeof value === "object") { + for (let [innerKey, innerValue] of Object.entries( + originalStepInput[key] + )) { + if (typeof innerValue === "string") { + originalStepInput[key][innerKey] = + automationUtils.substituteLoopStep( + innerValue, + `steps.${loopStepNumber}` + ) + } else if (typeof value === "object") { + for (let [innerObject, innerValue] of Object.entries( + originalStepInput[key][innerKey] + )) { + originalStepInput[key][innerKey][innerObject] = + automationUtils.substituteLoopStep( + innerValue as string, + `steps.${loopStepNumber}` + ) + } + } + } + } else { + if (typeof value === "string") { + originalStepInput[key] = + automationUtils.substituteLoopStep( + value, + `steps.${loopStepNumber}` + ) + } + } + } + + if ( + index === env.AUTOMATION_MAX_ITERATIONS || + index === parseInt(loopStep.inputs.iterations) + ) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.MAX_ITERATIONS, + success: true, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + + let isFailure = false + const currentItem = + this._context.steps[loopStepNumber]?.currentItem + if (currentItem && typeof currentItem === "object") { + isFailure = Object.keys(currentItem).some(value => { + return currentItem[value] === loopStep?.inputs.failure + }) + } else { + isFailure = + currentItem && currentItem === loopStep.inputs.failure + } + + if (isFailure) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.FAILURE_CONDITION, + success: false, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + } + + // execution stopped, record state for that + if (stopped) { + this.updateExecutionOutput( + step.id, + step.stepId, + {}, + STOPPED_STATUS + ) + continue + } + + // If it's a loop step, we need to manually add the bindings to the context + let stepFn = await this.getStepFunctionality(step.stepId) + let inputs = await processObject(originalStepInput, this._context) + inputs = automationUtils.cleanInputValues( + inputs, + step.schema.inputs + ) + + try { + // appId is always passed + const outputs = await stepFn({ + inputs: inputs, + appId: this._appId, + emitter: this._emitter, + context: this._context, + }) + + this._context.steps[stepCount] = outputs + // if filter causes us to stop execution don't break the loop, set a var + // so that we can finish iterating through the steps and record that it stopped + if (step.stepId === FILTER_STEP_ID && !outputs.result) { + stopped = true + this.updateExecutionOutput( + step.id, + step.stepId, + step.inputs, + { + ...outputs, + ...STOPPED_STATUS, + } + ) + continue + } + if (loopStep && loopSteps) { + loopSteps.push(outputs) + } else { + this.updateExecutionOutput( + step.id, + step.stepId, + step.inputs, + outputs + ) + } + } catch (err) { + console.error(`Automation error - ${step.stepId} - ${err}`) + return err + } + + if (loopStep) { + iterationCount++ + if (index === iterations - 1) { + loopStep = undefined + this._context.steps.splice(loopStepNumber, 1) + break + } + } + } + } finally { + stepSpan?.finish() + } + + if (loopStep && iterations === 0) { loopStep = undefined + this.executionOutput.steps.splice(loopStepNumber + 1, 0, { + id: step.id, + stepId: step.stepId, + outputs: { + status: AutomationStepStatus.NO_ITERATIONS, + success: true, + }, + inputs: {}, + }) + this._context.steps.splice(loopStepNumber, 1) - break + iterations = 1 + } + + // Delete the step after the loop step as it's irrelevant, since information is included + // in the loop step + if (wasLoopStep && !loopStep) { + this._context.steps.splice(loopStepNumber + 1, 1) + wasLoopStep = false + } + if (loopSteps && loopSteps.length) { + let tempOutput = { + success: true, + items: loopSteps, + iterations: iterationCount, + } + this.executionOutput.steps.splice(loopStepNumber + 1, 0, { + id: step.id, + stepId: step.stepId, + outputs: tempOutput, + inputs: step.inputs, + }) + this._context.steps[loopStepNumber] = tempOutput + + wasLoopStep = true + loopSteps = [] } } - } - if (loopStep && iterations === 0) { - loopStep = undefined - this.executionOutput.steps.splice(loopStepNumber + 1, 0, { - id: step.id, - stepId: step.stepId, - outputs: { - status: AutomationStepStatus.NO_ITERATIONS, - success: true, - }, - inputs: {}, - }) + const end = performance.now() + const executionTime = end - start - this._context.steps.splice(loopStepNumber, 1) - iterations = 1 - } + console.info( + `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`, + { + _logKey: "automation", + executionTime, + } + ) - // Delete the step after the loop step as it's irrelevant, since information is included - // in the loop step - if (wasLoopStep && !loopStep) { - this._context.steps.splice(loopStepNumber + 1, 1) - wasLoopStep = false - } - if (loopSteps && loopSteps.length) { - let tempOutput = { - success: true, - items: loopSteps, - iterations: iterationCount, + // store the logs for the automation run + try { + await storeLog(this._automation, this.executionOutput) + } catch (e: any) { + if (e.status === 413 && e.request?.data) { + // if content is too large we shouldn't log it + delete e.request.data + e.request.data = { message: "removed due to large size" } + } + logging.logAlert("Error writing automation log", e) } - this.executionOutput.steps.splice(loopStepNumber + 1, 0, { - id: step.id, - stepId: step.stepId, - outputs: tempOutput, - inputs: step.inputs, - }) - this._context.steps[loopStepNumber] = tempOutput - - wasLoopStep = true - loopSteps = [] - } - } - - const end = performance.now() - const executionTime = end - start - - console.info( - `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`, - { - _logKey: "automation", - executionTime, + if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { + await this.updateMetadata(metadata) + } + return this.executionOutput } ) - - // store the logs for the automation run - try { - await storeLog(this._automation, this.executionOutput) - } catch (e: any) { - if (e.status === 413 && e.request?.data) { - // if content is too large we shouldn't log it - delete e.request.data - e.request.data = { message: "removed due to large size" } - } - logging.logAlert("Error writing automation log", e) - } - if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { - await this.updateMetadata(metadata) - } - return this.executionOutput } } diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 22c099c58c..cafd366cae 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -11,6 +11,7 @@ import { Row, Table, } from "@budibase/types" +import tracer from "dd-trace" interface FormulaOpts { dynamic?: boolean @@ -50,35 +51,42 @@ export function processFormulas( inputRows: T, { dynamic, contextRows }: FormulaOpts = { dynamic: true } ): T { - const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - if (rows) { - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + return tracer.trace("processFormulas", {}, span => { + const numRows = Array.isArray(inputRows) ? inputRows.length : 1 + span?.addTags({ table_id: table._id, dynamic, numRows }) + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) { + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rows.length; i++) { - let row = rows[i] - let context = contextRows ? contextRows[i] : row - let formula = schema.formula - rows[i] = { - ...row, - [column]: processStringSync(formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + let formula = schema.formula + rows[i] = { + ...row, + [column]: tracer.trace("processStringSync", {}, span => { + span?.addTags({ table_id: table._id, column, static: isStatic }) + return processStringSync(formula, context) + }), + } } } } - } - return Array.isArray(inputRows) ? rows : rows[0] + return Array.isArray(inputRows) ? rows : rows[0] + }) } /** diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts index 82a7089b3f..0e6053a4db 100644 --- a/packages/types/src/sdk/locks.ts +++ b/packages/types/src/sdk/locks.ts @@ -21,6 +21,7 @@ export enum LockName { PERSIST_WRITETHROUGH = "persist_writethrough", QUOTA_USAGE_EVENT = "quota_usage_event", APP_MIGRATION = "app_migrations", + PROCESS_AUTO_COLUMNS = "process_auto_columns", } export type LockOptions = { diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts index 884625805c..56b7ca9f40 100644 --- a/packages/worker/src/api/routes/global/tests/scim.spec.ts +++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts @@ -1,6 +1,6 @@ import tk from "timekeeper" import _ from "lodash" -import { mocks, structures } from "@budibase/backend-core/tests" +import { generator, mocks, structures } from "@budibase/backend-core/tests" import { ScimCreateUserRequest, ScimGroupResponse, @@ -14,9 +14,14 @@ import { events } from "@budibase/backend-core" jest.retryTimes(2, { logErrorsBeforeRetry: true }) jest.setTimeout(30000) -mocks.licenses.useScimIntegration() - describe("scim", () => { + beforeAll(async () => { + tk.freeze(mocks.date.MOCK_DATE) + mocks.licenses.useScimIntegration() + + await config.setSCIMConfig(true) + }) + beforeEach(async () => { jest.resetAllMocks() tk.freeze(mocks.date.MOCK_DATE) @@ -570,8 +575,15 @@ describe("scim", () => { beforeAll(async () => { groups = [] - for (let i = 0; i < groupCount; i++) { - const body = structures.scim.createGroupRequest() + const groupNames = generator.unique( + () => generator.word(), + groupCount + ) + + for (const groupName of groupNames) { + const body = structures.scim.createGroupRequest({ + displayName: groupName, + }) groups.push(await config.api.scimGroupsAPI.post({ body })) } diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts index be5de649da..e670a7d091 100644 --- a/packages/worker/src/sdk/auth/auth.ts +++ b/packages/worker/src/sdk/auth/auth.ts @@ -85,6 +85,9 @@ export const resetUpdate = async (resetCode: string, password: string) => { user.password = password user = await userSdk.db.save(user) + await cache.passwordReset.invalidateCode(resetCode) + await sessions.invalidateSessions(userId) + // remove password from the user before sending events delete user.password await events.user.passwordReset(user) diff --git a/packages/worker/src/sdk/auth/tests/auth.spec.ts b/packages/worker/src/sdk/auth/tests/auth.spec.ts new file mode 100644 index 0000000000..e9f348f7c7 --- /dev/null +++ b/packages/worker/src/sdk/auth/tests/auth.spec.ts @@ -0,0 +1,70 @@ +import { cache, context, sessions, utils } from "@budibase/backend-core" +import { loginUser, resetUpdate } from "../auth" +import { generator, structures } from "@budibase/backend-core/tests" +import { TestConfiguration } from "../../../tests" + +describe("auth", () => { + const config = new TestConfiguration() + + describe("resetUpdate", () => { + it("providing a valid code will update the password", async () => { + await context.doInTenant(structures.tenant.id(), async () => { + const user = await config.createUser() + const previousPassword = user.password + + const code = await cache.passwordReset.createCode(user._id!, {}) + const newPassword = generator.hash() + + await resetUpdate(code, newPassword) + + const persistedUser = await config.getUser(user.email) + expect(persistedUser.password).not.toBe(previousPassword) + expect( + await utils.compare(newPassword, persistedUser.password!) + ).toBeTruthy() + }) + }) + + it("wrong code will not allow to reset the password", async () => { + await context.doInTenant(structures.tenant.id(), async () => { + const code = generator.hash() + const newPassword = generator.hash() + + await expect(resetUpdate(code, newPassword)).rejects.toThrow( + "Provided information is not valid, cannot reset password - please try again." + ) + }) + }) + + it("the same code cannot be used twice", async () => { + await context.doInTenant(structures.tenant.id(), async () => { + const user = await config.createUser() + + const code = await cache.passwordReset.createCode(user._id!, {}) + const newPassword = generator.hash() + + await resetUpdate(code, newPassword) + await expect(resetUpdate(code, newPassword)).rejects.toThrow( + "Provided information is not valid, cannot reset password - please try again." + ) + }) + }) + + it("updating the password will invalidate all the sessions", async () => { + await context.doInTenant(structures.tenant.id(), async () => { + const user = await config.createUser() + + await loginUser(user) + + expect(await sessions.getSessionsForUser(user._id!)).toHaveLength(1) + + const code = await cache.passwordReset.createCode(user._id!, {}) + const newPassword = generator.hash() + + await resetUpdate(code, newPassword) + + expect(await sessions.getSessionsForUser(user._id!)).toHaveLength(0) + }) + }) + }) +}) diff --git a/packages/worker/src/sdk/users/tests/users.spec.ts b/packages/worker/src/sdk/users/tests/users.spec.ts index df1aa74200..a02ca42c2f 100644 --- a/packages/worker/src/sdk/users/tests/users.spec.ts +++ b/packages/worker/src/sdk/users/tests/users.spec.ts @@ -1,6 +1,5 @@ import { structures, mocks } from "../../../tests" import { env, context } from "@budibase/backend-core" -import * as users from "../users" import { db as userDb } from "../" import { CloudAccount } from "@budibase/types"