Update on audit logs, getting the full write flow from events configured.

packages/backend-core/package.json

@@ -25,6 +25,7 @@
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/types": "2.3.14-alpha.0",
+    "@budibase/pro": "2.3.14-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/events/events.ts

@@ -1,6 +1,8 @@
-import { Event } from "@budibase/types"
+import { Event, IdentityType } from "@budibase/types"
+import { sdk } from "@budibase/pro"
 import { processors } from "./processors"
 import identification from "./identification"
+import { getAppId } from "../context"
 import * as backfill from "./backfill"
 
 export const publishEvent = async (
@@ -14,6 +16,13 @@ export const publishEvent = async (
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {
+    // only audit log actual events, don't include backfills
+    const userId = identity.type === IdentityType.USER ? identity.id : undefined
+    await sdk.auditLogs.write(event, properties, {
+      userId,
+      timestamp,
+      appId: getAppId(),
+    })
     await processors.processEvent(event, identity, properties, timestamp)
     return
   }

packages/backend-core/tsconfig.json

@@ -4,11 +4,13 @@
     "composite": true,
     "baseUrl": ".",
     "paths": {
-      "@budibase/types": ["../types/src"]
+      "@budibase/types": ["../types/src"],
+      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
     }
   },
   "references": [
-    { "path": "../types" }
+    { "path": "../types" },
+    { "path": "../../../budibase-pro/packages/pro" }
   ],
   "exclude": [
     "node_modules",

packages/backend-core/yarn.lock

@@ -475,6 +475,45 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.3.14-alpha.0":
+  version "2.3.14-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.14-alpha.0.tgz#8d60c93e9861301296907f61e770bdf1881b560c"
+  integrity sha512-lvRl6o9CCUx1VQ6ftu4FYuTp5mGgvbshNYoeaH0DbWFfhLNbibEHxx+pR+6M1ufhbu/EO6jKEr7DDeNiRBBMTw==
+  dependencies:
+    "@budibase/nano" "10.1.1"
+    "@budibase/pouchdb-replication-stream" "1.2.10"
+    "@budibase/types" "2.3.14-alpha.0"
+    "@shopify/jest-koa-mocks" "5.0.1"
+    "@techpass/passport-openidconnect" "0.3.2"
+    aws-cloudfront-sign "2.2.0"
+    aws-sdk "2.1030.0"
+    bcrypt "5.0.1"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dotenv "16.0.1"
+    emitter-listener "1.1.2"
+    ioredis "4.28.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.0"
+    koa-passport "4.1.4"
+    lodash "4.17.21"
+    lodash.isarguments "3.1.0"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-jwt "4.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    posthog-node "1.3.0"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "7.3.7"
+    tar-fs "2.1.1"
+    uuid "8.3.2"
+    zlib "1.0.5"
+
 "@budibase/nano@10.1.1":
   version "10.1.1"
   resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.1.tgz#36ccda4d9bb64b5ee14dd2b27a295b40739b1038"
@@ -500,6 +539,25 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@2.3.14-alpha.0":
+  version "2.3.14-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.14-alpha.0.tgz#db8ac1d0cacea21240c59458c4143e3e55663a4d"
+  integrity sha512-RJqXlvZmYrKCLxe7uVnc2KmDwE5OmoUZCA9M6b5Ne7ZHqrhtaNB/4Xlr1ESG79///epEWLv+Fde7mpHPzCvsZw==
+  dependencies:
+    "@budibase/backend-core" "2.3.14-alpha.0"
+    "@budibase/types" "2.3.14-alpha.0"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    joi "17.6.0"
+    jsonwebtoken "8.5.1"
+    lru-cache "^7.14.1"
+    node-fetch "^2.6.1"
+
+"@budibase/types@2.3.14-alpha.0":
+  version "2.3.14-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.14-alpha.0.tgz#03072a669c016c616278dc01c577119dd710e4fe"
+  integrity sha512-D5qOD7YJhHUrPy81C9OpTlVzsXBmZwbF9WdoxYo/M8JYmhTJfj7Fu5dGTyDpipPQVqknFpTsig5kQqG3r08ZZw==
+
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
@@ -826,6 +884,18 @@
     "@jridgewell/resolve-uri" "^3.0.3"
     "@jridgewell/sourcemap-codec" "^1.4.10"
 
+"@koa/router@8.0.8":
+  version "8.0.8"
+  resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.8.tgz#95f32d11373d03d89dcb63fabe9ac6f471095236"
+  integrity sha512-FnT93N4NUehnXr+juupDmG2yfi0JnWdCmNEuIXpCG4TtG+9xvtrLambBH3RclycopVUOEYAim2lydiNBI7IRVg==
+  dependencies:
+    debug "^4.1.1"
+    http-errors "^1.7.3"
+    koa-compose "^4.1.0"
+    methods "^1.1.2"
+    path-to-regexp "1.x"
+    urijs "^1.19.2"
+
 "@mapbox/node-pre-gyp@^1.0.0":
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.9.tgz#09a8781a3a036151cdebbe8719d6f8b25d4058bc"
@@ -3145,7 +3215,7 @@ http-cookie-agent@^4.0.2:
   dependencies:
     agent-base "^6.0.2"
 
-http-errors@^1.6.3, http-errors@~1.8.0:
+http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0:
   version "1.8.1"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c"
   integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==
@@ -3935,17 +4005,7 @@ jsonc-parser@^3.2.0:
   resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.0.tgz#31ff3f4c2b9793f89c67212627c51c6394f88e76"
   integrity sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==
 
-jsonwebtoken@9.0.0:
+jsonwebtoken@8.5.1, jsonwebtoken@^8.2.0:
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
-  integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
-  dependencies:
-    jws "^3.2.2"
-    lodash "^4.17.21"
-    ms "^2.1.1"
-    semver "^7.3.8"
-
-jsonwebtoken@^8.2.0:
   version "8.5.1"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
   integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==
@@ -3961,6 +4021,16 @@ jsonwebtoken@^8.2.0:
     ms "^2.1.1"
     semver "^5.6.0"
 
+jsonwebtoken@9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
+  integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
+  dependencies:
+    jws "^3.2.2"
+    lodash "^4.17.21"
+    ms "^2.1.1"
+    semver "^7.3.8"
+
 jsprim@^1.2.2:
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb"
@@ -4263,6 +4333,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+lru-cache@^7.14.1:
+  version "7.14.1"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea"
+  integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==
+
 ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"
@@ -4505,6 +4580,13 @@ node-fetch@2.6.7, node-fetch@^2.6.7:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.1:
+  version "2.6.9"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6"
+  integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-gyp-build-optional-packages@5.0.3:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
@@ -4837,6 +4919,13 @@ path-parse@^1.0.7:
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
   integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
 
+path-to-regexp@1.x:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a"
+  integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==
+  dependencies:
+    isarray "0.0.1"
+
 pause@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/pause/-/pause-0.0.1.tgz#1d408b3fdb76923b9543d96fb4c9dfd535d9cb5d"
@@ -6154,6 +6243,11 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
+urijs@^1.19.2:
+  version "1.19.11"
+  resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc"
+  integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==
+
 url-parse-lax@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c"

packages/server/package.json

@@ -141,6 +141,7 @@
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
+    "@types/supertest": "^2.0.12",
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -24,8 +24,7 @@ import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
-import { context } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
-import { removeKeyNumbering } from "./utils"
 import sdk from "../../../sdk"
 
 export interface ManyRelationship {
@@ -61,7 +60,7 @@ function buildFilters(
     let prefix = 1
     for (let operator of Object.values(filters)) {
       for (let field of Object.keys(operator || {})) {
-        if (removeKeyNumbering(field) === "_id") {
+        if (dbCore.removeKeyNumbering(field) === "_id") {
           if (primary) {
             const parts = breakRowIdField(operator[field])
             for (let field of primary) {

packages/types/src/documents/global/auditLogs.ts

@@ -1,6 +1,8 @@
 import { Document } from "../document"
 import { Event } from "../../sdk"
 
+export const AuditLogSystemUser = "SYSTEM"
+
 export interface AuditLogDoc extends Document {
   appId?: string
   event: Event

scripts/link-dependencies.sh

@@ -53,6 +53,10 @@ if [ -d "../budibase-pro" ]; then
   echo "Linking pro to server"
   cd packages/server && yarn link '@budibase/pro'
   cd -
+
+  echo "Linking pro to backend-core"
+  cd packages/backend-core && yarn link '@budibase/pro'
+  cd -
 fi
 
 if [ -d "../account-portal" ]; then