Merge branch 'master' into dean-fixes

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.18",
+  "version": "2.29.20",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/server/src/api/controllers/deploy/Deployment.ts

@@ -1,5 +1,4 @@
-import newid from "../../../db/newid"
-import { context } from "@budibase/backend-core"
+import { context, utils } from "@budibase/backend-core"
 
 /**
  * This is used to pass around information about the deployment that is occurring
@@ -12,7 +11,7 @@ export default class Deployment {
   appUrl?: string
 
   constructor(id = null) {
-    this._id = id || newid()
+    this._id = id || utils.newid()
   }
 
   setVerification(verification: any) {

packages/server/src/api/routes/tests/permissions.spec.ts

@@ -203,7 +203,7 @@ describe("/permission", () => {
       // replicate changes before checking permissions
       await config.publish()
 
-      await config.api.viewV2.publicSearch(view.id, undefined, { status: 403 })
+      await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
     })
 
     it("should ignore the view permissions if the flag is not on", async () => {
@@ -221,7 +221,7 @@ describe("/permission", () => {
       await config.publish()
 
       await config.api.viewV2.publicSearch(view.id, undefined, {
-        status: 403,
+        status: 401,
       })
     })
 
@@ -250,8 +250,8 @@ describe("/permission", () => {
         .send(basicRow(table._id))
         .set(config.publicHeaders())
         .expect("Content-Type", /json/)
-        .expect(403)
-      expect(res.status).toEqual(403)
+        .expect(401)
+      expect(res.status).toEqual(401)
     })
   })
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -1460,17 +1460,12 @@ describe.each([
         delete tableRequest.schema.id
 
         const table = await config.api.table.save(tableRequest)
+        const toCreate = generator
+          .unique(() => generator.integer({ min: 0, max: 10000 }), 10)
+          .map(number => ({ number, string: generator.word({ length: 30 }) }))
 
         const rows = await Promise.all(
-          generator
-            .unique(
-              () => ({
-                string: generator.word({ length: 30 }),
-                number: generator.integer({ min: 0, max: 10000 }),
-              }),
-              10
-            )
-            .map(d => config.api.row.save(table._id!, d))
+          toCreate.map(d => config.api.row.save(table._id!, d))
         )
 
         const res = await config.api.row.exportRows(table._id!, {

packages/server/src/api/routes/tests/utilities/TestFunctions.ts

@@ -151,7 +151,7 @@ export const checkPermissionsEndpoint = async ({
   await exports
     .createRequest(config.request, method, url, body)
     .set(failHeader)
-    .expect(403)
+    .expect(401)
 }
 
 export const getDB = () => {

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -1490,7 +1490,7 @@ describe.each([
       it("does not allow public users to fetch by default", async () => {
         await config.publish()
         await config.api.viewV2.publicSearch(view.id, undefined, {
-          status: 403,
+          status: 401,
         })
       })
 
@@ -1534,7 +1534,7 @@ describe.each([
         await config.publish()
 
         await config.api.viewV2.publicSearch(view.id, undefined, {
-          status: 403,
+          status: 401,
         })
       })
     })

packages/server/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import { automationQueue } from "./bullboard"
-import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes } from "../constants"
-import { db as dbCore, context } from "@budibase/backend-core"
+import { db as dbCore, context, utils } from "@budibase/backend-core"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
@@ -207,7 +206,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
       )
     }
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
-    const jobId = `${appId}_cron_${newid()}`
+    const jobId = `${appId}_cron_${utils.newid()}`
     const job: any = await automationQueue.add(
       {
         automation,

packages/server/src/db/inMemoryView.ts

@@ -1,9 +1,8 @@
-import newid from "./newid"
 import { Row, Document, DBView } from "@budibase/types"
 
 // bypass the main application db config
 // use in memory pouchdb directly
-import { db as dbCore } from "@budibase/backend-core"
+import { db as dbCore, utils } from "@budibase/backend-core"
 
 const Pouch = dbCore.getPouch({ inMemory: true })
 
@@ -16,7 +15,7 @@ export async function runView(
   // use a different ID each time for the DB, make sure they
   // are always unique for each query, don't want overlap
   // which could cause 409s
-  const db = new Pouch(newid())
+  const db = new Pouch(utils.newid())
   try {
     // write all the docs to the in memory Pouch (remove revs)
     await db.bulkDocs(

packages/server/src/db/newid.ts

@@ -1,5 +0,0 @@
-import { v4 } from "uuid"
-
-export default function (): string {
-  return v4().replace(/-/g, "")
-}

packages/server/src/db/utils.ts

@@ -1,5 +1,4 @@
-import newid from "./newid"
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, utils } from "@budibase/backend-core"
 import {
   DatabaseQueryOpts,
   Datasource,
@@ -15,6 +14,8 @@ import {
 
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
+const newid = utils.newid
+
 type Optional = string | null
 
 export const enum AppStatus {

packages/server/src/middleware/authorized.ts

@@ -96,7 +96,7 @@ const authorized =
     }
 
     if (!ctx.user) {
-      return ctx.throw(403, "No user info found")
+      return ctx.throw(401, "No user info found")
     }
 
     // get the resource roles
@@ -148,7 +148,7 @@ const authorized =
 
     // check authenticated
     if (!ctx.isAuthenticated) {
-      return ctx.throw(403, "Session not authenticated")
+      return ctx.throw(401, "Session not authenticated")
     }
 
     // check general builder stuff, this middleware is a good way

packages/server/src/middleware/tests/authorized.spec.ts

@@ -105,7 +105,7 @@ describe("Authorization middleware", () => {
     it("throws when no user data is present in context", async () => {
       await config.executeMiddleware()
 
-      expect(config.throw).toHaveBeenCalledWith(403, "No user info found")
+      expect(config.throw).toHaveBeenCalledWith(401, "No user info found")
     })
 
     it("passes on to next() middleware if user is an admin", async () => {
@@ -157,7 +157,7 @@ describe("Authorization middleware", () => {
 
       await config.executeMiddleware()
       expect(config.throw).toHaveBeenCalledWith(
-        403,
+        401,
         "Session not authenticated"
       )
     })

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -45,13 +45,10 @@ import {
   getTableIDList,
 } from "./filters"
 import { dataFilters } from "@budibase/shared-core"
-import { DEFAULT_TABLE_IDS } from "../../../../constants"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
-const USER_COLUMN_PREFIX_REGEX = new RegExp(
-  `no such column: .+${USER_COLUMN_PREFIX}`
-)
+const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 
 function buildInternalFieldList(
   table: Table,
@@ -240,10 +237,10 @@ async function runSqlQuery(
 function resyncDefinitionsRequired(status: number, message: string) {
   // pre data_ prefix on column names, need to resync
   return (
-    (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) ||
-    // default tables aren't included in definition
-    (status === 400 &&
-      DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) ||
+    // there are tables missing - try a resync
+    (status === 400 && message.match(MISSING_TABLE_REGX)) ||
+    // there are columns missing - try a resync
+    (status === 400 && message.match(MISSING_COLUMN_REGEX)) ||
     // no design document found, needs a full sync
     (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
   )
@@ -251,7 +248,8 @@ function resyncDefinitionsRequired(status: number, message: string) {
 
 export async function search(
   options: RowSearchParams,
-  table: Table
+  table: Table,
+  opts?: { retrying?: boolean }
 ): Promise<SearchResponse<Row>> {
   let { paginate, query, ...params } = options
 
@@ -376,9 +374,9 @@ export async function search(
     return response
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    if (resyncDefinitionsRequired(err.status, msg)) {
+    if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) {
       await sdk.tables.sqs.syncDefinition()
-      return search(options, table)
+      return search(options, table, { retrying: true })
     }
     // previously the internal table didn't error when a column didn't exist in search
     if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -127,9 +127,14 @@ function mapTable(table: Table): SQLiteTables {
 // nothing exists, need to iterate though existing tables
 async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
-  const defaultTables = DEFAULT_TABLES
+  for (const defaultTable of DEFAULT_TABLES) {
+    // the default table doesn't exist in Couch, use the in-memory representation
+    if (!tables.find(table => table._id === defaultTable._id)) {
+      tables.push(defaultTable)
+    }
+  }
   const definition = sql.designDoc.base("tableId")
-  for (let table of tables.concat(defaultTables)) {
+  for (let table of tables) {
     definition.sql.tables = {
       ...definition.sql.tables,
       ...mapTable(table),

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -26,6 +26,7 @@ import {
   roles,
   sessions,
   tenancy,
+  utils,
 } from "@budibase/backend-core"
 import {
   app as appController,
@@ -40,7 +41,6 @@ import {
 } from "./controllers"
 
 import { cleanup } from "../../utilities/fileSystem"
-import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
 import supertest from "supertest"
@@ -74,6 +74,8 @@ import { cloneDeep } from "lodash"
 import jwt, { Secret } from "jsonwebtoken"
 import { Server } from "http"
 
+const newid = utils.newid
+
 mocks.licenses.init(pro)
 
 // use unlimited license by default