Merge branch 'master' into fix/binding-drawer-evaluation

This commit is contained in:
Andrew Kingston 2025-02-26 09:05:10 +00:00 committed by GitHub
commit bf7bc619bf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 263 additions and 180 deletions

View File

@ -155,7 +155,18 @@ jobs:
strategy:
matrix:
datasource:
[mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
[
mssql,
mysql,
postgres,
postgres_legacy,
mongodb,
mariadb,
oracle,
sqs,
elasticsearch,
none,
]
steps:
- name: Checkout repo
uses: actions/checkout@v4
@ -192,6 +203,8 @@ jobs:
docker pull budibase/oracle-database:23.2-slim-faststart
elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
docker pull postgres:9.5.25
elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
fi
docker pull minio/minio &
docker pull redis &

View File

@ -4492,6 +4492,12 @@
}
]
},
{
"type": "text",
"label": "Zoom level",
"key": "defaultZoom",
"defaultValue": "1"
},
{
"type": "event",
"label": "On change",

View File

@ -28,7 +28,7 @@
"apexcharts": "^3.48.0",
"dayjs": "^1.10.8",
"downloadjs": "1.4.7",
"html5-qrcode": "^2.2.1",
"html5-qrcode": "^2.3.8",
"leaflet": "^1.7.1",
"sanitize-html": "^2.13.0",
"screenfull": "^6.0.1",

View File

@ -20,6 +20,7 @@
export let beepFrequency = 2637
export let customFrequency = 1046
export let preferredCamera = "environment"
export let defaultZoom = 1
export let validator
const dispatch = createEventDispatcher()
@ -58,6 +59,14 @@
html5QrCode
.start(cameraSetting, cameraConfig, onScanSuccess)
.then(() => {
if (defaultZoom > 1) {
const cameraOptions =
html5QrCode.getRunningTrackCameraCapabilities()
const zoom = cameraOptions.zoomFeature()
if (zoom.isSupported()) {
zoom.apply(defaultZoom)
}
}
resolve({ initialised: true })
})
.catch(err => {

View File

@ -17,6 +17,7 @@
export let beepFrequency
export let customFrequency
export let preferredCamera
export let defaultZoom
export let helpText = null
let fieldState
@ -56,6 +57,7 @@
{beepFrequency}
{customFrequency}
{preferredCamera}
{defaultZoom}
validator={fieldState.validator}
/>
{/if}

View File

@ -1,24 +0,0 @@
const elastic: any = {}
elastic.Client = function () {
this.index = jest.fn().mockResolvedValue({ body: [] })
this.search = jest.fn().mockResolvedValue({
body: {
hits: {
hits: [
{
_source: {
name: "test",
},
},
],
},
},
})
this.update = jest.fn().mockResolvedValue({ body: [] })
this.delete = jest.fn().mockResolvedValue({ body: [] })
this.close = jest.fn()
}
module.exports = elastic

View File

@ -3,3 +3,4 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb
POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0

View File

@ -165,7 +165,8 @@ describe("/datasources", () => {
})
const descriptions = datasourceDescribe({
exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
plus: true,
exclude: [DatabaseName.SQS],
})
if (descriptions.length) {
@ -590,7 +591,8 @@ if (descriptions.length) {
}
const datasources = datasourceDescribe({
exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE],
plus: true,
exclude: [DatabaseName.SQS, DatabaseName.ORACLE],
})
if (datasources.length) {

View File

@ -9,7 +9,8 @@ import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
const descriptions = datasourceDescribe({
exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
plus: true,
exclude: [DatabaseName.SQS],
})
if (descriptions.length) {

View File

@ -1,9 +1,6 @@
import * as setup from "./utilities"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { datasourceDescribe } from "../../../integrations/tests/utils"
import tk from "timekeeper"
import emitter from "../../../../src/events"
@ -80,7 +77,7 @@ function encodeJS(binding: string) {
return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
}
const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
const descriptions = datasourceDescribe({ plus: true })
if (descriptions.length) {
describe.each(descriptions)(

View File

@ -1,8 +1,5 @@
import { tableForDatasource } from "../../../tests/utilities/structures"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { datasourceDescribe } from "../../../integrations/tests/utils"
import {
context,
db as dbCore,
@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({
},
}))
const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
const descriptions = datasourceDescribe({ plus: true })
if (descriptions.length) {
describe.each(descriptions)(

View File

@ -1,11 +1,3 @@
// Directly mock the AWS SDK
jest.mock("@aws-sdk/s3-request-presigner", () => ({
getSignedUrl: jest.fn(() => {
return `http://example.com`
}),
}))
jest.mock("@aws-sdk/client-s3")
import { Datasource, SourceName } from "@budibase/types"
import { setEnv } from "../../../environment"
import { getRequest, getConfig, afterAll as _afterAll } from "./utilities"
@ -92,7 +84,17 @@ describe("/static", () => {
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.signedUrl).toEqual("http://example.com")
expect(res.body.signedUrl).toStartWith(
"https://foo.s3.eu-west-1.amazonaws.com/bar?"
)
expect(res.body.signedUrl).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256")
expect(res.body.signedUrl).toContain("X-Amz-Credential=bb")
expect(res.body.signedUrl).toContain("X-Amz-Date=")
expect(res.body.signedUrl).toContain("X-Amz-Signature=")
expect(res.body.signedUrl).toContain("X-Amz-Expires=900")
expect(res.body.signedUrl).toContain("X-Amz-SignedHeaders=host")
expect(res.body.publicUrl).toEqual(
`https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
)

View File

@ -28,17 +28,14 @@ import * as setup from "./utilities"
import * as uuid from "uuid"
import { generator } from "@budibase/backend-core/tests"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { datasourceDescribe } from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
import timekeeper from "timekeeper"
const { basicTable } = setup.structures
const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
const descriptions = datasourceDescribe({ plus: true })
if (descriptions.length) {
describe.each(descriptions)(

View File

@ -37,17 +37,14 @@ import {
ViewV2Type,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { datasourceDescribe } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import { context, db, events, roles, setEnv } from "@budibase/backend-core"
import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
import nock from "nock"
const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
const descriptions = datasourceDescribe({ plus: true })
if (descriptions.length) {
describe.each(descriptions)(

View File

@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests"
import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
const descriptions = datasourceDescribe({
exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
plus: true,
exclude: [DatabaseName.SQS],
})
if (descriptions.length) {

View File

@ -10,7 +10,7 @@ import {
import { Client, ClientOptions } from "@elastic/elasticsearch"
import { HOST_ADDRESS } from "./utils"
interface ElasticsearchConfig {
export interface ElasticsearchConfig {
url: string
ssl?: boolean
ca?: string
@ -99,9 +99,9 @@ const SCHEMA: Integration = {
},
}
class ElasticSearchIntegration implements IntegrationBase {
export class ElasticSearchIntegration implements IntegrationBase {
private config: ElasticsearchConfig
private client
private client: Client
constructor(config: ElasticsearchConfig) {
this.config = config
@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase {
}
}
async create(query: { index: string; json: object }) {
const { index, json } = query
async create(query: {
index: string
json: object
extra?: Record<string, string>
}) {
const { index, json, extra } = query
try {
const result = await this.client.index({
index,
body: json,
...extra,
})
return result.body
} catch (err) {
console.error("Error writing to elasticsearch", err)
throw err
} finally {
await this.client.close()
}
}
@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase {
} catch (err) {
console.error("Error querying elasticsearch", err)
throw err
} finally {
await this.client.close()
}
}
async update(query: { id: string; index: string; json: object }) {
const { id, index, json } = query
async update(query: {
id: string
index: string
json: object
extra?: Record<string, string>
}) {
const { id, index, json, extra } = query
try {
const result = await this.client.update({
id,
index,
body: json,
...extra,
})
return result.body
} catch (err) {
console.error("Error querying elasticsearch", err)
throw err
} finally {
await this.client.close()
}
}
async delete(query: { id: string; index: string }) {
const { id, index } = query
async delete(query: {
id: string
index: string
extra?: Record<string, string>
}) {
const { id, index, extra } = query
try {
const result = await this.client.delete({
id,
index,
...extra,
})
return result.body
} catch (err) {
console.error("Error deleting from elasticsearch", err)
throw err
} finally {
await this.client.close()
}
}
}

View File

@ -1,83 +1,81 @@
import { default as ElasticSearchIntegration } from "../elasticsearch"
import { Datasource } from "@budibase/types"
import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch"
import { generator } from "@budibase/backend-core/tests"
import { DatabaseName, datasourceDescribe } from "./utils"
jest.mock("@elastic/elasticsearch")
const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] })
class TestConfiguration {
integration: any
if (describes.length) {
describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => {
let datasource: Datasource
let integration: ElasticSearchIntegration
constructor(config: any = {}) {
this.integration = new ElasticSearchIntegration.integration(config)
}
let index: string
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
})
beforeEach(() => {
index = generator.guid()
integration = new ElasticSearchIntegration(
datasource.config! as ElasticsearchConfig
)
})
it("can create a record", async () => {
await integration.create({
index,
json: { name: "Hello" },
extra: { refresh: "true" },
})
const records = await integration.read({
index,
json: { query: { match_all: {} } },
})
expect(records).toEqual([{ name: "Hello" }])
})
it("can update a record", async () => {
const create = await integration.create({
index,
json: { name: "Hello" },
extra: { refresh: "true" },
})
await integration.update({
id: create._id,
index,
json: { doc: { name: "World" } },
extra: { refresh: "true" },
})
const records = await integration.read({
index,
json: { query: { match_all: {} } },
})
expect(records).toEqual([{ name: "World" }])
})
it("can delete a record", async () => {
const create = await integration.create({
index,
json: { name: "Hello" },
extra: { refresh: "true" },
})
await integration.delete({
id: create._id,
index,
extra: { refresh: "true" },
})
const records = await integration.read({
index,
json: { query: { match_all: {} } },
})
expect(records).toEqual([])
})
})
}
describe("Elasticsearch Integration", () => {
let config: any
let indexName = "Users"
beforeEach(() => {
config = new TestConfiguration()
})
it("calls the create method with the correct params", async () => {
const body = {
name: "Hello",
}
await config.integration.create({
index: indexName,
json: body,
})
expect(config.integration.client.index).toHaveBeenCalledWith({
index: indexName,
body,
})
})
it("calls the read method with the correct params", async () => {
const body = {
query: {
term: {
name: "kimchy",
},
},
}
const response = await config.integration.read({
index: indexName,
json: body,
})
expect(config.integration.client.search).toHaveBeenCalledWith({
index: indexName,
body,
})
expect(response).toEqual(expect.any(Array))
})
it("calls the update method with the correct params", async () => {
const body = {
name: "updated",
}
const response = await config.integration.update({
id: "1234",
index: indexName,
json: body,
})
expect(config.integration.client.update).toHaveBeenCalledWith({
id: "1234",
index: indexName,
body,
})
expect(response).toEqual(expect.any(Array))
})
it("calls the delete method with the correct params", async () => {
const body = {
id: "1234",
}
const response = await config.integration.delete(body)
expect(config.integration.client.delete).toHaveBeenCalledWith(body)
expect(response).toEqual(expect.any(Array))
})
})

View File

@ -0,0 +1,54 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait } from "testcontainers"
import { testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
import { ELASTICSEARCH_IMAGE } from "./images"
import { ElasticsearchConfig } from "../../elasticsearch"
let ports: Promise<testContainerUtils.Port[]>
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer(ELASTICSEARCH_IMAGE)
.withExposedPorts(9200)
.withEnvironment({
// We need to set the discovery type to single-node to avoid the
// cluster waiting for other nodes to join before starting up.
"discovery.type": "single-node",
// We disable security to avoid having to do any auth against the
// container, and to disable SSL. With SSL enabled it uses a self
// signed certificate that we'd have to ignore anyway.
"xpack.security.enabled": "false",
})
.withWaitStrategy(
Wait.forHttp(
// Single node clusters never reach status green, so we wait for
// yellow instead.
"/_cluster/health?wait_for_status=yellow&timeout=10s",
9200
).withStartupTimeout(60000)
)
// We gave the container a tmpfs data directory. Without this, I found
// that the default data directory was very small and the container
// easily filled it up. This caused the cluster to go into a red status
// and stop responding to requests.
.withTmpFs({ "/usr/share/elasticsearch/data": "rw" })
)
}
const port = (await ports).find(x => x.container === 9200)?.host
if (!port) {
throw new Error("Elasticsearch port not found")
}
const config: ElasticsearchConfig = {
url: `http://127.0.0.1:${port}`,
}
return {
type: "datasource",
source: SourceName.ELASTICSEARCH,
config,
}
}

View File

@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}`
export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25`
export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`
export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}`

View File

@ -6,6 +6,7 @@ import * as mysql from "./mysql"
import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import * as oracle from "./oracle"
import * as elasticsearch from "./elasticsearch"
import { testContainerUtils } from "@budibase/backend-core/tests"
import { Knex } from "knex"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -23,22 +24,32 @@ export enum DatabaseName {
MARIADB = "mariadb",
ORACLE = "oracle",
SQS = "sqs",
ELASTICSEARCH = "elasticsearch",
}
const DATASOURCE_PLUS = [
DatabaseName.POSTGRES,
DatabaseName.POSTGRES_LEGACY,
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
DatabaseName.SQS,
]
const providers: Record<DatabaseName, DatasourceProvider> = {
// datasource_plus entries
[DatabaseName.POSTGRES]: postgres.getDatasource,
[DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource,
[DatabaseName.MONGODB]: mongodb.getDatasource,
[DatabaseName.MYSQL]: mysql.getDatasource,
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
[DatabaseName.MARIADB]: mariadb.getDatasource,
[DatabaseName.ORACLE]: oracle.getDatasource,
[DatabaseName.SQS]: async () => undefined,
}
export interface DatasourceDescribeOpts {
only?: DatabaseName[]
exclude?: DatabaseName[]
// rest
[DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource,
[DatabaseName.MONGODB]: mongodb.getDatasource,
}
export interface DatasourceDescribeReturnPromise {
@ -103,6 +114,20 @@ function createDummyTest() {
})
}
interface OnlyOpts {
only: DatabaseName[]
}
interface PlusOpts {
plus: true
exclude?: DatabaseName[]
}
export type DatasourceDescribeOpts = OnlyOpts | PlusOpts
// If you ever want to rename this function, be mindful that you will also need
// to modify src/tests/filters/index.js to make sure that we're correctly
// filtering datasource/non-datasource tests in CI.
export function datasourceDescribe(opts: DatasourceDescribeOpts) {
// tests that call this need a lot longer timeouts
jest.setTimeout(120000)
@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) {
createDummyTest()
}
const { only, exclude } = opts
if (only && exclude) {
throw new Error("you can only supply one of 'only' or 'exclude'")
}
let databases = Object.values(DatabaseName)
if (only) {
databases = only
} else if (exclude) {
databases = databases.filter(db => !exclude.includes(db))
let databases: DatabaseName[] = []
if ("only" in opts) {
databases = opts.only
} else if ("plus" in opts) {
databases = Object.values(DatabaseName)
.filter(db => DATASOURCE_PLUS.includes(db))
.filter(db => !opts.exclude?.includes(db))
} else {
throw new Error("invalid options")
}
if (process.env.DATASOURCE) {
@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) {
isMSSQL: dbName === DatabaseName.SQL_SERVER,
isOracle: dbName === DatabaseName.ORACLE,
isMariaDB: dbName === DatabaseName.MARIADB,
isElasticsearch: dbName === DatabaseName.ELASTICSEARCH,
}))
}

View File

@ -10,16 +10,13 @@ import {
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
DatabaseName,
datasourceDescribe,
} from "../../../../../integrations/tests/utils"
import { datasourceDescribe } from "../../../../../integrations/tests/utils"
import { tableForDatasource } from "../../../../../tests/utilities/structures"
// These test cases are only for things that cannot be tested through the API
// (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead.
const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
const descriptions = datasourceDescribe({ plus: true })
if (descriptions.length) {
describe.each(descriptions)(

View File

@ -12789,10 +12789,10 @@ html-tag@^2.0.0:
is-self-closing "^1.0.1"
kind-of "^6.0.0"
html5-qrcode@^2.2.1:
version "2.3.7"
resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be"
integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA==
html5-qrcode@^2.3.8:
version "2.3.8"
resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d"
integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ==
htmlparser2@^8.0.0:
version "8.0.1"