Merge branch 'master' into ci/clean-unused-scripts

This commit is contained in:
Adria Navarro 2024-11-28 18:42:13 +01:00 committed by GitHub
commit 7408dbc199
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 77 additions and 125 deletions

View File

@ -6,6 +6,26 @@ import {
import { ContainerInfo } from "dockerode"
import path from "path"
import lockfile from "proper-lockfile"
import { execSync } from "child_process"
interface DockerContext {
Name: string
Description: string
DockerEndpoint: string
ContextType: string
Error: string
}
function getCurrentDockerContext(): DockerContext {
const out = execSync("docker context ls --format json")
for (const line of out.toString().split("\n")) {
const parsed = JSON.parse(line)
if (parsed.Current) {
return parsed as DockerContext
}
}
throw new Error("No current Docker context")
}
async function getBudibaseContainers() {
const client = await getContainerRuntimeClient()
@ -27,6 +47,14 @@ async function killContainers(containers: ContainerInfo[]) {
}
export default async function setup() {
// For whatever reason, testcontainers doesn't always use the correct current
// docker context. This bit of code forces the issue by finding the current
// context and setting it as the DOCKER_HOST environment
if (!process.env.DOCKER_HOST) {
const dockerContext = getCurrentDockerContext()
process.env.DOCKER_HOST = dockerContext.DockerEndpoint
}
const lockPath = path.resolve(__dirname, "globalSetup.ts")
// If you run multiple tests at the same time, it's possible for the CouchDB
// shared container to get started multiple times despite having an

View File

@ -25,7 +25,7 @@ function getTestcontainers(): ContainerInfo[] {
// We use --format json to make sure the output is nice and machine-readable,
// and we use --no-trunc so that the command returns full container IDs so we
// can filter on them correctly.
return execSync("docker ps --format json --no-trunc")
return execSync("docker ps --all --format json --no-trunc")
.toString()
.split("\n")
.filter(x => x.length > 0)
@ -37,6 +37,10 @@ function getTestcontainers(): ContainerInfo[] {
)
}
function removeContainer(container: ContainerInfo) {
execSync(`docker rm ${container.ID}`)
}
export function getContainerByImage(image: string) {
const containers = getTestcontainers().filter(x => x.Image.startsWith(image))
if (containers.length > 1) {
@ -49,6 +53,10 @@ export function getContainerByImage(image: string) {
return containers[0]
}
function getContainerByName(name: string) {
return getTestcontainers().find(x => x.Names === name)
}
export function getContainerById(id: string) {
return getTestcontainers().find(x => x.ID === id)
}
@ -70,7 +78,34 @@ export function getExposedV4Port(container: ContainerInfo, port: number) {
return getExposedV4Ports(container).find(x => x.container === port)?.host
}
interface DockerContext {
Name: string
Description: string
DockerEndpoint: string
ContextType: string
Error: string
}
function getCurrentDockerContext(): DockerContext {
const out = execSync("docker context ls --format json")
for (const line of out.toString().split("\n")) {
const parsed = JSON.parse(line)
if (parsed.Current) {
return parsed as DockerContext
}
}
throw new Error("No current Docker context")
}
export function setupEnv(...envs: any[]) {
// For whatever reason, testcontainers doesn't always use the correct current
// docker context. This bit of code forces the issue by finding the current
// context and setting it as the DOCKER_HOST environment
if (!process.env.DOCKER_HOST) {
const dockerContext = getCurrentDockerContext()
process.env.DOCKER_HOST = dockerContext.DockerEndpoint
}
// We start couchdb in globalSetup.ts, in the root of the monorepo, so it
// should be relatively safe to look for it by its image name.
const couch = getContainerByImage("budibase/couchdb")
@ -116,6 +151,16 @@ export async function startContainer(container: GenericContainer) {
key = imageName.split("@")[0]
}
key = key.replace(/\//g, "-").replace(/:/g, "-")
const name = `${key}_testcontainer`
// If a container has died it hangs around and future attempts to start a
// container with the same name will fail. What we do here is if we find a
// matching container and it has exited, we remove it before carrying on. This
// removes the need to do this removal manually.
const existingContainer = getContainerByName(name)
if (existingContainer?.State === "exited") {
removeContainer(existingContainer)
}
container = container
.withReuse()

@ -1 +1 @@
Subproject commit 6f38253253ee364aea636add990083ca5cda3bde
Subproject commit e2252498ddfade3c2592b1ec78f7bee4e3cf0d2f

View File

@ -23,13 +23,11 @@ import {
Table,
RowValue,
DynamicVariable,
QueryJsonRequest,
} from "@budibase/types"
import sdk from "../../sdk"
import { builderSocket } from "../../websockets"
import { isEqual } from "lodash"
import { processTable } from "../../sdk/app/tables/getters"
import { makeExternalQuery } from "../../integrations/base/query"
export async function fetch(ctx: UserCtx) {
ctx.body = await sdk.datasources.fetch()
@ -298,16 +296,6 @@ export async function find(ctx: UserCtx) {
ctx.body = await sdk.datasources.removeSecretSingle(datasource)
}
// dynamic query functionality
export async function query(ctx: UserCtx<QueryJsonRequest>) {
const queryJson = ctx.request.body
try {
ctx.body = await makeExternalQuery(queryJson)
} catch (err: any) {
ctx.throw(400, err)
}
}
export async function getExternalSchema(ctx: UserCtx) {
const datasource = await sdk.datasources.get(ctx.params.datasourceId)
const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(

View File

@ -2,10 +2,7 @@ import Router from "@koa/router"
import * as datasourceController from "../controllers/datasource"
import authorized from "../../middleware/authorized"
import { permissions } from "@budibase/backend-core"
import {
datasourceValidator,
datasourceQueryValidator,
} from "./utils/validators"
import { datasourceValidator } from "./utils/validators"
const router: Router = new Router()
@ -41,15 +38,6 @@ router
),
datasourceController.update
)
.post(
"/api/datasources/query",
authorized(
permissions.PermissionType.TABLE,
permissions.PermissionLevel.READ
),
datasourceQueryValidator(),
datasourceController.query
)
.post(
"/api/datasources/:datasourceId/schema",
authorized(permissions.BUILDER),

View File

@ -1,10 +1,4 @@
import {
Datasource,
Operation,
Query,
QueryPreview,
TableSourceType,
} from "@budibase/types"
import { Datasource, Query, QueryPreview } from "@budibase/types"
import {
DatabaseName,
datasourceDescribe,
@ -817,49 +811,6 @@ if (descriptions.length) {
})
describe("query through datasource", () => {
it("should be able to query the datasource", async () => {
const datasource = await config.api.datasource.create(rawDatasource)
const entityId = tableName
await config.api.datasource.update({
...datasource,
entities: {
[entityId]: {
name: entityId,
schema: {},
type: "table",
primary: ["id"],
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
},
},
})
const res = await config.api.datasource.query({
endpoint: {
datasourceId: datasource._id!,
operation: Operation.READ,
entityId,
},
resource: {
fields: ["id", "name"],
},
filters: {
string: {
name: "two",
},
},
})
expect(res).toHaveLength(1)
expect(res[0]).toEqual({
id: 2,
name: "two",
// the use of table.* introduces the possibility of nulls being returned
birthday: null,
number: null,
})
})
// this parameter really only impacts SQL queries
describe("confirm nullDefaultSupport", () => {
let queryParams: Partial<Query>

View File

@ -1,5 +1,4 @@
import { auth, permissions } from "@budibase/backend-core"
import { DataSourceOperation } from "../../../constants"
import {
AutomationActionStepId,
AutomationStep,
@ -231,30 +230,6 @@ export function externalSearchValidator() {
)
}
export function datasourceQueryValidator() {
return auth.joiValidator.body(
Joi.object({
endpoint: Joi.object({
datasourceId: Joi.string().required(),
operation: Joi.string()
.required()
.valid(...Object.values(DataSourceOperation)),
entityId: Joi.string().required(),
}).required(),
resource: Joi.object({
fields: Joi.array().items(Joi.string()).optional(),
}).optional(),
body: Joi.object().optional(),
sort: Joi.object().optional(),
filters: filterObject().optional(),
paginate: Joi.object({
page: Joi.string().alphanum().optional(),
limit: Joi.number().optional(),
}).optional(),
})
)
}
export function webhookValidator() {
return auth.joiValidator.body(
Joi.object({

View File

@ -45,17 +45,6 @@ export enum AuthTypes {
EXTERNAL = "external",
}
export enum DataSourceOperation {
CREATE = "CREATE",
READ = "READ",
UPDATE = "UPDATE",
DELETE = "DELETE",
BULK_CREATE = "BULK_CREATE",
CREATE_TABLE = "CREATE_TABLE",
UPDATE_TABLE = "UPDATE_TABLE",
DELETE_TABLE = "DELETE_TABLE",
}
export enum DatasourceAuthTypes {
GOOGLE = "google",
}

View File

@ -4,7 +4,6 @@ import {
Datasource,
FetchDatasourceInfoResponse,
FieldType,
QueryJsonRequest,
RelationshipType,
UpdateDatasourceRequest,
UpdateDatasourceResponse,
@ -69,13 +68,6 @@ export class DatasourceAPI extends TestAPI {
return await this._get<Datasource[]>(`/api/datasources`, { expectations })
}
query = async (query: QueryJsonRequest, expectations?: Expectations) => {
return await this._post<any>(`/api/datasources/query`, {
body: query,
expectations,
})
}
fetchSchema = async (
{
datasourceId,

View File

@ -190,10 +190,6 @@ export interface EnrichedQueryJson extends QueryJson {
datasource?: Datasource
}
export interface QueryJsonRequest extends Omit<QueryJson, "endpoint"> {
endpoint: QueryJson["endpoint"] & { datasourceId: string; entityId: string }
}
export interface QueryOptions {
disableReturning?: boolean
disableBindings?: boolean