Merge pull request #10264 from Budibase/qa-core-datasource-api-tests

Qa core datasource api tests
This commit is contained in:
Pedro Silva 2023-04-26 17:09:09 +01:00 committed by GitHub
commit e2937d985c
20 changed files with 704 additions and 33 deletions

View File

@ -12,7 +12,15 @@ import { getIntegration } from "../../integrations"
import { getDatasourceAndQuery } from "./row/utils"
import { invalidateDynamicVariables } from "../../threads/utils"
import { db as dbCore, context, events } from "@budibase/backend-core"
import { UserCtx, Datasource, Row } from "@budibase/types"
import {
UserCtx,
Datasource,
Row,
CreateDatasourceResponse,
UpdateDatasourceResponse,
UpdateDatasourceRequest,
CreateDatasourceRequest,
} from "@budibase/types"
import sdk from "../../sdk"
export async function fetch(ctx: UserCtx) {
@ -146,7 +154,7 @@ async function invalidateVariables(
await invalidateDynamicVariables(toInvalidate)
}
export async function update(ctx: UserCtx) {
export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
const db = context.getAppDB()
const datasourceId = ctx.params.datasourceId
let datasource = await sdk.datasources.get(datasourceId)
@ -187,15 +195,17 @@ export async function update(ctx: UserCtx) {
}
}
export async function save(ctx: UserCtx) {
export async function save(
ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
) {
const db = context.getAppDB()
const plus = ctx.request.body.datasource.plus
const fetchSchema = ctx.request.body.fetchSchema
const datasource = {
_id: generateDatasourceID({ plus }),
type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
...ctx.request.body.datasource,
type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
}
let schemaError = null
@ -218,7 +228,7 @@ export async function save(ctx: UserCtx) {
}
}
const response: any = {
const response: CreateDatasourceResponse = {
datasource: await sdk.datasources.removeSecretSingle(datasource),
}
if (schemaError) {

View File

@ -349,7 +349,7 @@ describe("row api - postgres", () => {
},
plus: true,
source: "POSTGRES",
type: "datasource",
type: "datasource_plus",
_id: expect.any(String),
_rev: expect.any(String),
createdAt: expect.any(String),

View File

@ -0,0 +1,19 @@
import { Datasource } from "../../../documents"
export interface CreateDatasourceResponse {
datasource: Datasource
error?: any
}
export interface UpdateDatasourceResponse {
datasource: Datasource
}
export interface CreateDatasourceRequest {
datasource: Datasource
fetchSchema?: boolean
}
export interface UpdateDatasourceRequest extends Datasource {
datasource: Datasource
}

View File

@ -1 +1,2 @@
export * from "./backup"
export * from "./datasource"

View File

@ -42,3 +42,10 @@ export interface PaginationValues {
page: string | number | null
limit: number | null
}
export interface PreviewQueryRequest extends Omit<Query, "parameters"> {
parameters: {}
flags?: {
urlName?: boolean
}
}

View File

@ -14,6 +14,20 @@ const env = {
ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
POSTGRES_HOST: process.env.POSTGRES_HOST,
POSTGRES_PORT: process.env.POSTGRES_PORT,
POSTGRES_DB: process.env.POSTGRES_DB,
POSTGRES_USER: process.env.POSTGRES_USER,
POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD,
MONGODB_CONNECTION_STRING: process.env.MONGODB_CONNECTION_STRING,
MONGODB_DB: process.env.MONGODB_DB,
REST_API_BASE_URL: process.env.REST_API_BASE_URL,
REST_API_KEY: process.env.REST_API_KEY,
MARIADB_HOST: process.env.MARIADB_HOST,
MARIADB_PORT: process.env.MARIADB_PORT,
MARIADB_DB: process.env.MARIADB_DB,
MARIADB_USER: process.env.MARIADB_USER,
MARIADB_PASSWORD: process.env.MARIADB_PASSWORD,
}
export = env

View File

@ -7,6 +7,10 @@ import ScreenAPI from "./apis/ScreenAPI"
import SelfAPI from "./apis/SelfAPI"
import TableAPI from "./apis/TableAPI"
import UserAPI from "./apis/UserAPI"
import DatasourcesAPI from "./apis/DatasourcesAPI"
import IntegrationsAPI from "./apis/IntegrationsAPI"
import QueriesAPI from "./apis/QueriesAPI"
import PermissionsAPI from "./apis/PermissionsAPI"
import BudibaseInternalAPIClient from "./BudibaseInternalAPIClient"
import { State } from "../../types"
@ -22,6 +26,10 @@ export default class BudibaseInternalAPI {
self: SelfAPI
tables: TableAPI
users: UserAPI
datasources: DatasourcesAPI
integrations: IntegrationsAPI
queries: QueriesAPI
permissions: PermissionsAPI
constructor(state: State) {
this.client = new BudibaseInternalAPIClient(state)
@ -35,5 +43,9 @@ export default class BudibaseInternalAPI {
this.self = new SelfAPI(this.client)
this.tables = new TableAPI(this.client)
this.users = new UserAPI(this.client)
this.datasources = new DatasourcesAPI(this.client)
this.integrations = new IntegrationsAPI(this.client)
this.queries = new QueriesAPI(this.client)
this.permissions = new PermissionsAPI(this.client)
}
}

View File

@ -0,0 +1,95 @@
import { Response } from "node-fetch"
import {
Datasource,
CreateDatasourceResponse,
UpdateDatasourceResponse,
} from "@budibase/types"
import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
export default class DatasourcesAPI {
client: BudibaseInternalAPIClient
constructor(client: BudibaseInternalAPIClient) {
this.client = client
}
async getIntegrations(): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/integrations`)
expect(response).toHaveStatusCode(200)
const integrationsCount = Object.keys(json).length
expect(integrationsCount).toBe(16)
return [response, json]
}
async getAll(): Promise<[Response, Datasource[]]> {
const [response, json] = await this.client.get(`/datasources`)
expect(response).toHaveStatusCode(200)
expect(json.length).toBeGreaterThan(0)
return [response, json]
}
async getTable(dataSourceId: string): Promise<[Response, Datasource]> {
const [response, json] = await this.client.get(
`/datasources/${dataSourceId}`
)
expect(response).toHaveStatusCode(200)
expect(json._id).toEqual(dataSourceId)
return [response, json]
}
async add(body: any): Promise<[Response, CreateDatasourceResponse]> {
const [response, json] = await this.client.post(`/datasources`, { body })
expect(response).toHaveStatusCode(200)
expect(json.datasource._id).toBeDefined()
expect(json.datasource._rev).toBeDefined()
return [response, json]
}
async update(body: any): Promise<[Response, UpdateDatasourceResponse]> {
const [response, json] = await this.client.put(`/datasources/${body._id}`, {
body,
})
expect(response).toHaveStatusCode(200)
expect(json.datasource._id).toBeDefined()
expect(json.datasource._rev).toBeDefined()
return [response, json]
}
async previewQuery(body: any): Promise<[Response, any]> {
const [response, json] = await this.client.post(`/queries/preview`, {
body,
})
expect(response).toHaveStatusCode(200)
return [response, json]
}
async saveQuery(body: any): Promise<[Response, any]> {
const [response, json] = await this.client.post(`/queries`, {
body,
})
expect(response).toHaveStatusCode(200)
return [response, json]
}
async getQuery(queryId: string): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/queries/${queryId}`)
expect(response).toHaveStatusCode(200)
return [response, json]
}
async getQueryPermissions(queryId: string): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/permissions/${queryId}`)
expect(response).toHaveStatusCode(200)
return [response, json]
}
async delete(dataSourceId: string, revId: string): Promise<Response> {
const [response, json] = await this.client.del(
`/datasources/${dataSourceId}/${revId}`
)
expect(response).toHaveStatusCode(200)
return response
}
}

View File

@ -0,0 +1,18 @@
import { Response } from "node-fetch"
import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
export default class IntegrationsAPI {
client: BudibaseInternalAPIClient
constructor(client: BudibaseInternalAPIClient) {
this.client = client
}
async getAll(): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/integrations`)
expect(response).toHaveStatusCode(200)
const integrationsCount = Object.keys(json).length
expect(integrationsCount).toBeGreaterThan(0)
return [response, json]
}
}

View File

@ -0,0 +1,16 @@
import { Response } from "node-fetch"
import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
export default class PermissionsAPI {
client: BudibaseInternalAPIClient
constructor(client: BudibaseInternalAPIClient) {
this.client = client
}
async getAll(id: string): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/permissions/${id}`)
expect(response).toHaveStatusCode(200)
return [response, json]
}
}

View File

@ -0,0 +1,33 @@
import { Response } from "node-fetch"
import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
import { PreviewQueryRequest, Query } from "@budibase/types"
export default class DatasourcesAPI {
client: BudibaseInternalAPIClient
constructor(client: BudibaseInternalAPIClient) {
this.client = client
}
async preview(body: PreviewQueryRequest): Promise<[Response, any]> {
const [response, json] = await this.client.post(`/queries/preview`, {
body,
})
expect(response).toHaveStatusCode(200)
return [response, json]
}
async save(body: Query): Promise<[Response, any]> {
const [response, json] = await this.client.post(`/queries`, {
body,
})
expect(response).toHaveStatusCode(200)
return [response, json]
}
async get(queryId: string): Promise<[Response, any]> {
const [response, json] = await this.client.get(`/queries/${queryId}`)
expect(response).toHaveStatusCode(200)
return [response, json]
}
}

View File

@ -0,0 +1,71 @@
// Add information about the data source to the fixtures file from 1password
export const mongoDB = () => {
return {
datasource: {
name: "MongoDB",
source: "MONGODB",
type: "datasource",
config: {
connectionString: process.env.MONGODB_CONNECTION_STRING,
db: process.env.MONGODB_DB,
},
},
fetchSchema: false,
}
}
export const postgresSQL = () => {
return {
datasource: {
name: "PostgresSQL",
plus: true,
source: "POSTGRES",
type: "datasource",
config: {
database: process.env.POSTGRES_DB,
host: process.env.POSTGRES_HOST,
password: process.env.POSTGRES_PASSWORD,
port: process.env.POSTGRES_PORT,
schema: "public",
user: process.env.POSTGRES_USER,
},
},
fetchSchema: true,
}
}
export const mariaDB = () => {
return {
datasource: {
name: "MariaDB",
plus: true,
source: "MYSQL",
type: "datasource",
config: {
database: process.env.MARIADB_DB,
host: process.env.MARIADB_HOST,
password: process.env.MARIADB_PASSWORD,
port: process.env.MARIADB_PORT,
schema: "public",
user: process.env.MARIADB_USER,
},
},
fetchSchema: true,
}
}
export const restAPI = () => {
return {
datasource: {
name: "RestAPI",
source: "REST",
type: "datasource",
config: {
defaultHeaders: {},
rejectUnauthorized: true,
url: process.env.REST_API_BASE_URL,
},
},
fetchSchema: false,
}
}

View File

@ -4,3 +4,5 @@ export * as rows from "./rows"
export * as screens from "./screens"
export * as tables from "./tables"
export * as users from "./users"
export * as datasources from "./datasources"
export * as queries from "./queries"

View File

@ -0,0 +1,123 @@
import { PreviewQueryRequest } from "@budibase/types"
const query = (datasourceId: string, fields: any): any => {
return {
datasourceId: datasourceId,
fields: fields,
name: "Query 1",
parameters: {},
queryVerb: "read",
schema: {},
transformer: "return data",
}
}
export const mariaDB = (datasourceId: string): PreviewQueryRequest => {
const fields = {
sql: "SELECT * FROM employees LIMIT 10;",
}
return query(datasourceId, fields)
}
export const mongoDB = (datasourceId: string): PreviewQueryRequest => {
const fields = {
extra: {
collection: "movies",
actionType: "find",
},
json: "",
}
return query(datasourceId, fields)
}
export const postgres = (datasourceId: string): PreviewQueryRequest => {
const fields = {
sql: "SELECT * FROM customers;",
}
return query(datasourceId, fields)
}
export const expectedSchemaFields = {
mariaDB: {
birth_date: "string",
emp_no: "number",
first_name: "string",
gender: "string",
hire_date: "string",
last_name: "string",
},
mongoDB: {
directors: "array",
genres: "array",
image: "string",
plot: "string",
rank: "number",
rating: "number",
release_date: "string",
running_time_secs: "number",
title: "string",
year: "number",
_id: "json",
},
postgres: {
address: "string",
city: "string",
company_name: "string",
contact_name: "string",
contact_title: "string",
country: "string",
customer_id: "string",
fax: "string",
phone: "string",
postal_code: "string",
region: "string",
},
restAPI: {
abilities: "array",
base_experience: "number",
forms: "array",
game_indices: "array",
height: "number",
held_items: "array",
id: "number",
is_default: "string",
location_area_encounters: "string",
moves: "array",
name: "string",
order: "number",
past_types: "array",
species: "json",
sprites: "json",
stats: "array",
types: "array",
weight: "number",
},
}
const request = (datasourceId: string, fields: any, flags: any): any => {
return {
datasourceId: datasourceId,
fields: fields,
flags: flags,
name: "Query 1",
parameters: {},
queryVerb: "read",
schema: {},
transformer: "return data",
}
}
export const restAPI = (datasourceId: string): PreviewQueryRequest => {
const fields = {
authConfigId: null,
bodyType: "none",
disabledHeaders: {},
headers: {},
pagination: {},
path: `${process.env.REST_API_BASE_URL}/pokemon/ditto`,
queryString: "",
}
const flags = {
urlName: true,
}
return request(datasourceId, fields, flags)
}

View File

@ -1,27 +0,0 @@
import TestConfiguration from "../../config/TestConfiguration"
import * as fixtures from "../../fixtures"
describe("Internal API - Data Sources", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
it("Create an app with a data source", async () => {
// Create app
await config.createApp()
// Create Screen
const roleArray = ["BASIC", "POWER", "ADMIN", "PUBLIC"]
for (let role in roleArray) {
const [response, screen] = await config.api.screens.create(
fixtures.screens.generateScreen(roleArray[role])
)
}
})
})

View File

@ -0,0 +1,69 @@
import TestConfiguration from "../../config/TestConfiguration"
import * as fixtures from "../../fixtures"
import { Query } from "@budibase/types"
describe("Internal API - Data Sources: MariaDB", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
it("Create an app with a data source - MariaDB", async () => {
// Create app
await config.createApp()
// Get all integrations
await config.api.integrations.getAll()
// Add data source
const [dataSourceResponse, dataSourceJson] =
await config.api.datasources.add(fixtures.datasources.mariaDB())
// Update data source
const newDataSourceInfo = {
...dataSourceJson.datasource,
name: "MariaDB2",
}
const [updatedDataSourceResponse, updatedDataSourceJson] =
await config.api.datasources.update(newDataSourceInfo)
// Query data source
const [queryResponse, queryJson] = await config.api.queries.preview(
fixtures.queries.mariaDB(updatedDataSourceJson.datasource._id!)
)
expect(queryJson.rows.length).toEqual(10)
expect(queryJson.schemaFields).toEqual(
fixtures.queries.expectedSchemaFields.mariaDB
)
// Save query
const datasourcetoSave: Query = {
...fixtures.queries.mariaDB(updatedDataSourceJson.datasource._id!),
parameters: [],
}
const [saveQueryResponse, saveQueryJson] = await config.api.queries.save(
datasourcetoSave
)
// Get Query
const [getQueryResponse, getQueryJson] = await config.api.queries.get(
<string>saveQueryJson._id
)
// Get Query permissions
const [getQueryPermissionsResponse, getQueryPermissionsJson] =
await config.api.permissions.getAll(saveQueryJson._id!)
// Delete data source
const deleteResponse = await config.api.datasources.delete(
updatedDataSourceJson.datasource._id!,
updatedDataSourceJson.datasource._rev!
)
})
})

View File

@ -0,0 +1,69 @@
import TestConfiguration from "../../config/TestConfiguration"
import * as fixtures from "../../fixtures"
import { Query } from "@budibase/types"
describe("Internal API - Data Sources: MongoDB", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
it("Create an app with a data source - MongoDB", async () => {
// Create app
await config.createApp()
// Get all integrations
await config.api.integrations.getAll()
// Add data source
const [dataSourceResponse, dataSourceJson] =
await config.api.datasources.add(fixtures.datasources.mongoDB())
// Update data source
const newDataSourceInfo = {
...dataSourceJson.datasource,
name: "MongoDB2",
}
const [updatedDataSourceResponse, updatedDataSourceJson] =
await config.api.datasources.update(newDataSourceInfo)
// Query data source
const [queryResponse, queryJson] = await config.api.queries.preview(
fixtures.queries.mongoDB(updatedDataSourceJson.datasource._id!)
)
expect(queryJson.rows.length).toBeGreaterThan(10)
expect(queryJson.schemaFields).toEqual(
fixtures.queries.expectedSchemaFields.mongoDB
)
// Save query
const datasourcetoSave: Query = {
...fixtures.queries.mongoDB(updatedDataSourceJson.datasource._id!),
parameters: [],
}
const [saveQueryResponse, saveQueryJson] = await config.api.queries.save(
datasourcetoSave
)
// Get Query
const [getQueryResponse, getQueryJson] = await config.api.queries.get(
<string>saveQueryJson._id
)
// Get Query permissions
const [getQueryPermissionsResponse, getQueryPermissionsJson] =
await config.api.permissions.getAll(saveQueryJson._id!)
// Delete data source
const deleteResponse = await config.api.datasources.delete(
updatedDataSourceJson.datasource._id!,
updatedDataSourceJson.datasource._rev!
)
})
})

View File

@ -0,0 +1,69 @@
import TestConfiguration from "../../config/TestConfiguration"
import * as fixtures from "../../fixtures"
import { Query } from "@budibase/types"
describe("Internal API - Data Sources: PostgresSQL", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
it("Create an app with a data source - PostgresSQL", async () => {
// Create app
await config.createApp()
// Get all integrations
await config.api.integrations.getAll()
// Add data source
const [dataSourceResponse, dataSourceJson] =
await config.api.datasources.add(fixtures.datasources.postgresSQL())
// Update data source
const newDataSourceInfo = {
...dataSourceJson.datasource,
name: "PostgresSQL2",
}
const [updatedDataSourceResponse, updatedDataSourceJson] =
await config.api.datasources.update(newDataSourceInfo)
// Query data source
const [queryResponse, queryJson] = await config.api.queries.preview(
fixtures.queries.postgres(updatedDataSourceJson.datasource._id!)
)
expect(queryJson.rows.length).toEqual(91)
expect(queryJson.schemaFields).toEqual(
fixtures.queries.expectedSchemaFields.postgres
)
// Save query
const datasourcetoSave: Query = {
...fixtures.queries.postgres(updatedDataSourceJson.datasource._id!),
parameters: [],
}
const [saveQueryResponse, saveQueryJson] = await config.api.queries.save(
datasourcetoSave
)
// Get Query
const [getQueryResponse, getQueryJson] = await config.api.queries.get(
saveQueryJson._id!
)
// Get Query permissions
const [getQueryPermissionsResponse, getQueryPermissionsJson] =
await config.api.permissions.getAll(saveQueryJson._id!)
// Delete data source
const deleteResponse = await config.api.datasources.delete(
updatedDataSourceJson.datasource._id!,
updatedDataSourceJson.datasource._rev!
)
})
})

View File

@ -0,0 +1,69 @@
import TestConfiguration from "../../config/TestConfiguration"
import * as fixtures from "../../fixtures"
import { Query } from "@budibase/types"
describe("Internal API - Data Sources: REST API", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
it("Create an app with a data source - REST API", async () => {
// Create app
await config.createApp()
// Get all integrations
await config.api.integrations.getAll()
// Add data source
const [dataSourceResponse, dataSourceJson] =
await config.api.datasources.add(fixtures.datasources.restAPI())
// Update data source
const newDataSourceInfo = {
...dataSourceJson.datasource,
name: "RestAPI - Updated",
}
const [updatedDataSourceResponse, updatedDataSourceJson] =
await config.api.datasources.update(newDataSourceInfo)
// Query data source
const [queryResponse, queryJson] = await config.api.queries.preview(
fixtures.queries.restAPI(updatedDataSourceJson.datasource._id!)
)
expect(queryJson.rows.length).toEqual(1)
expect(queryJson.schemaFields).toEqual(
fixtures.queries.expectedSchemaFields.restAPI
)
// Save query
const datasourcetoSave: Query = {
...fixtures.queries.postgres(updatedDataSourceJson.datasource._id!),
parameters: [],
}
const [saveQueryResponse, saveQueryJson] = await config.api.queries.save(
datasourcetoSave
)
// Get Query
const [getQueryResponse, getQueryJson] = await config.api.queries.get(
saveQueryJson._id!
)
// Get Query permissions
const [getQueryPermissionsResponse, getQueryPermissionsJson] =
await config.api.permissions.getAll(saveQueryJson._id!)
// Delete data source
const deleteResponse = await config.api.datasources.delete(
updatedDataSourceJson.datasource._id!,
updatedDataSourceJson.datasource._rev!
)
})
})

View File

@ -8,6 +8,7 @@ export * from "./responseMessage"
export * from "./routing"
export * from "./state"
export * from "./unpublishAppResponse"
export * from "./addedDatasource"
// re-export public api types
export * from "@budibase/server/api/controllers/public/mapping/types"