Moving all datasource types around - this is needed to make the SDK/Document types more accesible for the development datasource plugins.

This commit is contained in:
mike12345567 2022-08-11 11:48:58 +01:00
parent 47c3bc8c30
commit 5dce4fc16a
27 changed files with 374 additions and 378 deletions

View File

@ -1,70 +1,10 @@
export { Query, Datasource } from "./datasource" export { Query, Datasource } from "./datasource"
export { FieldSchema, TableSchema, Table, Document, Row } from "@budibase/types"
export interface Base { export interface Application extends Document {
_id?: string
_rev?: string
}
export interface Application extends Base {
appId?: string appId?: string
} }
export interface FieldSchema {
// TODO: replace with field types enum when done
type: string
externalType?: string
fieldName?: string
name: string
tableId?: string
relationshipType?: string
through?: string
foreignKey?: string
autocolumn?: boolean
subtype?: string
throughFrom?: string
throughTo?: string
formula?: string
formulaType?: string
main?: boolean
ignoreTimezones?: boolean
meta?: {
toTable: string
toKey: string
}
constraints?: {
type?: string
email?: boolean
inclusion?: string[]
length?: {
minimum?: string | number
maximum?: string | number
}
presence?: boolean
}
}
export interface TableSchema {
[key: string]: FieldSchema
}
export interface Table extends Base {
type?: string
views?: {}
name: string
primary?: string[]
schema: TableSchema
primaryDisplay?: string
sourceId?: string
relatedFormula?: string[]
constrained?: string[]
}
export interface Row extends Base {
type?: string
tableId?: string
[key: string]: any
}
interface JsonSchemaField { interface JsonSchemaField {
properties: { properties: {
[key: string]: { [key: string]: {
@ -94,7 +34,7 @@ export interface AutomationStep {
type: string type: string
} }
export interface Automation extends Base { export interface Automation extends Document {
name: string name: string
type: string type: string
appId?: string appId?: string

View File

@ -1,214 +1,42 @@
import { Row, Table, Base } from "./common" import { SortDirection, Operation, SourceNames } from "@budibase/types"
// export everything that used to be exported from here
export {
Operation,
SortDirection,
QueryTypes,
DatasourceFieldTypes,
SourceNames,
IncludeRelationships,
FilterTypes,
QueryDefinition,
ExtraQueryConfig,
Integration,
IntegrationBase,
QueryParameter,
PaginationConfig,
PaginationValues,
RestQueryFields,
Query,
Datasource,
SearchFilters,
SortJson,
PaginationJson,
RenameColumn,
RelationshipsJson,
QueryJson,
SqlQuery,
} from "@budibase/types"
export enum Operation { /********************************************
CREATE = "CREATE", * This file contains structures which are *
READ = "READ", * internal to the server and don't need to *
UPDATE = "UPDATE", * be exposed for use by other services. *
DELETE = "DELETE", ********************************************/
BULK_CREATE = "BULK_CREATE",
CREATE_TABLE = "CREATE_TABLE",
UPDATE_TABLE = "UPDATE_TABLE",
DELETE_TABLE = "DELETE_TABLE",
}
export enum SortDirection {
ASCENDING = "ASCENDING",
DESCENDING = "DESCENDING",
}
export enum QueryTypes {
SQL = "sql",
JSON = "json",
FIELDS = "fields",
}
export enum DatasourceFieldTypes {
STRING = "string",
LONGFORM = "longForm",
BOOLEAN = "boolean",
NUMBER = "number",
PASSWORD = "password",
LIST = "list",
OBJECT = "object",
JSON = "json",
FILE = "file",
}
export enum SourceNames {
POSTGRES = "POSTGRES",
DYNAMODB = "DYNAMODB",
MONGODB = "MONGODB",
ELASTICSEARCH = "ELASTICSEARCH",
COUCHDB = "COUCHDB",
SQL_SERVER = "SQL_SERVER",
S3 = "S3",
AIRTABLE = "AIRTABLE",
MYSQL = "MYSQL",
ARANGODB = "ARANGODB",
REST = "REST",
ORACLE = "ORACLE",
GOOGLE_SHEETS = "GOOGLE_SHEETS",
FIRESTORE = "FIRESTORE",
REDIS = "REDIS",
SNOWFLAKE = "SNOWFLAKE",
}
export enum IncludeRelationships {
INCLUDE = 1,
EXCLUDE = 0,
}
export enum FilterTypes {
STRING = "string",
FUZZY = "fuzzy",
RANGE = "range",
EQUAL = "equal",
NOT_EQUAL = "notEqual",
EMPTY = "empty",
NOT_EMPTY = "notEmpty",
ONE_OF = "oneOf",
}
export interface QueryDefinition {
type: QueryTypes
displayName?: string
readable?: boolean
customisable?: boolean
fields?: object
urlDisplay?: boolean
}
export interface ExtraQueryConfig {
[key: string]: {
displayName: string
type: string
required: boolean
data?: object
}
}
export interface Integration {
docs: string
plus?: boolean
auth?: { type: string }
relationships?: boolean
description: string
friendlyName: string
type?: string
datasource: {}
query: {
[key: string]: QueryDefinition
}
extra?: ExtraQueryConfig
}
export interface SearchFilters {
allOr?: boolean
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any
}
}
export interface SortJson {
[key: string]: SortDirection
}
export interface PaginationJson {
limit: number
page?: string | number
}
export interface RenameColumn {
old: string
updated: string
}
export interface RelationshipsJson {
through?: string
from?: string
to?: string
fromPrimary?: string
toPrimary?: string
tableName: string
column: string
}
export interface QueryJson {
endpoint: {
datasourceId: string
entityId: string
operation: Operation
schema?: string
}
resource: {
fields: string[]
}
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
body?: Row | Row[]
table?: Table
meta?: {
table?: Table
tables?: Record<string, Table>
renamed: RenameColumn
}
extra?: {
idFilter?: SearchFilters
}
relationships?: RelationshipsJson[]
}
export interface SqlQuery {
sql: string
bindings?: string[]
}
export interface QueryOptions { export interface QueryOptions {
disableReturning?: boolean disableReturning?: boolean
} }
export interface Datasource extends Base {
type: string
name: string
source: SourceNames
// the config is defined by the schema
config: {
[key: string]: string | number | boolean
}
plus: boolean
entities?: {
[key: string]: Table
}
}
export enum AuthType { export enum AuthType {
BASIC = "basic", BASIC = "basic",
BEARER = "bearer", BEARER = "bearer",
@ -230,25 +58,6 @@ export interface BearerAuthConfig {
token: string token: string
} }
export interface QueryParameter {
name: string
default: string
}
export interface RestQueryFields {
path: string
queryString?: string
headers: { [key: string]: any }
disabledHeaders: { [key: string]: any }
requestBody: any
bodyType: string
json: object
method: string
authConfigId: string
pagination: PaginationConfig | null
paginationValues: PaginationValues | null
}
export interface RestConfig { export interface RestConfig {
url: string url: string
defaultHeaders: { defaultHeaders: {
@ -266,28 +75,3 @@ export interface RestConfig {
} }
] ]
} }
export interface PaginationConfig {
type: string
location: string
pageParam: string
sizeParam: string | null
responseParam: string | null
}
export interface PaginationValues {
page: string | number | null
limit: number | null
}
export interface Query {
_id?: string
datasourceId: string
name: string
parameters: QueryParameter[]
fields: RestQueryFields | any
transformer: string | null
schema: any
readable: boolean
queryVerb: string
}

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
module AirtableModule { module AirtableModule {
const Airtable = require("airtable") const Airtable = require("airtable")

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
module ArangoModule { module ArangoModule {
const { Database, aql } = require("arangojs") const { Database, aql } = require("arangojs")

View File

@ -1,6 +0,0 @@
export interface IntegrationBase {
create?(query: any): Promise<any[] | any>
read?(query: any): Promise<any[] | any>
update?(query: any): Promise<any[] | any>
delete?(query: any): Promise<any[] | any>
}

View File

@ -1,13 +0,0 @@
import { Table } from "../../definitions/common"
import { IntegrationBase } from "./IntegrationBase"
export interface DatasourcePlus extends IntegrationBase {
tables: Record<string, Table>
schemaErrors: Record<string, string>
// if the datasource supports the use of bindings directly (to protect against SQL injection)
// this returns the format of the identifier
getBindingIdentifier(): string
getStringConcat(parts: string[]): string
buildSchema(datasourceId: string, entities: Record<string, Table>): any
}

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
module CouchDBModule { module CouchDBModule {
const PouchDB = require("pouchdb") const PouchDB = require("pouchdb")

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
module DynamoModule { module DynamoModule {
const AWS = require("aws-sdk") const AWS = require("aws-sdk")

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
module ElasticsearchModule { module ElasticsearchModule {
const { Client } = require("@elastic/elasticsearch") const { Client } = require("@elastic/elasticsearch")

View File

@ -2,8 +2,8 @@ import {
DatasourceFieldTypes, DatasourceFieldTypes,
Integration, Integration,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
import { Firestore, WhereFilterOp } from "@google-cloud/firestore" import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
module Firebase { module Firebase {

View File

@ -1,12 +1,13 @@
import { import {
DatasourceFieldTypes, DatasourceFieldTypes,
Integration, Integration,
QueryJson,
QueryTypes, QueryTypes,
} from "../definitions/datasource" Table,
TableSchema,
QueryJson,
DatasourcePlus,
} from "@budibase/types"
import { OAuth2Client } from "google-auth-library" import { OAuth2Client } from "google-auth-library"
import { DatasourcePlus } from "./base/datasourcePlus"
import { Table, TableSchema } from "../definitions/common"
import { buildExternalTableId } from "./utils" import { buildExternalTableId } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants" import { DataSourceOperation, FieldTypes } from "../constants"
import { GoogleSpreadsheet } from "google-spreadsheet" import { GoogleSpreadsheet } from "google-spreadsheet"

View File

@ -2,10 +2,13 @@ import {
DatasourceFieldTypes, DatasourceFieldTypes,
Integration, Integration,
Operation, Operation,
Table,
TableSchema,
QueryJson, QueryJson,
QueryTypes, QueryTypes,
SqlQuery, SqlQuery,
} from "../definitions/datasource" DatasourcePlus,
} from "@budibase/types"
import { import {
getSqlQuery, getSqlQuery,
buildExternalTableId, buildExternalTableId,
@ -13,8 +16,6 @@ import {
finaliseExternalTables, finaliseExternalTables,
SqlClients, SqlClients,
} from "./utils" } from "./utils"
import { DatasourcePlus } from "./base/datasourcePlus"
import { Table, TableSchema } from "../definitions/common"
module MSSQLModule { module MSSQLModule {
const sqlServer = require("mssql") const sqlServer = require("mssql")

View File

@ -2,8 +2,8 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
} from "../definitions/datasource" IntegrationBase,
import { IntegrationBase } from "./base/IntegrationBase" } from "@budibase/types"
import { import {
MongoClient, MongoClient,
ObjectID, ObjectID,

View File

@ -4,8 +4,10 @@ import {
QueryTypes, QueryTypes,
QueryJson, QueryJson,
SqlQuery, SqlQuery,
} from "../definitions/datasource" Table,
import { Table, TableSchema } from "../definitions/common" TableSchema,
DatasourcePlus,
} from "@budibase/types"
import { import {
getSqlQuery, getSqlQuery,
SqlClients, SqlClients,
@ -13,7 +15,6 @@ import {
convertSqlType, convertSqlType,
finaliseExternalTables, finaliseExternalTables,
} from "./utils" } from "./utils"
import { DatasourcePlus } from "./base/datasourcePlus"
import dayjs from "dayjs" import dayjs from "dayjs"
const { NUMBER_REGEX } = require("../utilities") const { NUMBER_REGEX } = require("../utilities")

View File

@ -5,7 +5,9 @@ import {
QueryJson, QueryJson,
QueryTypes, QueryTypes,
SqlQuery, SqlQuery,
} from "../definitions/datasource" Table,
DatasourcePlus,
} from "@budibase/types"
import { import {
buildExternalTableId, buildExternalTableId,
convertSqlType, convertSqlType,
@ -21,8 +23,6 @@ import oracledb, {
Result, Result,
} from "oracledb" } from "oracledb"
import Sql from "./base/sql" import Sql from "./base/sql"
import { Table } from "../definitions/common"
import { DatasourcePlus } from "./base/datasourcePlus"
import { FieldTypes } from "../constants" import { FieldTypes } from "../constants"
module OracleModule { module OracleModule {

View File

@ -4,8 +4,9 @@ import {
QueryTypes, QueryTypes,
QueryJson, QueryJson,
SqlQuery, SqlQuery,
} from "../definitions/datasource" Table,
import { Table } from "../definitions/common" DatasourcePlus,
} from "@budibase/types"
import { import {
getSqlQuery, getSqlQuery,
buildExternalTableId, buildExternalTableId,
@ -13,7 +14,6 @@ import {
finaliseExternalTables, finaliseExternalTables,
SqlClients, SqlClients,
} from "./utils" } from "./utils"
import { DatasourcePlus } from "./base/datasourcePlus"
module PostgresModule { module PostgresModule {
const { Client, types } = require("pg") const { Client, types } = require("pg")

View File

@ -1,6 +1,5 @@
import { findHBSBlocks, processStringSync } from "@budibase/string-templates" import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
import { Integration } from "../../definitions/datasource" import { DatasourcePlus } from "@budibase/types"
import { DatasourcePlus } from "../base/datasourcePlus"
const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")

View File

@ -1,8 +1,4 @@
import { import { DatasourceFieldTypes, Integration, QueryTypes } from "@budibase/types"
DatasourceFieldTypes,
Integration,
QueryTypes,
} from "../definitions/datasource"
import Redis from "ioredis" import Redis from "ioredis"
module RedisModule { module RedisModule {

View File

@ -2,15 +2,17 @@ import {
Integration, Integration,
DatasourceFieldTypes, DatasourceFieldTypes,
QueryTypes, QueryTypes,
PaginationConfig,
IntegrationBase,
PaginationValues,
} from "@budibase/types"
import {
RestConfig, RestConfig,
RestQueryFields as RestQuery, RestQueryFields as RestQuery,
PaginationConfig,
AuthType, AuthType,
BasicAuthConfig, BasicAuthConfig,
BearerAuthConfig, BearerAuthConfig,
PaginationValues,
} from "../definitions/datasource" } from "../definitions/datasource"
import { IntegrationBase } from "./base/IntegrationBase"
import { get } from "lodash" import { get } from "lodash"
const BodyTypes = { const BodyTypes = {

View File

@ -1,5 +1,4 @@
import { Integration, QueryTypes } from "../definitions/datasource" import { Integration, QueryTypes, IntegrationBase } from "@budibase/types"
import { IntegrationBase } from "./base/IntegrationBase"
module S3Module { module S3Module {
const AWS = require("aws-sdk") const AWS = require("aws-sdk")

View File

@ -1,4 +1,4 @@
import { Integration, QueryTypes, SqlQuery } from "../definitions/datasource" import { Integration, QueryTypes, SqlQuery } from "@budibase/types"
import { Snowflake } from "snowflake-promise" import { Snowflake } from "snowflake-promise"
module SnowflakeModule { module SnowflakeModule {

View File

@ -1,5 +1,17 @@
import { Document } from "../document" import { Document } from "../document"
import { SourceNames } from "../../sdk"
import { Table } from "./table"
export interface Datasource extends Document { export interface Datasource extends Document {
source: string type: string
name: string
source: SourceNames
// the config is defined by the schema
config: {
[key: string]: string | number | boolean
}
plus: boolean
entities?: {
[key: string]: Table
}
} }

View File

@ -1,3 +1,23 @@
import { Document } from "../document" import { Document } from "../document"
export interface Row extends Document {} export enum FieldTypes {
STRING = "string",
LONGFORM = "longform",
OPTIONS = "options",
NUMBER = "number",
BOOLEAN = "boolean",
ARRAY = "array",
DATETIME = "datetime",
ATTACHMENT = "attachment",
LINK = "link",
FORMULA = "formula",
AUTO = "auto",
JSON = "json",
INTERNAL = "internal",
}
export interface Row extends Document {
type?: string
tableId?: string
[key: string]: any
}

View File

@ -1,6 +1,52 @@
import { Document } from "../document" import { Document } from "../document"
import { View } from "./view" import { View } from "./view"
export interface Table extends Document { export interface FieldSchema {
views: { [key: string]: View } // TODO: replace with field types enum when done
type: string
externalType?: string
fieldName?: string
name: string
tableId?: string
relationshipType?: string
through?: string
foreignKey?: string
autocolumn?: boolean
subtype?: string
throughFrom?: string
throughTo?: string
formula?: string
formulaType?: string
main?: boolean
ignoreTimezones?: boolean
meta?: {
toTable: string
toKey: string
}
constraints?: {
type?: string
email?: boolean
inclusion?: string[]
length?: {
minimum?: string | number
maximum?: string | number
}
presence?: boolean
}
}
export interface TableSchema {
[key: string]: FieldSchema
}
export interface Table extends Document {
type?: string
views?: { [key: string]: View }
name: string
primary?: string[]
schema: TableSchema
primaryDisplay?: string
sourceId?: string
relatedFormula?: string[]
constrained?: string[]
} }

View File

@ -0,0 +1,121 @@
import { Table } from "../documents"
export enum Operation {
CREATE = "CREATE",
READ = "READ",
UPDATE = "UPDATE",
DELETE = "DELETE",
BULK_CREATE = "BULK_CREATE",
CREATE_TABLE = "CREATE_TABLE",
UPDATE_TABLE = "UPDATE_TABLE",
DELETE_TABLE = "DELETE_TABLE",
}
export enum SortDirection {
ASCENDING = "ASCENDING",
DESCENDING = "DESCENDING",
}
export enum QueryTypes {
SQL = "sql",
JSON = "json",
FIELDS = "fields",
}
export enum DatasourceFieldTypes {
STRING = "string",
LONGFORM = "longForm",
BOOLEAN = "boolean",
NUMBER = "number",
PASSWORD = "password",
LIST = "list",
OBJECT = "object",
JSON = "json",
FILE = "file",
}
export enum SourceNames {
POSTGRES = "POSTGRES",
DYNAMODB = "DYNAMODB",
MONGODB = "MONGODB",
ELASTICSEARCH = "ELASTICSEARCH",
COUCHDB = "COUCHDB",
SQL_SERVER = "SQL_SERVER",
S3 = "S3",
AIRTABLE = "AIRTABLE",
MYSQL = "MYSQL",
ARANGODB = "ARANGODB",
REST = "REST",
ORACLE = "ORACLE",
GOOGLE_SHEETS = "GOOGLE_SHEETS",
FIRESTORE = "FIRESTORE",
REDIS = "REDIS",
SNOWFLAKE = "SNOWFLAKE",
}
export enum IncludeRelationships {
INCLUDE = 1,
EXCLUDE = 0,
}
export enum FilterTypes {
STRING = "string",
FUZZY = "fuzzy",
RANGE = "range",
EQUAL = "equal",
NOT_EQUAL = "notEqual",
EMPTY = "empty",
NOT_EMPTY = "notEmpty",
ONE_OF = "oneOf",
}
export interface QueryDefinition {
type: QueryTypes
displayName?: string
readable?: boolean
customisable?: boolean
fields?: object
urlDisplay?: boolean
}
export interface ExtraQueryConfig {
[key: string]: {
displayName: string
type: string
required: boolean
data?: object
}
}
export interface Integration {
docs: string
plus?: boolean
auth?: { type: string }
relationships?: boolean
description: string
friendlyName: string
type?: string
datasource: {}
query: {
[key: string]: QueryDefinition
}
extra?: ExtraQueryConfig
}
export interface IntegrationBase {
create?(query: any): Promise<any[] | any>
read?(query: any): Promise<any[] | any>
update?(query: any): Promise<any[] | any>
delete?(query: any): Promise<any[] | any>
}
export interface DatasourcePlus extends IntegrationBase {
tables: Record<string, Table>
schemaErrors: Record<string, string>
// if the datasource supports the use of bindings directly (to protect against SQL injection)
// this returns the format of the identifier
getBindingIdentifier(): string
getStringConcat(parts: string[]): string
buildSchema(datasourceId: string, entities: Record<string, Table>): any
}

View File

@ -3,3 +3,5 @@ export * from "./context"
export * from "./events" export * from "./events"
export * from "./licensing" export * from "./licensing"
export * from "./migrations" export * from "./migrations"
export * from "./datasources"
export * from "./search"

View File

@ -0,0 +1,91 @@
import { Operation, SortDirection } from "./datasources"
import { Row, Table } from "../documents"
export interface SearchFilters {
allOr?: boolean
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any
}
}
export interface SortJson {
[key: string]: SortDirection
}
export interface PaginationJson {
limit: number
page?: string | number
}
export interface RenameColumn {
old: string
updated: string
}
export interface RelationshipsJson {
through?: string
from?: string
to?: string
fromPrimary?: string
toPrimary?: string
tableName: string
column: string
}
export interface QueryJson {
endpoint: {
datasourceId: string
entityId: string
operation: Operation
schema?: string
}
resource: {
fields: string[]
}
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
body?: Row | Row[]
table?: Table
meta?: {
table?: Table
tables?: Record<string, Table>
renamed: RenameColumn
}
extra?: {
idFilter?: SearchFilters
}
relationships?: RelationshipsJson[]
}
export interface SqlQuery {
sql: string
bindings?: string[]
}