From aa9931b52186db723e320f8ece88b3ecf9cb11f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 10:59:07 +0100 Subject: [PATCH 01/16] yarn.lock --- yarn.lock | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index b15c549640..de64f7d4bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2489,6 +2489,11 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@fontsource/source-sans-pro@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@fontsource/source-sans-pro/-/source-sans-pro-5.0.3.tgz#7d6e84a8169ba12fa5e6ce70757aa2ca7e74d855" + integrity sha512-mQnjuif/37VxwRloHZ+wQdoozd2VPWutbFSt1AuSkk7nFXIBQxHJLw80rgCF/osL0t7N/3Gx1V7UJuOX2zxzhQ== + "@fortawesome/fontawesome-common-types@6.3.0": version "6.3.0" resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.3.0.tgz#51f734e64511dbc3674cd347044d02f4dd26e86b" @@ -8406,7 +8411,7 @@ chmodr@1.2.0: resolved "https://registry.yarnpkg.com/chmodr/-/chmodr-1.2.0.tgz#720e96caa09b7f1cdbb01529b7d0ab6bc5e118b9" integrity sha512-Y5uI7Iq/Az6HgJEL6pdw7THVd7jbVOTPwsmcPOBjQL8e3N+pz872kzK5QxYGEy21iRys+iHWV0UZQXDFJo1hyA== -chokidar@3.5.3, chokidar@^3.0.0, chokidar@^3.5.1, chokidar@^3.5.2: +chokidar@3.5.3, chokidar@^3.0.0, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -11850,7 +11855,7 @@ fast-glob@3.2.7: merge2 "^1.3.0" micromatch "^4.0.4" -fast-glob@^3.0.3: +fast-glob@^3.0.3, fast-glob@^3.2.11: version "3.2.12" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80" integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== @@ -25423,6 +25428,16 @@ vite-node@0.29.8: picocolors "^1.0.0" vite "^3.0.0 || ^4.0.0" +vite-plugin-static-copy@^0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/vite-plugin-static-copy/-/vite-plugin-static-copy-0.16.0.tgz#2f65227037f17fc99c0782fd0b344e962935e69e" + integrity sha512-dMVEg5Z2SwYRgQnHZaeokvSKB4p/TOTf65JU4sP3U6ccSBsukqdtDOjpmT+xzTFHAA8WJjcS31RMLjUdWQCBzw== + dependencies: + chokidar "^3.5.3" + fast-glob "^3.2.11" + fs-extra "^11.1.0" + picocolors "^1.0.0" + "vite@^3.0.0 || ^4.0.0": version "4.2.2" resolved "https://registry.yarnpkg.com/vite/-/vite-4.2.2.tgz#014c30e5163844f6e96d7fe18fbb702236516dc6" From e7026d4aed263c290cad240c8ae108651034fda7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 11:16:27 +0100 Subject: [PATCH 02/16] Get schema function --- packages/server/src/integrations/postgres.ts | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index b1f20f97ec..49c2c147a6 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -21,6 +21,7 @@ import { PostgresColumn } from "./base/types" import { escapeDangerousCharacters } from "../utilities" import { Client, ClientConfig, types } from "pg" +import { exec } from "child_process" // Return "date" and "timestamp" types as plain strings. // This lets us reference the original stored timezone. @@ -381,6 +382,34 @@ class PostgresIntegration extends Sql implements DatasourcePlus { return response.rows.length ? response.rows : [{ [operation]: true }] } } + + async getSchema() { + const dumpCommandParts = [ + `PGPASSWORD="${this.config.password}"`, + `pg_dump -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} -d ${this.config.database} --schema-only`, + ] + + const dumpCommand = dumpCommandParts.join(" ") + + return new Promise((res, rej) => { + exec(dumpCommand, (error, stdout, stderr) => { + if (error) { + console.error(`Error generating dump: ${error.message}`) + rej(error.message) + return + } + + if (stderr) { + console.error(`pg_dump error: ${stderr}`) + rej(stderr) + return + } + + res(stdout) + console.log("SQL dump generated successfully!") + }) + }) + } } export default { From dbcf7814a8231b080ba4e09ded82755159eccb3a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 11:46:22 +0100 Subject: [PATCH 03/16] Support ssl --- packages/server/src/integrations/postgres.ts | 33 +++++++++++++++++-- .../src/utilities/fileSystem/filesystem.ts | 4 ++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 49c2c147a6..451da5fa8f 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -1,3 +1,4 @@ +import fs from "fs" import { Integration, DatasourceFieldType, @@ -22,6 +23,7 @@ import { escapeDangerousCharacters } from "../utilities" import { Client, ClientConfig, types } from "pg" import { exec } from "child_process" +import { storeTempFile } from "../utilities/fileSystem" // Return "date" and "timestamp" types as plain strings. // This lets us reference the original stored timezone. @@ -385,11 +387,36 @@ class PostgresIntegration extends Sql implements DatasourcePlus { async getSchema() { const dumpCommandParts = [ - `PGPASSWORD="${this.config.password}"`, - `pg_dump -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} -d ${this.config.database} --schema-only`, + `user=${this.config.user}`, + `host=${this.config.host}`, + `port=${this.config.port}`, + `dbname=${this.config.database}`, ] - const dumpCommand = dumpCommandParts.join(" ") + if (this.config.ssl) { + dumpCommandParts.push("sslmode=verify-ca") + if (this.config.ca) { + const caFilePath = storeTempFile(this.config.ca) + fs.chmodSync(caFilePath, "0600") + dumpCommandParts.push(`sslrootcert=${caFilePath}`) + } + + if (this.config.clientCert) { + const clientCertFilePath = storeTempFile(this.config.clientCert) + fs.chmodSync(clientCertFilePath, "0600") + dumpCommandParts.push(`sslcert=${clientCertFilePath}`) + } + + if (this.config.clientKey) { + const clientKeyFilePath = storeTempFile(this.config.clientKey) + fs.chmodSync(clientKeyFilePath, "0600") + dumpCommandParts.push(`sslkey=${clientKeyFilePath}`) + } + } + + const dumpCommand = `PGPASSWORD="${ + this.config.password + }" pg_dump --schema-only "${dumpCommandParts.join(" ")}"` return new Promise((res, rej) => { exec(dumpCommand, (error, stdout, stderr) => { diff --git a/packages/server/src/utilities/fileSystem/filesystem.ts b/packages/server/src/utilities/fileSystem/filesystem.ts index 9434f071d4..a44fa03c28 100644 --- a/packages/server/src/utilities/fileSystem/filesystem.ts +++ b/packages/server/src/utilities/fileSystem/filesystem.ts @@ -81,7 +81,9 @@ export const streamFile = (path: string) => { * @param {string} fileContents contents which will be written to a temp file. * @return {string} the path to the temp file. */ -export const storeTempFile = (fileContents: any) => { +export const storeTempFile = ( + fileContents: string | NodeJS.ArrayBufferView +) => { const path = join(budibaseTempDir(), uuid()) fs.writeFileSync(path, fileContents) return path From 7a4eb3113daeeffc1c61f1f24ced5adde62ad3ba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:02:18 +0100 Subject: [PATCH 04/16] Add external schema endpoint --- .../server/src/api/controllers/datasource.ts | 16 ++++++++++++++++ packages/server/src/api/routes/datasource.ts | 5 +++++ packages/server/src/integrations/postgres.ts | 3 ++- packages/types/src/sdk/datasources.ts | 1 + 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index bbbcf96538..1d7643dd7b 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -26,6 +26,7 @@ import { IntegrationBase, DatasourcePlus, SourceName, + Ctx, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" @@ -441,3 +442,18 @@ export async function query(ctx: UserCtx) { ctx.throw(400, err) } } + +export async function getExternalSchema(ctx: Ctx) { + const { datasource } = ctx.request.body + const enrichedDatasource = await getAndMergeDatasource(datasource) + const connector = await getConnector(enrichedDatasource) + + if (!connector.getExternalSchema) { + ctx.throw(400, "Datasource does not support exporting external schema") + } + const response = await connector.getExternalSchema() + + ctx.body = { + schema: response, + } +} diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index f874d3c4aa..c05d1753a2 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -66,5 +66,10 @@ router authorized(permissions.BUILDER), datasourceController.destroy ) + .get( + "/api/datasources/:datasourceId/external-schema", + authorized(permissions.BUILDER), + datasourceController.getExternalSchema + ) export default router diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 451da5fa8f..54870cf0ec 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -181,6 +181,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { const response: ConnectionInfo = { connected: false, } + try { await this.openConnection() response.connected = true @@ -385,7 +386,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { } } - async getSchema() { + async getExternalSchema() { const dumpCommandParts = [ `user=${this.config.user}`, `host=${this.config.host}`, diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 50ea063ca3..bccb7c04d9 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -140,6 +140,7 @@ export interface IntegrationBase { update?(query: any): Promise delete?(query: any): Promise testConnection?(): Promise + getExternalSchema?(): Promise } export interface DatasourcePlus extends IntegrationBase { From d337c52adf3581e7dcd46dff17a1f6b5a016948f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:02:27 +0100 Subject: [PATCH 05/16] Add feature flags --- packages/server/src/integrations/postgres.ts | 1 + packages/types/src/sdk/datasources.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 54870cf0ec..8c18e6266a 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -60,6 +60,7 @@ const SCHEMA: Integration = { features: { [DatasourceFeature.CONNECTION_CHECKING]: true, [DatasourceFeature.FETCH_TABLE_NAMES]: true, + [DatasourceFeature.EXPORT_SCHEMA]: true, }, datasource: { host: { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index bccb7c04d9..f3001a971d 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -76,6 +76,7 @@ export enum FilterType { export enum DatasourceFeature { CONNECTION_CHECKING = "connection", FETCH_TABLE_NAMES = "fetch_table_names", + EXPORT_SCHEMA = "export_schema", } export interface StepDefinition { From c27573a9fb326d48b66272efe55ef13d7914c84f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:17:17 +0100 Subject: [PATCH 06/16] Add basic test --- .../postgres.integration.spec.ts | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 qa-core/src/integrations/external-schema/postgres.integration.spec.ts diff --git a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts new file mode 100644 index 0000000000..dcfab9a499 --- /dev/null +++ b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts @@ -0,0 +1,63 @@ +import { GenericContainer } from "testcontainers" +import postgres from "../../../../packages/server/src/integrations/postgres" + +jest.unmock("pg") + +describe("getExternalSchema", () => { + describe("postgres", () => { + let host: string + let port: number + let config: any + + beforeAll(async () => { + const container = await new GenericContainer("postgres") + .withExposedPorts(5432) + .withEnv("POSTGRES_PASSWORD", "password") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(5432) + + config = { + host, + port, + database: "postgres", + user: "postgres", + password: "password", + schema: "public", + ssl: false, + rejectUnauthorized: false, + } + }) + + it("can export an empty database", async () => { + const integration = new postgres.integration(config) + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "-- + -- PostgreSQL database dump + -- + + -- Dumped from database version 15.3 (Debian 15.3-1.pgdg120+1) + -- Dumped by pg_dump version 15.3 + + SET statement_timeout = 0; + SET lock_timeout = 0; + SET idle_in_transaction_session_timeout = 0; + SET client_encoding = 'UTF8'; + SET standard_conforming_strings = on; + SELECT pg_catalog.set_config('search_path', '', false); + SET check_function_bodies = false; + SET xmloption = content; + SET client_min_messages = warning; + SET row_security = off; + + -- + -- PostgreSQL database dump complete + -- + + " + `) + }) + }) +}) From c605ecdf0c2ebe80e3f457de4e42ccecd7fb1e6e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:20:07 +0100 Subject: [PATCH 07/16] Test export with database --- .../postgres.integration.spec.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts index dcfab9a499..4d4f8d9c24 100644 --- a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts +++ b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts @@ -52,6 +52,101 @@ describe("getExternalSchema", () => { SET client_min_messages = warning; SET row_security = off; + -- + -- PostgreSQL database dump complete + -- + + " + `) + }) + + it("can export a database with tables", async () => { + const integration = new postgres.integration(config) + + integration.internalQuery({ + sql: ` + CREATE TABLE IF NOT EXISTS "users" ( + "id" SERIAL, + "name" VARCHAR(100) NOT NULL, + "role" VARCHAR(15) NOT NULL, + PRIMARY KEY ("id") + );`, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "-- + -- PostgreSQL database dump + -- + + -- Dumped from database version 15.3 (Debian 15.3-1.pgdg120+1) + -- Dumped by pg_dump version 15.3 + + SET statement_timeout = 0; + SET lock_timeout = 0; + SET idle_in_transaction_session_timeout = 0; + SET client_encoding = 'UTF8'; + SET standard_conforming_strings = on; + SELECT pg_catalog.set_config('search_path', '', false); + SET check_function_bodies = false; + SET xmloption = content; + SET client_min_messages = warning; + SET row_security = off; + + SET default_tablespace = ''; + + SET default_table_access_method = heap; + + -- + -- Name: users; Type: TABLE; Schema: public; Owner: postgres + -- + + CREATE TABLE public.users ( + id integer NOT NULL, + name character varying(100) NOT NULL, + role character varying(15) NOT NULL + ); + + + ALTER TABLE public.users OWNER TO postgres; + + -- + -- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres + -- + + CREATE SEQUENCE public.users_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + ALTER TABLE public.users_id_seq OWNER TO postgres; + + -- + -- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres + -- + + ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; + + + -- + -- Name: users id; Type: DEFAULT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); + + + -- + -- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + -- -- PostgreSQL database dump complete -- From c30c9b319dd58860e8e75b67c47e0dabdfc54c95 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:21:42 +0100 Subject: [PATCH 08/16] Test not exporting data --- .../postgres.integration.spec.ts | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts index 4d4f8d9c24..e8264c99cb 100644 --- a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts +++ b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts @@ -139,6 +139,95 @@ describe("getExternalSchema", () => { ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); + -- + -- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + + -- + -- PostgreSQL database dump complete + -- + + " + `) + }) + + it("does not export a data", async () => { + const integration = new postgres.integration(config) + + integration.internalQuery({ + sql: `INSERT INTO "users" ("name", "role") VALUES ('John Doe', 'Administrator');`, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "-- + -- PostgreSQL database dump + -- + + -- Dumped from database version 15.3 (Debian 15.3-1.pgdg120+1) + -- Dumped by pg_dump version 15.3 + + SET statement_timeout = 0; + SET lock_timeout = 0; + SET idle_in_transaction_session_timeout = 0; + SET client_encoding = 'UTF8'; + SET standard_conforming_strings = on; + SELECT pg_catalog.set_config('search_path', '', false); + SET check_function_bodies = false; + SET xmloption = content; + SET client_min_messages = warning; + SET row_security = off; + + SET default_tablespace = ''; + + SET default_table_access_method = heap; + + -- + -- Name: users; Type: TABLE; Schema: public; Owner: postgres + -- + + CREATE TABLE public.users ( + id integer NOT NULL, + name character varying(100) NOT NULL, + role character varying(15) NOT NULL + ); + + + ALTER TABLE public.users OWNER TO postgres; + + -- + -- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres + -- + + CREATE SEQUENCE public.users_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + ALTER TABLE public.users_id_seq OWNER TO postgres; + + -- + -- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres + -- + + ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; + + + -- + -- Name: users id; Type: DEFAULT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); + + -- -- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- From 702f59a90e1e18941b03daa750fc8520e83041b4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 15:36:19 +0100 Subject: [PATCH 09/16] Improve tests --- .../postgres.integration.spec.ts | 117 +++++++++++++++++- 1 file changed, 112 insertions(+), 5 deletions(-) diff --git a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts index e8264c99cb..2ac812ad70 100644 --- a/qa-core/src/integrations/external-schema/postgres.integration.spec.ts +++ b/qa-core/src/integrations/external-schema/postgres.integration.spec.ts @@ -63,12 +63,18 @@ describe("getExternalSchema", () => { it("can export a database with tables", async () => { const integration = new postgres.integration(config) - integration.internalQuery({ + await integration.internalQuery({ sql: ` - CREATE TABLE IF NOT EXISTS "users" ( + CREATE TABLE "users" ( + "id" SERIAL, + "name" VARCHAR(100) NOT NULL, + "role" VARCHAR(15) NOT NULL, + PRIMARY KEY ("id") + ); + CREATE TABLE "products" ( "id" SERIAL, "name" VARCHAR(100) NOT NULL, - "role" VARCHAR(15) NOT NULL, + "price" DECIMAL NOT NULL, PRIMARY KEY ("id") );`, }) @@ -97,6 +103,41 @@ describe("getExternalSchema", () => { SET default_table_access_method = heap; + -- + -- Name: products; Type: TABLE; Schema: public; Owner: postgres + -- + + CREATE TABLE public.products ( + id integer NOT NULL, + name character varying(100) NOT NULL, + price numeric NOT NULL + ); + + + ALTER TABLE public.products OWNER TO postgres; + + -- + -- Name: products_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres + -- + + CREATE SEQUENCE public.products_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + ALTER TABLE public.products_id_seq OWNER TO postgres; + + -- + -- Name: products_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres + -- + + ALTER SEQUENCE public.products_id_seq OWNED BY public.products.id; + + -- -- Name: users; Type: TABLE; Schema: public; Owner: postgres -- @@ -132,6 +173,13 @@ describe("getExternalSchema", () => { ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; + -- + -- Name: products id; Type: DEFAULT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.products ALTER COLUMN id SET DEFAULT nextval('public.products_id_seq'::regclass); + + -- -- Name: users id; Type: DEFAULT; Schema: public; Owner: postgres -- @@ -139,6 +187,14 @@ describe("getExternalSchema", () => { ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); + -- + -- Name: products products_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.products + ADD CONSTRAINT products_pkey PRIMARY KEY (id); + + -- -- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -158,8 +214,9 @@ describe("getExternalSchema", () => { it("does not export a data", async () => { const integration = new postgres.integration(config) - integration.internalQuery({ - sql: `INSERT INTO "users" ("name", "role") VALUES ('John Doe', 'Administrator');`, + await integration.internalQuery({ + sql: `INSERT INTO "users" ("name", "role") VALUES ('John Doe', 'Administrator'); + INSERT INTO "products" ("name", "price") VALUES ('Book', 7.68);`, }) const result = await integration.getExternalSchema() @@ -186,6 +243,41 @@ describe("getExternalSchema", () => { SET default_table_access_method = heap; + -- + -- Name: products; Type: TABLE; Schema: public; Owner: postgres + -- + + CREATE TABLE public.products ( + id integer NOT NULL, + name character varying(100) NOT NULL, + price numeric NOT NULL + ); + + + ALTER TABLE public.products OWNER TO postgres; + + -- + -- Name: products_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres + -- + + CREATE SEQUENCE public.products_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + + ALTER TABLE public.products_id_seq OWNER TO postgres; + + -- + -- Name: products_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres + -- + + ALTER SEQUENCE public.products_id_seq OWNED BY public.products.id; + + -- -- Name: users; Type: TABLE; Schema: public; Owner: postgres -- @@ -221,6 +313,13 @@ describe("getExternalSchema", () => { ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; + -- + -- Name: products id; Type: DEFAULT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.products ALTER COLUMN id SET DEFAULT nextval('public.products_id_seq'::regclass); + + -- -- Name: users id; Type: DEFAULT; Schema: public; Owner: postgres -- @@ -228,6 +327,14 @@ describe("getExternalSchema", () => { ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); + -- + -- Name: products products_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres + -- + + ALTER TABLE ONLY public.products + ADD CONSTRAINT products_pkey PRIMARY KEY (id); + + -- -- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- From 76ac28f550deb6ed4891af1fac379c8d066b13b3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:53:02 +0100 Subject: [PATCH 10/16] Do not rquire min length for all password fields --- .../builder/src/components/start/ExportAppModal.svelte | 2 +- packages/builder/src/helpers/validation/yup/index.js | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 4a69aaef74..bc170c47be 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -16,7 +16,7 @@ let password = null const validation = createValidationStore() - validation.addValidatorType("password", "password", true) + validation.addValidatorType("password", "password", true, { minLength: 8 }) $: validation.observe("password", password) const Step = { CONFIG: "config", SET_PASSWORD: "set_password" } diff --git a/packages/builder/src/helpers/validation/yup/index.js b/packages/builder/src/helpers/validation/yup/index.js index 4e84975eb7..b10ed2369e 100644 --- a/packages/builder/src/helpers/validation/yup/index.js +++ b/packages/builder/src/helpers/validation/yup/index.js @@ -21,7 +21,7 @@ export const createValidationStore = () => { validator[propertyName] = propertyValidator } - const addValidatorType = (propertyName, type, required) => { + const addValidatorType = (propertyName, type, required, options) => { if (!type || !propertyName) { return } @@ -45,11 +45,8 @@ export const createValidationStore = () => { propertyValidator = propertyValidator.required() } - // We want to do this after the possible required validation, to prioritise the required error - switch (type) { - case "password": - propertyValidator = propertyValidator.min(8) - break + if (options?.minLength) { + propertyValidator = propertyValidator.min(options.minLength) } validator[propertyName] = propertyValidator From ab5b8716c080cc1fa99fa6704ffa8b4b580cf84b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 14:57:39 +0100 Subject: [PATCH 11/16] Get mysql schema --- packages/server/src/integrations/mysql.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c3bb5e066f..46bd97836b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -39,6 +39,7 @@ const SCHEMA: Integration = { features: { [DatasourceFeature.CONNECTION_CHECKING]: true, [DatasourceFeature.FETCH_TABLE_NAMES]: true, + [DatasourceFeature.EXPORT_SCHEMA]: true, }, datasource: { host: { @@ -324,6 +325,14 @@ class MySQLIntegration extends Sql implements DatasourcePlus { await this.disconnect() } } + + async getExternalSchema() { + const [result] = await this.internalQuery({ + sql: `SHOW CREATE DATABASE ${this.config.database}`, + }) + const schema = result["Create Database"] + return schema + } } export default { From e21dca55823b1b919ffe995febd5ebbf0d45db66 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 15:32:49 +0100 Subject: [PATCH 12/16] Implement and test mysql sql dump --- packages/server/src/integrations/mysql.ts | 26 ++++- .../external-schema/mysql.integration.spec.ts | 108 ++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 qa-core/src/integrations/external-schema/mysql.integration.spec.ts diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 46bd97836b..4bcecb0b44 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -327,11 +327,33 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } async getExternalSchema() { - const [result] = await this.internalQuery({ + try { + const [databaseResult] = await this.internalQuery({ sql: `SHOW CREATE DATABASE ${this.config.database}`, }) - const schema = result["Create Database"] + let dumpContent = [databaseResult["Create Database"]] + + const tablesResult = await this.internalQuery({ + sql: `SHOW TABLES`, + }) + + for (const row of tablesResult) { + const tableName = row[`Tables_in_${this.config.database}`] + + const createTableResults = await this.internalQuery({ + sql: `SHOW CREATE TABLE \`${tableName}\``, + }) + + const createTableStatement = createTableResults[0]["Create Table"] + + dumpContent.push(createTableStatement) + } + + const schema = dumpContent.join("\n") return schema + } finally { + this.disconnect() + } } } diff --git a/qa-core/src/integrations/external-schema/mysql.integration.spec.ts b/qa-core/src/integrations/external-schema/mysql.integration.spec.ts new file mode 100644 index 0000000000..c34651ea0e --- /dev/null +++ b/qa-core/src/integrations/external-schema/mysql.integration.spec.ts @@ -0,0 +1,108 @@ +import { GenericContainer } from "testcontainers" +import mysql from "../../../../packages/server/src/integrations/mysql" + +jest.unmock("mysql2/promise") + +describe("datasource validators", () => { + describe("mysql", () => { + let config: any + + beforeAll(async () => { + const container = await new GenericContainer("mysql") + .withExposedPorts(3306) + .withEnv("MYSQL_ROOT_PASSWORD", "admin") + .withEnv("MYSQL_DATABASE", "db") + .withEnv("MYSQL_USER", "user") + .withEnv("MYSQL_PASSWORD", "password") + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(3306) + config = { + host, + port, + user: "user", + database: "db", + password: "password", + rejectUnauthorized: true, + } + }) + + it("can export an empty database", async () => { + const integration = new mysql.integration(config) + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot( + `"CREATE DATABASE \`db\` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */"` + ) + }) + + it("can export a database with tables", async () => { + const integration = new mysql.integration(config) + + await integration.internalQuery({ + sql: ` + CREATE TABLE users ( + id INT AUTO_INCREMENT, + name VARCHAR(100) NOT NULL, + role VARCHAR(15) NOT NULL, + PRIMARY KEY (id) + ); + + + CREATE TABLE products ( + id INT AUTO_INCREMENT, + name VARCHAR(100) NOT NULL, + price DECIMAL, + PRIMARY KEY (id) + ); + `, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "CREATE DATABASE \`db\` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */ + CREATE TABLE \`products\` ( + \`id\` int NOT NULL AUTO_INCREMENT, + \`name\` varchar(100) NOT NULL, + \`price\` decimal(10,0) DEFAULT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + CREATE TABLE \`users\` ( + \`id\` int NOT NULL AUTO_INCREMENT, + \`name\` varchar(100) NOT NULL, + \`role\` varchar(15) NOT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci" + `) + }) + + it("does not export a data", async () => { + const integration = new mysql.integration(config) + + await integration.internalQuery({ + sql: `INSERT INTO users (name, role) VALUES ('John Doe', 'Administrator');`, + }) + + await integration.internalQuery({ + sql: `INSERT INTO products (name, price) VALUES ('Book', 7.68);`, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "CREATE DATABASE \`db\` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */ + CREATE TABLE \`products\` ( + \`id\` int NOT NULL AUTO_INCREMENT, + \`name\` varchar(100) NOT NULL, + \`price\` decimal(10,0) DEFAULT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + CREATE TABLE \`users\` ( + \`id\` int NOT NULL AUTO_INCREMENT, + \`name\` varchar(100) NOT NULL, + \`role\` varchar(15) NOT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci" + `) + }) + }) +}) From c44b10eadf3e087c0011c645ce84fcfebf0a229c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 15:33:10 +0100 Subject: [PATCH 13/16] Lint --- packages/server/src/integrations/mysql.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 4bcecb0b44..b376b31d0b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -329,8 +329,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async getExternalSchema() { try { const [databaseResult] = await this.internalQuery({ - sql: `SHOW CREATE DATABASE ${this.config.database}`, - }) + sql: `SHOW CREATE DATABASE ${this.config.database}`, + }) let dumpContent = [databaseResult["Create Database"]] const tablesResult = await this.internalQuery({ @@ -350,7 +350,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } const schema = dumpContent.join("\n") - return schema + return schema } finally { this.disconnect() } From 718fe1efc65c0cc81c736c73865654690e3f99da Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Jun 2023 16:11:02 +0100 Subject: [PATCH 14/16] Implement mssql and tests --- .../src/integrations/microsoftSqlServer.ts | 76 ++++++++++++ .../external-schema/mssql.integration.spec.ts | 112 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 qa-core/src/integrations/external-schema/mssql.integration.spec.ts diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 291aad8631..6038b842c4 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -43,6 +43,7 @@ const SCHEMA: Integration = { features: { [DatasourceFeature.CONNECTION_CHECKING]: true, [DatasourceFeature.FETCH_TABLE_NAMES]: true, + [DatasourceFeature.EXPORT_SCHEMA]: true, }, datasource: { user: { @@ -336,6 +337,81 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { result.recordset ? result.recordset : [{ [operation]: true }] return this.queryWithReturning(json, queryFn, processFn) } + + async getExternalSchema() { + // Query to retrieve table schema + const query = ` + SELECT + t.name AS TableName, + c.name AS ColumnName, + ty.name AS DataType, + c.max_length AS MaxLength, + c.is_nullable AS IsNullable, + c.is_identity AS IsIdentity + FROM + sys.tables t + INNER JOIN sys.columns c ON t.object_id = c.object_id + INNER JOIN sys.types ty ON c.system_type_id = ty.system_type_id + WHERE + t.is_ms_shipped = 0 + ORDER BY + t.name, c.column_id +` + + await this.connect() + + const result = await this.internalQuery({ + sql: query, + }) + + const scriptParts = [] + const tables: any = {} + for (const row of result.recordset) { + const { + TableName, + ColumnName, + DataType, + MaxLength, + IsNullable, + IsIdentity, + } = row + + if (!tables[TableName]) { + tables[TableName] = { + columns: [], + } + } + + const columnDefinition = `${ColumnName} ${DataType}${ + MaxLength ? `(${MaxLength})` : "" + }${IsNullable ? " NULL" : " NOT NULL"}` + + tables[TableName].columns.push(columnDefinition) + + if (IsIdentity) { + tables[TableName].identityColumn = ColumnName + } + } + + // Generate SQL statements for table creation + for (const tableName in tables) { + const { columns, identityColumn } = tables[tableName] + + let createTableStatement = `CREATE TABLE [${tableName}] (\n` + createTableStatement += columns.join(",\n") + + if (identityColumn) { + createTableStatement += `,\n CONSTRAINT [PK_${tableName}] PRIMARY KEY (${identityColumn})` + } + + createTableStatement += "\n);" + + scriptParts.push(createTableStatement) + } + + const schema = scriptParts.join("\n") + return schema + } } export default { diff --git a/qa-core/src/integrations/external-schema/mssql.integration.spec.ts b/qa-core/src/integrations/external-schema/mssql.integration.spec.ts new file mode 100644 index 0000000000..450b093cf7 --- /dev/null +++ b/qa-core/src/integrations/external-schema/mssql.integration.spec.ts @@ -0,0 +1,112 @@ +import { GenericContainer, Wait } from "testcontainers" +import { Duration, TemporalUnit } from "node-duration" +import mssql from "../../../../packages/server/src/integrations/microsoftSqlServer" + +jest.unmock("mssql") + +describe("getExternalSchema", () => { + describe("postgres", () => { + let config: any + + beforeAll(async () => { + const password = "Str0Ng_p@ssW0rd!" + const container = await new GenericContainer( + "mcr.microsoft.com/mssql/server" + ) + .withExposedPorts(1433) + .withEnv("ACCEPT_EULA", "Y") + .withEnv("MSSQL_SA_PASSWORD", password) + .withEnv("MSSQL_PID", "Developer") + .withWaitStrategy(Wait.forHealthCheck()) + .withHealthCheck({ + test: `/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${password}" -Q "SELECT 1" -b -o /dev/null`, + interval: new Duration(1000, TemporalUnit.MILLISECONDS), + timeout: new Duration(3, TemporalUnit.SECONDS), + retries: 20, + startPeriod: new Duration(100, TemporalUnit.MILLISECONDS), + }) + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(1433) + config = { + user: "sa", + password, + server: host, + port: port, + database: "master", + schema: "dbo", + } + }) + + it("can export an empty database", async () => { + const integration = new mssql.integration(config) + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(`""`) + }) + + it("can export a database with tables", async () => { + const integration = new mssql.integration(config) + + await integration.connect() + await integration.internalQuery({ + sql: ` + CREATE TABLE users ( + id INT IDENTITY(1,1) PRIMARY KEY, + name VARCHAR(100) NOT NULL, + role VARCHAR(15) NOT NULL + ); + + CREATE TABLE products ( + id INT IDENTITY(1,1) PRIMARY KEY, + name VARCHAR(100) NOT NULL, + price DECIMAL(10, 2) NOT NULL + ); + `, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "CREATE TABLE [products] ( + id int(4) NOT NULL, + name varchar(100) NOT NULL, + price decimal(9) NOT NULL, + CONSTRAINT [PK_products] PRIMARY KEY (id) + ); + CREATE TABLE [users] ( + id int(4) NOT NULL, + name varchar(100) NOT NULL, + role varchar(15) NOT NULL, + CONSTRAINT [PK_users] PRIMARY KEY (id) + );" + `) + }) + + it("does not export a data", async () => { + const integration = new mssql.integration(config) + + await integration.connect() + await integration.internalQuery({ + sql: `INSERT INTO [users] ([name], [role]) VALUES ('John Doe', 'Administrator'); + INSERT INTO [products] ([name], [price]) VALUES ('Book', 7.68); + `, + }) + + const result = await integration.getExternalSchema() + expect(result).toMatchInlineSnapshot(` + "CREATE TABLE [products] ( + id int(4) NOT NULL, + name varchar(100) NOT NULL, + price decimal(9) NOT NULL, + CONSTRAINT [PK_products] PRIMARY KEY (id) + ); + CREATE TABLE [users] ( + id int(4) NOT NULL, + name varchar(100) NOT NULL, + role varchar(15) NOT NULL, + CONSTRAINT [PK_users] PRIMARY KEY (id) + );" + `) + }) + }) +}) From 6ed5894441ff370cbc6ae0bf893b895b1fdb6132 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Jun 2023 11:14:23 +0100 Subject: [PATCH 15/16] Type --- packages/server/src/api/controllers/datasource.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 1d7643dd7b..86669845e6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -26,7 +26,6 @@ import { IntegrationBase, DatasourcePlus, SourceName, - Ctx, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" @@ -443,7 +442,7 @@ export async function query(ctx: UserCtx) { } } -export async function getExternalSchema(ctx: Ctx) { +export async function getExternalSchema(ctx: UserCtx) { const { datasource } = ctx.request.body const enrichedDatasource = await getAndMergeDatasource(datasource) const connector = await getConnector(enrichedDatasource) From 8ab5913eb9a7bbd0bbbcc3529a9ffdf4c05129b3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Jun 2023 11:33:38 +0100 Subject: [PATCH 16/16] Change url --- packages/server/src/api/routes/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index c05d1753a2..7b4945806a 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -67,7 +67,7 @@ router datasourceController.destroy ) .get( - "/api/datasources/:datasourceId/external-schema", + "/api/datasources/:datasourceId/schema/external", authorized(permissions.BUILDER), datasourceController.getExternalSchema )