diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 7dd3c1ed22..dfcfe566bd 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -64,11 +64,6 @@ RUN mkdir -p /var/log/nginx && \ touch /var/run/nginx.pid && \ usermod -a -G tty www-data -WORKDIR / -RUN mkdir -p scripts/integrations/oracle -COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle -RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh - # setup minio WORKDIR /minio COPY scripts/install-minio.sh ./install.sh diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a4b924bf54..a67da7bc10 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -435,6 +435,11 @@ class InternalBuilder { )} = ? THEN 1 ELSE 0 END = 1`, [value] ) + } else if (this.client === SqlClient.ORACLE) { + query = query[fnc]( + `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`, + [value] + ) } else { query = query[fnc]( `COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`, @@ -454,6 +459,11 @@ class InternalBuilder { )} = ? THEN 1 ELSE 0 END = 0`, [value] ) + } else if (this.client === SqlClient.ORACLE) { + query = query[fnc]( + `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`, + [value] + ) } else { query = query[fnc]( `COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`, diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 7c0e6e59bc..9a8a358bac 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -21,11 +21,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends g++ make python3 jq RUN yarn global add pm2 -# Install client for oracle datasource -RUN apt-get install unzip libaio1 -COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/ -RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh - # Install postgres client for pg_dump utils RUN apt update && apt upgrade -y \ && apt install software-properties-common apt-transport-https curl gpg -y \ diff --git a/packages/server/package.json b/packages/server/package.json index 2aa7099a0d..9054e1a89f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -101,6 +101,7 @@ "object-sizeof": "2.6.1", "openai": "^4.52.1", "openapi-types": "9.3.1", + "oracledb": "6.5.1", "pg": "8.10.0", "pouchdb": "7.3.0", "pouchdb-all-dbs": "1.1.1", @@ -111,12 +112,12 @@ "snowflake-promise": "^4.5.0", "socket.io": "4.7.5", "tar": "6.2.1", + "tmp": "0.2.3", "to-json-schema": "0.2.5", "uuid": "^8.3.2", "validate.js": "0.13.1", "worker-farm": "1.7.0", - "xml2js": "0.5.0", - "tmp": "0.2.3" + "xml2js": "0.5.0" }, "devDependencies": { "@babel/preset-env": "7.16.11", @@ -131,13 +132,13 @@ "@types/lodash": "4.14.200", "@types/mssql": "9.1.4", "@types/node-fetch": "2.6.4", - "@types/oracledb": "5.2.2", + "@types/oracledb": "6.5.1", "@types/pg": "8.6.6", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.14", "@types/tar": "6.1.5", - "@types/uuid": "8.3.4", "@types/tmp": "0.2.6", + "@types/uuid": "8.3.4", "copyfiles": "2.4.1", "docker-compose": "0.23.17", "jest": "29.7.0", @@ -156,9 +157,6 @@ "update-dotenv": "1.1.1", "yargs": "13.2.4" }, - "optionalDependencies": { - "oracledb": "5.3.0" - }, "nx": { "targets": { "dev": { diff --git a/packages/server/scripts/integrations/oracle/docker-compose.yml b/packages/server/scripts/integrations/oracle/docker-compose.yml index 15b4c16eee..586f0b683d 100644 --- a/packages/server/scripts/integrations/oracle/docker-compose.yml +++ b/packages/server/scripts/integrations/oracle/docker-compose.yml @@ -16,4 +16,4 @@ services: - oracle_data:/opt/oracle/oradata volumes: - oracle_data: + oracle_data: \ No newline at end of file diff --git a/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip b/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip deleted file mode 100644 index a6ab63b68e..0000000000 Binary files a/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip and /dev/null differ diff --git a/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/install.sh b/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/install.sh deleted file mode 100755 index 7b7402673d..0000000000 --- a/packages/server/scripts/integrations/oracle/instantclient/linux/arm64/install.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Must be root to continue -if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi - -# Allow for re-runs -rm -rf /opt/oracle - -echo "Installing oracle instant client" - -# copy and unzip package -mkdir -p /opt/oracle -cp scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip /opt/oracle -cd /opt/oracle -unzip -qq basiclite-19.10.zip -d . -rm *.zip -mv instantclient* instantclient - -# update runtime link path -sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf" -ldconfig /etc/ld.so.conf.d - -echo "Installation complete" diff --git a/packages/server/scripts/integrations/oracle/instantclient/linux/install.sh b/packages/server/scripts/integrations/oracle/instantclient/linux/install.sh deleted file mode 100755 index 4662996a44..0000000000 --- a/packages/server/scripts/integrations/oracle/instantclient/linux/install.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" -if [[ $TARGETARCH == arm* ]] ; -then - echo "Installing ARM Oracle instant client..." - $SCRIPT_DIR/arm64/install.sh -else - echo "Installing x86-64 Oracle instant client..." - $SCRIPT_DIR/x86-64/install.sh -fi diff --git a/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip b/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip deleted file mode 100644 index 777bccbe0f..0000000000 Binary files a/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip and /dev/null differ diff --git a/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh b/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh deleted file mode 100755 index 1225860a1a..0000000000 --- a/packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Must be root to continue -if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi - -# Allow for re-runs -rm -rf /opt/oracle - -echo "Installing oracle instant client" - -# copy and unzip package -mkdir -p /opt/oracle -cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle -cd /opt/oracle -unzip -qq basiclite-21.4.zip -d . -rm *.zip -mv instantclient* instantclient - -# update runtime link path -sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf" -ldconfig /etc/ld.so.conf.d - -echo "Installation complete" \ No newline at end of file diff --git a/packages/server/scripts/integrations/oracle/oracle.md b/packages/server/scripts/integrations/oracle/oracle.md index 58d2a7dfbf..fccbc9c00f 100644 --- a/packages/server/scripts/integrations/oracle/oracle.md +++ b/packages/server/scripts/integrations/oracle/oracle.md @@ -10,16 +10,16 @@ To install oracle express edition simply run `docker-compose up` - A single instance pluggable database (PDB) will be created named `xepdb1` - The default password is configured in the compose file as `oracle` - - The `system` and `pdbadmin` users share this password + - The `system` and `pdbadmin` users share this password -## Instant Client +## Instant Client Before oracle can be connected to from nodejs, the oracle client must be installed. For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html **Important** - Oracle client is supported only on **x86-64 architecture** -- Oracle client is **not supported on Mac ARM architecture** +- Oracle client is **not supported on Mac ARM architecture** ### Linux Run the provided install script for linux from the `server` root path: @@ -29,7 +29,7 @@ sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install ``` For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst -### Mac +### Mac **This has not yet been tested** See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst @@ -64,7 +64,7 @@ grant create session, to &USERNAME; ``` -To set the password for the sales schema use: +To set the password for the sales schema use: ```sql define USERNAME = sales @@ -80,11 +80,10 @@ docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1 ## HR Schema -The `HR` schema is populated with dummy data by default in oracle for testing purposes. +The `HR` schema is populated with dummy data by default in oracle for testing purposes. To connect to the HR schema first update the user password and unlock the account by performing ```sql ALTER USER hr ACCOUNT UNLOCK; ALTER USER hr IDENTIFIED BY hr; ``` You should now be able to connect to the hr schema using the credentials hr/hr - diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 8cbc29251b..fb892dcc79 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -41,7 +41,7 @@ const DEFINITIONS: Record = { [SourceName.GOOGLE_SHEETS]: googlesheets.schema, [SourceName.REDIS]: redis.schema, [SourceName.SNOWFLAKE]: snowflake.schema, - [SourceName.ORACLE]: undefined, + [SourceName.ORACLE]: oracle.schema, [SourceName.BUDIBASE]: undefined, } @@ -64,20 +64,10 @@ const INTEGRATIONS: Record = [SourceName.GOOGLE_SHEETS]: googlesheets.integration, [SourceName.REDIS]: redis.integration, [SourceName.SNOWFLAKE]: snowflake.integration, - [SourceName.ORACLE]: undefined, + [SourceName.ORACLE]: oracle.integration, [SourceName.BUDIBASE]: undefined, } -// optionally add oracle integration if the oracle binary can be installed -if ( - process.arch && - !process.arch.startsWith("arm") && - oracle.integration.isInstalled() -) { - DEFINITIONS[SourceName.ORACLE] = oracle.schema - INTEGRATIONS[SourceName.ORACLE] = oracle.integration -} - export async function getDefinition( source: SourceName ): Promise { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 55e62260b8..9f40372546 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -24,7 +24,7 @@ import { getSqlQuery, HOST_ADDRESS, } from "./utils" -import { +import oracledb, { BindParameters, Connection, ConnectionAttributes, @@ -36,13 +36,7 @@ import { sql } from "@budibase/backend-core" const Sql = sql.Sql -let oracledb: any -try { - oracledb = require("oracledb") - oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT -} catch (err) { - console.log("ORACLEDB is not installed") -} +oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT interface OracleConfig { host: string @@ -158,10 +152,6 @@ class OracleIntegration extends Sql implements DatasourcePlus { return parts.join(" || ") } - static isInstalled() { - return oracledb != null - } - /** * Map the flat tabular columns and constraints data into a nested object */ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index b595508093..fe7ab761ca 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -208,4 +208,38 @@ describe("SQL query builder", () => { sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`, }) }) + + it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => { + let query = new Sql(SqlClient.ORACLE, limit)._query( + generateReadJson({ + filters: { + equal: { + name: "John", + }, + }, + }) + ) + + expect(query).toEqual({ + bindings: ["John", limit, 5000], + sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`, + }) + }) + + it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => { + let query = new Sql(SqlClient.ORACLE, limit)._query( + generateReadJson({ + filters: { + notEqual: { + name: "John", + }, + }, + }) + ) + + expect(query).toEqual({ + bindings: ["John", limit, 5000], + sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`, + }) + }) }) diff --git a/yarn.lock b/yarn.lock index 4d22c9f069..db8336b7a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2038,7 +2038,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.23.12": +"@budibase/backend-core@2.29.22": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2119,14 +2119,14 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "2.23.12" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.23.12.tgz#b2e813c547a5ed22b5bd86b1158159fe4b918260" - integrity sha512-DMtfkrJDSIF9V7AL6brpuWWw7Ot5XxO4YQ32ggmr0264uU9KYsTFvlFXFP3MSF2H+247ZYUouSJU76+XeC13qQ== + version "2.29.22" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.22.tgz#2608b2a76be0426879068e5a61100d6b8dde6f3a" + integrity sha512-flMVIpWQb9w3f4aiBSM73aLcYmfoPIf+kP8JXgRWO0k3nGrUGaMQNBKzXwC7soTkTrJCZjBh8uaY75AxTP2RdA== dependencies: - "@budibase/backend-core" "2.23.12" - "@budibase/shared-core" "2.23.12" - "@budibase/string-templates" "2.23.12" - "@budibase/types" "2.23.12" + "@budibase/backend-core" "2.29.22" + "@budibase/shared-core" "2.29.22" + "@budibase/string-templates" "2.29.22" + "@budibase/types" "2.29.22" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -2137,13 +2137,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.23.12": +"@budibase/shared-core@2.29.22": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@2.23.12": +"@budibase/string-templates@2.29.22": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2151,7 +2151,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@2.23.12": +"@budibase/types@2.29.22": version "0.0.0" dependencies: scim-patch "^0.8.1" @@ -5731,13 +5731,12 @@ dependencies: "@types/node" "*" -"@types/oracledb@5.2.2": - version "5.2.2" - resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-5.2.2.tgz#ae7ba795969e3bbd8d57ab141873a1aa012b86cd" - integrity sha512-aYb2DdZOQVIgSCSXjXNikQuyiHAY09SkRA4cjwoj+F/mhLJDahdjNeBmvQvfFojyChCKLuupSJHqoAXPExgV5w== +"@types/oracledb@6.5.1": + version "6.5.1" + resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-6.5.1.tgz#17d021cabc9d216dfa6d3d65ae3ee585c33baab3" + integrity sha512-Ll0bKGXmCZVngBL3juSaytA8Jeocx0VghDHTt+FEC2bs8fdl9pzoaBXYWXjBUxCCT8Y/69m5AzuTgBd79j24WA== dependencies: "@types/node" "*" - dotenv "^8.2.0" "@types/passport-google-oauth@^1.0.42": version "1.0.45" @@ -16660,10 +16659,10 @@ ora@^5.4.1: strip-ansi "^6.0.0" wcwidth "^1.0.1" -oracledb@5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-5.3.0.tgz#a15e6cd16757d8711a2c006a28bd7ecd3b8466f7" - integrity sha512-HMJzQ6lCf287ztvvehTEmjCWA21FQ3RMvM+mgoqd4i8pkREuqFWO+y3ovsGR9moJUg4T0xjcwS8rl4mggWPxmg== +oracledb@6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-6.5.1.tgz#814d985035acdb1a6470b1152af0ca3767569ede" + integrity sha512-JzoSGei1wnvmqgKnAZK1W650mzHTZXx+7hClV4mwsbY/ZjUtrpnojNJMYJ2jkOhj7XG5oJPfXc4GqDKaNzkxqg== os-locale@^3.1.0: version "3.1.0"