Merge pull request #14217 from Budibase/fix-oracle-parse

Fix oracle parsing for CRUD operations, remove need to install oracle client

hosting/single/Dockerfile

@@ -64,11 +64,6 @@ RUN mkdir -p /var/log/nginx && \
   touch /var/run/nginx.pid && \
   usermod -a -G tty www-data
 
-WORKDIR /
-RUN mkdir -p scripts/integrations/oracle
-COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
-RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
-
 # setup minio
 WORKDIR /minio
 COPY scripts/install-minio.sh ./install.sh

packages/backend-core/src/sql/sql.ts

@@ -435,6 +435,11 @@ class InternalBuilder {
             )} = ? THEN 1 ELSE 0 END = 1`,
             [value]
           )
+        } else if (this.client === SqlClient.ORACLE) {
+          query = query[fnc](
+            `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
+            [value]
+          )
         } else {
           query = query[fnc](
             `COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`,
@@ -454,6 +459,11 @@ class InternalBuilder {
             )} = ? THEN 1 ELSE 0 END = 0`,
             [value]
           )
+        } else if (this.client === SqlClient.ORACLE) {
+          query = query[fnc](
+            `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
+            [value]
+          )
         } else {
           query = query[fnc](
             `COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`,

packages/server/Dockerfile

@@ -21,11 +21,6 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends g++ make python3 jq
 RUN yarn global add pm2
 
-# Install client for oracle datasource
-RUN apt-get install unzip libaio1
-COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/
-RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
-
 # Install postgres client for pg_dump utils
 RUN apt update && apt upgrade -y \
     && apt install software-properties-common apt-transport-https curl gpg -y \

packages/server/package.json

@@ -101,6 +101,7 @@
     "object-sizeof": "2.6.1",
     "openai": "^4.52.1",
     "openapi-types": "9.3.1",
+    "oracledb": "6.5.1",
     "pg": "8.10.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
@@ -111,12 +112,12 @@
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.7.5",
     "tar": "6.2.1",
+    "tmp": "0.2.3",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
+    "xml2js": "0.5.0"
-    "tmp": "0.2.3"
   },
   "devDependencies": {
     "@babel/preset-env": "7.16.11",
@@ -131,13 +132,13 @@
     "@types/lodash": "4.14.200",
     "@types/mssql": "9.1.4",
     "@types/node-fetch": "2.6.4",
-    "@types/oracledb": "5.2.2",
+    "@types/oracledb": "6.5.1",
     "@types/pg": "8.6.6",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
-    "@types/uuid": "8.3.4",
     "@types/tmp": "0.2.6",
+    "@types/uuid": "8.3.4",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "jest": "29.7.0",
@@ -156,9 +157,6 @@
     "update-dotenv": "1.1.1",
     "yargs": "13.2.4"
   },
-  "optionalDependencies": {
-    "oracledb": "5.3.0"
-  },
   "nx": {
     "targets": {
       "dev": {

packages/server/scripts/integrations/oracle/docker-compose.yml

@@ -16,4 +16,4 @@ services:
       - oracle_data:/opt/oracle/oradata
 
 volumes:
- oracle_data:
+ oracle_data:

packages/server/scripts/integrations/oracle/instantclient/linux/arm64/install.sh

@@ -1,23 +0,0 @@
-#!/bin/bash 
-
-# Must be root to continue
-if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
-
-# Allow for re-runs
-rm -rf /opt/oracle
-
-echo "Installing oracle instant client"
-
-# copy and unzip package
-mkdir -p /opt/oracle
-cp scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip /opt/oracle
-cd /opt/oracle
-unzip -qq basiclite-19.10.zip -d .
-rm *.zip
-mv instantclient* instantclient
-
-# update runtime link path
-sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
-ldconfig /etc/ld.so.conf.d
-
-echo "Installation complete"

packages/server/scripts/integrations/oracle/instantclient/linux/install.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
-if [[ $TARGETARCH == arm* ]] ;
-then
-  echo "Installing ARM Oracle instant client..."
-  $SCRIPT_DIR/arm64/install.sh
-else
-  echo "Installing x86-64 Oracle instant client..."
-  $SCRIPT_DIR/x86-64/install.sh
-fi

packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh

@@ -1,23 +0,0 @@
-#!/bin/bash 
-
-# Must be root to continue
-if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
-
-# Allow for re-runs
-rm -rf /opt/oracle
-
-echo "Installing oracle instant client"
-
-# copy and unzip package
-mkdir -p /opt/oracle
-cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
-cd /opt/oracle
-unzip -qq basiclite-21.4.zip -d .
-rm *.zip
-mv instantclient* instantclient
-
-# update runtime link path
-sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
-ldconfig /etc/ld.so.conf.d
-
-echo "Installation complete"

packages/server/scripts/integrations/oracle/oracle.md

@@ -10,16 +10,16 @@ To install oracle express edition simply run `docker-compose up`
 
 - A single instance pluggable database (PDB) will be created named `xepdb1`
 - The default password is configured in the compose file as `oracle`
-  - The `system` and `pdbadmin` users share this password
+    - The `system` and `pdbadmin` users share this password
 
-## Instant Client 
+## Instant Client
 
 Before oracle can be connected to from nodejs, the oracle client must be installed.
 For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html
 
 **Important**
 - Oracle client is supported only on **x86-64 architecture**
-- Oracle client is **not supported on Mac ARM architecture** 
+- Oracle client is **not supported on Mac ARM architecture**
 
 ### Linux
 Run the provided install script for linux from the `server` root path:
@@ -29,7 +29,7 @@ sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install
 ```
 For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst
 
-### Mac 
+### Mac
 **This has not yet been tested**
 
 See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst
@@ -64,7 +64,7 @@ grant create session,
     to &USERNAME;
 ```
 
-To set the password for the sales schema use: 
+To set the password for the sales schema use:
 
 ```sql
 define USERNAME = sales
@@ -80,11 +80,10 @@ docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1
 
 ## HR Schema
 
-The `HR` schema is populated with dummy data by default in oracle for testing purposes. 
+The `HR` schema is populated with dummy data by default in oracle for testing purposes.
 To connect to the HR schema first update the user password and unlock the account by performing
 ```sql
 ALTER USER hr ACCOUNT UNLOCK;
 ALTER USER hr IDENTIFIED BY hr;
 ```
 You should now be able to connect to the hr schema using the credentials hr/hr
-

packages/server/src/integrations/index.ts

@@ -41,7 +41,7 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.GOOGLE_SHEETS]: googlesheets.schema,
   [SourceName.REDIS]: redis.schema,
   [SourceName.SNOWFLAKE]: snowflake.schema,
-  [SourceName.ORACLE]: undefined,
+  [SourceName.ORACLE]: oracle.schema,
   [SourceName.BUDIBASE]: undefined,
 }
 
@@ -64,20 +64,10 @@ const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
     [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
     [SourceName.REDIS]: redis.integration,
     [SourceName.SNOWFLAKE]: snowflake.integration,
-    [SourceName.ORACLE]: undefined,
+    [SourceName.ORACLE]: oracle.integration,
     [SourceName.BUDIBASE]: undefined,
   }
 
-// optionally add oracle integration if the oracle binary can be installed
-if (
-  process.arch &&
-  !process.arch.startsWith("arm") &&
-  oracle.integration.isInstalled()
-) {
-  DEFINITIONS[SourceName.ORACLE] = oracle.schema
-  INTEGRATIONS[SourceName.ORACLE] = oracle.integration
-}
-
 export async function getDefinition(
   source: SourceName
 ): Promise<Integration | undefined> {

packages/server/src/integrations/oracle.ts

@@ -24,7 +24,7 @@ import {
   getSqlQuery,
   HOST_ADDRESS,
 } from "./utils"
-import {
+import oracledb, {
   BindParameters,
   Connection,
   ConnectionAttributes,
@@ -36,13 +36,7 @@ import { sql } from "@budibase/backend-core"
 
 const Sql = sql.Sql
 
-let oracledb: any
+oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
-try {
-  oracledb = require("oracledb")
-  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
-} catch (err) {
-  console.log("ORACLEDB is not installed")
-}
 
 interface OracleConfig {
   host: string
@@ -158,10 +152,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     return parts.join(" || ")
   }
 
-  static isInstalled() {
-    return oracledb != null
-  }
-
   /**
    * Map the flat tabular columns and constraints data into a nested object
    */

packages/server/src/integrations/tests/sql.spec.ts

@@ -208,4 +208,38 @@ describe("SQL query builder", () => {
       sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
     })
   })
+
+  it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
+    let query = new Sql(SqlClient.ORACLE, limit)._query(
+      generateReadJson({
+        filters: {
+          equal: {
+            name: "John",
+          },
+        },
+      })
+    )
+
+    expect(query).toEqual({
+      bindings: ["John", limit, 5000],
+      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+    })
+  })
+
+  it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
+    let query = new Sql(SqlClient.ORACLE, limit)._query(
+      generateReadJson({
+        filters: {
+          notEqual: {
+            name: "John",
+          },
+        },
+      })
+    )
+
+    expect(query).toEqual({
+      bindings: ["John", limit, 5000],
+      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+    })
+  })
 })

yarn.lock

@@ -2038,7 +2038,7 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.23.12":
+"@budibase/backend-core@2.29.22":
   version "0.0.0"
   dependencies:
     "@budibase/nano" "10.1.5"
@@ -2119,14 +2119,14 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "2.23.12"
+  version "2.29.22"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.23.12.tgz#b2e813c547a5ed22b5bd86b1158159fe4b918260"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.22.tgz#2608b2a76be0426879068e5a61100d6b8dde6f3a"
-  integrity sha512-DMtfkrJDSIF9V7AL6brpuWWw7Ot5XxO4YQ32ggmr0264uU9KYsTFvlFXFP3MSF2H+247ZYUouSJU76+XeC13qQ==
+  integrity sha512-flMVIpWQb9w3f4aiBSM73aLcYmfoPIf+kP8JXgRWO0k3nGrUGaMQNBKzXwC7soTkTrJCZjBh8uaY75AxTP2RdA==
   dependencies:
-    "@budibase/backend-core" "2.23.12"
+    "@budibase/backend-core" "2.29.22"
-    "@budibase/shared-core" "2.23.12"
+    "@budibase/shared-core" "2.29.22"
-    "@budibase/string-templates" "2.23.12"
+    "@budibase/string-templates" "2.29.22"
-    "@budibase/types" "2.23.12"
+    "@budibase/types" "2.29.22"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -2137,13 +2137,13 @@
     scim-patch "^0.8.1"
     scim2-parse-filter "^0.2.8"
 
-"@budibase/shared-core@2.23.12":
+"@budibase/shared-core@2.29.22":
   version "0.0.0"
   dependencies:
     "@budibase/types" "0.0.0"
     cron-validate "1.4.5"
 
-"@budibase/string-templates@2.23.12":
+"@budibase/string-templates@2.29.22":
   version "0.0.0"
   dependencies:
     "@budibase/handlebars-helpers" "^0.13.2"
@@ -2151,7 +2151,7 @@
     handlebars "^4.7.8"
     lodash.clonedeep "^4.5.0"
 
-"@budibase/types@2.23.12":
+"@budibase/types@2.29.22":
   version "0.0.0"
   dependencies:
     scim-patch "^0.8.1"
@@ -5731,13 +5731,12 @@
   dependencies:
     "@types/node" "*"
 
-"@types/oracledb@5.2.2":
+"@types/oracledb@6.5.1":
-  version "5.2.2"
+  version "6.5.1"
-  resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-5.2.2.tgz#ae7ba795969e3bbd8d57ab141873a1aa012b86cd"
+  resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-6.5.1.tgz#17d021cabc9d216dfa6d3d65ae3ee585c33baab3"
-  integrity sha512-aYb2DdZOQVIgSCSXjXNikQuyiHAY09SkRA4cjwoj+F/mhLJDahdjNeBmvQvfFojyChCKLuupSJHqoAXPExgV5w==
+  integrity sha512-Ll0bKGXmCZVngBL3juSaytA8Jeocx0VghDHTt+FEC2bs8fdl9pzoaBXYWXjBUxCCT8Y/69m5AzuTgBd79j24WA==
   dependencies:
     "@types/node" "*"
-    dotenv "^8.2.0"
 
 "@types/passport-google-oauth@^1.0.42":
   version "1.0.45"
@@ -16660,10 +16659,10 @@ ora@^5.4.1:
     strip-ansi "^6.0.0"
     wcwidth "^1.0.1"
 
-oracledb@5.3.0:
+oracledb@6.5.1:
-  version "5.3.0"
+  version "6.5.1"
-  resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-5.3.0.tgz#a15e6cd16757d8711a2c006a28bd7ecd3b8466f7"
+  resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-6.5.1.tgz#814d985035acdb1a6470b1152af0ca3767569ede"
-  integrity sha512-HMJzQ6lCf287ztvvehTEmjCWA21FQ3RMvM+mgoqd4i8pkREuqFWO+y3ovsGR9moJUg4T0xjcwS8rl4mggWPxmg==
+  integrity sha512-JzoSGei1wnvmqgKnAZK1W650mzHTZXx+7hClV4mwsbY/ZjUtrpnojNJMYJ2jkOhj7XG5oJPfXc4GqDKaNzkxqg==
 
 os-locale@^3.1.0:
   version "3.1.0"