MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14217 from Budibase/fix-oracle-parse 

Fix oracle parsing for CRUD operations, remove need to install oracle client
This commit is contained in:
Martin McKeaveney 2024-07-23 11:55:30 +01:00 committed by GitHub
parent 09135be8b0 add8691090
commit 4ec7d49d2f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 79 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hosting/single/Dockerfile
View File

@ -64,11 +64,6 @@ RUN mkdir -p /var/log/nginx && \
touch /var/run/nginx.pid && \
usermod -a -G tty www-data
WORKDIR /
RUN mkdir -p scripts/integrations/oracle
COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
# setup minio
WORKDIR /minio
COPY scripts/install-minio.sh ./install.sh

10
packages/backend-core/src/sql/sql.ts
View File

@ -435,6 +435,11 @@ class InternalBuilder {
)} = ? THEN 1 ELSE 0 END = 1`,
[value]
)
} else if (this.client === SqlClient.ORACLE) {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
[value]
)
} else {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`,
@ -454,6 +459,11 @@ class InternalBuilder {
)} = ? THEN 1 ELSE 0 END = 0`,
[value]
)
} else if (this.client === SqlClient.ORACLE) {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
[value]
)
} else {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`,

5
packages/server/Dockerfile
View File

@ -21,11 +21,6 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends g++ make python3 jq
RUN yarn global add pm2
# Install client for oracle datasource
RUN apt-get install unzip libaio1
COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/
RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
# Install postgres client for pg_dump utils
RUN apt update && apt upgrade -y \
&& apt install software-properties-common apt-transport-https curl gpg -y \

12
packages/server/package.json
View File

@ -101,6 +101,7 @@
"object-sizeof": "2.6.1",
"openai": "^4.52.1",
"openapi-types": "9.3.1",
"oracledb": "6.5.1",
"pg": "8.10.0",
"pouchdb": "7.3.0",
"pouchdb-all-dbs": "1.1.1",
@ -111,12 +112,12 @@
"snowflake-promise": "^4.5.0",
"socket.io": "4.7.5",
"tar": "6.2.1",
"tmp": "0.2.3",
"to-json-schema": "0.2.5",
"uuid": "^8.3.2",
"validate.js": "0.13.1",
"worker-farm": "1.7.0",
"xml2js": "0.5.0",
"tmp": "0.2.3"
"xml2js": "0.5.0"
},
"devDependencies": {
"@babel/preset-env": "7.16.11",
@ -131,13 +132,13 @@
"@types/lodash": "4.14.200",
"@types/mssql": "9.1.4",
"@types/node-fetch": "2.6.4",
"@types/oracledb": "5.2.2",
"@types/oracledb": "6.5.1",
"@types/pg": "8.6.6",
"@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.14",
"@types/tar": "6.1.5",
"@types/uuid": "8.3.4",
"@types/tmp": "0.2.6",
"@types/uuid": "8.3.4",
"copyfiles": "2.4.1",
"docker-compose": "0.23.17",
"jest": "29.7.0",
@ -156,9 +157,6 @@
"update-dotenv": "1.1.1",
"yargs": "13.2.4"
},
"optionalDependencies": {
"oracledb": "5.3.0"
},
"nx": {
"targets": {
"dev": {

BIN
packages/server/scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip
View File

Binary file not shown.

23
packages/server/scripts/integrations/oracle/instantclient/linux/arm64/install.sh
View File

@ -1,23 +0,0 @@
#!/bin/bash
# Must be root to continue
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
# Allow for re-runs
rm -rf /opt/oracle
echo "Installing oracle instant client"
# copy and unzip package
mkdir -p /opt/oracle
cp scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip /opt/oracle
cd /opt/oracle
unzip -qq basiclite-19.10.zip -d .
rm *.zip
mv instantclient* instantclient
# update runtime link path
sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
ldconfig /etc/ld.so.conf.d
echo "Installation complete"

10
packages/server/scripts/integrations/oracle/instantclient/linux/install.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
if [[ $TARGETARCH == arm* ]] ;
then
echo "Installing ARM Oracle instant client..."
$SCRIPT_DIR/arm64/install.sh
else
echo "Installing x86-64 Oracle instant client..."
$SCRIPT_DIR/x86-64/install.sh
fi

BIN
packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip
View File

Binary file not shown.

23
packages/server/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
View File

@ -1,23 +0,0 @@
#!/bin/bash
# Must be root to continue
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
# Allow for re-runs
rm -rf /opt/oracle
echo "Installing oracle instant client"
# copy and unzip package
mkdir -p /opt/oracle
cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
cd /opt/oracle
unzip -qq basiclite-21.4.zip -d .
rm *.zip
mv instantclient* instantclient
# update runtime link path
sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
ldconfig /etc/ld.so.conf.d
echo "Installation complete"

1
packages/server/scripts/integrations/oracle/oracle.md
View File

@ -87,4 +87,3 @@ ALTER USER hr ACCOUNT UNLOCK;
ALTER USER hr IDENTIFIED BY hr;
```
You should now be able to connect to the hr schema using the credentials hr/hr

14
packages/server/src/integrations/index.ts
View File

@ -41,7 +41,7 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
[SourceName.GOOGLE_SHEETS]: googlesheets.schema,
[SourceName.REDIS]: redis.schema,
[SourceName.SNOWFLAKE]: snowflake.schema,
[SourceName.ORACLE]: undefined,
[SourceName.ORACLE]: oracle.schema,
[SourceName.BUDIBASE]: undefined,
}
@ -64,20 +64,10 @@ const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.ORACLE]: oracle.integration,
[SourceName.BUDIBASE]: undefined,
}
// optionally add oracle integration if the oracle binary can be installed
if (
process.arch &&
!process.arch.startsWith("arm") &&
oracle.integration.isInstalled()
) {
DEFINITIONS[SourceName.ORACLE] = oracle.schema
INTEGRATIONS[SourceName.ORACLE] = oracle.integration
}
export async function getDefinition(
source: SourceName
): Promise<Integration | undefined> {

14
packages/server/src/integrations/oracle.ts
View File

@ -24,7 +24,7 @@ import {
getSqlQuery,
HOST_ADDRESS,
} from "./utils"
import {
import oracledb, {
BindParameters,
Connection,
ConnectionAttributes,
@ -36,13 +36,7 @@ import { sql } from "@budibase/backend-core"
const Sql = sql.Sql
let oracledb: any
try {
oracledb = require("oracledb")
oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
} catch (err) {
console.log("ORACLEDB is not installed")
}
oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
interface OracleConfig {
host: string
@ -158,10 +152,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
return parts.join(" || ")
}
static isInstalled() {
return oracledb != null
}
/**
* Map the flat tabular columns and constraints data into a nested object
*/

34
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -208,4 +208,38 @@ describe("SQL query builder", () => {
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
filters: {
equal: {
name: "John",
},
},
})
)
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
filters: {
notEqual: {
name: "John",
},
},
})
)
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
})

39
yarn.lock
View File

@ -2038,7 +2038,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.23.12":
"@budibase/backend-core@2.29.22":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
@ -2119,14 +2119,14 @@
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.23.12"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.23.12.tgz#b2e813c547a5ed22b5bd86b1158159fe4b918260"
integrity sha512-DMtfkrJDSIF9V7AL6brpuWWw7Ot5XxO4YQ32ggmr0264uU9KYsTFvlFXFP3MSF2H+247ZYUouSJU76+XeC13qQ==
version "2.29.22"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.22.tgz#2608b2a76be0426879068e5a61100d6b8dde6f3a"
integrity sha512-flMVIpWQb9w3f4aiBSM73aLcYmfoPIf+kP8JXgRWO0k3nGrUGaMQNBKzXwC7soTkTrJCZjBh8uaY75AxTP2RdA==
dependencies:
"@budibase/backend-core" "2.23.12"
"@budibase/shared-core" "2.23.12"
"@budibase/string-templates" "2.23.12"
"@budibase/types" "2.23.12"
"@budibase/backend-core" "2.29.22"
"@budibase/shared-core" "2.29.22"
"@budibase/string-templates" "2.29.22"
"@budibase/types" "2.29.22"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -2137,13 +2137,13 @@
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.23.12":
"@budibase/shared-core@2.29.22":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.23.12":
"@budibase/string-templates@2.29.22":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
@ -2151,7 +2151,7 @@
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.23.12":
"@budibase/types@2.29.22":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"
@ -5731,13 +5731,12 @@
dependencies:
"@types/node" "*"
"@types/oracledb@5.2.2":
version "5.2.2"
resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-5.2.2.tgz#ae7ba795969e3bbd8d57ab141873a1aa012b86cd"
integrity sha512-aYb2DdZOQVIgSCSXjXNikQuyiHAY09SkRA4cjwoj+F/mhLJDahdjNeBmvQvfFojyChCKLuupSJHqoAXPExgV5w==
"@types/oracledb@6.5.1":
version "6.5.1"
resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-6.5.1.tgz#17d021cabc9d216dfa6d3d65ae3ee585c33baab3"
integrity sha512-Ll0bKGXmCZVngBL3juSaytA8Jeocx0VghDHTt+FEC2bs8fdl9pzoaBXYWXjBUxCCT8Y/69m5AzuTgBd79j24WA==
dependencies:
"@types/node" "*"
dotenv "^8.2.0"
"@types/passport-google-oauth@^1.0.42":
version "1.0.45"
@ -16660,10 +16659,10 @@ ora@^5.4.1:
strip-ansi "^6.0.0"
wcwidth "^1.0.1"
oracledb@5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-5.3.0.tgz#a15e6cd16757d8711a2c006a28bd7ecd3b8466f7"
integrity sha512-HMJzQ6lCf287ztvvehTEmjCWA21FQ3RMvM+mgoqd4i8pkREuqFWO+y3ovsGR9moJUg4T0xjcwS8rl4mggWPxmg==
oracledb@6.5.1:
version "6.5.1"
resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-6.5.1.tgz#814d985035acdb1a6470b1152af0ca3767569ede"
integrity sha512-JzoSGei1wnvmqgKnAZK1W650mzHTZXx+7hClV4mwsbY/ZjUtrpnojNJMYJ2jkOhj7XG5oJPfXc4GqDKaNzkxqg==
os-locale@^3.1.0:
version "3.1.0"