From 1ed32f0ce4a983574b04cb0476343ec31409985e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 8 Aug 2022 13:51:02 +0100 Subject: [PATCH 1/4] charset encoding --- packages/server/src/utilities/fileSystem/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index f4aebd11a8..e421d19275 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -106,8 +106,10 @@ exports.loadHandlebarsFile = path => { */ exports.apiFileReturn = contents => { const path = join(budibaseTempDir(), uuid()) - fs.writeFileSync(path, contents) - return fs.createReadStream(path) + fs.writeFileSync(path, "\ufeff" + contents) + let readerStream = fs.createReadStream(path) + readerStream.setEncoding("binary") + return readerStream } exports.defineFilter = excludeRows => { From 9a4eebcc3753e56866a84d315dec7006688d432e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 8 Aug 2022 13:54:11 +0100 Subject: [PATCH 2/4] changing readstream let to const --- packages/server/src/utilities/fileSystem/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index e421d19275..8e93d33ed9 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -107,7 +107,7 @@ exports.loadHandlebarsFile = path => { exports.apiFileReturn = contents => { const path = join(budibaseTempDir(), uuid()) fs.writeFileSync(path, "\ufeff" + contents) - let readerStream = fs.createReadStream(path) + const readerStream = fs.createReadStream(path) readerStream.setEncoding("binary") return readerStream } From 903d259fc4b06bb826e30cf9e51073d6d82899aa Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 29 Nov 2022 15:23:01 +0000 Subject: [PATCH 3/4] Removing old JS file. --- .../server/src/utilities/fileSystem/index.js | 333 ------------------ 1 file changed, 333 deletions(-) delete mode 100644 packages/server/src/utilities/fileSystem/index.js diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js deleted file mode 100644 index 8e93d33ed9..0000000000 --- a/packages/server/src/utilities/fileSystem/index.js +++ /dev/null @@ -1,333 +0,0 @@ -const { budibaseTempDir } = require("../budibaseDir") -const fs = require("fs") -const { join } = require("path") -const uuid = require("uuid/v4") -const { - doWithDB, - dangerousGetDB, - closeDB, -} = require("@budibase/backend-core/db") -const { ObjectStoreBuckets } = require("../../constants") -const { - upload, - retrieve, - retrieveToTmp, - streamUpload, - deleteFolder, - downloadTarball, -} = require("./utilities") -const { updateClientLibrary } = require("./clientLibrary") -const env = require("../../environment") -const { - USER_METDATA_PREFIX, - LINK_USER_METADATA_PREFIX, - TABLE_ROW_PREFIX, -} = require("../../db/utils") -const MemoryStream = require("memorystream") -const { getAppId } = require("@budibase/backend-core/context") - -const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") -const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") - -/** - * The single stack system (Cloud and Builder) should not make use of the file system where possible, - * this file handles all of the file access for the system with the intention of limiting it all to one - * place. Keeping all of this logic in one place means that when we need to do file system access (like - * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't - * be done through an object store instead. - */ - -/** - * Upon first startup of instance there may not be everything we need in tmp directory, set it up. - */ -exports.init = () => { - const tempDir = budibaseTempDir() - if (!fs.existsSync(tempDir)) { - fs.mkdirSync(tempDir) - } - const clientLibPath = join(budibaseTempDir(), "budibase-client.js") - if (env.isTest() && !fs.existsSync(clientLibPath)) { - fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath) - } -} - -/** - * Checks if the system is currently in development mode and if it is makes sure - * everything required to function is ready. - */ -exports.checkDevelopmentEnvironment = () => { - if (!env.isDev() || env.isTest()) { - return - } - if (!fs.existsSync(budibaseTempDir())) { - fs.mkdirSync(budibaseTempDir()) - } - let error - if (!fs.existsSync(join(process.cwd(), ".env"))) { - error = "Must run via yarn once to generate environment." - } - if (error) { - console.error(error) - process.exit(-1) - } -} - -/** - * This function manages temporary template files which are stored by Koa. - * @param {Object} template The template object retrieved from the Koa context object. - * @returns {Object} Returns an fs read stream which can be loaded into the database. - */ -exports.getTemplateStream = async template => { - if (template.file) { - return fs.createReadStream(template.file.path) - } else { - const [type, name] = template.key.split("/") - const tmpPath = await exports.downloadTemplate(type, name) - return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) - } -} - -/** - * Used to retrieve a handlebars file from the system which will be used as a template. - * This is allowable as the template handlebars files should be static and identical across - * the cluster. - * @param {string} path The path to the handlebars file which is to be loaded. - * @returns {string} The loaded handlebars file as a string - loaded as utf8. - */ -exports.loadHandlebarsFile = path => { - return fs.readFileSync(path, "utf8") -} - -/** - * When return a file from the API need to write the file to the system temporarily so we - * can create a read stream to send. - * @param {string} contents the contents of the file which is to be returned from the API. - * @return {Object} the read stream which can be put into the koa context body. - */ -exports.apiFileReturn = contents => { - const path = join(budibaseTempDir(), uuid()) - fs.writeFileSync(path, "\ufeff" + contents) - const readerStream = fs.createReadStream(path) - readerStream.setEncoding("binary") - return readerStream -} - -exports.defineFilter = excludeRows => { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] - if (excludeRows) { - ids.push(TABLE_ROW_PREFIX) - } - return doc => - !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) -} - -/** - * Local utility to back up the database state for an app, excluding global user - * data or user relationships. - * @param {string} appId The app to backup - * @param {object} config Config to send to export DB - * @param {boolean} includeRows Flag to state whether the export should include data. - * @returns {*} either a string or a stream of the backup - */ -const backupAppData = async (appId, config, includeRows) => { - return await exports.exportDB(appId, { - ...config, - filter: exports.defineFilter(includeRows), - }) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -exports.performBackup = async (appId, backupName) => { - return await backupAppData(appId, { exportName: backupName }) -} - -/** - * Streams a backup of the database state for an app - * @param {string} appId The ID of the app which is to be backed up. - * @param {boolean} includeRows Flag to state whether the export should include data. - * @returns {*} a readable stream of the backup which is written in real time - */ -exports.streamBackup = async (appId, includeRows) => { - return await backupAppData(appId, { stream: true }, includeRows) -} - -/** - * Exports a DB to either file or a variable (memory). - * @param {string} dbName the DB which is to be exported. - * @param {string} exportName optional - provide a filename to write the backup to a file - * @param {boolean} stream optional - whether to perform a full backup - * @param {function} filter optional - a filter function to clear out any un-wanted docs. - * @return {*} either a readable stream or a string - */ -exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - // streaming a DB dump is a bit more complicated, can't close DB - if (stream) { - const db = dangerousGetDB(dbName) - const memStream = new MemoryStream() - memStream.on("end", async () => { - await closeDB(db) - }) - db.dump(memStream, { filter }) - return memStream - } - - return doWithDB(dbName, async db => { - // Write the dump to file if required - if (exportName) { - const path = join(budibaseTempDir(), exportName) - const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter }) - - // Upload the dump to the object store if self hosted - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, exportName), - fs.createReadStream(path) - ) - } - - return fs.createReadStream(path) - } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", chunk => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter }) - return appString - }) -} - -/** - * Writes the provided contents to a temporary file, which can be used briefly. - * @param {string} fileContents contents which will be written to a temp file. - * @return {string} the path to the temp file. - */ -exports.storeTempFile = fileContents => { - const path = join(budibaseTempDir(), uuid()) - fs.writeFileSync(path, fileContents) - return path -} - -/** - * Utility function for getting a file read stream - a simple in memory buffered read - * stream doesn't work for pouchdb. - */ -exports.stringToFileStream = contents => { - const path = exports.storeTempFile(contents) - return fs.createReadStream(path) -} - -/** - * Creates a temp file and returns it from the API. - * @param {string} fileContents the contents to be returned in file. - */ -exports.sendTempFile = fileContents => { - const path = exports.storeTempFile(fileContents) - return fs.createReadStream(path) -} - -/** - * Uploads the latest client library to the object store. - * @param {string} appId The ID of the app which is being created. - * @return {Promise} once promise completes app resources should be ready in object store. - */ -exports.createApp = async appId => { - await updateClientLibrary(appId) -} - -/** - * Removes all of the assets created for an app in the object store. - * @param {string} appId The ID of the app which is being deleted. - * @return {Promise} once promise completes the app resources will be removed from object store. - */ -exports.deleteApp = async appId => { - await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) -} - -/** - * Retrieves a template and pipes it to minio as well as making it available temporarily. - * @param {string} type The type of template which is to be retrieved. - * @param name - * @return {Promise<*>} - */ -exports.downloadTemplate = async (type, name) => { - const DEFAULT_TEMPLATES_BUCKET = - "prod-budi-templates.s3-eu-west-1.amazonaws.com" - const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` - return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) -} - -/** - * Retrieves component libraries from object store (or tmp symlink if in local) - */ -exports.getComponentLibraryManifest = async library => { - const appId = getAppId() - const filename = "manifest.json" - /* istanbul ignore next */ - // when testing in cypress and so on we need to get the package - // as the environment may not be fully fleshed out for dev or prod - if (env.isTest()) { - library = library.replace("standard-components", "client") - const lib = library.split("/")[1] - const path = require.resolve(library).split(lib)[0] - return require(join(path, lib, filename)) - } else if (env.isDev()) { - const path = join(NODE_MODULES_PATH, "@budibase", "client", filename) - // always load from new so that updates are refreshed - delete require.cache[require.resolve(path)] - return require(path) - } - - let resp - try { - // Try to load the manifest from the new file location - const path = join(appId, filename) - resp = await retrieve(ObjectStoreBuckets.APPS, path) - } catch (error) { - // Fallback to loading it from the old location for old apps - const path = join(appId, "node_modules", library, "package", filename) - resp = await retrieve(ObjectStoreBuckets.APPS, path) - } - if (typeof resp !== "string") { - resp = resp.toString("utf8") - } - return JSON.parse(resp) -} - -/** - * All file reads come through here just to make sure all of them make sense - * allows a centralised location to check logic is all good. - */ -exports.readFileSync = (filepath, options = "utf8") => { - return fs.readFileSync(filepath, options) -} - -/** - * Given a set of app IDs makes sure file system is cleared of any of their temp info. - */ -exports.cleanup = appIds => { - for (let appId of appIds) { - const path = join(budibaseTempDir(), appId) - if (fs.existsSync(path)) { - fs.rmdirSync(path, { recursive: true }) - } - } -} - -/** - * Full function definition for below can be found in the utilities. - */ -exports.upload = upload -exports.retrieve = retrieve -exports.retrieveToTmp = retrieveToTmp -exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH -exports.NODE_MODULES_PATH = NODE_MODULES_PATH From de0b16397150fc37145f9dd43261a52f3832bea2 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 29 Nov 2022 16:03:22 +0000 Subject: [PATCH 4/4] Adding test case for char encoding and being explicit about utf8 export. --- packages/server/src/api/routes/tests/view.spec.js | 6 +++--- packages/server/src/utilities/fileSystem/index.ts | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/view.spec.js b/packages/server/src/api/routes/tests/view.spec.js index 2ea90ce32d..bd5177b905 100644 --- a/packages/server/src/api/routes/tests/view.spec.js +++ b/packages/server/src/api/routes/tests/view.spec.js @@ -347,7 +347,7 @@ describe("/views", () => { const setupExport = async () => { const table = await config.createTable() - await config.createRow({ name: "test-name", description: "test-desc" }) + await config.createRow({ name: "test-name", description: "ùúûü" }) return table } @@ -362,11 +362,11 @@ describe("/views", () => { const rows = JSON.parse(res.text) expect(rows.length).toBe(1) expect(rows[0].name).toBe("test-name") - expect(rows[0].description).toBe("test-desc") + expect(rows[0].description).toBe("ùúûü") } const assertCSVExport = (res) => { - expect(res.text).toBe("\"name\",\"description\"\n\"test-name\",\"test-desc\"") + expect(res.text).toBe(`"name","description"\n"test-name","ùúûü"`) } it("should be able to export a table as JSON", async () => { diff --git a/packages/server/src/utilities/fileSystem/index.ts b/packages/server/src/utilities/fileSystem/index.ts index a888c72b57..58a687c31b 100644 --- a/packages/server/src/utilities/fileSystem/index.ts +++ b/packages/server/src/utilities/fileSystem/index.ts @@ -80,14 +80,16 @@ export function loadHandlebarsFile(path: string) { * When return a file from the API need to write the file to the system temporarily so we * can create a read stream to send. * @param {string} contents the contents of the file which is to be returned from the API. + * @param {string} encoding the encoding of the file to return (utf8 default) * @return {Object} the read stream which can be put into the koa context body. */ -export function apiFileReturn(contents: string) { +export function apiFileReturn( + contents: string, + encoding: BufferEncoding = "utf8" +) { const path = join(budibaseTempDir(), uuid()) - fs.writeFileSync(path, "\ufeff" + contents) - const readerStream = fs.createReadStream(path) - readerStream.setEncoding("binary") - return readerStream + fs.writeFileSync(path, contents, { encoding }) + return fs.createReadStream(path, { encoding }) } export function streamFile(path: string) {