diff --git a/packages/server/.gitignore b/packages/server/.gitignore index b42fc06f06..22397018be 100644 --- a/packages/server/.gitignore +++ b/packages/server/.gitignore @@ -1,7 +1,6 @@ node_modules/ myapps/ .env -dev.env /builder/* !/builder/assets/ !/builder/pickr.min.js diff --git a/packages/server/dev.env b/packages/server/dev.env new file mode 100644 index 0000000000..45e34f8be4 --- /dev/null +++ b/packages/server/dev.env @@ -0,0 +1 @@ +PORT=4001 diff --git a/packages/server/package.json b/packages/server/package.json index a17da46c73..84b46e051f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -137,6 +137,7 @@ "devDependencies": { "@budibase/standard-components": "^0.8.9", "@jest/test-sequencer": "^24.8.0", + "docker-compose": "^0.23.6", "electron": "10.1.3", "electron-builder": "^22.9.1", "electron-builder-notarize": "^1.1.2", diff --git a/packages/server/src/api/controllers/deploy/awsDeploy.js b/packages/server/src/api/controllers/deploy/awsDeploy.js index 18c9279515..2d34bc1b04 100644 --- a/packages/server/src/api/controllers/deploy/awsDeploy.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -66,12 +66,7 @@ exports.deploy = async function(deployment) { const appId = deployment.getAppId() const { bucket, accountId } = deployment.getVerification() const metadata = { accountId } - const s3Client = new AWS.S3({ - params: { - Bucket: bucket, - }, - }) - await deployToObjectStore(appId, s3Client, metadata) + await deployToObjectStore(appId, bucket, metadata) } exports.replicateDb = async function(deployment) { diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index 81fa72cae5..444e7cd873 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -7,7 +7,6 @@ const { const { getWorkerUrl, getCouchUrl, - getMinioUrl, getSelfHostKey, } = require("../../../utilities/builder/hosting") @@ -45,17 +44,9 @@ exports.postDeployment = async function() { exports.deploy = async function(deployment) { const appId = deployment.getAppId() const verification = deployment.getVerification() - const objClient = new AWS.S3({ - endpoint: await getMinioUrl(), - s3ForcePathStyle: true, // needed with minio? - signatureVersion: "v4", - params: { - Bucket: verification.bucket, - }, - }) // no metadata, aws has account ID in metadata const metadata = {} - await deployToObjectStore(appId, objClient, metadata) + await deployToObjectStore(appId, verification.bucket, metadata) } exports.replicateDb = async function(deployment) { diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index 3536a6f630..eb9f85eed8 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -1,17 +1,10 @@ -const fs = require("fs") -const sanitize = require("sanitize-s3-objectkey") const { walkDir } = require("../../../utilities") const { join } = require("../../../utilities/centralPath") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const fetch = require("node-fetch") const PouchDB = require("../../../db") const CouchDB = require("pouchdb") - -const CONTENT_TYPE_MAP = { - html: "text/html", - css: "text/css", - js: "application/javascript", -} +const { upload } = require("../../../utilities/fileSystem") exports.fetchCredentials = async function(url, body) { const response = await fetch(url, { @@ -34,30 +27,25 @@ exports.fetchCredentials = async function(url, body) { return json } -exports.prepareUpload = async function({ s3Key, metadata, client, file }) { - const extension = [...file.name.split(".")].pop() - const fileBytes = fs.readFileSync(file.path) - - const upload = await client - .upload({ - // windows file paths need to be converted to forward slashes for s3 - Key: sanitize(s3Key).replace(/\\/g, "/"), - Body: fileBytes, - ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], - Metadata: metadata, - }) - .promise() +exports.prepareUpload = async function({ s3Key, bucket, metadata, file }) { + const response = await upload({ + bucket, + metadata, + filename: s3Key, + path: file.path, + type: file.type, + }) return { size: file.size, name: file.name, - extension, - url: upload.Location, - key: upload.Key, + extension: [...file.name.split(".")].pop(), + url: response.Location, + key: response.Key, } } -exports.deployToObjectStore = async function(appId, objectClient, metadata) { +exports.deployToObjectStore = async function(appId, bucket, metadata) { const appAssetsPath = join(budibaseAppsDir(), appId, "public") let uploads = [] @@ -66,12 +54,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) { walkDir(appAssetsPath, function(filePath) { const filePathParts = filePath.split("/") const appAssetUpload = exports.prepareUpload({ + bucket, file: { path: filePath, name: filePathParts.pop(), }, s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), - client: objectClient, metadata, }) uploads.push(appAssetUpload) @@ -92,7 +80,7 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) { const attachmentUpload = exports.prepareUpload({ file, s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - client: objectClient, + bucket, metadata, }) diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index 40bfdf8729..a5978d5d2e 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -4,7 +4,6 @@ const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") const fetch = require("node-fetch") const uuid = require("uuid") -const AWS = require("aws-sdk") const { prepareUpload } = require("../deploy/utils") const { processString } = require("@budibase/string-templates") const { @@ -56,12 +55,6 @@ exports.uploadFile = async function(ctx) { ? Array.from(ctx.request.files.file) : [ctx.request.files.file] - const s3 = new AWS.S3({ - params: { - Bucket: "prod-budi-app-assets", - }, - }) - const uploads = files.map(async file => { const fileExtension = [...file.name.split(".")].pop() // filenames converted to UUIDs so they are unique @@ -76,7 +69,7 @@ exports.uploadFile = async function(ctx) { return prepareUpload({ file, s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, - s3, + bucket: "prod-budi-app-assets", }) }) diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 8a46f37f0b..cec04ad699 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -5,7 +5,7 @@ const { join } = require("path") const uuid = require("uuid/v4") const CouchDB = require("../../db") const { ObjectStoreBuckets } = require("../../constants") -const { streamUpload, deleteFolder, downloadTarball } = require("./utilities") +const { upload, streamUpload, deleteFolder, downloadTarball } = require("./utilities") const { downloadLibraries, newAppPublicPath } = require("./newApp") /** @@ -104,3 +104,8 @@ exports.downloadTemplate = async (type, name) => { exports.readFileSync = (filepath, options = "utf8") => { return fs.readFileSync(filepath, options) } + +/** + * Full function definition provided in the utilities. + */ +exports.upload = upload diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 01d956dac6..aa58668b45 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -9,9 +9,16 @@ const { join } = require("path") const { streamUpload } = require("./utilities") const fs = require("fs") const { budibaseTempDir } = require("../budibaseDir") +const env = require("../../environment") const streamPipeline = promisify(stream.pipeline) +const CONTENT_TYPE_MAP = { + html: "text/html", + css: "text/css", + js: "application/javascript", +} + /** * Gets a connection to the object store using the S3 SDK. * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. @@ -20,6 +27,10 @@ const streamPipeline = promisify(stream.pipeline) */ exports.ObjectStore = bucket => { return new AWS.S3({ + // TODO: need to deal with endpoint properly + endpoint: env.MINIO_URL, + s3ForcePathStyle: true, // needed with minio? + signatureVersion: "v4", params: { Bucket: bucket, }, @@ -47,6 +58,34 @@ exports.makeSureBucketExists = async (client, bucketName) => { } } +/** + * Uploads the contents of a file given the required parameters, useful when + * temp files in use (for example file uploaded as an attachment). + * @param {string} bucket The name of the bucket to be uploaded to. + * @param {string} filename The name/path of the file in the object store. + * @param {string} path The path to the file (ideally a temporary file). + * @param {string} type If the content type is known can be specified. + * @param {object} metadata If there is metadata for the object it can be passed as well. + * @return {Promise} The file has been uploaded to the object store successfully when + * promise completes. + */ +exports.upload = async ({ bucket, filename, path, type, metadata }) => { + const extension = [...filename.split(".")].pop() + const fileBytes = fs.readFileSync(path) + + const objectStore = exports.ObjectStore(bucket) + const config = { + // windows file paths need to be converted to forward slashes for s3 + Key: sanitize(filename).replace(/\\/g, "/"), + Body: fileBytes, + ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()], + } + if (metadata) { + config.Metadata = metadata + } + return objectStore.upload(config).promise() +} + exports.streamUpload = async (bucket, filename, stream) => { const objectStore = exports.ObjectStore(bucket) await exports.makeSureBucketExists(objectStore, bucket)