diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14c0cc06a3..792191af7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,8 +52,8 @@ jobs: mac_certs: ${{ secrets.mac_certs }} mac_certs_password: ${{ secrets.mac_certs_password }} - # windows_certs: ${{ secrets.windows_certs }} - # windows_certs_password: ${{ secrets.windows_certs_password }} + windows_certs: ${{ secrets.windows_certs }} + windows_certs_password: ${{ secrets.windows_certs_password }} # release the app after building release: ${{ startsWith(github.ref, 'refs/tags/v') }} diff --git a/.gitignore b/.gitignore index b6cfa424ad..a84d25d46d 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,7 @@ typings/ .DS_Store # Nova Editor -.nova \ No newline at end of file +.nova + +# swap files (linux) +*.swp diff --git a/hosting/build/docker-compose.yaml b/hosting/build/docker-compose.yaml deleted file mode 100644 index 6988e3841b..0000000000 --- a/hosting/build/docker-compose.yaml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3" - -services: - app-service: - build: ./server - volumes: - - ./server:/app - environment: - SELF_HOSTED: 1 - PORT: 4002 - - worker-service: - build: ./worker - environment: - SELF_HOSTED: 1, - PORT: 4003 diff --git a/hosting/build/server b/hosting/build/server deleted file mode 120000 index c40730cce5..0000000000 --- a/hosting/build/server +++ /dev/null @@ -1 +0,0 @@ -../../packages/server/ \ No newline at end of file diff --git a/hosting/build/worker b/hosting/build/worker deleted file mode 120000 index 8582fefbee..0000000000 --- a/hosting/build/worker +++ /dev/null @@ -1 +0,0 @@ -../../packages/worker/ \ No newline at end of file diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml new file mode 100644 index 0000000000..8ee1753e11 --- /dev/null +++ b/hosting/docker-compose.dev.yaml @@ -0,0 +1,73 @@ +version: "3" + +# optional ports are specified throughout for more advanced use cases. + +services: + minio-service: + container_name: budi-minio-dev + restart: always + image: minio/minio + volumes: + - minio_data:/data + ports: + - "${MINIO_PORT}:9000" + environment: + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BROWSER: "off" + command: server /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + proxy-service: + container_name: budi-envoy-dev + restart: always + image: envoyproxy/envoy:v1.16-latest + volumes: + - ./envoy.dev.yaml:/etc/envoy/envoy.yaml + ports: + - "${MAIN_PORT}:10000" + depends_on: + - minio-service + - couchdb-service + + couchdb-service: + container_name: budi-couchdb-dev + restart: always + image: ibmcom/couchdb3 + environment: + - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} + - COUCHDB_USER=${COUCH_DB_USER} + ports: + - "${COUCH_DB_PORT}:5984" + volumes: + - couchdb3_data:/opt/couchdb/data + + couch-init: + container_name: budi-couchdb-init-dev + image: curlimages/curl + environment: + PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" + depends_on: + - couchdb-service + command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] + + redis-service: + container_name: budi-redis-dev + restart: always + image: redis + ports: + - "${REDIS_PORT}:6379" + volumes: + - redis_data:/data + +volumes: + couchdb3_data: + driver: local + minio_data: + driver: local + redis_data: + driver: local diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index e8408d9a7d..8de5e9fcdd 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -11,13 +11,18 @@ services: - "${APP_PORT}:4002" environment: SELF_HOSTED: 1 - CLOUD: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 WORKER_URL: http://worker-service:4003 + MINIO_URL: http://minio-service:9000 + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} HOSTING_KEY: ${HOSTING_KEY} BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} PORT: 4002 JWT_SECRET: ${JWT_SECRET} + LOG_LEVEL: info + SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 + ENABLE_ANALYTICS: true depends_on: - worker-service @@ -28,7 +33,7 @@ services: ports: - "${WORKER_PORT}:4003" environment: - SELF_HOSTED: 1, + SELF_HOSTED: 1 PORT: 4003 MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} @@ -66,7 +71,6 @@ services: - ./envoy.yaml:/etc/envoy/envoy.yaml ports: - "${MAIN_PORT}:10000" - #- "9901:9901" depends_on: - minio-service - worker-service @@ -75,16 +79,14 @@ services: couchdb-service: restart: always - image: apache/couchdb:3.0 + image: ibmcom/couchdb3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - #- "4369:4369" - #- "9100:9100" volumes: - - couchdb_data:/opt/couchdb/data + - couchdb3_data:/opt/couchdb/data couch-init: image: curlimages/curl @@ -93,9 +95,19 @@ services: depends_on: - couchdb-service command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] + + redis-service: + restart: always + image: redis + ports: + - "${REDIS_PORT}:6379" + volumes: + - redis_data:/data volumes: - couchdb_data: + couchdb3_data: driver: local minio_data: driver: local + redis_data: + driver: local diff --git a/hosting/envoy.dev.yaml b/hosting/envoy.dev.yaml new file mode 100644 index 0000000000..e12bc2c0e0 --- /dev/null +++ b/hosting/envoy.dev.yaml @@ -0,0 +1,79 @@ +static_resources: + listeners: + - name: main_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress + codec_type: auto + route_config: + name: local_route + virtual_hosts: + - name: local_services + domains: ["*"] + routes: + - match: { prefix: "/db/" } + route: + cluster: couchdb-service + prefix_rewrite: "/" + + - match: { prefix: "/cache/" } + route: + cluster: redis-service + prefix_rewrite: "/" + + # minio is on the default route because this works + # best, minio + AWS SDK doesn't handle path proxy + - match: { prefix: "/" } + route: + cluster: minio-service + + http_filters: + - name: envoy.filters.http.router + + clusters: + - name: minio-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: minio-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: minio-service + port_value: 9000 + + - name: couchdb-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: couchdb-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: couchdb-service + port_value: 5984 + + - name: redis-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: redis-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: redis-service + port_value: 6379 diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 11f5c81b99..8c6081d1a7 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -36,6 +36,11 @@ static_resources: cluster: worker-service prefix_rewrite: "/" + - match: { prefix: "/cache/" } + route: + cluster: redis-service + prefix_rewrite: "/" + - match: { prefix: "/db/" } route: cluster: couchdb-service @@ -107,3 +112,18 @@ static_resources: address: couchdb-service port_value: 5984 + - name: redis-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: redis-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: redis-service + port_value: 6379 + + diff --git a/hosting/hosting.properties b/hosting/hosting.properties index ad047a3826..138e66d629 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -18,4 +18,5 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +REDIS_PORT=6379 BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/packages/builder/cypress.json b/packages/builder/cypress.json index 09b38ae985..84c50ae863 100644 --- a/packages/builder/cypress.json +++ b/packages/builder/cypress.json @@ -3,6 +3,7 @@ "video": true, "projectId": "bmbemn", "env": { - "PORT": "4001" + "PORT": "4001", + "JWT_SECRET": "test" } } diff --git a/packages/builder/cypress/setup.js b/packages/builder/cypress/setup.js index 6848d38cba..7b849eb887 100644 --- a/packages/builder/cypress/setup.js +++ b/packages/builder/cypress/setup.js @@ -1,25 +1,26 @@ -// What this script does: -// 1. Removes the old test folder if it exists (.budibase) -// 2. Initialises using `.budibase` -// 3. Runs the server using said folder - -const { join, resolve } = require("path") -const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase") const cypressConfig = require("../cypress.json") +const path = require("path") -const homedir = join(require("os").homedir(), ".budibase") +const tmpdir = path.join(require("os").tmpdir(), ".budibase") process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" process.env.NODE_ENV = "cypress" process.env.ENABLE_ANALYTICS = "false" process.env.PORT = cypressConfig.env.PORT +process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET +process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` +process.env.SELF_HOSTED = 1 +process.env.MINIO_URL = "http://localhost:10000/" +process.env.MINIO_ACCESS_KEY = "budibase" +process.env.MINIO_SECRET_KEY = "budibase" +process.env.COUCH_DB_USER = "budibase" +process.env.COUCH_DB_PASSWORD = "budibase" // Stop info logs polluting test outputs process.env.LOG_LEVEL = "error" -async function run(dir) { - process.env.BUDIBASE_DIR = resolve(dir) - require("dotenv").config({ path: resolve(dir, ".env") }) +async function run() { + // require("dotenv").config({ path: resolve(dir, ".env") }) // dont make this a variable or top level require // it will cause environment module to be loaded prematurely @@ -27,12 +28,15 @@ async function run(dir) { server.on("close", () => console.log("Server Closed")) } -initialiseBudibase({ dir: homedir, clientId: "cypress-test" }) - .then(() => { - delete require.cache[require.resolve("../../server/src/environment")] - const xPlatHomeDir = homedir.startsWith("~") - ? join(homedir(), homedir.substring(1)) - : homedir - run(xPlatHomeDir) - }) - .catch(e => console.error(e)) +run() + +// TODO: ensure that this still works +// initialiseBudibase({ dir: homedir, clientId: "cypress-test" }) +// .then(() => { +// delete require.cache[require.resolve("../../server/src/environment")] +// const xPlatHomeDir = homedir.startsWith("~") +// ? join(homedir(), homedir.substring(1)) +// : homedir +// run(xPlatHomeDir) +// }) +// .catch(e => console.error(e)) diff --git a/packages/builder/src/builderStore/store/backend.js b/packages/builder/src/builderStore/store/backend.js new file mode 100644 index 0000000000..5d0e722650 --- /dev/null +++ b/packages/builder/src/builderStore/store/backend.js @@ -0,0 +1,362 @@ +import { writable, get } from "svelte/store" +import { cloneDeep } from "lodash/fp" +import api from "../api" + +const INITIAL_BACKEND_UI_STATE = { + tables: [], + views: [], + users: [], + roles: [], + datasources: [], + queries: [], + integrations: {}, + selectedDatabase: {}, + selectedTable: {}, + draftTable: {}, +} + +export const getBackendUiStore = () => { + const store = writable({ ...INITIAL_BACKEND_UI_STATE }) + + store.actions = { + reset: () => store.set({ ...INITIAL_BACKEND_UI_STATE }), + database: { + select: async db => { + const tablesResponse = await api.get(`/api/tables`) + const tables = await tablesResponse.json() + const datasourcesResponse = await api.get(`/api/datasources`) + const datasources = await datasourcesResponse.json() + const queriesResponse = await api.get(`/api/queries`) + const queries = await queriesResponse.json() + const integrationsResponse = await api.get("/api/integrations") + const integrations = await integrationsResponse.json() + const permissionLevels = await store.actions.permissions.fetchLevels() + + store.update(state => { + state.selectedDatabase = db + state.tables = tables + state.datasources = datasources + state.queries = queries + state.integrations = integrations + state.permissionLevels = permissionLevels + return state + }) + }, + }, + rows: { + save: () => + store.update(state => { + state.selectedView = state.selectedView + return state + }), + delete: () => + store.update(state => { + state.selectedView = state.selectedView + return state + }), + select: row => + store.update(state => { + state.selectedRow = row + return state + }), + }, + datasources: { + fetch: async () => { + const response = await api.get(`/api/datasources`) + const json = await response.json() + store.update(state => { + state.datasources = json + return state + }) + return json + }, + select: async datasourceId => { + store.update(state => { + state.selectedDatasourceId = datasourceId + state.selectedQueryId = null + return state + }) + }, + save: async datasource => { + const response = await api.post("/api/datasources", datasource) + const json = await response.json() + store.update(state => { + const currentIdx = state.datasources.findIndex( + ds => ds._id === json._id + ) + + if (currentIdx >= 0) { + state.datasources.splice(currentIdx, 1, json) + } else { + state.datasources.push(json) + } + + state.datasources = state.datasources + state.selectedDatasourceId = json._id + return state + }) + return json + }, + delete: async datasource => { + await api.delete( + `/api/datasources/${datasource._id}/${datasource._rev}` + ) + store.update(state => { + state.datasources = state.datasources.filter( + existing => existing._id !== datasource._id + ) + if (datasource._id === state.selectedDatasourceId) { + state.selectedDatasourceId = null + } + return state + }) + }, + }, + queries: { + fetch: async () => { + const response = await api.get(`/api/queries`) + const json = await response.json() + store.update(state => { + state.queries = json + return state + }) + return json + }, + save: async (datasourceId, query) => { + const integrations = get(store).integrations + const dataSource = get(store).datasources.filter( + ds => ds._id === datasourceId + ) + // check if readable attribute is found + if (dataSource.length !== 0) { + const integration = integrations[dataSource[0].source] + const readable = integration.query[query.queryVerb].readable + if (readable) { + query.readable = readable + } + } + query.datasourceId = datasourceId + const response = await api.post(`/api/queries`, query) + if (response.status !== 200) { + throw new Error("Failed saving query.") + } + const json = await response.json() + store.update(state => { + const currentIdx = state.queries.findIndex( + query => query._id === json._id + ) + + if (currentIdx >= 0) { + state.queries.splice(currentIdx, 1, json) + } else { + state.queries.push(json) + } + + state.queries = state.queries + state.selectedQueryId = json._id + return state + }) + return json + }, + select: query => + store.update(state => { + state.selectedDatasourceId = query.datasourceId + state.selectedQueryId = query._id + return state + }), + delete: async query => { + await api.delete(`/api/queries/${query._id}/${query._rev}`) + store.update(state => { + state.queries = state.queries.filter( + existing => existing._id !== query._id + ) + if (state.selectedQueryId === query._id) { + state.selectedQueryId = null + } + + return state + }) + }, + }, + tables: { + fetch: async () => { + const tablesResponse = await api.get(`/api/tables`) + const tables = await tablesResponse.json() + store.update(state => { + state.tables = tables + return state + }) + }, + select: table => + store.update(state => { + state.selectedTable = table + state.draftTable = cloneDeep(table) + state.selectedView = { name: `all_${table._id}` } + return state + }), + save: async table => { + const updatedTable = cloneDeep(table) + const oldTable = get(store).tables.filter(t => t._id === table._id)[0] + + const fieldNames = [] + // update any renamed schema keys to reflect their names + for (let key of Object.keys(updatedTable.schema)) { + // if field name has been seen before remove it + if (fieldNames.indexOf(key.toLowerCase()) !== -1) { + delete updatedTable.schema[key] + continue + } + const field = updatedTable.schema[key] + const oldField = oldTable?.schema[key] + // if the type has changed then revert back to the old field + if (oldField != null && oldField.type !== field.type) { + updatedTable.schema[key] = oldField + } + // field has been renamed + if (field.name && field.name !== key) { + updatedTable.schema[field.name] = field + updatedTable._rename = { old: key, updated: field.name } + delete updatedTable.schema[key] + } + // finally record this field has been used + fieldNames.push(key.toLowerCase()) + } + + const SAVE_TABLE_URL = `/api/tables` + const response = await api.post(SAVE_TABLE_URL, updatedTable) + const savedTable = await response.json() + await store.actions.tables.fetch() + store.actions.tables.select(savedTable) + return savedTable + }, + delete: async table => { + await api.delete(`/api/tables/${table._id}/${table._rev}`) + store.update(state => { + state.tables = state.tables.filter( + existing => existing._id !== table._id + ) + if (table._id === state.selectedTable._id) { + state.selectedTable = {} + } + return state + }) + }, + saveField: ({ originalName, field, primaryDisplay = false, indexes }) => { + store.update(state => { + // delete the original if renaming + // need to handle if the column had no name, empty string + if (originalName || originalName === "") { + delete state.draftTable.schema[originalName] + state.draftTable._rename = { + old: originalName, + updated: field.name, + } + } + + // Optionally set display column + if (primaryDisplay) { + state.draftTable.primaryDisplay = field.name + } + + if (indexes) { + state.draftTable.indexes = indexes + } + + state.draftTable.schema[field.name] = cloneDeep(field) + store.actions.tables.save(state.draftTable) + return state + }) + }, + deleteField: field => { + store.update(state => { + delete state.draftTable.schema[field.name] + store.actions.tables.save(state.draftTable) + return state + }) + }, + }, + views: { + select: view => + store.update(state => { + state.selectedView = view + state.selectedTable = {} + return state + }), + delete: async view => { + await api.delete(`/api/views/${view}`) + await store.actions.tables.fetch() + }, + save: async view => { + const response = await api.post(`/api/views`, view) + const json = await response.json() + + const viewMeta = { + name: view.name, + ...json, + } + + store.update(state => { + const viewTable = state.tables.find( + table => table._id === view.tableId + ) + + if (view.originalName) delete viewTable.views[view.originalName] + viewTable.views[view.name] = viewMeta + + state.tables = state.tables + state.selectedView = viewMeta + return state + }) + }, + }, + users: { + create: user => + store.update(state => { + state.users.push(user) + state.users = state.users + return state + }), + }, + roles: { + fetch: async () => { + const response = await api.get("/api/roles") + const roles = await response.json() + store.update(state => { + state.roles = roles + return state + }) + }, + delete: async role => { + const response = await api.delete(`/api/roles/${role._id}/${role._rev}`) + await store.actions.roles.fetch() + return response + }, + save: async role => { + const response = await api.post("/api/roles", role) + await store.actions.roles.fetch() + return response + }, + }, + permissions: { + fetchLevels: async () => { + const response = await api.get("/api/permission/levels") + const json = await response.json() + return json + }, + forResource: async resourceId => { + const response = await api.get(`/api/permission/${resourceId}`) + const json = await response.json() + return json + }, + save: async ({ role, resource, level }) => { + const response = await api.post( + `/api/permission/${role}/${resource}/${level}` + ) + const json = await response.json() + return json + }, + }, + } + + return store +} diff --git a/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte b/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte index cf5560ed26..df72b7feba 100644 --- a/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte @@ -1,4 +1,4 @@ - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte index 53ad3b90d6..fe3998e137 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte @@ -22,9 +22,13 @@ } async function deleteDatasource() { + const wasSelectedSource = $datasources.selected await datasources.delete(datasource) notifier.success("Datasource deleted") - $goto('./datasource') + // navigate to first index page if the source you are deleting is selected + if (wasSelectedSource === datasource._id) { + $goto("./datasource") + } hideEditor() } diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index 108b61d168..fab7d4a3ee 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -1,4 +1,5 @@ - + - \ No newline at end of file diff --git a/packages/builder/src/pages/[application]/automate/index.svelte b/packages/builder/src/pages/[application]/automate/index.svelte index 98e5b80a18..f6568417a8 100644 --- a/packages/builder/src/pages/[application]/automate/index.svelte +++ b/packages/builder/src/pages/[application]/automate/index.svelte @@ -1,18 +1,19 @@ Create your first automation to get started @@ -23,4 +24,4 @@ color: var(--grey-5); margin-top: 2px; } - \ No newline at end of file + diff --git a/packages/builder/src/pages/[application]/data/_layout.svelte b/packages/builder/src/pages/[application]/data/_layout.svelte index 0674c93c88..5a31d7db92 100644 --- a/packages/builder/src/pages/[application]/data/_layout.svelte +++ b/packages/builder/src/pages/[application]/data/_layout.svelte @@ -17,11 +17,11 @@ }, ] - let tab = $isActive('./datasource') ? "datasource" : "table" + let tab = $isActive("./datasource") ? "datasource" : "table" function selectFirstTableOrSource({ detail }) { const type = detail.heading.key - if (type === 'datasource') { + if (type === "datasource") { $goto("./datasource") } else { $goto("./table") @@ -34,7 +34,10 @@
- +
\ No newline at end of file + diff --git a/packages/builder/src/stores/backend/tests/views.spec.js b/packages/builder/src/stores/backend/tests/views.spec.js index 4ba4d6ffed..f5156dfe7d 100644 --- a/packages/builder/src/stores/backend/tests/views.spec.js +++ b/packages/builder/src/stores/backend/tests/views.spec.js @@ -15,7 +15,8 @@ describe("Tables Store", () => { await store.init() }) - it("Initialises correctly", async () => { + it("Selects the correct view", async () => { + expect(get(store)).toEqual({ list: SOME_TABLES, selected: {}, draft: {}}) }) }) \ No newline at end of file diff --git a/packages/builder/src/stores/backend/views.js b/packages/builder/src/stores/backend/views.js index 0f2837bfbd..f3891c22aa 100644 --- a/packages/builder/src/stores/backend/views.js +++ b/packages/builder/src/stores/backend/views.js @@ -29,7 +29,6 @@ export function createViewsStore() { name: view.name, ...json, } - console.log('JSON: ', json) update(state => { const viewTable = get(tables).list.find( diff --git a/packages/builder/yarn.lock b/packages/builder/yarn.lock index 5518beb45d..72c0698fd3 100644 --- a/packages/builder/yarn.lock +++ b/packages/builder/yarn.lock @@ -988,64 +988,11 @@ svelte-portal "^1.0.0" turndown "^7.0.0" -"@budibase/client@^0.8.9": - version "0.8.12" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.8.12.tgz#1186d95509e28bb660f0c801b37b3cc52b0f3864" - integrity sha512-68eh1QvF443OgMsjpDEgSY9D/MUutnaKFruP98PHJgrRkVhQDY7F2EKn2FWgC5NJuCY3aJ1emRPdXUq4MapWuA== - dependencies: - "@budibase/string-templates" "^0.8.12" - regexparam "^1.3.0" - shortid "^2.2.15" - svelte-spa-router "^3.0.5" - "@budibase/colorpicker@1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@budibase/colorpicker/-/colorpicker-1.1.2.tgz#f7436924ee746d7be9b2009c2fa193e710c30f89" integrity sha512-2PlZBVkATDqDC4b4Ri8Xi8X3OxhuHOGfmZwtXbZL38lNIeofaQT3Qyc1ECzEY5N+HrdGrWhY9EnliF6QM+LIuA== -"@budibase/handlebars-helpers@^0.11.3": - version "0.11.3" - resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.11.3.tgz#b6e5c91b83e8906e7d7ff10ddde277a3d561016e" - integrity sha512-MS1ptZEYq8o9J3tNLM7cZ2RGSSJIer4GiMIUHtbBI3sC9UKqZebao1JYNfmZKpNjntuqhZKgjqc5GfnVIEjsYQ== - dependencies: - arr-flatten "^1.1.0" - array-sort "^0.1.4" - define-property "^1.0.0" - extend-shallow "^3.0.2" - "falsey" "^0.3.2" - for-in "^1.0.2" - for-own "^1.0.0" - get-object "^0.2.0" - get-value "^2.0.6" - handlebars "^4.0.11" - handlebars-utils "^1.0.6" - has-value "^1.0.0" - helper-date "^1.0.1" - helper-markdown "^1.0.0" - helper-md "^0.2.2" - html-tag "^2.0.0" - is-even "^1.0.0" - is-glob "^4.0.0" - is-number "^4.0.0" - kind-of "^6.0.0" - logging-helpers "^1.0.0" - micromatch "^3.1.4" - relative "^3.0.2" - striptags "^3.1.0" - to-gfm-code-block "^0.1.1" - year "^0.2.1" - -"@budibase/string-templates@^0.8.12", "@budibase/string-templates@^0.8.9": - version "0.8.12" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.8.12.tgz#bc08d441ac90b2f71c89b4b15c40c7c5e934a6fb" - integrity sha512-FkTm3XRBozhrJqJA4YdDxbLbcY3me7sAbJ+dgO7mZkTybRcrdbxRwL1XjYfRw9DL7+8pucN2Y6OW8gOQgyaTtw== - dependencies: - "@budibase/handlebars-helpers" "^0.11.3" - dayjs "^1.10.4" - handlebars "^4.7.6" - handlebars-utils "^1.0.6" - lodash "^4.17.20" - "@budibase/svelte-ag-grid@^1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@budibase/svelte-ag-grid/-/svelte-ag-grid-1.0.4.tgz#41cceec4bde2c4aea8b9da8f610fe36055c7709f" @@ -1756,130 +1703,6 @@ ajv@^6.12.3: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ansi-bgblack@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgblack/-/ansi-bgblack-0.1.1.tgz#a68ba5007887701b6aafbe3fa0dadfdfa8ee3ca2" - integrity sha1-poulAHiHcBtqr74/oNrf36juPKI= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgblue@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgblue/-/ansi-bgblue-0.1.1.tgz#67bdc04edc9b9b5278969da196dea3d75c8c3613" - integrity sha1-Z73ATtybm1J4lp2hlt6j11yMNhM= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgcyan@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgcyan/-/ansi-bgcyan-0.1.1.tgz#58489425600bde9f5507068dd969ebfdb50fe768" - integrity sha1-WEiUJWAL3p9VBwaN2Wnr/bUP52g= - dependencies: - ansi-wrap "0.1.0" - -ansi-bggreen@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bggreen/-/ansi-bggreen-0.1.1.tgz#4e3191248529943f4321e96bf131d1c13816af49" - integrity sha1-TjGRJIUplD9DIelr8THRwTgWr0k= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgmagenta@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgmagenta/-/ansi-bgmagenta-0.1.1.tgz#9b28432c076eaa999418672a3efbe19391c2c7a1" - integrity sha1-myhDLAduqpmUGGcqPvvhk5HCx6E= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgred@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgred/-/ansi-bgred-0.1.1.tgz#a76f92838382ba43290a6c1778424f984d6f1041" - integrity sha1-p2+Sg4OCukMpCmwXeEJPmE1vEEE= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgwhite@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgwhite/-/ansi-bgwhite-0.1.1.tgz#6504651377a58a6ececd0331994e480258e11ba8" - integrity sha1-ZQRlE3elim7OzQMxmU5IAljhG6g= - dependencies: - ansi-wrap "0.1.0" - -ansi-bgyellow@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bgyellow/-/ansi-bgyellow-0.1.1.tgz#c3fe2eb08cd476648029e6874d15a0b38f61d44f" - integrity sha1-w/4usIzUdmSAKeaHTRWgs49h1E8= - dependencies: - ansi-wrap "0.1.0" - -ansi-black@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-black/-/ansi-black-0.1.1.tgz#f6185e889360b2545a1ec50c0bf063fc43032453" - integrity sha1-9hheiJNgslRaHsUMC/Bj/EMDJFM= - dependencies: - ansi-wrap "0.1.0" - -ansi-blue@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-blue/-/ansi-blue-0.1.1.tgz#15b804990e92fc9ca8c5476ce8f699777c21edbf" - integrity sha1-FbgEmQ6S/JyoxUds6PaZd3wh7b8= - dependencies: - ansi-wrap "0.1.0" - -ansi-bold@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-bold/-/ansi-bold-0.1.1.tgz#3e63950af5acc2ae2e670e6f67deb115d1a5f505" - integrity sha1-PmOVCvWswq4uZw5vZ96xFdGl9QU= - dependencies: - ansi-wrap "0.1.0" - -ansi-colors@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-0.2.0.tgz#72c31de2a0d9a2ccd0cac30cc9823eeb2f6434b5" - integrity sha1-csMd4qDZoszQysMMyYI+6y9kNLU= - dependencies: - ansi-bgblack "^0.1.1" - ansi-bgblue "^0.1.1" - ansi-bgcyan "^0.1.1" - ansi-bggreen "^0.1.1" - ansi-bgmagenta "^0.1.1" - ansi-bgred "^0.1.1" - ansi-bgwhite "^0.1.1" - ansi-bgyellow "^0.1.1" - ansi-black "^0.1.1" - ansi-blue "^0.1.1" - ansi-bold "^0.1.1" - ansi-cyan "^0.1.1" - ansi-dim "^0.1.1" - ansi-gray "^0.1.1" - ansi-green "^0.1.1" - ansi-grey "^0.1.1" - ansi-hidden "^0.1.1" - ansi-inverse "^0.1.1" - ansi-italic "^0.1.1" - ansi-magenta "^0.1.1" - ansi-red "^0.1.1" - ansi-reset "^0.1.1" - ansi-strikethrough "^0.1.1" - ansi-underline "^0.1.1" - ansi-white "^0.1.1" - ansi-yellow "^0.1.1" - lazy-cache "^2.0.1" - -ansi-cyan@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-cyan/-/ansi-cyan-0.1.1.tgz#538ae528af8982f28ae30d86f2f17456d2609873" - integrity sha1-U4rlKK+JgvKK4w2G8vF0VtJgmHM= - dependencies: - ansi-wrap "0.1.0" - -ansi-dim@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-dim/-/ansi-dim-0.1.1.tgz#40de4c603aa8086d8e7a86b8ff998d5c36eefd6c" - integrity sha1-QN5MYDqoCG2Oeoa4/5mNXDbu/Ww= - dependencies: - ansi-wrap "0.1.0" - ansi-escapes@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b" @@ -1892,62 +1715,6 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.21.3" -ansi-gray@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-gray/-/ansi-gray-0.1.1.tgz#2962cf54ec9792c48510a3deb524436861ef7251" - integrity sha1-KWLPVOyXksSFEKPetSRDaGHvclE= - dependencies: - ansi-wrap "0.1.0" - -ansi-green@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-green/-/ansi-green-0.1.1.tgz#8a5d9a979e458d57c40e33580b37390b8e10d0f7" - integrity sha1-il2al55FjVfEDjNYCzc5C44Q0Pc= - dependencies: - ansi-wrap "0.1.0" - -ansi-grey@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-grey/-/ansi-grey-0.1.1.tgz#59d98b6ac2ba19f8a51798e9853fba78339a33c1" - integrity sha1-WdmLasK6GfilF5jphT+6eDOaM8E= - dependencies: - ansi-wrap "0.1.0" - -ansi-hidden@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-hidden/-/ansi-hidden-0.1.1.tgz#ed6a4c498d2bb7cbb289dbf2a8d1dcc8567fae0f" - integrity sha1-7WpMSY0rt8uyidvyqNHcyFZ/rg8= - dependencies: - ansi-wrap "0.1.0" - -ansi-inverse@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-inverse/-/ansi-inverse-0.1.1.tgz#b6af45826fe826bfb528a6c79885794355ccd269" - integrity sha1-tq9Fgm/oJr+1KKbHmIV5Q1XM0mk= - dependencies: - ansi-wrap "0.1.0" - -ansi-italic@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-italic/-/ansi-italic-0.1.1.tgz#104743463f625c142a036739cf85eda688986f23" - integrity sha1-EEdDRj9iXBQqA2c5z4XtpoiYbyM= - dependencies: - ansi-wrap "0.1.0" - -ansi-magenta@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-magenta/-/ansi-magenta-0.1.1.tgz#063b5ba16fb3f23e1cfda2b07c0a89de11e430ae" - integrity sha1-BjtboW+z8j4c/aKwfAqJ3hHkMK4= - dependencies: - ansi-wrap "0.1.0" - -ansi-red@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-red/-/ansi-red-0.1.1.tgz#8c638f9d1080800a353c9c28c8a81ca4705d946c" - integrity sha1-jGOPnRCAgAo1PJwoyKgcpHBdlGw= - dependencies: - ansi-wrap "0.1.0" - ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" @@ -1963,20 +1730,6 @@ ansi-regex@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== -ansi-reset@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-reset/-/ansi-reset-0.1.1.tgz#e7e71292c3c7ddcd4d62ef4a6c7c05980911c3b7" - integrity sha1-5+cSksPH3c1NYu9KbHwFmAkRw7c= - dependencies: - ansi-wrap "0.1.0" - -ansi-strikethrough@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-strikethrough/-/ansi-strikethrough-0.1.1.tgz#d84877140b2cff07d1c93ebce69904f68885e568" - integrity sha1-2Eh3FAss/wfRyT685pkE9oiF5Wg= - dependencies: - ansi-wrap "0.1.0" - ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" @@ -1996,32 +1749,6 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -ansi-underline@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-underline/-/ansi-underline-0.1.1.tgz#dfc920f4c97b5977ea162df8ffb988308aaa71a4" - integrity sha1-38kg9Ml7WXfqFi34/7mIMIqqcaQ= - dependencies: - ansi-wrap "0.1.0" - -ansi-white@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-white/-/ansi-white-0.1.1.tgz#9c77b7c193c5ee992e6011d36ec4c921b4578944" - integrity sha1-nHe3wZPF7pkuYBHTbsTJIbRXiUQ= - dependencies: - ansi-wrap "0.1.0" - -ansi-wrap@0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/ansi-wrap/-/ansi-wrap-0.1.0.tgz#a82250ddb0015e9a27ca82e82ea603bbfa45efaf" - integrity sha1-qCJQ3bABXponyoLoLqYDu/pF768= - -ansi-yellow@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ansi-yellow/-/ansi-yellow-0.1.1.tgz#cb9356f2f46c732f0e3199e6102955a77da83c1d" - integrity sha1-y5NW8vRscy8OMZnmEClVp32oPB0= - dependencies: - ansi-wrap "0.1.0" - any-observable@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/any-observable/-/any-observable-0.3.0.tgz#af933475e5806a67d0d7df090dd5e8bef65d119b" @@ -2048,7 +1775,7 @@ arch@^2.1.2: resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== -argparse@^1.0.10, argparse@^1.0.7: +argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== @@ -2083,15 +1810,6 @@ arr-union@^3.1.0: resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= -array-sort@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/array-sort/-/array-sort-0.1.4.tgz#662855eaeb671b4188df4451b2f24a0753992b23" - integrity sha512-BNcM+RXxndPxiZ2rd76k6nyQLRZr2/B/sdi8pQ+Joafr5AH279L40dfokSUTp8O+AaqYjXWhblBWa2st2nc4fQ== - dependencies: - default-compare "^1.0.0" - get-value "^2.0.6" - kind-of "^5.0.2" - array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" @@ -2154,13 +1872,6 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== -autolinker@~0.28.0: - version "0.28.1" - resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-0.28.1.tgz#0652b491881879f0775dace0cdca3233942a4e47" - integrity sha1-BlK0kYgYefB3XazgzcoyM5QqTkc= - dependencies: - gulp-header "^1.7.1" - aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2800,13 +2511,6 @@ concat-stream@^1.4.4, concat-stream@^1.6.2: readable-stream "^2.2.2" typedarray "^0.0.6" -concat-with-sourcemaps@*: - version "1.1.0" - resolved "https://registry.yarnpkg.com/concat-with-sourcemaps/-/concat-with-sourcemaps-1.1.0.tgz#d4ea93f05ae25790951b99e7b3b09e3908a4082e" - integrity sha512-4gEjHJFT9e+2W/77h/DS5SGUgwDaOwprX8L/gl5+3ixnzkVJJsZWDSelmN3Oilw3LNDZjZV0yqH1hLG3k6nghg== - dependencies: - source-map "^0.6.1" - console-clear@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/console-clear/-/console-clear-1.1.1.tgz#995e20cbfbf14dd792b672cde387bd128d674bf7" @@ -3030,18 +2734,6 @@ date-fns@^1.27.2: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw== -date.js@^0.3.1: - version "0.3.3" - resolved "https://registry.yarnpkg.com/date.js/-/date.js-0.3.3.tgz#ef1e92332f507a638795dbb985e951882e50bbda" - integrity sha512-HgigOS3h3k6HnW011nAb43c5xx5rBXk8P2v/WIT9Zv4koIaVXiH2BURguI78VVp+5Qc076T7OR378JViCnZtBw== - dependencies: - debug "~3.1.0" - -dayjs@^1.10.4: - version "1.10.4" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.4.tgz#8e544a9b8683f61783f570980a8a80eaf54ab1e2" - integrity sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw== - debug@4.3.1, debug@^4.1.0, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" @@ -3063,13 +2755,6 @@ debug@^3.1.0: dependencies: ms "^2.1.1" -debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -3107,13 +2792,6 @@ deepmerge@^4.2.2: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== -default-compare@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/default-compare/-/default-compare-1.0.0.tgz#cb61131844ad84d84788fb68fd01681ca7781a2f" - integrity sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ== - dependencies: - kind-of "^5.0.2" - deferred-leveldown@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-0.2.0.tgz#2cef1f111e1c57870d8bbb8af2650e587cd2f5b4" @@ -3269,11 +2947,6 @@ end-of-stream@^1.1.0: dependencies: once "^1.4.0" -ent@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" - integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= - entities@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" @@ -3293,11 +2966,6 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -error-symbol@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/error-symbol/-/error-symbol-0.1.0.tgz#0a4dae37d600d15a29ba453d8ef920f1844333f6" - integrity sha1-Ck2uN9YA0VopukU9jvkg8YRDM/Y= - escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -3565,13 +3233,6 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= -"falsey@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/falsey/-/falsey-0.3.2.tgz#b21c90c5c34660fc192bf909575db95b6880d597" - integrity sha512-lxEuefF5MBIVDmE6XeqCdM4BWk1+vYmGZtkbKZ/VFcg6uBBw6fXNEbWmxCjDdQlFc9hy450nkiWwM3VAW6G1qg== - dependencies: - kind-of "^5.0.2" - fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -3685,18 +3346,11 @@ follow-redirects@^1.10.0: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.2.tgz#dd73c8effc12728ba5cf4259d760ea5fb83e3147" integrity sha512-6mPTgLxYm3r6Bkkg0vNM0HTjfGrOEtsfbhagQvbxDEsEkpNhw582upBaoRZylzen6krEmxXJgt9Ju6HiI4O7BA== -for-in@^1.0.1, for-in@^1.0.2: +for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= -for-own@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" - integrity sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs= - dependencies: - for-in "^1.0.1" - foreach@~2.0.1: version "2.0.5" resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" @@ -3728,11 +3382,6 @@ from@~0: resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe" integrity sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4= -fs-exists-sync@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add" - integrity sha1-mC1ok6+RjnLQjeyehnP/K1qNat0= - fs-extra@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" @@ -3793,14 +3442,6 @@ get-intrinsic@^1.0.2: has "^1.0.3" has-symbols "^1.0.1" -get-object@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/get-object/-/get-object-0.2.0.tgz#d92ff7d5190c64530cda0543dac63a3d47fe8c0c" - integrity sha1-2S/31RkMZFMM2gVD2sY6PUf+jAw= - dependencies: - is-number "^2.0.2" - isobject "^0.2.0" - get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -3899,35 +3540,6 @@ growly@^1.3.0: resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081" integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE= -gulp-header@^1.7.1: - version "1.8.12" - resolved "https://registry.yarnpkg.com/gulp-header/-/gulp-header-1.8.12.tgz#ad306be0066599127281c4f8786660e705080a84" - integrity sha512-lh9HLdb53sC7XIZOYzTXM4lFuXElv3EVkSDhsd7DoJBj7hm+Ni7D3qYbb+Rr8DuM8nRanBvkVO9d7askreXGnQ== - dependencies: - concat-with-sourcemaps "*" - lodash.template "^4.4.0" - through2 "^2.0.0" - -handlebars-utils@^1.0.2, handlebars-utils@^1.0.4, handlebars-utils@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/handlebars-utils/-/handlebars-utils-1.0.6.tgz#cb9db43362479054782d86ffe10f47abc76357f9" - integrity sha512-d5mmoQXdeEqSKMtQQZ9WkiUcO1E3tPbWxluCK9hVgIDPzQa9WsKo3Lbe/sGflTe7TomHEeZaOgwIkyIr1kfzkw== - dependencies: - kind-of "^6.0.0" - typeof-article "^0.1.1" - -handlebars@^4.0.11, handlebars@^4.7.6: - version "4.7.7" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" - integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA== - dependencies: - minimist "^1.2.5" - neo-async "^2.6.0" - source-map "^0.6.1" - wordwrap "^1.0.0" - optionalDependencies: - uglify-js "^3.1.4" - har-schema@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" @@ -4028,39 +3640,6 @@ he@1.2.x: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== -helper-date@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/helper-date/-/helper-date-1.0.1.tgz#12fedea3ad8e44a7ca4c4efb0ff4104a5120cffb" - integrity sha512-wU3VOwwTJvGr/w5rZr3cprPHO+hIhlblTJHD6aFBrKLuNbf4lAmkawd2iK3c6NbJEvY7HAmDpqjOFSI5/+Ey2w== - dependencies: - date.js "^0.3.1" - handlebars-utils "^1.0.4" - moment "^2.18.1" - -helper-markdown@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/helper-markdown/-/helper-markdown-1.0.0.tgz#ee7e9fc554675007d37eb90f7853b13ce74f3e10" - integrity sha512-AnDqMS4ejkQK0MXze7pA9TM3pu01ZY+XXsES6gEE0RmCGk5/NIfvTn0NmItfyDOjRAzyo9z6X7YHbHX4PzIvOA== - dependencies: - handlebars-utils "^1.0.2" - highlight.js "^9.12.0" - remarkable "^1.7.1" - -helper-md@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/helper-md/-/helper-md-0.2.2.tgz#c1f59d7e55bbae23362fd8a0e971607aec69d41f" - integrity sha1-wfWdflW7riM2L9ig6XFgeuxp1B8= - dependencies: - ent "^2.2.0" - extend-shallow "^2.0.1" - fs-exists-sync "^0.1.0" - remarkable "^1.6.2" - -highlight.js@^9.12.0: - version "9.18.5" - resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.18.5.tgz#d18a359867f378c138d6819edfc2a8acd5f29825" - integrity sha512-a5bFyofd/BHCX52/8i8uJkjr9DYwXIPnM/plwI6W7ezItLGqzt7X2G2nXuYSfsIJdkwwj/g9DG1LkcGJI/dDoA== - hmac-drbg@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1" @@ -4100,14 +3679,6 @@ html-minifier@^3.0.2: relateurl "0.2.x" uglify-js "3.4.x" -html-tag@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/html-tag/-/html-tag-2.0.0.tgz#36c3bc8d816fd30b570d5764a497a641640c2fed" - integrity sha512-XxzooSo6oBoxBEUazgjdXj7VwTn/iSTSZzTYKzYY6I916tkaYzypHxy+pbVU1h+0UQ9JlVf5XkNQyxOAiiQO1g== - dependencies: - is-self-closing "^1.0.1" - kind-of "^6.0.0" - http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -4190,11 +3761,6 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -info-symbol@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/info-symbol/-/info-symbol-0.1.0.tgz#27841d72867ddb4242cd612d79c10633881c6a78" - integrity sha1-J4QdcoZ920JCzWEtecEGM4gcang= - inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" @@ -4304,13 +3870,6 @@ is-docker@^2.0.0: resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.1.1.tgz#4125a88e44e450d384e09047ede71adc2d144156" integrity sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw== -is-even@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-even/-/is-even-1.0.0.tgz#76b5055fbad8d294a86b6a949015e1c97b717c06" - integrity sha1-drUFX7rY0pSoa2qUkBXhyXtxfAY= - dependencies: - is-odd "^0.1.2" - is-extendable@^0.1.0, is-extendable@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" @@ -4350,7 +3909,7 @@ is-generator-fn@^2.0.0: resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118" integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ== -is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1: +is-glob@^4.0.1, is-glob@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== @@ -4370,13 +3929,6 @@ is-module@^1.0.0: resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE= -is-number@^2.0.2: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f" - integrity sha1-Afy7s5NGOlSPL0ZszhbezknbkI8= - dependencies: - kind-of "^3.0.2" - is-number@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" @@ -4384,11 +3936,6 @@ is-number@^3.0.0: dependencies: kind-of "^3.0.2" -is-number@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-4.0.0.tgz#0026e37f5454d73e356dfe6564699867c6a7f0ff" - integrity sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ== - is-number@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" @@ -4406,13 +3953,6 @@ is-observable@^1.1.0: dependencies: symbol-observable "^1.1.0" -is-odd@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/is-odd/-/is-odd-0.1.2.tgz#bc573b5ce371ef2aad6e6f49799b72bef13978a7" - integrity sha1-vFc7XONx7yqtbm9JeZtyvvE5eKc= - dependencies: - is-number "^3.0.0" - is-path-inside@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" @@ -4455,13 +3995,6 @@ is-regex@^1.0.4: call-bind "^1.0.2" has-symbols "^1.0.1" -is-self-closing@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-self-closing/-/is-self-closing-1.0.1.tgz#5f406b527c7b12610176320338af0fa3896416e4" - integrity sha512-E+60FomW7Blv5GXTlYee2KDrnG6srxF7Xt1SjrhWUGUEsTFIqY/nq2y3DaftCsgUMdh89V07IVfhY9KIJhLezg== - dependencies: - self-closing-tags "^1.0.1" - is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" @@ -4514,11 +4047,6 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= -isobject@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-0.2.0.tgz#a3432192f39b910b5f02cc989487836ec70aa85e" - integrity sha1-o0MhkvObkQtfAsyYlIeDbscKqF4= - isobject@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" @@ -5106,7 +4634,7 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" -kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.1.0, kind-of@^3.2.0: +kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= @@ -5120,7 +4648,7 @@ kind-of@^4.0.0: dependencies: is-buffer "^1.1.5" -kind-of@^5.0.0, kind-of@^5.0.2: +kind-of@^5.0.0: version "5.1.0" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw== @@ -5140,13 +4668,6 @@ lazy-ass@1.6.0, lazy-ass@^1.6.0: resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= -lazy-cache@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-2.0.2.tgz#b9190a4f913354694840859f8a8f7084d8822264" - integrity sha1-uRkKT5EzVGlIQIWfio9whNiCImQ= - dependencies: - set-getter "^0.1.0" - level-blobs@^0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/level-blobs/-/level-blobs-0.1.7.tgz#9ab9b97bb99f1edbf9f78a3433e21ed56386bdaf" @@ -5333,11 +4854,6 @@ lodash-es@^4.17.11: resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.20.tgz#29f6332eefc60e849f869c264bc71126ad61e8f7" integrity sha512-JD1COMZsq8maT6mnuz1UMV0jvYD0E0aUsSOdrr1/nAG3dhqQXwRRgeW0cSqH1U43INKcqxaiVIQNOUDld7gRDA== -lodash._reinterpolate@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" - integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0= - lodash.once@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" @@ -5348,21 +4864,6 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= -lodash.template@^4.4.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab" - integrity sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A== - dependencies: - lodash._reinterpolate "^3.0.0" - lodash.templatesettings "^4.0.0" - -lodash.templatesettings@^4.0.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz#e481310f049d3cf6d47e912ad09313b154f0fb33" - integrity sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ== - dependencies: - lodash._reinterpolate "^3.0.0" - lodash@4.17.13: version "4.17.13" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.13.tgz#0bdc3a6adc873d2f4e0c4bac285df91b64fc7b93" @@ -5373,14 +4874,6 @@ lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== -log-ok@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334" - integrity sha1-vqPdNqzQuKckDXhza1uXxlREozQ= - dependencies: - ansi-green "^0.1.1" - success-symbol "^0.1.0" - log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -5411,27 +4904,6 @@ log-update@^2.3.0: cli-cursor "^2.0.0" wrap-ansi "^3.0.1" -log-utils@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/log-utils/-/log-utils-0.2.1.tgz#a4c217a0dd9a50515d9b920206091ab3d4e031cf" - integrity sha1-pMIXoN2aUFFdm5ICBgkas9TgMc8= - dependencies: - ansi-colors "^0.2.0" - error-symbol "^0.1.0" - info-symbol "^0.1.0" - log-ok "^0.1.1" - success-symbol "^0.1.0" - time-stamp "^1.0.1" - warning-symbol "^0.1.0" - -logging-helpers@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/logging-helpers/-/logging-helpers-1.0.0.tgz#b5a37b32ad53eb0137c58c7898a47b175ddb7c36" - integrity sha512-qyIh2goLt1sOgQQrrIWuwkRjUx4NUcEqEGAcYqD8VOnOC6ItwkrVE8/tA4smGpjzyp4Svhc6RodDp9IO5ghpyA== - dependencies: - isobject "^3.0.0" - log-utils "^0.2.1" - lower-case@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac" @@ -5643,7 +5115,7 @@ mkdirp@^0.5.1, mkdirp@^0.5.4: dependencies: minimist "^1.2.5" -moment@^2.18.1, moment@^2.27.0: +moment@^2.27.0: version "2.29.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== @@ -5700,11 +5172,6 @@ ncp@^2.0.0: resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M= -neo-async@^2.6.0: - version "2.6.2" - resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" - integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== - nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" @@ -6294,7 +5761,7 @@ readable-stream@^1.0.26-4: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.2.2, readable-stream@~2.3.6: +readable-stream@^2.2.2: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -6381,11 +5848,6 @@ regexp.prototype.flags@^1.2.0: call-bind "^1.0.2" define-properties "^1.1.3" -regexparam@1.3.0, regexparam@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f" - integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g== - regexpu-core@^4.7.1: version "4.7.1" resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6" @@ -6415,21 +5877,6 @@ relateurl@0.2.x: resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= -relative@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f" - integrity sha1-Dc2OxUpdNaPBXhBFA9ZTdbWlNn8= - dependencies: - isobject "^2.0.0" - -remarkable@^1.6.2, remarkable@^1.7.1: - version "1.7.4" - resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-1.7.4.tgz#19073cb960398c87a7d6546eaa5e50d2022fcd00" - integrity sha512-e6NKUXgX95whv7IgddywbeN/ItCkWbISmc2DiqHJb0wTrqZIexqdco5b8Z3XZoo/48IdNVKM9ZCvTPJ4F5uvhg== - dependencies: - argparse "^1.0.10" - autolinker "~0.28.0" - remixicon@2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/remixicon/-/remixicon-2.5.0.tgz#b5e245894a1550aa23793f95daceadbf96ad1a41" @@ -6779,11 +6226,6 @@ saxes@^5.0.0, saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -self-closing-tags@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/self-closing-tags/-/self-closing-tags-1.0.1.tgz#6c5fa497994bb826b484216916371accee490a5d" - integrity sha512-7t6hNbYMxM+VHXTgJmxwgZgLGktuXtVVD5AivWzNTdJBM4DBjnDKDzkf2SrNjihaArpeJYNjxkELBu1evI4lQA== - "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.5.1: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" @@ -6823,13 +6265,6 @@ set-blocking@^2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= -set-getter@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/set-getter/-/set-getter-0.1.0.tgz#d769c182c9d5a51f409145f2fba82e5e86e80376" - integrity sha1-12nBgsnVpR9AkUXy+6guXoboA3Y= - dependencies: - to-object-path "^0.3.0" - set-value@^2.0.0, set-value@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" @@ -6884,13 +6319,6 @@ shortid@2.2.15: dependencies: nanoid "^2.1.0" -shortid@^2.2.15: - version "2.2.16" - resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608" - integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g== - dependencies: - nanoid "^2.1.0" - signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -7224,16 +6652,6 @@ strip-indent@^3.0.0: dependencies: min-indent "^1.0.0" -striptags@^3.1.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.1.1.tgz#c8c3e7fdd6fb4bb3a32a3b752e5b5e3e38093ebd" - integrity sha1-yMPn/db7S7OjKjt1LltePjgJPr0= - -success-symbol@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/success-symbol/-/success-symbol-0.1.0.tgz#24022e486f3bf1cdca094283b769c472d3b72897" - integrity sha1-JAIuSG878c3KCUKDt2nEctO3KJc= - supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -7295,13 +6713,6 @@ svelte-portal@^1.0.0: resolved "https://registry.yarnpkg.com/svelte-portal/-/svelte-portal-1.0.0.tgz#36a47c5578b1a4d9b4dc60fa32a904640ec4cdd3" integrity sha512-nHf+DS/jZ6jjnZSleBMSaZua9JlG5rZv9lOGKgJuaZStfevtjIlUJrkLc3vbV8QdBvPPVmvcjTlazAzfKu0v3Q== -svelte-spa-router@^3.0.5: - version "3.1.0" - resolved "https://registry.yarnpkg.com/svelte-spa-router/-/svelte-spa-router-3.1.0.tgz#a929f0def7e12c41f32bc356f91685aeadcd75bf" - integrity sha512-jlM/xwjn57mylr+pzHYCOOy+IPQauT46gOucNGTBu6jHcFXu3F+oaojN4PXC1LYizRGxFB6QA0qnYbZnRfX7Sg== - dependencies: - regexparam "1.3.0" - svelte@^3.30.0: version "3.32.3" resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.32.3.tgz#db0c50c65573ecffe4e2f4924e4862d8f9feda74" @@ -7358,24 +6769,11 @@ throttleit@^1.0.0: resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw= -through2@^2.0.0: - version "2.0.5" - resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" - integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ== - dependencies: - readable-stream "~2.3.6" - xtend "~4.0.1" - through@2, through@~2.3, through@~2.3.1: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= -time-stamp@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/time-stamp/-/time-stamp-1.1.0.tgz#764a5a11af50561921b133f3b44e618687e0f5c3" - integrity sha1-dkpaEa9QVhkhsTPztE5hhofg9cM= - tinydate@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/tinydate/-/tinydate-1.3.0.tgz#e6ca8e5a22b51bb4ea1c3a2a4fd1352dbd4c57fb" @@ -7398,11 +6796,6 @@ to-fast-properties@^2.0.0: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= -to-gfm-code-block@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/to-gfm-code-block/-/to-gfm-code-block-0.1.1.tgz#25d045a5fae553189e9637b590900da732d8aa82" - integrity sha1-JdBFpfrlUxielje1kJANpzLYqoI= - to-object-path@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af" @@ -7541,13 +6934,6 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typeof-article@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/typeof-article/-/typeof-article-0.1.1.tgz#9f07e733c3fbb646ffa9e61c08debacd460e06af" - integrity sha1-nwfnM8P7tkb/qeYcCN66zUYOBq8= - dependencies: - kind-of "^3.1.0" - uc.micro@^1.0.1, uc.micro@^1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" @@ -7561,11 +6947,6 @@ uglify-js@3.4.x: commander "~2.19.0" source-map "~0.6.1" -uglify-js@^3.1.4: - version "3.13.2" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.2.tgz#fe10319861bccc8682bfe2e8151fbdd8aa921c44" - integrity sha512-SbMu4D2Vo95LMC/MetNaso1194M1htEA+JrqE9Hk+G2DhI+itfS9TRu9ZKeCahLDNa/J3n4MqUJ/fOHMzQpRWw== - unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -7742,11 +7123,6 @@ walker@^1.0.7, walker@~1.0.5: dependencies: makeerror "1.0.x" -warning-symbol@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/warning-symbol/-/warning-symbol-0.1.0.tgz#bb31dd11b7a0f9d67ab2ed95f457b65825bbad21" - integrity sha1-uzHdEbeg+dZ6su2V9Fe2WCW7rSE= - webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -7802,11 +7178,6 @@ word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wordwrap@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" - integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= - wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba" @@ -7891,11 +7262,6 @@ xtend@~3.0.0: resolved "https://registry.yarnpkg.com/xtend/-/xtend-3.0.0.tgz#5cce7407baf642cba7becda568111c493f59665a" integrity sha1-XM50B7r2Qsunvs2laBEcST9ZZlo= -xtend@~4.0.1: - version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - y18n@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.1.tgz#8db2b83c31c5d75099bb890b23f3094891e247d4" @@ -7944,11 +7310,6 @@ yauzl@^2.10.0: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" -year@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/year/-/year-0.2.1.tgz#4083ae520a318b23ec86037f3000cb892bdf9bb0" - integrity sha1-QIOuUgoxiyPshgN/MADLiSvfm7A= - yup@0.29.2: version "0.29.2" resolved "https://registry.yarnpkg.com/yup/-/yup-0.29.2.tgz#5302abd9024cca335b987793f8df868e410b7b67" diff --git a/packages/cli/package.json b/packages/cli/package.json index 5c58c80560..f369111372 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,9 +1,11 @@ { - "name": "cli", - "version": "0.8.7", + "name": "@budibase/cli", + "version": "0.8.10", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", - "bin": "src/index.js", + "bin": { + "budi": "src/index.js" + }, "author": "Budibase", "license": "AGPL-3.0-or-later", "scripts": { diff --git a/packages/cli/src/constants.js b/packages/cli/src/constants.js index a601909232..c8da56fa5c 100644 --- a/packages/cli/src/constants.js +++ b/packages/cli/src/constants.js @@ -2,3 +2,8 @@ exports.CommandWords = { HOSTING: "hosting", HELP: "help", } + +exports.InitTypes = { + QUICK: "quick", + DIGITAL_OCEAN: "do", +} diff --git a/packages/cli/src/hosting/index.js b/packages/cli/src/hosting/index.js index a8e77f49d7..710397f301 100644 --- a/packages/cli/src/hosting/index.js +++ b/packages/cli/src/hosting/index.js @@ -1,11 +1,18 @@ const Command = require("../structures/Command") -const { CommandWords } = require("../constants") +const { CommandWords, InitTypes } = require("../constants") const { lookpath } = require("lookpath") -const { downloadFile, logErrorToFile, success, info } = require("../utils") +const { + downloadFile, + logErrorToFile, + success, + info, + parseEnv, +} = require("../utils") const { confirmation } = require("../questions") const fs = require("fs") const compose = require("docker-compose") -const envFile = require("./makeEnv") +const makeEnv = require("./makeEnv") +const axios = require("axios") const BUDIBASE_SERVICES = ["app-service", "worker-service"] const ERROR_FILE = "docker-error.log" @@ -13,6 +20,7 @@ const FILE_URLS = [ "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml", "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml", ] +const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data" async function downloadFiles() { const promises = [] @@ -34,7 +42,7 @@ async function checkDockerConfigured() { } function checkInitComplete() { - if (!fs.existsSync(envFile.filePath)) { + if (!fs.existsSync(makeEnv.filePath)) { throw "Please run the hosting --init command before any other hosting command." } } @@ -50,24 +58,41 @@ async function handleError(func) { } } -async function init() { +async function init(type) { + const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN await checkDockerConfigured() - const shouldContinue = await confirmation( - "This will create multiple files in current directory, should continue?" - ) - if (!shouldContinue) { - console.log("Stopping.") - return + if (!isQuick) { + const shouldContinue = await confirmation( + "This will create multiple files in current directory, should continue?" + ) + if (!shouldContinue) { + console.log("Stopping.") + return + } } await downloadFiles() - await envFile.make() + const config = isQuick ? makeEnv.QUICK_CONFIG : {} + if (type === InitTypes.DIGITAL_OCEAN) { + try { + const output = await axios.get(DO_USER_DATA_URL) + const response = parseEnv(output.data) + for (let [key, value] of Object.entries(makeEnv.ConfigMap)) { + if (response[key]) { + config[value] = response[key] + } + } + } catch (err) { + // don't need to handle error, just don't do anything + } + } + await makeEnv.make(config) } async function start() { await checkDockerConfigured() checkInitComplete() console.log(info("Starting services, this may take a moment.")) - const port = envFile.get("MAIN_PORT") + const port = makeEnv.get("MAIN_PORT") await handleError(async () => { await compose.upAll({ cwd: "./", log: false }) }) @@ -128,8 +153,8 @@ async function update() { const command = new Command(`${CommandWords.HOSTING}`) .addHelp("Controls self hosting on the Budibase platform.") .addSubOption( - "--init", - "Configure a self hosted platform in current directory.", + "--init [type]", + "Configure a self hosted platform in current directory, type can be unspecified or 'quick'.", init ) .addSubOption( diff --git a/packages/cli/src/hosting/makeEnv.js b/packages/cli/src/hosting/makeEnv.js index b7d3b9e849..318a72def1 100644 --- a/packages/cli/src/hosting/makeEnv.js +++ b/packages/cli/src/hosting/makeEnv.js @@ -26,19 +26,32 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +REDIS_PORT=6379 BUDIBASE_ENVIRONMENT=PRODUCTION` } module.exports.filePath = FILE_PATH +module.exports.ConfigMap = { + HOSTING_KEY: "key", + MAIN_PORT: "port", +} +module.exports.QUICK_CONFIG = { + key: "budibase", + port: 10000, +} -module.exports.make = async () => { - const hostingKey = await string( - "Please input the password you'd like to use as your hosting key: " - ) - const hostingPort = await number( - "Please enter the port on which you want your installation to run: ", - 10000 - ) +module.exports.make = async (inputs = {}) => { + const hostingKey = + inputs.key || + (await string( + "Please input the password you'd like to use as your hosting key: " + )) + const hostingPort = + inputs.port || + (await number( + "Please enter the port on which you want your installation to run: ", + 10000 + )) const fileContents = getContents(hostingPort, hostingKey) fs.writeFileSync(FILE_PATH, fileContents) console.log( diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index 526f259fc4..6693446b39 100644 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -1,3 +1,4 @@ +#!/usr/bin/env node const { getCommands } = require("./options") const { Command } = require("commander") const { getHelpDescription } = require("./utils") diff --git a/packages/cli/src/structures/Command.js b/packages/cli/src/structures/Command.js index 5ca0ce67ee..a8d24566be 100644 --- a/packages/cli/src/structures/Command.js +++ b/packages/cli/src/structures/Command.js @@ -13,8 +13,8 @@ class Command { return this } - addSubOption(command, help, func) { - this.opts.push({ command, help, func }) + addSubOption(command, help, func, extras = []) { + this.opts.push({ command, help, func, extras }) return this } @@ -37,13 +37,10 @@ class Command { command.action(async options => { try { let executed = false - if (thisCmd.func) { - await thisCmd.func(options) - executed = true - } for (let opt of thisCmd.opts) { - if (options[opt.command.replace("--", "")]) { - await opt.func(options) + const lookup = opt.command.split(" ")[0].replace("--", "") + if (options[lookup]) { + await opt.func(options[lookup]) executed = true } } diff --git a/packages/cli/src/utils.js b/packages/cli/src/utils.js index 1605c277c0..05bb8d4991 100644 --- a/packages/cli/src/utils.js +++ b/packages/cli/src/utils.js @@ -44,3 +44,15 @@ exports.info = info => { exports.logErrorToFile = (file, error) => { fs.writeFileSync(path.resolve(`./${file}`), `Budiase Error\n${error}`) } + +exports.parseEnv = env => { + const lines = env.toString().split("\n") + let result = {} + for (const line of lines) { + const match = line.match(/^([^=:#]+?)[=:](.*)/) + if (match) { + result[match[1].trim()] = match[2].trim() + } + } + return result +} diff --git a/packages/cli/yarn.lock b/packages/cli/yarn.lock index c03030576c..298fb78061 100644 --- a/packages/cli/yarn.lock +++ b/packages/cli/yarn.lock @@ -15,30 +15,30 @@ integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== "@babel/highlight@^7.10.4": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.12.13.tgz#8ab538393e00370b26271b01fa08f7f27f2e795c" - integrity sha512-kocDQvIbgMKlWxXe9fof3TQ+gkIPOUSEYhJjqUjvKMez3krV7vbzYCDq39Oj11UAVK7JqPVGQPlgE85dPNlQww== + version "7.13.10" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1" + integrity sha512-5aPpe5XQPzflQrFwL1/QoeHkP2MsA4JCntcXHRhEsdsfPVkvPi2w7Qix4iV7t5S/oC9OodGrggd8aco1g3SZFg== dependencies: "@babel/helper-validator-identifier" "^7.12.11" chalk "^2.0.0" js-tokens "^4.0.0" "@babel/parser@^7.9.4": - version "7.13.4" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.4.tgz#340211b0da94a351a6f10e63671fa727333d13ab" - integrity sha512-uvoOulWHhI+0+1f9L4BoozY7U5cIkZ9PgJqvb041d6vypgUmtVPG4vmGm4pSggjl8BELzvHyUeJSUyEMY6b+qA== + version "7.13.11" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.11.tgz#f93ebfc99d21c1772afbbaa153f47e7ce2f50b88" + integrity sha512-PhuoqeHoO9fc4ffMEVk4qb/w/s2iOSWohvbHxLtxui0eBg3Lg5gN1U8wp1V1u61hOWkPQJJyJzGH6Y+grwkq8Q== "@babel/runtime@^7.9.2": - version "7.13.7" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.7.tgz#d494e39d198ee9ca04f4dcb76d25d9d7a1dc961a" - integrity sha512-h+ilqoX998mRVM5FtB5ijRuHUDVt5l3yfoOi2uh18Z/O3hvyaHQ39NpxVkCIG5yFs+mLq/ewFp8Bss6zmWv6ZA== + version "7.13.10" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.10.tgz#47d42a57b6095f4468da440388fdbad8bebf0d7d" + integrity sha512-4QPkjJq6Ns3V/RgpEahRk+AGfL0eO6RHHtTWoNNr5mO49G6B5+X6d6THgWEAvTrznU5xYpbAlVKRYcsCgh/Akw== dependencies: regenerator-runtime "^0.13.4" -"@eslint/eslintrc@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.3.0.tgz#d736d6963d7003b6514e6324bec9c602ac340318" - integrity sha512-1JTKgrOKAHVivSvOYw+sJOunkBjUOvjqWk1DPja7ZFhIS2mX/4EgTT8M7eTK9jrKhL/FvXXEbQwIs3pg1xp3dg== +"@eslint/eslintrc@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.0.tgz#99cc0a0584d72f1df38b900fb062ba995f395547" + integrity sha512-2ZPCc+uNbjV5ERJr+aKSPRwZgKd2z11x0EgLvb1PURmUrn9QNRXFqje0Ldq454PfAVyaJYyrDvvIKSFP4NnBog== dependencies: ajv "^6.12.4" debug "^4.1.1" @@ -47,7 +47,6 @@ ignore "^4.0.6" import-fresh "^3.2.1" js-yaml "^3.13.1" - lodash "^4.17.20" minimatch "^3.0.4" strip-json-comments "^3.1.1" @@ -93,9 +92,9 @@ ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: uri-js "^4.2.2" ajv@^7.0.2: - version "7.1.1" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.1.1.tgz#1e6b37a454021fa9941713f38b952fc1c8d32a84" - integrity sha512-ga/aqDYnUy/o7vbsRTFhhTsNeXiYb5JWDIcRIeZfwRNCefwjNTVYCGdGSUrEmiu3yDK3vFvNbgJxvrQW4JXrYQ== + version "7.2.1" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.2.1.tgz#a5ac226171912447683524fa2f1248fcf8bac83d" + integrity sha512-+nu0HDv7kNSOua9apAVc979qd932rrZeb3WOvoiD31A/p1mIE5/9bN2027pE2rOPYEdS3UHzsvof4hY+lM9/WQ== dependencies: fast-deep-equal "^3.1.1" json-schema-traverse "^1.0.0" @@ -434,12 +433,12 @@ eslint-visitor-keys@^2.0.0: integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== eslint@^7.20.0: - version "7.20.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.20.0.tgz#db07c4ca4eda2e2316e7aa57ac7fc91ec550bdc7" - integrity sha512-qGi0CTcOGP2OtCQBgWZlQjcTuP0XkIpYFj25XtRTQSHC+umNnp7UMshr2G8SLsRFYDdAPFeHOsiteadmMH02Yw== + version "7.22.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.22.0.tgz#07ecc61052fec63661a2cab6bd507127c07adc6f" + integrity sha512-3VawOtjSJUQiiqac8MQc+w457iGLfuNGLFn8JmF051tTKbh5/x/0vlcEj8OgDCaw7Ysa2Jn8paGshV7x2abKXg== dependencies: "@babel/code-frame" "7.12.11" - "@eslint/eslintrc" "^0.3.0" + "@eslint/eslintrc" "^0.4.0" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -452,10 +451,10 @@ eslint@^7.20.0: espree "^7.3.1" esquery "^1.4.0" esutils "^2.0.2" - file-entry-cache "^6.0.0" + file-entry-cache "^6.0.1" functional-red-black-tree "^1.0.1" glob-parent "^5.0.0" - globals "^12.1.0" + globals "^13.6.0" ignore "^4.0.6" import-fresh "^3.0.0" imurmurhash "^0.1.4" @@ -463,7 +462,7 @@ eslint@^7.20.0: js-yaml "^3.13.1" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" - lodash "^4.17.20" + lodash "^4.17.21" minimatch "^3.0.4" natural-compare "^1.4.0" optionator "^0.9.1" @@ -589,7 +588,7 @@ figures@^3.0.0: dependencies: escape-string-regexp "^1.0.5" -file-entry-cache@^6.0.0: +file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== @@ -617,9 +616,9 @@ flatted@^3.1.0: integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA== follow-redirects@^1.10.0: - version "1.13.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.2.tgz#dd73c8effc12728ba5cf4259d760ea5fb83e3147" - integrity sha512-6mPTgLxYm3r6Bkkg0vNM0HTjfGrOEtsfbhagQvbxDEsEkpNhw582upBaoRZylzen6krEmxXJgt9Ju6HiI4O7BA== + version "1.13.3" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267" + integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA== forever-agent@~0.6.1: version "0.6.1" @@ -675,9 +674,9 @@ getpass@^0.1.1: assert-plus "^1.0.0" glob-parent@^5.0.0, glob-parent@^5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" - integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ== + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" @@ -700,6 +699,13 @@ globals@^12.1.0: dependencies: type-fest "^0.8.1" +globals@^13.6.0: + version "13.7.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.7.0.tgz#aed3bcefd80ad3ec0f0be2cf0c895110c0591795" + integrity sha512-Aipsz6ZKRxa/xQkZhNg0qIWXT6x6rD46f6x/PCnBomlttdIyAPak4YD9jTmKpZ72uROSMU87qJtcgpgHaVchiA== + dependencies: + type-fest "^0.20.2" + globby@^11.0.0: version "11.0.2" resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.2.tgz#1af538b766a3b540ebfb58a32b2e2d5897321d83" @@ -957,9 +963,9 @@ lodash@^4.17.20, lodash@^4.17.21: integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== lookpath@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/lookpath/-/lookpath-1.1.0.tgz#932d68371a2f0b4a5644f03d6a2b4728edba96d2" - integrity sha512-B9NM7XpVfkyWqfOBI/UW0kVhGw7pJztsduch+1wkbYDi90mYK6/InFul3lG0hYko/VEcVMARVBJ5daFRc5aKCw== + version "1.2.0" + resolved "https://registry.yarnpkg.com/lookpath/-/lookpath-1.2.0.tgz#5fccf91497acec085e66d98cb12446c21fe665ae" + integrity sha512-cUl+R2bGJcSJiHLVKzGHRTYTBhudbHIgd7s63gfGHteaz0BBKEEz2yw2rgbxZAFze92KlbkiWzL1ylYOmqIPVA== lru-cache@^6.0.0: version "6.0.0" @@ -1417,9 +1423,9 @@ stream-meter@^1.0.4: readable-stream "^2.1.4" string-width@^4.1.0, string-width@^4.2.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.1.tgz#1933ce1f470973d224368009bd1316cad81d5f4f" - integrity sha512-LL0OLyN6AnfV9xqGQpDBwedT2Rt63737LxvsRxbcwpa2aIeynBApG2Sm//F3TaLHIR1aJBN52DWklc06b94o5Q== + version "4.2.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" + integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" @@ -1541,6 +1547,11 @@ type-fest@^0.11.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== + type-fest@^0.8.1: version "0.8.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" @@ -1583,9 +1594,9 @@ uuid@^3.3.2: integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== v8-compile-cache@^2.0.3: - version "2.2.0" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132" - integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q== + version "2.3.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" + integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== verror@1.10.0: version "1.10.0" diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index ce06019b54..248e1516c2 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -21,14 +21,16 @@ export const fetchTableData = async tableId => { * Perform a mango query against an internal table * @param {String} tableId - id of the table to search * @param {Object} search - Mango Compliant search object + * @param {Object} pagination - the pagination controls */ export const searchTableData = async ({ tableId, search, pagination }) => { - const rows = await API.post({ + const output = await API.post({ url: `/api/${tableId}/rows/search`, body: { query: search, pagination, }, }) - return await enrichRows(rows, tableId) + output.rows = await enrichRows(output.rows, tableId) + return output } diff --git a/packages/server/.env.template b/packages/server/.env.template deleted file mode 100644 index b2ff5be3f4..0000000000 --- a/packages/server/.env.template +++ /dev/null @@ -1,17 +0,0 @@ -# url of couch db, including username and password -# http://admin:password@localhost:5984 -COUCH_DB_URL={{couchDbUrl}} - -# identifies a client database - i.e. group of apps -CLIENT_ID={{clientId}} - -# used to create cookie hashes -JWT_SECRET={{cookieKey1}} - -# error level for koa-pino -LOG_LEVEL=info - -DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/" -DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984" -SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 -ENABLE_ANALYTICS="true" diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index d75fe1f5d0..c13022c2d6 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -2,7 +2,6 @@ FROM node:12-alpine WORKDIR /app -ENV CLOUD=1 ENV PORT=4001 ENV COUCH_DB_URL=https://couchdb.budi.live:5984 ENV BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/packages/server/__mocks__/node-fetch.js b/packages/server/__mocks__/node-fetch.js index 3cc412b1c6..d023802582 100644 --- a/packages/server/__mocks__/node-fetch.js +++ b/packages/server/__mocks__/node-fetch.js @@ -30,6 +30,17 @@ module.exports = async (url, opts) => { }, 404 ) + } else if (url.includes("_search")) { + return json({ + rows: [ + { + doc: { + _id: "test", + }, + }, + ], + bookmark: "test", + }) } return fetch(url, opts) } diff --git a/packages/server/package.json b/packages/server/package.json index 6746b01c89..36cd215423 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -36,7 +36,10 @@ "test:integration": "jest --coverage --detectOpenHandles", "test:watch": "jest --watch", "run:docker": "node src/index", - "dev:builder": "cross-env PORT=4001 nodemon src/index.js", + "dev:stack:up": "node scripts/dev/manage.js up", + "dev:stack:down": "node scripts/dev/manage.js down", + "dev:stack:nuke": "node scripts/dev/manage.js nuke", + "dev:builder": "npm run dev:stack:up && nodemon src/index.js", "electron": "electron src/electron.js", "build:electron": "electron-builder --dir", "publish:electron": "electron-builder -mwl --publish always", @@ -63,7 +66,7 @@ "!src/tests/**/*", "!src/automations/tests/**/*", "!src/utilities/fileProcessor.js", - "!src/utilities/initialiseBudibase.js" + "!src/utilities/fileSystem/**/*" ], "coverageReporters": [ "lcov", @@ -134,7 +137,7 @@ "devDependencies": { "@budibase/standard-components": "^0.8.9", "@jest/test-sequencer": "^24.8.0", - "cross-env": "^7.0.3", + "docker-compose": "^0.23.6", "electron": "10.1.3", "electron-builder": "^22.9.1", "electron-builder-notarize": "^1.1.2", diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js new file mode 100644 index 0000000000..af4f67d23a --- /dev/null +++ b/packages/server/scripts/dev/manage.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node +const compose = require("docker-compose") +const path = require("path") +const fs = require("fs") + +// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands. +const CONFIG = { + cwd: path.resolve(process.cwd(), "../../hosting"), + config: "docker-compose.dev.yaml", + log: true, +} + +const Commands = { + Up: "up", + Down: "down", + Nuke: "nuke", +} + +async function init() { + const envFilePath = path.join(process.cwd(), ".env") + if (fs.existsSync(envFilePath)) { + return + } + const envFileJson = { + PORT: 4001, + MINIO_URL: "http://localhost:10000/", + COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", + WORKER_URL: "http://localhost:4002", + JWT_SECRET: "testsecret", + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + COUCH_DB_PASSWORD: "budibase", + COUCH_DB_USER: "budibase", + SELF_HOSTED: 1, + } + let envFile = "" + Object.keys(envFileJson).forEach(key => { + envFile += `${key}=${envFileJson[key]}\n` + }) + fs.writeFileSync(envFilePath, envFile) +} + +async function up() { + console.log("Spinning up your budibase dev environment... 🔧✨") + await init() + await compose.upAll(CONFIG) +} + +async function down() { + console.log("Spinning down your budibase dev environment... 🌇") + await compose.stop(CONFIG) +} + +async function nuke() { + console.log( + "Clearing down your budibase dev environment, including all containers and volumes... 💥" + ) + await compose.down(CONFIG) +} + +const managementCommand = process.argv.slice(2)[0] + +if ( + !managementCommand || + !Object.values(Commands).some(command => managementCommand === command) +) { + throw new Error( + "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment." + ) +} + +let command +switch (managementCommand) { + case Commands.Up: + command = up + break + case Commands.Down: + command = down + break + case Commands.Nuke: + command = nuke + break + default: + command = up +} + +command() + .then(() => { + console.log("Done! 🎉") + }) + .catch(err => { + console.error( + "Something went wrong while managing budibase dev environment:", + err.message + ) + }) diff --git a/packages/server/scripts/exportAppTemplate.js b/packages/server/scripts/exportAppTemplate.js index e896917d5b..34988cda37 100755 --- a/packages/server/scripts/exportAppTemplate.js +++ b/packages/server/scripts/exportAppTemplate.js @@ -1,6 +1,10 @@ #!/usr/bin/env node -const { exportTemplateFromApp } = require("../src/utilities/templates") const yargs = require("yargs") +const fs = require("fs") +const { join } = require("path") +const CouchDB = require("../src/db") +// load environment +const env = require("../src/environment") // Script to export a chosen budibase app into a package // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId @@ -22,18 +26,26 @@ yargs }, }, async args => { + if (!env.isDev()) { + throw "Only works in dev" + } + const name = args.name, + appId = args.appId console.log("Exporting app..") - if (args.name == null || args.appId == null) { + if (name == null || appId == null) { console.error( "Unable to export without a name and app ID being specified, check help for more info." ) return } - const exportPath = await exportTemplateFromApp({ - templateName: args.name, - appId: args.appId, - }) - console.log(`Template ${args.name} exported to ${exportPath}`) + const exportPath = join(process.cwd(), name, "db") + fs.ensureDirSync(exportPath) + const writeStream = fs.createWriteStream(join(exportPath, "dump.text")) + // perform couch dump + + const instanceDb = new CouchDB(appId) + await instanceDb.dump(writeStream, {}) + console.log(`Template ${name} exported to ${exportPath}`) } ) .help() diff --git a/packages/server/scripts/initialise.js b/packages/server/scripts/initialise.js deleted file mode 100644 index 485ebd56a0..0000000000 --- a/packages/server/scripts/initialise.js +++ /dev/null @@ -1,17 +0,0 @@ -const { join } = require("path") -const { homedir } = require("os") - -const initialiseBudibase = require("../src/utilities/initialiseBudibase") -const DIRECTORY = "~/.budibase" - -function run() { - let opts = {} - let dir = DIRECTORY - opts.quiet = true - opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir - return initialiseBudibase(opts) -} - -run().then(() => { - console.log("Init complete.") -}) diff --git a/packages/server/src/api/controllers/apikeys.js b/packages/server/src/api/controllers/apikeys.js index 96754f17cc..1c8caba1cb 100644 --- a/packages/server/src/api/controllers/apikeys.js +++ b/packages/server/src/api/controllers/apikeys.js @@ -1,56 +1,32 @@ -const fs = require("fs") -const { join } = require("../../utilities/centralPath") -const readline = require("readline") -const { budibaseAppsDir } = require("../../utilities/budibaseDir") -const env = require("../../environment") -const ENV_FILE_PATH = "/.env" +const builderDB = require("../../db/builder") exports.fetch = async function(ctx) { - ctx.status = 200 - ctx.body = { - budibase: env.BUDIBASE_API_KEY, - userId: env.USERID_API_KEY, + try { + const mainDoc = await builderDB.getBuilderMainDoc() + ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {} + } catch (err) { + /* istanbul ignore next */ + ctx.throw(400, err) } } exports.update = async function(ctx) { - const key = `${ctx.params.key.toUpperCase()}_API_KEY` + const key = ctx.params.key const value = ctx.request.body.value - // set environment variables - env._set(key, value) - - // Write to file - await updateValues([key, value]) - - ctx.status = 200 - ctx.message = `Updated ${ctx.params.key} API key succesfully.` - ctx.body = { [ctx.params.key]: ctx.request.body.value } -} - -async function updateValues([key, value]) { - let newContent = "" - let keyExists = false - let envPath = join(budibaseAppsDir(), ENV_FILE_PATH) - const readInterface = readline.createInterface({ - input: fs.createReadStream(envPath), - output: process.stdout, - console: false, - }) - readInterface.on("line", function(line) { - // Mutate lines and change API Key - if (line.startsWith(key)) { - line = `${key}=${value}` - keyExists = true + try { + const mainDoc = await builderDB.getBuilderMainDoc() + if (mainDoc.apiKeys == null) { + mainDoc.apiKeys = {} } - newContent = `${newContent}\n${line}` - }) - readInterface.on("close", function() { - // Write file here - if (!keyExists) { - // Add API Key if it doesn't exist in the file at all - newContent = `${newContent}\n${key}=${value}` + mainDoc.apiKeys[key] = value + const resp = await builderDB.setBuilderMainDoc(mainDoc) + ctx.body = { + _id: resp.id, + _rev: resp.rev, } - fs.writeFileSync(envPath, newContent) - }) + } catch (err) { + /* istanbul ignore next */ + ctx.throw(400, err) + } } diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 9e50319d5d..00678b85a0 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -1,15 +1,17 @@ const CouchDB = require("../../db") -const compileStaticAssets = require("../../utilities/builder/compileStaticAssets") const env = require("../../environment") -const { existsSync } = require("fs-extra") -const { budibaseAppsDir } = require("../../utilities/budibaseDir") const setBuilderToken = require("../../utilities/builder/setBuilderToken") -const fs = require("fs-extra") -const { join, resolve } = require("../../utilities/centralPath") const packageJson = require("../../../package.json") -const { createLinkView } = require("../../db/linkedRows") -const { createRoutingView } = require("../../utilities/routing") -const { downloadTemplate } = require("../../utilities/templates") +const { + createLinkView, + createRoutingView, + createAllSearchIndex, +} = require("../../db/views/staticViews") +const { + getTemplateStream, + createApp, + deleteApp, +} = require("../../utilities/fileSystem") const { generateAppID, getLayoutParams, @@ -20,9 +22,6 @@ const { BUILTIN_ROLE_IDS, AccessController, } = require("../../utilities/security/roles") -const { - downloadExtractComponentLibraries, -} = require("../../utilities/createAppPackage") const { BASE_LAYOUTS } = require("../../constants/layouts") const { createHomeScreen, @@ -32,11 +31,7 @@ const { cloneDeep } = require("lodash/fp") const { processObject } = require("@budibase/string-templates") const { getAllApps } = require("../../utilities") const { USERS_TABLE_SCHEMA } = require("../../constants") -const { - getDeployedApps, - getHostingInfo, - HostingTypes, -} = require("../../utilities/builder/hosting") +const { getDeployedApps } = require("../../utilities/builder/hosting") const URL_REGEX_SLASH = /\/|\\/g @@ -75,8 +70,7 @@ async function getAppUrlIfNotInUse(ctx) { url = encodeURI(`${ctx.request.body.name}`) } url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase() - const hostingInfo = await getHostingInfo() - if (hostingInfo.type === HostingTypes.CLOUD) { + if (!env.SELF_HOSTED) { return url } const deployedApps = await getDeployedApps() @@ -101,21 +95,13 @@ async function createInstance(template) { // add view for linked rows await createLinkView(appId) await createRoutingView(appId) + await createAllSearchIndex(appId) // replicate the template data to the instance DB // this is currently very hard to test, downloading and importing template files /* istanbul ignore next */ if (template && template.useTemplate === "true") { - let dbDumpReadStream - if (template.file) { - dbDumpReadStream = fs.createReadStream(template.file.path) - } else { - const templatePath = await downloadTemplate(...template.key.split("/")) - dbDumpReadStream = fs.createReadStream( - join(templatePath, "db", "dump.txt") - ) - } - const { ok } = await db.load(dbDumpReadStream) + const { ok } = await db.load(await getTemplateStream(template)) if (!ok) { throw "Error loading database dump from template." } @@ -190,10 +176,10 @@ exports.create = async function(ctx) { const instanceDb = new CouchDB(appId) await instanceDb.put(newApplication) - const newAppFolder = await createEmptyAppPackage(ctx, newApplication) + await createEmptyAppPackage(ctx, newApplication) /* istanbul ignore next */ - if (env.NODE_ENV !== "jest") { - await downloadExtractComponentLibraries(newAppFolder) + if (!env.isTest()) { + await createApp(appId) } await setBuilderToken(ctx, appId, version) @@ -222,11 +208,10 @@ exports.delete = async function(ctx) { const db = new CouchDB(ctx.params.appId) const app = await db.get(ctx.params.appId) const result = await db.destroy() - - // remove top level directory - await fs.rmdir(join(budibaseAppsDir(), ctx.params.appId), { - recursive: true, - }) + /* istanbul ignore next */ + if (!env.isTest()) { + await deleteApp(ctx.params.appId) + } ctx.status = 200 ctx.message = `Application ${app.name} deleted successfully.` @@ -234,17 +219,8 @@ exports.delete = async function(ctx) { } const createEmptyAppPackage = async (ctx, app) => { - const appsFolder = budibaseAppsDir() - const newAppFolder = resolve(appsFolder, app._id) - const db = new CouchDB(app._id) - if (existsSync(newAppFolder)) { - ctx.throw(400, "App folder already exists for this application") - } - - fs.mkdirpSync(newAppFolder) - let screensAndLayouts = [] for (let layout of BASE_LAYOUTS) { const cloned = cloneDeep(layout) @@ -260,6 +236,4 @@ const createEmptyAppPackage = async (ctx, app) => { screensAndLayouts.push(loginScreen) await db.bulkDocs(screensAndLayouts) - await compileStaticAssets(app._id) - return newAppFolder } diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index e5c0f9a029..fc486bcb50 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -45,9 +45,9 @@ exports.authenticate = async ctx => { roleId: dbUser.roleId, version: app.version, } - // if in cloud add the user api key, unless self hosted + // if in prod add the user api key, unless self hosted /* istanbul ignore next */ - if (env.CLOUD && !env.SELF_HOSTED) { + if (env.isProd() && !env.SELF_HOSTED) { const { apiKey } = await getAPIKey(ctx.user.appId) payload.apiKey = apiKey } diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.js index a83f96165b..02be10bbec 100644 --- a/packages/server/src/api/controllers/backup.js +++ b/packages/server/src/api/controllers/backup.js @@ -1,28 +1,10 @@ -const { performDump } = require("../../utilities/templates") -const path = require("path") -const os = require("os") -const fs = require("fs-extra") +const { performBackup } = require("../../utilities/fileSystem") exports.exportAppDump = async function(ctx) { const { appId } = ctx.query - const appname = decodeURI(ctx.query.appname) - - const backupsDir = path.join(os.homedir(), ".budibase", "backups") - fs.ensureDirSync(backupsDir) - const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` - await performDump({ - dir: backupsDir, - appId, - name: backupIdentifier, - }) - - ctx.status = 200 - - const backupFile = path.join(backupsDir, backupIdentifier) - ctx.attachment(backupIdentifier) - ctx.body = fs.createReadStream(backupFile) + ctx.body = await performBackup(appId, backupIdentifier) } diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.js index 092d154817..a9b88a5bff 100644 --- a/packages/server/src/api/controllers/component.js +++ b/packages/server/src/api/controllers/component.js @@ -1,44 +1,30 @@ const CouchDB = require("../../db") -const { resolve, join } = require("../../utilities/centralPath") -const { - budibaseTempDir, - budibaseAppsDir, -} = require("../../utilities/budibaseDir") +const { getComponentLibraryManifest } = require("../../utilities/fileSystem") exports.fetchAppComponentDefinitions = async function(ctx) { const appId = ctx.params.appId || ctx.appId const db = new CouchDB(appId) const app = await db.get(appId) - ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => { - let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules") + let componentManifests = await Promise.all( + app.componentLibraries.map(async library => { + let manifest = await getComponentLibraryManifest(appId, library) - if (ctx.isDev) { - appDirectory = budibaseTempDir() - } - - const componentJson = require(join( - appDirectory, - componentLibrary, - ctx.isDev ? "" : "package", - "manifest.json" - )) - - const result = {} - - // map over the components.json and add the library identifier as a key - // button -> @budibase/standard-components/button - for (let key of Object.keys(componentJson)) { - const fullComponentName = `${componentLibrary}/${key}`.toLowerCase() - result[fullComponentName] = { + return { + manifest, + library, + } + }) + ) + const definitions = {} + for (let { manifest, library } of componentManifests) { + for (let key of Object.keys(manifest)) { + const fullComponentName = `${library}/${key}`.toLowerCase() + definitions[fullComponentName] = { component: fullComponentName, - ...componentJson[key], + ...manifest[key], } } - - return { - ...acc, - ...result, - } - }, {}) + } + ctx.body = definitions } diff --git a/packages/server/src/api/controllers/deploy/awsDeploy.js b/packages/server/src/api/controllers/deploy/awsDeploy.js index 18c9279515..2d34bc1b04 100644 --- a/packages/server/src/api/controllers/deploy/awsDeploy.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -66,12 +66,7 @@ exports.deploy = async function(deployment) { const appId = deployment.getAppId() const { bucket, accountId } = deployment.getVerification() const metadata = { accountId } - const s3Client = new AWS.S3({ - params: { - Bucket: bucket, - }, - }) - await deployToObjectStore(appId, s3Client, metadata) + await deployToObjectStore(appId, bucket, metadata) } exports.replicateDb = async function(deployment) { diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index 81fa72cae5..444e7cd873 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -7,7 +7,6 @@ const { const { getWorkerUrl, getCouchUrl, - getMinioUrl, getSelfHostKey, } = require("../../../utilities/builder/hosting") @@ -45,17 +44,9 @@ exports.postDeployment = async function() { exports.deploy = async function(deployment) { const appId = deployment.getAppId() const verification = deployment.getVerification() - const objClient = new AWS.S3({ - endpoint: await getMinioUrl(), - s3ForcePathStyle: true, // needed with minio? - signatureVersion: "v4", - params: { - Bucket: verification.bucket, - }, - }) // no metadata, aws has account ID in metadata const metadata = {} - await deployToObjectStore(appId, objClient, metadata) + await deployToObjectStore(appId, verification.bucket, metadata) } exports.replicateDb = async function(deployment) { diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index 3536a6f630..de608acfb1 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -1,16 +1,24 @@ -const fs = require("fs") -const sanitize = require("sanitize-s3-objectkey") -const { walkDir } = require("../../../utilities") const { join } = require("../../../utilities/centralPath") +const fs = require("fs") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const fetch = require("node-fetch") const PouchDB = require("../../../db") const CouchDB = require("pouchdb") +const { upload } = require("../../../utilities/fileSystem") -const CONTENT_TYPE_MAP = { - html: "text/html", - css: "text/css", - js: "application/javascript", +// TODO: everything in this file is to be removed + +function walkDir(dirPath, callback) { + for (let filename of fs.readdirSync(dirPath)) { + const filePath = `${dirPath}/${filename}` + const stat = fs.lstatSync(filePath) + + if (stat.isFile()) { + callback(filePath) + } else { + walkDir(filePath, callback) + } + } } exports.fetchCredentials = async function(url, body) { @@ -34,30 +42,25 @@ exports.fetchCredentials = async function(url, body) { return json } -exports.prepareUpload = async function({ s3Key, metadata, client, file }) { - const extension = [...file.name.split(".")].pop() - const fileBytes = fs.readFileSync(file.path) - - const upload = await client - .upload({ - // windows file paths need to be converted to forward slashes for s3 - Key: sanitize(s3Key).replace(/\\/g, "/"), - Body: fileBytes, - ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], - Metadata: metadata, - }) - .promise() +exports.prepareUpload = async function({ s3Key, bucket, metadata, file }) { + const response = await upload({ + bucket, + metadata, + filename: s3Key, + path: file.path, + type: file.type, + }) return { size: file.size, name: file.name, - extension, - url: upload.Location, - key: upload.Key, + extension: [...file.name.split(".")].pop(), + url: response.Location, + key: response.Key, } } -exports.deployToObjectStore = async function(appId, objectClient, metadata) { +exports.deployToObjectStore = async function(appId, bucket, metadata) { const appAssetsPath = join(budibaseAppsDir(), appId, "public") let uploads = [] @@ -66,12 +69,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) { walkDir(appAssetsPath, function(filePath) { const filePathParts = filePath.split("/") const appAssetUpload = exports.prepareUpload({ + bucket, file: { path: filePath, name: filePathParts.pop(), }, s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), - client: objectClient, metadata, }) uploads.push(appAssetUpload) @@ -92,7 +95,7 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) { const attachmentUpload = exports.prepareUpload({ file, s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - client: objectClient, + bucket, metadata, }) diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js index 1d1884eb52..4b070cf75b 100644 --- a/packages/server/src/api/controllers/hosting.js +++ b/packages/server/src/api/controllers/hosting.js @@ -1,11 +1,11 @@ const CouchDB = require("../../db") -const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") const { getHostingInfo, getDeployedApps, HostingTypes, getAppUrl, } = require("../../utilities/builder/hosting") +const { StaticDatabases } = require("../../db/utils") exports.fetchInfo = async ctx => { ctx.body = { @@ -14,17 +14,17 @@ exports.fetchInfo = async ctx => { } exports.save = async ctx => { - const db = new CouchDB(BUILDER_CONFIG_DB) + const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name) const { type } = ctx.request.body if (type === HostingTypes.CLOUD && ctx.request.body._rev) { ctx.body = await db.remove({ ...ctx.request.body, - _id: HOSTING_DOC, + _id: StaticDatabases.BUILDER_HOSTING.baseDoc, }) } else { ctx.body = await db.put({ ...ctx.request.body, - _id: HOSTING_DOC, + _id: StaticDatabases.BUILDER_HOSTING.baseDoc, }) } } diff --git a/packages/server/src/api/controllers/query.js b/packages/server/src/api/controllers/query.js index a2badb0d0d..b9b7c85427 100644 --- a/packages/server/src/api/controllers/query.js +++ b/packages/server/src/api/controllers/query.js @@ -93,7 +93,7 @@ exports.find = async function(ctx) { const db = new CouchDB(ctx.user.appId) const query = enrichQueries(await db.get(ctx.params.queryId)) // remove properties that could be dangerous in real app - if (env.CLOUD) { + if (env.isProd()) { delete query.fields delete query.parameters delete query.schema diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index bf985fe55d..7540cc1894 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -17,6 +17,7 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") +const { QueryBuilder, search } = require("./search/utils") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -259,39 +260,46 @@ exports.search = async function(ctx) { const db = new CouchDB(appId) const { query, - pagination: { pageSize = 10, page }, + pagination: { pageSize = 10, bookmark }, } = ctx.request.body + const tableId = ctx.params.tableId - // make all strings a starts with operation rather than pure equality - for (const [key, queryVal] of Object.entries(query)) { - if (typeof queryVal === "string") { - query[key] = { - $gt: queryVal, - $lt: `${queryVal}\uffff`, - } - } + const queryBuilder = new QueryBuilder(appId) + .setLimit(pageSize) + .addTable(tableId) + if (bookmark) { + queryBuilder.setBookmark(bookmark) } - // pure equality for table - query.tableId = ctx.params.tableId - const response = await db.find({ - selector: query, - limit: pageSize, - skip: pageSize * page, - }) + let searchString + if (ctx.query && ctx.query.raw && ctx.query.raw !== "") { + searchString = queryBuilder.complete(query["RAW"]) + } else { + // make all strings a starts with operation rather than pure equality + for (const [key, queryVal] of Object.entries(query)) { + if (typeof queryVal === "string") { + queryBuilder.addString(key, queryVal) + } else { + queryBuilder.addEqual(key, queryVal) + } + } + searchString = queryBuilder.complete() + } - const rows = response.docs + const response = await search(searchString) // delete passwords from users - if (query.tableId === ViewNames.USERS) { - for (let row of rows) { + if (tableId === ViewNames.USERS) { + for (let row of response.rows) { delete row.password } } - const table = await db.get(ctx.params.tableId) - - ctx.body = await outputProcessing(appId, table, rows) + const table = await db.get(tableId) + ctx.body = { + rows: await outputProcessing(appId, table, response.rows), + bookmark: response.bookmark, + } } exports.fetchTableRows = async function(ctx) { diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js new file mode 100644 index 0000000000..1810f07198 --- /dev/null +++ b/packages/server/src/api/controllers/search/index.js @@ -0,0 +1,18 @@ +const { QueryBuilder, buildSearchUrl, search } = require("./utils") + +exports.rowSearch = async ctx => { + // this can't be done through pouch, have to reach for trusty node-fetch + const appId = ctx.user.appId + const bookmark = ctx.params.bookmark + let url + if (ctx.params.query) { + url = new QueryBuilder(appId, ctx.params.query, bookmark).complete() + } else if (ctx.params.raw) { + url = buildSearchUrl({ + appId, + query: ctx.params.raw, + bookmark, + }) + } + ctx.body = await search(url) +} diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js new file mode 100644 index 0000000000..d3ffb26be7 --- /dev/null +++ b/packages/server/src/api/controllers/search/utils.js @@ -0,0 +1,137 @@ +const { SearchIndexes } = require("../../../db/utils") +const { checkSlashesInUrl } = require("../../../utilities") +const env = require("../../../environment") +const fetch = require("node-fetch") + +/** + * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB. + * @param {string} appId The ID of the app which we will be searching within. + * @param {string} query The lucene query string which is to be used for searching. + * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was + * returned with query for next set of search results. + * @param {number} limit The number of entries to return per query. + * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled. + * @return {string} The URL which a GET can be performed on to receive results. + */ +function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) { + let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` + url += `/${SearchIndexes.ROWS}?q=${query}` + url += `&limit=${limit}` + if (!excludeDocs) { + url += "&include_docs=true" + } + if (bookmark) { + url += `&bookmark=${bookmark}` + } + return checkSlashesInUrl(url) +} + +class QueryBuilder { + constructor(appId, base) { + this.appId = appId + this.query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + ...base, + } + this.limit = 50 + this.bookmark = null + } + + setLimit(limit) { + this.limit = limit + return this + } + + setBookmark(bookmark) { + this.bookmark = bookmark + return this + } + + addString(key, partial) { + this.query.string[key] = partial + return this + } + + addFuzzy(key, fuzzy) { + this.query.fuzzy[key] = fuzzy + return this + } + + addRange(key, low, high) { + this.query.range = { + low, + high, + } + return this + } + + addEqual(key, value) { + this.query.equal[key] = value + return this + } + + addTable(tableId) { + this.query.equal.tableId = tableId + return this + } + + complete(rawQuery = null) { + let output = "" + function build(structure, queryFn) { + for (let [key, value] of Object.entries(structure)) { + if (output.length !== 0) { + output += " AND " + } + output += queryFn(key, value) + } + } + + if (this.query.string) { + build(this.query.string, (key, value) => `${key}:${value}*`) + } + if (this.query.range) { + build( + this.query.range, + (key, value) => `${key}:[${value.low} TO ${value.high}]` + ) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key, value) => `${key}:${value}~`) + } + if (this.query.equal) { + build(this.query.equal, (key, value) => `${key}:${value}`) + } + if (rawQuery) { + output = output.length === 0 ? rawQuery : `&${rawQuery}` + } + return buildSearchUrl({ + appId: this.appId, + query: output, + bookmark: this.bookmark, + limit: this.limit, + }) + } +} + +exports.search = async query => { + const response = await fetch(query, { + method: "GET", + }) + const json = await response.json() + let output = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map(row => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +exports.QueryBuilder = QueryBuilder +exports.buildSearchUrl = buildSearchUrl diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index ff1ecf5b2e..7caf6d0f7f 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -2,32 +2,47 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") +const { checkSlashesInUrl } = require("../../../utilities") const fetch = require("node-fetch") -const fs = require("fs-extra") const uuid = require("uuid") -const AWS = require("aws-sdk") const { prepareUpload } = require("../deploy/utils") const { processString } = require("@budibase/string-templates") -const { - budibaseAppsDir, - budibaseTempDir, -} = require("../../../utilities/budibaseDir") +const { budibaseTempDir } = require("../../../utilities/budibaseDir") const { getDeployedApps } = require("../../../utilities/builder/hosting") const CouchDB = require("../../../db") const setBuilderToken = require("../../../utilities/builder/setBuilderToken") -const fileProcessor = require("../../../utilities/fileProcessor") +const { loadHandlebarsFile } = require("../../../utilities/fileSystem") const env = require("../../../environment") const { OBJ_STORE_DIRECTORY } = require("../../../constants") +const fileProcessor = require("../../../utilities/fileSystem/processor") + +const BB_CDN = "https://cdn.app.budi.live/assets" function objectStoreUrl() { if (env.SELF_HOSTED) { // can use a relative url for this as all goes through the proxy (this is hosted in minio) return OBJ_STORE_DIRECTORY } else { - return "https://cdn.app.budi.live/assets" + return BB_CDN } } +function internalObjectStoreUrl() { + if (env.SELF_HOSTED) { + return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY) + } else { + return BB_CDN + } +} + +async function returnObjectStoreFile(ctx, path) { + const S3_URL = `${internalObjectStoreUrl()}/${path}` + const response = await fetch(S3_URL) + const body = await response.text() + ctx.set("Content-Type", response.headers.get("Content-Type")) + ctx.body = body +} + async function checkForSelfHostedURL(ctx) { // the "appId" component of the URL may actually be a specific self hosted URL let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}` @@ -50,106 +65,31 @@ exports.serveBuilder = async function(ctx) { await send(ctx, ctx.file, { root: ctx.devPath || builderPath }) } -exports.serveSelfHostPage = async function(ctx) { - const logo = fs.readFileSync(resolve(__dirname, "selfhost/logo.svg"), "utf8") - const hostingHbs = fs.readFileSync( - resolve(__dirname, "selfhost/index.hbs"), - "utf8" - ) - ctx.body = await processString(hostingHbs, { - logo, - }) -} - exports.uploadFile = async function(ctx) { - let files - files = + let files = ctx.request.files.file.length > 1 ? Array.from(ctx.request.files.file) : [ctx.request.files.file] - const attachmentsPath = resolve( - budibaseAppsDir(), - ctx.user.appId, - "attachments" - ) - - if (env.CLOUD) { - // remote upload - const s3 = new AWS.S3({ - params: { - Bucket: "prod-budi-app-assets", - }, - }) - - const uploads = files.map(file => { - const fileExtension = [...file.name.split(".")].pop() - const processedFileName = `${uuid.v4()}.${fileExtension}` - - return prepareUpload({ - file, - s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, - s3, - }) - }) - - ctx.body = await Promise.all(uploads) - return - } - - ctx.body = await processLocalFileUploads({ - files, - outputPath: attachmentsPath, - appId: ctx.user.appId, - }) -} - -async function processLocalFileUploads({ files, outputPath, appId }) { - // create attachments dir if it doesnt exist - !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true }) - - const filesToProcess = files.map(file => { + const uploads = files.map(async file => { const fileExtension = [...file.name.split(".")].pop() // filenames converted to UUIDs so they are unique const processedFileName = `${uuid.v4()}.${fileExtension}` - return { - name: file.name, - path: file.path, - size: file.size, - type: file.type, - processedFileName, + // need to handle image processing + await fileProcessor.process({ + ...file, extension: fileExtension, - outputPath: join(outputPath, processedFileName), - url: join("/attachments", processedFileName), - } + }) + + return prepareUpload({ + file, + s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, + bucket: "prod-budi-app-assets", + }) }) - const fileProcessOperations = filesToProcess.map(fileProcessor.process) - - const processedFiles = await Promise.all(fileProcessOperations) - - let pendingFileUploads - // local document used to track which files need to be uploaded - // db.get throws an error if the document doesn't exist - // need to use a promise to default - const db = new CouchDB(appId) - await db - .get("_local/fileuploads") - .then(data => { - pendingFileUploads = data - }) - .catch(() => { - pendingFileUploads = { _id: "_local/fileuploads", uploads: [] } - }) - - pendingFileUploads.uploads = [ - ...processedFiles, - ...pendingFileUploads.uploads, - ] - await db.put(pendingFileUploads) - - return processedFiles + ctx.body = await Promise.all(uploads) } exports.serveApp = async function(ctx) { @@ -163,12 +103,12 @@ exports.serveApp = async function(ctx) { const { head, html, css } = App.render({ title: appInfo.name, - production: env.CLOUD, + production: env.isProd(), appId, objectStoreUrl: objectStoreUrl(), }) - const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8") + const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`) ctx.body = await processString(appHbs, { head, body: html, @@ -178,71 +118,47 @@ exports.serveApp = async function(ctx) { } exports.serveAttachment = async function(ctx) { - const appId = ctx.user.appId - const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments") - - // Serve from object store - if (env.CLOUD) { - const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file) - const response = await fetch(S3_URL) - const body = await response.text() - ctx.set("Content-Type", response.headers.get("Content-Type")) - ctx.body = body - return - } - - await send(ctx, ctx.file, { root: attachmentsPath }) + await returnObjectStoreFile( + ctx, + join(ctx.user.appId, "attachments", ctx.file) + ) } exports.serveAppAsset = async function(ctx) { - // default to homedir - const appPath = resolve(budibaseAppsDir(), ctx.user.appId, "public") - - await send(ctx, ctx.file, { root: ctx.devPath || appPath }) + if (env.isDev() || env.isTest()) { + return send(ctx, ctx.file, { root: budibaseTempDir() }) + } + await returnObjectStoreFile(ctx, join(ctx.user.appId, "public", ctx.file)) } exports.serveComponentLibrary = async function(ctx) { const appId = ctx.query.appId || ctx.appId - // default to homedir - let componentLibraryPath = resolve( - budibaseAppsDir(), - appId, - "node_modules", - decodeURI(ctx.query.library), - "package", - "dist" - ) - if (ctx.isDev) { - componentLibraryPath = join( + if (env.isDev() || env.isTest()) { + const componentLibraryPath = join( budibaseTempDir(), decodeURI(ctx.query.library), "dist" ) + return send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) } - - if (env.CLOUD) { - let componentLib = "componentlibrary" - if (ctx.user.version) { - componentLib += `-${ctx.user.version}` - } else { - componentLib += `-${COMP_LIB_BASE_APP_VERSION}` - } - const S3_URL = encodeURI( - join( - objectStoreUrl(appId), - componentLib, - ctx.query.library, - "dist", - "index.js" - ) + let componentLib = "componentlibrary" + if (ctx.user.version) { + componentLib += `-${ctx.user.version}` + } else { + componentLib += `-${COMP_LIB_BASE_APP_VERSION}` + } + const S3_URL = encodeURI( + join( + objectStoreUrl(appId), + componentLib, + ctx.query.library, + "dist", + "index.js" ) - const response = await fetch(S3_URL) - const body = await response.text() - ctx.type = "application/javascript" - ctx.body = body - return - } - - await send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) + ) + const response = await fetch(S3_URL) + const body = await response.text() + ctx.type = "application/javascript" + ctx.body = body } diff --git a/packages/server/src/api/controllers/static/selfhost/index.hbs b/packages/server/src/api/controllers/static/selfhost/index.hbs deleted file mode 100644 index cbf1b8f3e5..0000000000 --- a/packages/server/src/api/controllers/static/selfhost/index.hbs +++ /dev/null @@ -1,173 +0,0 @@ - - - - - Budibase self hosting️ - - - -
- -

Get started with Budibase Self Hosting

-

Use the address in your Builder

-
-
-

📚Documentation

-

- Find out more about your self hosted platform. -

- - Documentation - -
-
-

💻Next steps

-

- Find out how to make use of your self hosted Budibase platform. -

- - Next steps - -
-
-
- - - - - - -

A Hosting Key will also be required, this can be found in your hosting properties, info found here.

-
-
- - - \ No newline at end of file diff --git a/packages/server/src/api/controllers/static/selfhost/logo.svg b/packages/server/src/api/controllers/static/selfhost/logo.svg deleted file mode 100644 index 37bfb4ff83..0000000000 --- a/packages/server/src/api/controllers/static/selfhost/logo.svg +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/packages/server/src/api/controllers/templates.js b/packages/server/src/api/controllers/templates.js index c3cfa28706..4d55bc5957 100644 --- a/packages/server/src/api/controllers/templates.js +++ b/packages/server/src/api/controllers/templates.js @@ -1,10 +1,5 @@ const fetch = require("node-fetch") -const { - downloadTemplate, - exportTemplateFromApp, - getLocalTemplates, -} = require("../../utilities/templates") -const env = require("../../environment") +const { downloadTemplate } = require("../../utilities/fileSystem") // development flag, can be used to test against templates exported locally const DEFAULT_TEMPLATES_BUCKET = @@ -12,16 +7,11 @@ const DEFAULT_TEMPLATES_BUCKET = exports.fetch = async function(ctx) { const { type = "app" } = ctx.query - - if (env.LOCAL_TEMPLATES) { - ctx.body = Object.values(getLocalTemplates()[type]) - } else { - const response = await fetch( - `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json` - ) - const json = await response.json() - ctx.body = Object.values(json.templates[type]) - } + const response = await fetch( + `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json` + ) + const json = await response.json() + ctx.body = Object.values(json.templates[type]) } // can't currently test this, have to ignore from coverage @@ -29,26 +19,9 @@ exports.fetch = async function(ctx) { exports.downloadTemplate = async function(ctx) { const { type, name } = ctx.params - if (!env.LOCAL_TEMPLATES) { - await downloadTemplate(type, name) - } + await downloadTemplate(type, name) ctx.body = { message: `template ${type}:${name} downloaded successfully.`, } } - -exports.exportTemplateFromApp = async function(ctx) { - const { appId } = ctx.user - const { templateName } = ctx.request.body - - await exportTemplateFromApp({ - appId, - templateName, - }) - - ctx.status = 200 - ctx.body = { - message: `Created template: ${templateName}`, - } -} diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index f482f3f2a6..0f6f008a1b 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -1,8 +1,6 @@ const CouchDB = require("../../../db") const viewTemplate = require("./viewBuilder") -const fs = require("fs") -const { join } = require("../../../utilities/centralPath") -const os = require("os") +const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") const { fetchView } = require("../row") const { ViewNames } = require("../../../db/utils") @@ -120,12 +118,10 @@ const controller = { // Export part let headers = Object.keys(schema) const exporter = exporters[format] - const exportedFile = exporter(headers, ctx.body) const filename = `${viewName}.${format}` - fs.writeFileSync(join(os.tmpdir(), filename), exportedFile) - + // send down the file ctx.attachment(filename) - ctx.body = fs.createReadStream(join(os.tmpdir(), filename)) + ctx.body = apiFileReturn(exporter(headers, ctx.body)) }, } diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 7628fa2077..aeceb65039 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -2,8 +2,6 @@ const Router = require("@koa/router") const authenticated = require("../middleware/authenticated") const compress = require("koa-compress") const zlib = require("zlib") -const { budibaseAppsDir } = require("../utilities/budibaseDir") -const { isDev } = require("../utilities") const { mainRoutes, authRoutes, staticRoutes } = require("./routes") const pkg = require("../../package.json") @@ -25,11 +23,9 @@ router ) .use(async (ctx, next) => { ctx.config = { - latestPackagesFolder: budibaseAppsDir(), jwtSecret: env.JWT_SECRET, useAppRootPath: true, } - ctx.isDev = isDev() await next() }) .use("/health", ctx => (ctx.status = 200)) @@ -68,8 +64,6 @@ for (let route of mainRoutes) { router.use(staticRoutes.routes()) router.use(staticRoutes.allowedMethods()) -if (!env.SELF_HOSTED && !env.CLOUD) { - router.redirect("/", "/_builder") -} +router.redirect("/", "/_builder") module.exports = router diff --git a/packages/server/src/api/routes/search.js b/packages/server/src/api/routes/search.js new file mode 100644 index 0000000000..8858a72d6e --- /dev/null +++ b/packages/server/src/api/routes/search.js @@ -0,0 +1,8 @@ +const Router = require("@koa/router") +const controller = require("../controllers/search") + +const router = Router() + +router.get("/api/search/rows", controller.rowSearch) + +module.exports = router diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index 30701d578b..64500f078d 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -1,34 +1,32 @@ const Router = require("@koa/router") const controller = require("../controllers/static") const { budibaseTempDir } = require("../../utilities/budibaseDir") -const env = require("../../environment") const authorized = require("../../middleware/authorized") const { BUILDER } = require("../../utilities/security/permissions") const usage = require("../../middleware/usageQuota") +const env = require("../../environment") const router = Router() /* istanbul ignore next */ router.param("file", async (file, ctx, next) => { ctx.file = file && file.includes(".") ? file : "index.html" - - // Serving the client library from your local dir in dev - if (ctx.isDev && ctx.file.startsWith("budibase-client")) { + if (!ctx.file.startsWith("budibase-client")) { + return next() + } + // test serves from require + if (env.isTest()) { + ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0] + } else if (env.isDev()) { + // Serving the client library from your local dir in dev ctx.devPath = budibaseTempDir() } - - await next() + return next() }) -if (env.NODE_ENV !== "production") { - router.get("/_builder/:file*", controller.serveBuilder) -} - -if (env.SELF_HOSTED) { - router.get("/", controller.serveSelfHostPage) -} - router + // TODO: for now this _builder endpoint is not authorized/secured, will need to be + .get("/_builder/:file*", controller.serveBuilder) .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile) .post("/api/attachments/upload", usage, controller.uploadFile) .get("/componentlibrary", controller.serveComponentLibrary) diff --git a/packages/server/src/api/routes/templates.js b/packages/server/src/api/routes/templates.js index 05882a22ea..6a427e8383 100644 --- a/packages/server/src/api/routes/templates.js +++ b/packages/server/src/api/routes/templates.js @@ -12,6 +12,5 @@ router authorized(BUILDER), controller.downloadTemplate ) - .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp) module.exports = router diff --git a/packages/server/src/api/routes/tests/apikeys.spec.js b/packages/server/src/api/routes/tests/apikeys.spec.js index dbee57c8b0..24402a8794 100644 --- a/packages/server/src/api/routes/tests/apikeys.spec.js +++ b/packages/server/src/api/routes/tests/apikeys.spec.js @@ -1,8 +1,5 @@ const setup = require("./utilities") const { checkBuilderEndpoint } = require("./utilities/TestFunctions") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") -const fs = require("fs") -const path = require("path") describe("/api/keys", () => { let request = setup.getRequest() @@ -16,12 +13,14 @@ describe("/api/keys", () => { describe("fetch", () => { it("should allow fetching", async () => { - const res = await request - .get(`/api/keys`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body).toBeDefined() + await setup.switchToSelfHosted(async () => { + const res = await request + .get(`/api/keys`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body).toBeDefined() + }) }) it("should check authorization for builder", async () => { @@ -35,17 +34,18 @@ describe("/api/keys", () => { describe("update", () => { it("should allow updating a value", async () => { - fs.writeFileSync(path.join(budibaseAppsDir(), ".env"), "TEST_API_KEY=thing") - const res = await request - .put(`/api/keys/TEST`) - .send({ - value: "test" - }) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body["TEST"]).toEqual("test") - expect(process.env.TEST_API_KEY).toEqual("test") + await setup.switchToSelfHosted(async () => { + const res = await request + .put(`/api/keys/TEST`) + .send({ + value: "test" + }) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body._id).toBeDefined() + expect(res.body._rev).toBeDefined() + }) }) it("should check authorization for builder", async () => { diff --git a/packages/server/src/api/routes/tests/auth.spec.js b/packages/server/src/api/routes/tests/auth.spec.js index 0eb0b6d851..13695d596d 100644 --- a/packages/server/src/api/routes/tests/auth.spec.js +++ b/packages/server/src/api/routes/tests/auth.spec.js @@ -1,4 +1,3 @@ -const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") describe("/authenticate", () => { diff --git a/packages/server/src/api/routes/tests/backup.spec.js b/packages/server/src/api/routes/tests/backup.spec.js index d603990294..4e586bfd08 100644 --- a/packages/server/src/api/routes/tests/backup.spec.js +++ b/packages/server/src/api/routes/tests/backup.spec.js @@ -1,6 +1,8 @@ const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") +jest.mock("../../../utilities/fileSystem/utilities") + describe("/backups", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -14,7 +16,7 @@ describe("/backups", () => { describe("exportAppDump", () => { it("should be able to export app", async () => { const res = await request - .get(`/api/backups/export?appId=${config.getAppId()}`) + .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`) .set(config.defaultHeaders()) .expect(200) expect(res.text).toBeDefined() diff --git a/packages/server/src/api/routes/tests/cloud.spec.js b/packages/server/src/api/routes/tests/cloud.spec.js deleted file mode 100644 index 3cb65ed819..0000000000 --- a/packages/server/src/api/routes/tests/cloud.spec.js +++ /dev/null @@ -1,16 +0,0 @@ -const setup = require("./utilities") - -describe("test things in the Cloud/Self hosted", () => { - describe("test self hosted static page", () => { - it("should be able to load the static page", async () => { - await setup.switchToCloudForFunction(async () => { - let request = setup.getRequest() - let config = setup.getConfig() - await config.init() - const res = await request.get(`/`).expect(200) - expect(res.text.includes("Budibase self hosting️")).toEqual(true) - setup.afterAll() - }) - }) - }) -}) diff --git a/packages/server/src/api/routes/tests/component.spec.js b/packages/server/src/api/routes/tests/component.spec.js index cabf9f8223..35394a1537 100644 --- a/packages/server/src/api/routes/tests/component.spec.js +++ b/packages/server/src/api/routes/tests/component.spec.js @@ -1,8 +1,5 @@ const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") -const fs = require("fs") -const { resolve, join } = require("path") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") describe("/component", () => { let request = setup.getRequest() @@ -14,23 +11,8 @@ describe("/component", () => { await config.init() }) - function mock() { - const manifestFile = "manifest.json" - const appId = config.getAppId() - const libraries = [join("@budibase", "standard-components")] - for (let library of libraries) { - let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules", library, "package") - fs.mkdirSync(appDirectory, { recursive: true }) - - const file = require.resolve(library).split(join("dist", "index.js"))[0] + manifestFile - fs.copyFileSync(file, join(appDirectory, manifestFile)) - } - } - describe("fetch definitions", () => { it("should be able to fetch definitions", async () => { - // have to "mock" the files required - mock() const res = await request .get(`/${config.getAppId()}/components/definitions`) .set(config.defaultHeaders()) diff --git a/packages/server/src/api/routes/tests/hosting.spec.js b/packages/server/src/api/routes/tests/hosting.spec.js index 2da5b11778..99a44640bc 100644 --- a/packages/server/src/api/routes/tests/hosting.spec.js +++ b/packages/server/src/api/routes/tests/hosting.spec.js @@ -107,17 +107,16 @@ describe("/hosting", () => { }) describe("getDeployedApps", () => { - it("should get apps when in builder", async () => { - const res = await request + it("should fail when not self hosted", async () => { + await request .get(`/api/hosting/apps`) .set(config.defaultHeaders()) .expect("Content-Type", /json/) - .expect(200) - expect(res.body.app1).toEqual({url: "/app1"}) + .expect(400) }) it("should get apps when in cloud", async () => { - await setup.switchToCloudForFunction(async () => { + await setup.switchToSelfHosted(async () => { const res = await request .get(`/api/hosting/apps`) .set(config.defaultHeaders()) diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index 87938c6a37..2755c0230e 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -89,7 +89,7 @@ describe("/queries", () => { }) it("should find a query in cloud", async () => { - await setup.switchToCloudForFunction(async () => { + await setup.switchToSelfHosted(async () => { const query = await config.createQuery() const res = await request .get(`/api/queries/${query._id}`) diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.js index 652a17366d..6a1c309c39 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.js @@ -2,6 +2,9 @@ const { outputProcessing } = require("../../../utilities/rowProcessor") const setup = require("./utilities") const { basicRow } = setup.structures +// mock the fetch for the search system +jest.mock("node-fetch") + describe("/rows", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -303,25 +306,19 @@ describe("/rows", () => { describe("search", () => { it("should run a search on the table", async () => { - const row = await config.createRow() - // add another row that shouldn't be found - await config.createRow({ - ...basicRow(), - name: "Other Contact", - }) const res = await request .post(`/api/${table._id}/rows/search`) .send({ query: { name: "Test", }, - pagination: { pageSize: 25, page: 0 } + pagination: { pageSize: 25 } }) .set(config.defaultHeaders()) .expect('Content-Type', /json/) .expect(200) - expect(res.body.length).toEqual(1) - expect(res.body[0]._id).toEqual(row._id) + expect(res.body.rows.length).toEqual(1) + expect(res.body.bookmark).toBeDefined() }) }) @@ -410,7 +407,7 @@ describe("/rows", () => { tableId: table._id, }) // the environment needs configured for this - await setup.switchToCloudForFunction(async () => { + await setup.switchToSelfHosted(async () => { const enriched = await outputProcessing(config.getAppId(), table, [row]) expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${config.getAppId()}/test/thing`) }) diff --git a/packages/server/src/api/routes/tests/templates.spec.js b/packages/server/src/api/routes/tests/templates.spec.js index f0d26bc7db..30e337e855 100644 --- a/packages/server/src/api/routes/tests/templates.spec.js +++ b/packages/server/src/api/routes/tests/templates.spec.js @@ -1,7 +1,4 @@ const setup = require("./utilities") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") -const fs = require("fs") -const { join } = require("path") describe("/templates", () => { let request = setup.getRequest() @@ -24,26 +21,4 @@ describe("/templates", () => { expect(Array.isArray(res.body)).toEqual(true) }) }) - - describe("export", () => { - it("should be able to export the basic app", async () => { - const res = await request - .post(`/api/templates`) - .send({ - templateName: "test", - }) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body.message).toEqual("Created template: test") - const dir = join( - budibaseAppsDir(), - "templates", - "app", - "test", - "db" - ) - expect(fs.existsSync(dir)).toEqual(true) - }) - }) }) \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/utilities/index.js b/packages/server/src/api/routes/tests/utilities/index.js index ed5c98cc48..3bd3886a31 100644 --- a/packages/server/src/api/routes/tests/utilities/index.js +++ b/packages/server/src/api/routes/tests/utilities/index.js @@ -35,18 +35,18 @@ exports.getConfig = () => { return config } -exports.switchToCloudForFunction = async func => { +exports.switchToSelfHosted = async func => { // self hosted stops any attempts to Dynamo - env.CLOUD = true - env.SELF_HOSTED = true + env._set("NODE_ENV", "production") + env._set("SELF_HOSTED", true) let error try { await func() } catch (err) { error = err } - env.CLOUD = false - env.SELF_HOSTED = false + env._set("NODE_ENV", "jest") + env._set("SELF_HOSTED", false) // don't throw error until after reset if (error) { throw error diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 8bbea00474..e5e9b77084 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -9,6 +9,7 @@ const env = require("./environment") const eventEmitter = require("./events") const automations = require("./automations/index") const Sentry = require("@sentry/node") +const fileSystem = require("./utilities/fileSystem") const app = new Koa() @@ -65,6 +66,7 @@ module.exports = server.listen(env.PORT || 0, async () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) env._set("PORT", server.address().port) eventEmitter.emitPort(env.PORT) + fileSystem.init() await automations.init() }) diff --git a/packages/server/src/automations/actions.js b/packages/server/src/automations/actions.js index ee57f5a109..9675568808 100644 --- a/packages/server/src/automations/actions.js +++ b/packages/server/src/automations/actions.js @@ -5,17 +5,12 @@ const deleteRow = require("./steps/deleteRow") const createUser = require("./steps/createUser") const outgoingWebhook = require("./steps/outgoingWebhook") const env = require("../environment") -const download = require("download") -const fetch = require("node-fetch") -const { join } = require("../utilities/centralPath") -const os = require("os") -const fs = require("fs") const Sentry = require("@sentry/node") +const { + automationInit, + getExternalAutomationStep, +} = require("../utilities/fileSystem") -const DEFAULT_BUCKET = - "https://prod-budi-automations.s3-eu-west-1.amazonaws.com" -const DEFAULT_DIRECTORY = ".budibase-automations" -const AUTOMATION_MANIFEST = "manifest.json" const BUILTIN_ACTIONS = { SEND_EMAIL: sendEmail.run, CREATE_ROW: createRow.run, @@ -33,8 +28,6 @@ const BUILTIN_DEFINITIONS = { OUTGOING_WEBHOOK: outgoingWebhook.definition, } -let AUTOMATION_BUCKET = env.AUTOMATION_BUCKET -let AUTOMATION_DIRECTORY = env.AUTOMATION_DIRECTORY let MANIFEST = null /* istanbul ignore next */ @@ -42,22 +35,13 @@ function buildBundleName(pkgName, version) { return `${pkgName}@${version}.min.js` } -/* istanbul ignore next */ -async function downloadPackage(name, version, bundleName) { - await download( - `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`, - AUTOMATION_DIRECTORY - ) - return require(join(AUTOMATION_DIRECTORY, bundleName)) -} - /* istanbul ignore next */ module.exports.getAction = async function(actionName) { if (BUILTIN_ACTIONS[actionName] != null) { return BUILTIN_ACTIONS[actionName] } // worker pools means that a worker may not have manifest - if (env.CLOUD && MANIFEST == null) { + if (env.isProd() && MANIFEST == null) { MANIFEST = await module.exports.init() } // env setup to get async packages @@ -66,28 +50,12 @@ module.exports.getAction = async function(actionName) { } const pkg = MANIFEST.packages[actionName] const bundleName = buildBundleName(pkg.stepId, pkg.version) - try { - return require(join(AUTOMATION_DIRECTORY, bundleName)) - } catch (err) { - return downloadPackage(pkg.stepId, pkg.version, bundleName) - } + return getExternalAutomationStep(pkg.stepId, pkg.version, bundleName) } module.exports.init = async function() { - // set defaults - if (!AUTOMATION_DIRECTORY) { - AUTOMATION_DIRECTORY = join(os.homedir(), DEFAULT_DIRECTORY) - } - if (!AUTOMATION_BUCKET) { - AUTOMATION_BUCKET = DEFAULT_BUCKET - } - if (!fs.existsSync(AUTOMATION_DIRECTORY)) { - fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true }) - } - // env setup to get async packages try { - let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`) - MANIFEST = await response.json() + MANIFEST = await automationInit() module.exports.DEFINITIONS = MANIFEST && MANIFEST.packages ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS) diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 9aba399133..d67227e6ac 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -34,10 +34,10 @@ module.exports.init = async function() { await actions.init() triggers.automationQueue.process(async job => { try { - if (env.CLOUD && job.data.automation && !env.SELF_HOSTED) { + if (env.USE_QUOTAS) { job.data.automation.apiKey = await updateQuota(job.data.automation) } - if (env.BUDIBASE_ENVIRONMENT === "PRODUCTION") { + if (env.isProd()) { await runWorker(job) } else { await singleThread(job) diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index ef136e1131..aa910dbb42 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -85,7 +85,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) { inputs.row.tableId, inputs.row ) - if (env.CLOUD) { + if (env.isProd()) { await usage.update(apiKey, usage.Properties.ROW, 1) } await rowController.save(ctx) diff --git a/packages/server/src/automations/steps/createUser.js b/packages/server/src/automations/steps/createUser.js index 8496967105..147a3f7868 100644 --- a/packages/server/src/automations/steps/createUser.js +++ b/packages/server/src/automations/steps/createUser.js @@ -72,7 +72,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) { } try { - if (env.CLOUD) { + if (env.isProd()) { await usage.update(apiKey, usage.Properties.USER, 1) } await userController.create(ctx) diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index ea4d60a04e..57555ddaad 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -70,7 +70,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) { } try { - if (env.CLOUD) { + if (env.isProd()) { await usage.update(apiKey, usage.Properties.ROW, -1) } await rowController.destroy(ctx) diff --git a/packages/server/src/automations/tests/automation.spec.js b/packages/server/src/automations/tests/automation.spec.js index f4d3b4c865..2e9bb16e55 100644 --- a/packages/server/src/automations/tests/automation.spec.js +++ b/packages/server/src/automations/tests/automation.spec.js @@ -47,27 +47,23 @@ describe("Run through some parts of the automations system", () => { expect(thread).toHaveBeenCalled() }) - it("should be able to init in cloud", async () => { - env.CLOUD = true - env.BUDIBASE_ENVIRONMENT = "PRODUCTION" - await triggers.externalTrigger(basicAutomation(), { a: 1 }) - await wait(100) - // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined - expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1) - expect(workerJob).toBeDefined() - env.BUDIBASE_ENVIRONMENT = "JEST" - env.CLOUD = false + it("should be able to init in prod", async () => { + await setup.runInProd(async () => { + await triggers.externalTrigger(basicAutomation(), { a: 1 }) + await wait(100) + // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined + expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1) + expect(workerJob).toBeDefined() + }) }) it("try error scenario", async () => { - env.CLOUD = true - env.BUDIBASE_ENVIRONMENT = "PRODUCTION" - // the second call will throw an error - await triggers.externalTrigger(basicAutomation(), { a: 1 }) - await wait(100) - expect(console.error).toHaveBeenCalled() - env.BUDIBASE_ENVIRONMENT = "JEST" - env.CLOUD = false + await setup.runInProd(async () => { + // the second call will throw an error + await triggers.externalTrigger(basicAutomation(), { a: 1 }) + await wait(100) + expect(console.error).toHaveBeenCalled() + }) }) it("should be able to check triggering row filling", async () => { diff --git a/packages/server/src/automations/tests/createRow.spec.js b/packages/server/src/automations/tests/createRow.spec.js index 0be2803e47..c01d630bed 100644 --- a/packages/server/src/automations/tests/createRow.spec.js +++ b/packages/server/src/automations/tests/createRow.spec.js @@ -42,12 +42,12 @@ describe("test the create row action", () => { }) it("check usage quota attempts", async () => { - env.CLOUD = true - await setup.runStep(setup.actions.CREATE_ROW.stepId, { - row + await setup.runInProd(async () => { + await setup.runStep(setup.actions.CREATE_ROW.stepId, { + row + }) + expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1) }) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1) - env.CLOUD = false }) it("should check invalid inputs return an error", async () => { diff --git a/packages/server/src/automations/tests/createUser.spec.js b/packages/server/src/automations/tests/createUser.spec.js index 5f65e260a9..f188c31aa4 100644 --- a/packages/server/src/automations/tests/createUser.spec.js +++ b/packages/server/src/automations/tests/createUser.spec.js @@ -35,9 +35,9 @@ describe("test the create user action", () => { }) it("check usage quota attempts", async () => { - env.CLOUD = true - await setup.runStep(setup.actions.CREATE_USER.stepId, user) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1) - env.CLOUD = false + await setup.runInProd(async () => { + await setup.runStep(setup.actions.CREATE_USER.stepId, user) + expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1) + }) }) }) diff --git a/packages/server/src/automations/tests/deleteRow.spec.js b/packages/server/src/automations/tests/deleteRow.spec.js index 0d5ff47ed8..2a300cbd8c 100644 --- a/packages/server/src/automations/tests/deleteRow.spec.js +++ b/packages/server/src/automations/tests/deleteRow.spec.js @@ -36,10 +36,10 @@ describe("test the delete row action", () => { }) it("check usage quota attempts", async () => { - env.CLOUD = true - await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1) - env.CLOUD = false + await setup.runInProd(async () => { + await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) + expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1) + }) }) it("should check invalid inputs return an error", async () => { diff --git a/packages/server/src/automations/tests/utilities/index.js b/packages/server/src/automations/tests/utilities/index.js index ad149d6bde..ab9de55430 100644 --- a/packages/server/src/automations/tests/utilities/index.js +++ b/packages/server/src/automations/tests/utilities/index.js @@ -2,6 +2,7 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration") const actions = require("../../actions") const logic = require("../../logic") const emitter = require("../../../events/index") +const env = require("../../../environment") let config @@ -16,6 +17,22 @@ exports.afterAll = () => { config.end() } +exports.runInProd = async fn => { + env._set("NODE_ENV", "production") + env._set("USE_QUOTAS", 1) + let error + try { + await fn() + } catch (err) { + error = err + } + env._set("NODE_ENV", "jest") + env._set("USE_QUOTAS", null) + if (error) { + throw error + } +} + exports.runStep = async function runStep(stepId, inputs) { let step if ( diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 46fb5cb649..379cfb8aa5 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -80,8 +80,6 @@ exports.AutoFieldSubTypes = { AUTO_ID: "autoID", } -exports.BUILDER_CONFIG_DB = "builder-config-db" -exports.HOSTING_DOC = "hosting-doc" exports.OBJ_STORE_DIRECTORY = "/app-assets/assets" exports.BaseQueryVerbs = { CREATE: "create", @@ -89,3 +87,9 @@ exports.BaseQueryVerbs = { UPDATE: "update", DELETE: "delete", } + +exports.ObjectStoreBuckets = { + BACKUPS: "backups", + APPS: "prod-budi-app-assets", + TEMPLATES: "templates", +} diff --git a/packages/server/src/db/builder.js b/packages/server/src/db/builder.js new file mode 100644 index 0000000000..d2bbcd404b --- /dev/null +++ b/packages/server/src/db/builder.js @@ -0,0 +1,38 @@ +const CouchDB = require("./index") +const { StaticDatabases } = require("./utils") +const env = require("../environment") + +const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted." +const BUILDER_DB = StaticDatabases.BUILDER + +/** + * This is the builder database, right now this is a single, static database + * that is present across the whole system and determines some core functionality + * for the builder (e.g. storage of API keys). This has been limited to self hosting + * as it doesn't make as much sense against the currently design Cloud system. + */ + +exports.getBuilderMainDoc = async () => { + if (!env.SELF_HOSTED) { + throw SELF_HOST_ERR + } + const db = new CouchDB(BUILDER_DB.name) + try { + return await db.get(BUILDER_DB.baseDoc) + } catch (err) { + // doesn't exist yet, nothing to get + return { + _id: BUILDER_DB.baseDoc, + } + } +} + +exports.setBuilderMainDoc = async doc => { + if (!env.SELF_HOSTED) { + throw SELF_HOST_ERR + } + // make sure to override the ID + doc._id = BUILDER_DB.baseDoc + const db = new CouchDB(BUILDER_DB.name) + return db.put(doc) +} diff --git a/packages/server/src/db/client.js b/packages/server/src/db/client.js index f6dea33a40..3e3a4f50fe 100644 --- a/packages/server/src/db/client.js +++ b/packages/server/src/db/client.js @@ -1,12 +1,10 @@ const PouchDB = require("pouchdb") const replicationStream = require("pouchdb-replication-stream") const allDbs = require("pouchdb-all-dbs") -const { budibaseAppsDir } = require("../utilities/budibaseDir") const find = require("pouchdb-find") const env = require("../environment") -const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/` -const isInMemory = env.NODE_ENV === "jest" +const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/" PouchDB.plugin(replicationStream.plugin) PouchDB.plugin(find) @@ -14,10 +12,10 @@ PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) let POUCH_DB_DEFAULTS = { prefix: COUCH_DB_URL, - skip_setup: !!env.CLOUD, + skip_setup: env.isProd(), } -if (isInMemory) { +if (env.isTest()) { PouchDB.plugin(require("pouchdb-adapter-memory")) POUCH_DB_DEFAULTS = { prefix: undefined, diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js index fcba726f84..19924b1a7e 100644 --- a/packages/server/src/db/dynamoClient.js +++ b/packages/server/src/db/dynamoClient.js @@ -1,4 +1,4 @@ -let _ = require("lodash") +let { merge } = require("lodash") let env = require("../environment") const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1" @@ -38,7 +38,7 @@ class Table { params.Key[this._sort] = sort } if (otherProps) { - params = _.merge(params, otherProps) + params = merge(params, otherProps) } let response = await docClient.get(params).promise() return response.Item @@ -77,7 +77,7 @@ class Table { params.ConditionExpression += "attribute_exists(#PRIMARY)" } if (otherProps) { - params = _.merge(params, otherProps) + params = merge(params, otherProps) } return docClient.update(params).promise() } @@ -94,7 +94,7 @@ class Table { Item: item, } if (otherProps) { - params = _.merge(params, otherProps) + params = merge(params, otherProps) } return docClient.put(params).promise() } @@ -119,7 +119,7 @@ exports.init = endpoint => { exports.apiKeyTable = new Table(TableInfo.API_KEYS) exports.userTable = new Table(TableInfo.USERS) -if (env.CLOUD) { +if (env.isProd()) { exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`) } else { env._set("AWS_ACCESS_KEY_ID", "KEY_ID") diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index 052386ba86..8de2093fb2 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -27,7 +27,7 @@ const EventType = { } exports.EventType = EventType -// re-export utils here for ease of use +// re-export search here for ease of use exports.IncludeDocs = IncludeDocs exports.getLinkDocuments = getLinkDocuments exports.createLinkView = createLinkView diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 2d0722d83a..4c31f0398e 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -3,6 +3,18 @@ const newid = require("./newid") const UNICODE_MAX = "\ufff0" const SEPARATOR = "_" +const StaticDatabases = { + BUILDER: { + name: "builder-db", + baseDoc: "builder-doc", + }, + // TODO: needs removed + BUILDER_HOSTING: { + name: "builder-config-db", + baseDoc: "hosting-doc", + }, +} + const DocumentTypes = { TABLE: "ta", ROW: "ro", @@ -25,10 +37,16 @@ const ViewNames = { USERS: "ta_users", } +const SearchIndexes = { + ROWS: "rows", +} + +exports.StaticDatabases = StaticDatabases exports.ViewNames = ViewNames exports.DocumentTypes = DocumentTypes exports.SEPARATOR = SEPARATOR exports.UNICODE_MAX = UNICODE_MAX +exports.SearchIndexes = SearchIndexes exports.getQueryIndex = viewName => { return `database/${viewName}` diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index a0d0fbb239..305d042217 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -1,5 +1,10 @@ const CouchDB = require("../index") -const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils") +const { + DocumentTypes, + SEPARATOR, + ViewNames, + SearchIndexes, +} = require("../utils") const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR /************************************************** @@ -72,3 +77,42 @@ exports.createRoutingView = async appId => { } await db.put(designDoc) } + +async function searchIndex(appId, indexName, fnString) { + const db = new CouchDB(appId) + const designDoc = await db.get("_design/database") + designDoc.indexes = { + [indexName]: { + index: fnString, + }, + } + await db.put(designDoc) +} + +exports.createAllSearchIndex = async appId => { + await searchIndex( + appId, + SearchIndexes.ROWS, + function(doc) { + function idx(input, prev) { + for (let key of Object.keys(input)) { + const idxKey = prev != null ? `${prev}.${key}` : key + if (key === "_id" || key === "_rev") { + continue + } + if (typeof input[key] !== "object") { + // eslint-disable-next-line no-undef + index(idxKey, input[key], { store: true }) + } else { + idx(input[key], idxKey) + } + } + } + if (doc._id.startsWith("ro_")) { + // eslint-disable-next-line no-undef + index("default", doc._id) + idx(doc) + } + }.toString() + ) +} diff --git a/packages/server/src/electron.js b/packages/server/src/electron.js index 290c13e6a0..49ef8758c3 100644 --- a/packages/server/src/electron.js +++ b/packages/server/src/electron.js @@ -1,112 +1,112 @@ -const { app, BrowserWindow, shell, dialog } = require("electron") -const { join } = require("./utilities/centralPath") -const isDev = require("electron-is-dev") -const { autoUpdater } = require("electron-updater") -const unhandled = require("electron-unhandled") -const { existsSync } = require("fs-extra") -const initialiseBudibase = require("./utilities/initialiseBudibase") -const { budibaseAppsDir } = require("./utilities/budibaseDir") -const { openNewGitHubIssue, debugInfo } = require("electron-util") -const eventEmitter = require("./events") +// const { app, BrowserWindow, shell, dialog } = require("electron") +// const { join } = require("./utilities/centralPath") +// const isDev = require("electron-is-dev") +// const { autoUpdater } = require("electron-updater") +// const unhandled = require("electron-unhandled") +// const { existsSync } = require("fs-extra") +// const initialiseBudibase = require("./utilities/initialiseBudibase") +// const { budibaseAppsDir } = require("./utilities/budibaseDir") +// const { openNewGitHubIssue, debugInfo } = require("electron-util") +// const eventEmitter = require("./events") -const budibaseDir = budibaseAppsDir() -const envFile = join(budibaseDir, ".env") +// const budibaseDir = budibaseAppsDir() +// const envFile = join(budibaseDir, ".env") -async function startApp() { - if (!existsSync(envFile)) { - await initialiseBudibase({ dir: budibaseDir }) - } - // evict environment from cache, so it reloads when next asked - delete require.cache[require.resolve("./environment")] - // store the port incase its going to get overridden - const port = process.env.PORT - require("dotenv").config({ path: envFile }) - // overwrite the port - don't want to use dotenv for the port - require("./environment")._set("PORT", port) +// async function startApp() { +// if (!existsSync(envFile)) { +// await initialiseBudibase({ dir: budibaseDir }) +// } +// // evict environment from cache, so it reloads when next asked +// delete require.cache[require.resolve("./environment")] +// // store the port incase its going to get overridden +// const port = process.env.PORT +// require("dotenv").config({ path: envFile }) +// // overwrite the port - don't want to use dotenv for the port +// require("./environment")._set("PORT", port) - unhandled({ - showDialog: true, - reportButton: error => { - openNewGitHubIssue({ - title: error.message, - user: "Budibase", - labels: ["error-report"], - repo: "budibase", - body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${ - error.stack - }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`, - }) - }, - }) +// unhandled({ +// showDialog: true, +// reportButton: error => { +// openNewGitHubIssue({ +// title: error.message, +// user: "Budibase", +// labels: ["error-report"], +// repo: "budibase", +// body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${ +// error.stack +// }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`, +// }) +// }, +// }) - let win +// let win - function handleRedirect(e, url) { - e.preventDefault() - shell.openExternal(url) - } +// function handleRedirect(e, url) { +// e.preventDefault() +// shell.openExternal(url) +// } - async function createWindow() { - app.server = require("./app") - eventEmitter.on("internal:port", port => { - const APP_URL = `http://localhost:${port}/_builder` - const APP_TITLE = "Budibase Builder" - win = new BrowserWindow({ - width: 1920, - height: 1080, - icon: join(__dirname, "..", "build", "icons", "512x512.png"), - }) - win.setTitle(APP_TITLE) - win.loadURL(APP_URL) - if (isDev) { - win.webContents.openDevTools() - } else { - autoUpdater.checkForUpdatesAndNotify() - } +// async function createWindow() { +// app.server = require("./app") +// eventEmitter.on("internal:port", port => { +// const APP_URL = `http://localhost:${port}/_builder` +// const APP_TITLE = "Budibase Builder" +// win = new BrowserWindow({ +// width: 1920, +// height: 1080, +// icon: join(__dirname, "..", "build", "icons", "512x512.png"), +// }) +// win.setTitle(APP_TITLE) +// win.loadURL(APP_URL) +// if (isDev) { +// win.webContents.openDevTools() +// } else { +// autoUpdater.checkForUpdatesAndNotify() +// } - // open _blank in default browser - win.webContents.on("new-window", handleRedirect) - win.webContents.on("will-navigate", handleRedirect) - }) - } +// // open _blank in default browser +// win.webContents.on("new-window", handleRedirect) +// win.webContents.on("will-navigate", handleRedirect) +// }) +// } - app.whenReady().then(createWindow) +// app.whenReady().then(createWindow) - // Quit when all windows are closed. - app.on("window-all-closed", () => { - // On macOS it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q - if (process.platform !== "darwin") { - app.server.close() - app.quit() - } - }) +// // Quit when all windows are closed. +// app.on("window-all-closed", () => { +// // On macOS it is common for applications and their menu bar +// // to stay active until the user quits explicitly with Cmd + Q +// if (process.platform !== "darwin") { +// app.server.close() +// app.quit() +// } +// }) - app.on("activate", () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) createWindow() - }) -} +// app.on("activate", () => { +// // On macOS it's common to re-create a window in the app when the +// // dock icon is clicked and there are no other windows open. +// if (win === null) createWindow() +// }) +// } -autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => { - const dialogOpts = { - type: "info", - buttons: ["Restart", "Later"], - title: "Budibase Update Available", - message: process.platform === "win32" ? releaseNotes : releaseName, - detail: - "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.", - } +// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => { +// const dialogOpts = { +// type: "info", +// buttons: ["Restart", "Later"], +// title: "Budibase Update Available", +// message: process.platform === "win32" ? releaseNotes : releaseName, +// detail: +// "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.", +// } - dialog.showMessageBox(dialogOpts).then(returnValue => { - if (returnValue.response === 0) autoUpdater.quitAndInstall() - }) -}) +// dialog.showMessageBox(dialogOpts).then(returnValue => { +// if (returnValue.response === 0) autoUpdater.quitAndInstall() +// }) +// }) -autoUpdater.on("error", message => { - console.error("There was a problem updating the application") - console.error(message) -}) +// autoUpdater.on("error", message => { +// console.error("There was a problem updating the application") +// console.error(message) +// }) -startApp() +// startApp() diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 4faaabe6ab..dc15bc8a9a 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -1,45 +1,64 @@ -const { resolve, join } = require("./utilities/centralPath") -const { homedir } = require("os") -const { app } = require("electron") +function isTest() { + return ( + process.env.NODE_ENV === "jest" || + process.env.NODE_ENV === "cypress" || + process.env.JEST_WORKER_ID != null + ) +} + +function isDev() { + return ( + process.env.NODE_ENV !== "production" && + process.env.BUDIBASE_ENVIRONMENT !== "production" + ) +} let LOADED = false - -if (!LOADED) { - const homeDir = app ? app.getPath("home") : homedir() - const budibaseDir = join(homeDir, ".budibase") - process.env.BUDIBASE_DIR = budibaseDir - require("dotenv").config({ path: resolve(budibaseDir, ".env") }) +if (!LOADED && isDev() && !isTest()) { + require("dotenv").config() LOADED = true } module.exports = { - CLIENT_ID: process.env.CLIENT_ID, - NODE_ENV: process.env.NODE_ENV, - JWT_SECRET: process.env.JWT_SECRET, - BUDIBASE_DIR: process.env.BUDIBASE_DIR, + // important PORT: process.env.PORT, + JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL, + MINIO_URL: process.env.MINIO_URL, + WORKER_URL: process.env.WORKER_URL, + SELF_HOSTED: process.env.SELF_HOSTED, + AWS_REGION: process.env.AWS_REGION, + ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, + MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, + MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, + USE_QUOTAS: process.env.USE_QUOTAS, + // environment + NODE_ENV: process.env.NODE_ENV, + JEST_WORKER_ID: process.env.JEST_WORKER_ID, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, + // minor SALT_ROUNDS: process.env.SALT_ROUNDS, LOGGER: process.env.LOGGER, LOG_LEVEL: process.env.LOG_LEVEL, AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY, AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET, - BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, - CLOUD: process.env.CLOUD, - SELF_HOSTED: process.env.SELF_HOSTED, - WORKER_URL: process.env.WORKER_URL, - HOSTING_KEY: process.env.HOSTING_KEY, DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, - AWS_REGION: process.env.AWS_REGION, - DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, + // old - to remove + CLIENT_ID: process.env.CLIENT_ID, + BUDIBASE_DIR: process.env.BUDIBASE_DIR, + DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY, USERID_API_KEY: process.env.USERID_API_KEY, - ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, - DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, - LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, + DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, + HOSTING_KEY: process.env.HOSTING_KEY, _set(key, value) { process.env[key] = value module.exports[key] = value }, + isTest, + isDev, + isProd: () => { + return !isDev() + }, } diff --git a/packages/server/src/index.js b/packages/server/src/index.js index f5506aa2a3..3aa210f96c 100644 --- a/packages/server/src/index.js +++ b/packages/server/src/index.js @@ -1,17 +1,10 @@ -const { budibaseTempDir } = require("./utilities/budibaseDir") -const { isDev } = require("./utilities") - const fixPath = require("fix-path") -const fs = require("fs") - -async function runServer() { - if (isDev() && !fs.existsSync(budibaseTempDir())) { - console.error( - "Please run a build before attempting to run server independently to fill 'tmp' directory." - ) - process.exit(-1) - } +const { checkDevelopmentEnvironment } = require("./utilities/fileSystem") +function runServer() { + // this will shutdown the system if development environment not ready + // will print an error explaining what to do + checkDevelopmentEnvironment() fixPath() require("./app") } diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index 2a1caef2a2..564896080e 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -13,19 +13,12 @@ const { AuthTypes } = require("../constants") const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER] -const LOCAL_PASS = new RegExp(["webhooks/trigger"].join("|")) - function hasResource(ctx) { return ctx.resourceId != null } module.exports = (permType, permLevel = null) => async (ctx, next) => { - // webhooks can pass locally - if (!env.CLOUD && LOCAL_PASS.test(ctx.request.url)) { - return next() - } - - if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) { + if (env.isProd() && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) { // api key header passed by external webhook if (await isAPIKeyValid(ctx.headers["x-api-key"])) { ctx.auth = { @@ -41,20 +34,23 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { return ctx.throw(403, "API key invalid") } - // don't expose builder endpoints in the cloud - if (env.CLOUD && permType === PermissionTypes.BUILDER) return - if (!ctx.user) { return ctx.throw(403, "No user info found") } const role = ctx.user.role + const isBuilder = role._id === BUILTIN_ROLE_IDS.BUILDER + const isAdmin = ADMIN_ROLES.includes(role._id) + const isAuthed = ctx.auth.authenticated + + if (permType === PermissionTypes.BUILDER && isBuilder) { + return next() + } + const { basePermissions, permissions } = await getUserPermissions( ctx.appId, role._id ) - const isAdmin = ADMIN_ROLES.includes(role._id) - const isAuthed = ctx.auth.authenticated // this may need to change in the future, right now only admins // can have access to builder features, this is hard coded into diff --git a/packages/server/src/middleware/selfhost.js b/packages/server/src/middleware/selfhost.js index 1c96cee33c..1e7117c83d 100644 --- a/packages/server/src/middleware/selfhost.js +++ b/packages/server/src/middleware/selfhost.js @@ -1,14 +1,8 @@ const env = require("../environment") -const hosting = require("../utilities/builder/hosting") // if added as a middleware will stop requests unless builder is in self host mode // or cloud is in self host module.exports = async (ctx, next) => { - if (env.CLOUD && env.SELF_HOSTED) { - await next() - return - } - const hostingInfo = await hosting.getHostingInfo() - if (hostingInfo.type === hosting.HostingTypes.SELF) { + if (env.SELF_HOSTED) { await next() return } diff --git a/packages/server/src/middleware/tests/authorized.spec.js b/packages/server/src/middleware/tests/authorized.spec.js index d3e5e52d2d..7968a8a939 100644 --- a/packages/server/src/middleware/tests/authorized.spec.js +++ b/packages/server/src/middleware/tests/authorized.spec.js @@ -3,8 +3,15 @@ const env = require("../../environment") const apiKey = require("../../utilities/security/apikey") const { AuthTypes } = require("../../constants") const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions") -const { Test } = require("supertest") -jest.mock("../../environment") +jest.mock("../../environment", () => ({ + prod: false, + isTest: () => true, + isProd: () => this.prod, + _set: (key, value) => { + this.prod = value === "production" + } + }) +) jest.mock("../../utilities/security/apikey") class TestConfiguration { @@ -47,8 +54,8 @@ class TestConfiguration { this.ctx.request.url = url } - setCloudEnv(isCloud) { - env.CLOUD = isCloud + setEnvironment(isProd) { + env._set("NODE_ENV", isProd ? "production" : "jest") } setRequestHeaders(headers) { @@ -71,12 +78,6 @@ describe("Authorization middleware", () => { beforeEach(() => { config = new TestConfiguration() - }) - - it("passes the middleware for local webhooks", async () => { - config.setRequestUrl("https://something/webhooks/trigger") - await config.executeMiddleware() - expect(config.next).toHaveBeenCalled() }) describe("external web hook call", () => { @@ -85,7 +86,7 @@ describe("Authorization middleware", () => { beforeEach(() => { config = new TestConfiguration() - config.setCloudEnv(true) + config.setEnvironment(true) config.setRequestHeaders({ "x-api-key": "abc123", "x-instanceid": "instance123", @@ -121,7 +122,7 @@ describe("Authorization middleware", () => { beforeEach(() => { config = new TestConfiguration() - config.setCloudEnv(true) + config.setEnvironment(true) config.setAuthenticated(true) }) @@ -144,7 +145,7 @@ describe("Authorization middleware", () => { }) it("throws if the user has only builder permissions", async () => { - config.setCloudEnv(false) + config.setEnvironment(false) config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER) config.setUser({ role: { diff --git a/packages/server/src/middleware/tests/selfhost.spec.js b/packages/server/src/middleware/tests/selfhost.spec.js index 061da17f9c..6ce61c60ef 100644 --- a/packages/server/src/middleware/tests/selfhost.spec.js +++ b/packages/server/src/middleware/tests/selfhost.spec.js @@ -1,6 +1,5 @@ -const selfHostMiddleware = require("../selfhost"); +const selfHostMiddleware = require("../selfhost") const env = require("../../environment") -const hosting = require("../../utilities/builder/hosting"); jest.mock("../../environment") jest.mock("../../utilities/builder/hosting") @@ -20,16 +19,6 @@ class TestConfiguration { return this.middleware(this.ctx, this.next) } - setCloudHosted() { - env.CLOUD = 1 - env.SELF_HOSTED = 0 - } - - setSelfHosted() { - env.CLOUD = 0 - env.SELF_HOSTED = 1 - } - afterEach() { jest.clearAllMocks() } @@ -46,30 +35,10 @@ describe("Self host middleware", () => { config.afterEach() }) - it("calls next() when CLOUD and SELF_HOSTED env vars are set", async () => { - env.CLOUD = 1 + it("calls next() when SELF_HOSTED env var is set", async () => { env.SELF_HOSTED = 1 await config.executeMiddleware() expect(config.next).toHaveBeenCalled() }) - - it("throws when hostingInfo type is cloud", async () => { - config.setSelfHosted() - - hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.CLOUD })) - - await config.executeMiddleware() - expect(config.throw).toHaveBeenCalledWith(400, "Endpoint unavailable in cloud hosting.") - expect(config.next).not.toHaveBeenCalled() - }) - - it("calls the self hosting middleware to pass through to next() when the hostingInfo type is self", async () => { - config.setSelfHosted() - - hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.SELF })) - - await config.executeMiddleware() - expect(config.next).toHaveBeenCalled() - }) }) diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index 395f14c1ed..9ab17ef992 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -5,7 +5,12 @@ const env = require("../../environment") jest.mock("../../db") jest.mock("../../utilities/usageQuota") -jest.mock("../../environment") +jest.mock("../../environment", () => ({ + isTest: () => true, + isProd: () => false, + isDev: () => true, + _set: () => {}, +})) class TestConfiguration { constructor() { @@ -32,12 +37,14 @@ class TestConfiguration { return this.middleware(this.ctx, this.next) } - cloudHosted(bool) { + setProd(bool) { if (bool) { - env.CLOUD = 1 + env.isDev = () => false + env.isProd = () => true this.ctx.auth = { apiKey: "test" } } else { - env.CLOUD = 0 + env.isDev = () => true + env.isProd = () => false } } @@ -102,7 +109,7 @@ describe("usageQuota middleware", () => { it("calculates and persists the correct usage quota for the relevant action", async () => { config.setUrl("/rows") - config.cloudHosted(true) + config.setProd(true) await config.executeMiddleware() @@ -112,7 +119,7 @@ describe("usageQuota middleware", () => { it("calculates the correct file size from a file upload call and adds it to quota", async () => { config.setUrl("/upload") - config.cloudHosted(true) + config.setProd(true) config.setFiles([ { size: 100 diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 1b809868be..1bc829fbcf 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -44,8 +44,8 @@ module.exports = async (ctx, next) => { } } - // if running in builder or a self hosted cloud usage quotas should not be executed - if (!env.CLOUD || env.SELF_HOSTED) { + // if in development or a self hosted cloud usage quotas should not be executed + if (env.isDev() || env.SELF_HOSTED) { return next() } // update usage for uploads to be the total size diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index a12d596534..433cec4a0a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -14,9 +14,7 @@ const { } = require("./structures") const controllers = require("./controllers") const supertest = require("supertest") -const fs = require("fs") -const { budibaseAppsDir } = require("../../utilities/budibaseDir") -const { join } = require("path") +const { cleanup } = require("../../utilities/fileSystem") const EMAIL = "babs@babs.com" const PASSWORD = "babs_password" @@ -66,13 +64,7 @@ class TestConfiguration { if (this.server) { this.server.close() } - const appDir = budibaseAppsDir() - const files = fs.readdirSync(appDir) - for (let file of files) { - if (this.allApps.some(app => file.includes(app._id))) { - fs.rmdirSync(join(appDir, file), { recursive: true }) - } - } + cleanup(this.allApps.map(app => app._id)) } defaultHeaders() { @@ -81,9 +73,11 @@ class TestConfiguration { roleId: BUILTIN_ROLE_IDS.BUILDER, } const builderToken = jwt.sign(builderUser, env.JWT_SECRET) + // can be "production" for test case + const type = env.isProd() ? "cloud" : "local" const headers = { Accept: "application/json", - Cookie: [`budibase:builder:local=${builderToken}`], + Cookie: [`budibase:builder:${type}=${builderToken}`], } if (this.appId) { headers["x-budibase-app-id"] = this.appId diff --git a/packages/server/src/utilities/builder/compileStaticAssets.js b/packages/server/src/utilities/builder/compileStaticAssets.js deleted file mode 100644 index 0389c920ee..0000000000 --- a/packages/server/src/utilities/builder/compileStaticAssets.js +++ /dev/null @@ -1,35 +0,0 @@ -const { ensureDir, constants, copyFile } = require("fs-extra") -const { join } = require("../centralPath") -const { budibaseAppsDir } = require("../budibaseDir") - -/** - * Compile all the non-db static web assets that are required for the running of - * a budibase application. This includes the JSON structure of the DOM and - * the client library, a script responsible for reading the JSON structure - * and rendering the application. - * @param {string} appId id of the application we want to compile static assets for - */ -module.exports = async appId => { - const publicPath = join(budibaseAppsDir(), appId, "public") - await ensureDir(publicPath) - await copyClientLib(publicPath) -} - -/** - * Copy the budibase client library and sourcemap from NPM to /public/. - * The client library is then served as a static asset when the budibase application - * is running in preview or prod - * @param {String} publicPath - path to write the client library to - */ -const copyClientLib = async publicPath => { - const sourcepath = require.resolve("@budibase/client") - const destPath = join(publicPath, "budibase-client.js") - - await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE) - - await copyFile( - sourcepath + ".map", - destPath + ".map", - constants.COPYFILE_FICLONE - ) -} diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js index c265c26dd0..f852cefec1 100644 --- a/packages/server/src/utilities/builder/hosting.js +++ b/packages/server/src/utilities/builder/hosting.js @@ -1,5 +1,5 @@ const CouchDB = require("../../db") -const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") +const { StaticDatabases } = require("../../db/utils") const fetch = require("node-fetch") const env = require("../../environment") @@ -23,16 +23,16 @@ exports.HostingTypes = { } exports.getHostingInfo = async () => { - const db = new CouchDB(BUILDER_CONFIG_DB) + const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name) let doc try { - doc = await db.get(HOSTING_DOC) + doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc) } catch (err) { // don't write this doc, want to be able to update these default props // for our servers with a new release without needing to worry about state of // PouchDB in peoples installations doc = { - _id: HOSTING_DOC, + _id: StaticDatabases.BUILDER_HOSTING.baseDoc, type: exports.HostingTypes.CLOUD, hostingUrl: PROD_HOSTING_URL, selfHostKey: "", @@ -85,15 +85,11 @@ exports.getTemplatesUrl = async (appId, type, name) => { } exports.getDeployedApps = async () => { - const hostingInfo = await exports.getHostingInfo() - if ( - (!env.CLOUD && hostingInfo.type === exports.HostingTypes.CLOUD) || - (env.CLOUD && !env.SELF_HOSTED) - ) { + if (!env.SELF_HOSTED) { throw "Can only check apps for self hosted environments" } - const workerUrl = !env.CLOUD ? await exports.getWorkerUrl() : env.WORKER_URL - const hostingKey = !env.CLOUD ? hostingInfo.selfHostKey : env.HOSTING_KEY + const workerUrl = env.WORKER_URL + const hostingKey = env.HOSTING_KEY try { const response = await fetch(`${workerUrl}/api/apps`, { method: "GET", @@ -102,12 +98,14 @@ exports.getDeployedApps = async () => { }, }) const json = await response.json() - for (let value of Object.values(json)) { + const apps = {} + for (let [key, value] of Object.entries(json)) { if (value.url) { value.url = value.url.toLowerCase() + apps[key] = value } } - return json + return apps } catch (err) { // error, cannot determine deployed apps, don't stop app creation - sort this later return {} diff --git a/packages/server/src/utilities/createAppPackage.js b/packages/server/src/utilities/createAppPackage.js deleted file mode 100644 index 9500554227..0000000000 --- a/packages/server/src/utilities/createAppPackage.js +++ /dev/null @@ -1,29 +0,0 @@ -const stream = require("stream") -const fetch = require("node-fetch") -const tar = require("tar-fs") -const zlib = require("zlib") -const { promisify } = require("util") -const packageJson = require("../../package.json") - -const streamPipeline = promisify(stream.pipeline) - -// can't really test this due to the downloading nature of it, wouldn't be a great test case -/* istanbul ignore next */ -exports.downloadExtractComponentLibraries = async appFolder => { - const LIBRARIES = ["standard-components"] - - // Need to download tarballs directly from NPM as our users may not have node on their machine - for (let lib of LIBRARIES) { - // download tarball - const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` - const response = await fetch(registryUrl) - if (!response.ok) - throw new Error(`unexpected response ${response.statusText}`) - - await streamPipeline( - response.body, - zlib.Unzip(), - tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) - ) - } -} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js new file mode 100644 index 0000000000..7ef99eda41 --- /dev/null +++ b/packages/server/src/utilities/fileSystem/index.js @@ -0,0 +1,232 @@ +const { budibaseTempDir } = require("../budibaseDir") +const { isDev } = require("../index") +const fs = require("fs") +const { join } = require("path") +const uuid = require("uuid/v4") +const CouchDB = require("../../db") +const { ObjectStoreBuckets } = require("../../constants") +const { + upload, + retrieve, + retrieveToTmp, + streamUpload, + deleteFolder, + downloadTarball, +} = require("./utilities") +const { downloadLibraries, newAppPublicPath } = require("./newApp") +const download = require("download") +const env = require("../../environment") +const { homedir } = require("os") +const fetch = require("node-fetch") + +const DEFAULT_AUTOMATION_BUCKET = + "https://prod-budi-automations.s3-eu-west-1.amazonaws.com" +const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations" + +/** + * The single stack system (Cloud and Builder) should not make use of the file system where possible, + * this file handles all of the file access for the system with the intention of limiting it all to one + * place. Keeping all of this logic in one place means that when we need to do file system access (like + * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't + * be done through an object store instead. + */ + +/** + * Upon first startup of instance there may not be everything we need in tmp directory, set it up. + */ +exports.init = () => { + const tempDir = budibaseTempDir() + if (!fs.existsSync(tempDir)) { + fs.mkdirSync(tempDir) + } + const clientLibPath = join(budibaseTempDir(), "budibase-client.js") + if (env.isTest() && !fs.existsSync(clientLibPath)) { + fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath) + } +} + +/** + * Checks if the system is currently in development mode and if it is makes sure + * everything required to function is ready. + */ +exports.checkDevelopmentEnvironment = () => { + if (!isDev()) { + return + } + let error + if (!fs.existsSync(budibaseTempDir())) { + error = + "Please run a build before attempting to run server independently to fill 'tmp' directory." + } + if (!fs.existsSync(join(process.cwd(), ".env"))) { + error = "Must run via yarn once to generate environment." + } + if (error) { + console.error(error) + process.exit(-1) + } +} + +/** + * This function manages temporary template files which are stored by Koa. + * @param {Object} template The template object retrieved from the Koa context object. + * @returns {Object} Returns an fs read stream which can be loaded into the database. + */ +exports.getTemplateStream = async template => { + if (template.file) { + return fs.createReadStream(template.file.path) + } else { + const tmpPath = await exports.downloadTemplate(...template.key.split("/")) + return fs.createReadStream(join(tmpPath, "db", "dump.txt")) + } +} + +/** + * Used to retrieve a handlebars file from the system which will be used as a template. + * This is allowable as the template handlebars files should be static and identical across + * the cluster. + * @param {string} path The path to the handlebars file which is to be loaded. + * @returns {string} The loaded handlebars file as a string - loaded as utf8. + */ +exports.loadHandlebarsFile = path => { + return fs.readFileSync(path, "utf8") +} + +/** + * When return a file from the API need to write the file to the system temporarily so we + * can create a read stream to send. + * @param {string} contents the contents of the file which is to be returned from the API. + * @return {Object} the read stream which can be put into the koa context body. + */ +exports.apiFileReturn = contents => { + const path = join(budibaseTempDir(), uuid()) + fs.writeFileSync(path, contents) + return fs.createReadStream(path) +} + +/** + * Takes a copy of the database state for an app to the object store. + * @param {string} appId The ID of the app which is to be backed up. + * @param {string} backupName The name of the backup located in the object store. + * @return The backup has been completed when this promise completes and returns a file stream + * to the temporary backup file (to return via API if required). + */ +exports.performBackup = async (appId, backupName) => { + const path = join(budibaseTempDir(), backupName) + const writeStream = fs.createWriteStream(path) + // perform couch dump + const instanceDb = new CouchDB(appId) + await instanceDb.dump(writeStream, {}) + // write the file to the object store + await streamUpload( + ObjectStoreBuckets.BACKUPS, + join(appId, backupName), + fs.createReadStream(path) + ) + return fs.createReadStream(path) +} + +/** + * Downloads required libraries and creates a new path in the object store. + * @param {string} appId The ID of the app which is being created. + * @return {Promise} once promise completes app resources should be ready in object store. + */ +exports.createApp = async appId => { + await downloadLibraries(appId) + await newAppPublicPath(appId) +} + +/** + * Removes all of the assets created for an app in the object store. + * @param {string} appId The ID of the app which is being deleted. + * @return {Promise} once promise completes the app resources will be removed from object store. + */ +exports.deleteApp = async appId => { + await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) +} + +/** + * Retrieves a template and pipes it to minio as well as making it available temporarily. + * @param {string} type The type of template which is to be retrieved. + * @param name + * @return {Promise<*>} + */ +exports.downloadTemplate = async (type, name) => { + const DEFAULT_TEMPLATES_BUCKET = + "prod-budi-templates.s3-eu-west-1.amazonaws.com" + const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` + return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) +} + +/** + * Retrieves component libraries from object store (or tmp symlink if in local) + */ +exports.getComponentLibraryManifest = async (appId, library) => { + const filename = "manifest.json" + /* istanbul ignore next */ + // when testing in cypress and so on we need to get the package + // as the environment may not be fully fleshed out for dev or prod + if (env.isTest()) { + const lib = library.split("/")[1] + const path = require.resolve(library).split(lib)[0] + return require(join(path, lib, filename)) + } + const devPath = join(budibaseTempDir(), library, filename) + if (env.isDev() && fs.existsSync(devPath)) { + return require(devPath) + } + const path = join(appId, "node_modules", library, "package", filename) + let resp = await retrieve(ObjectStoreBuckets.APPS, path) + if (typeof resp !== "string") { + resp = resp.toString("utf8") + } + return JSON.parse(resp) +} + +exports.automationInit = async () => { + const directory = + env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) + const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }) + } + // env setup to get async packages + let response = await fetch(`${bucket}/manifest.json`) + return response.json() +} + +exports.getExternalAutomationStep = async (name, version, bundleName) => { + const directory = + env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) + const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET + try { + return require(join(directory, bundleName)) + } catch (err) { + await download(`${bucket}/${name}/${version}/${bundleName}`, directory) + return require(join(directory, bundleName)) + } +} + +/** + * All file reads come through here just to make sure all of them make sense + * allows a centralised location to check logic is all good. + */ +exports.readFileSync = (filepath, options = "utf8") => { + return fs.readFileSync(filepath, options) +} + +/** + * Given a set of app IDs makes sure file system is cleared of any of their temp info. + */ +exports.cleanup = appIds => { + for (let appId of appIds) { + fs.rmdirSync(join(budibaseTempDir(), appId), { recursive: true }) + } +} + +/** + * Full function definition for below can be found in the utilities. + */ +exports.upload = upload +exports.retrieve = retrieve +exports.retrieveToTmp = retrieveToTmp diff --git a/packages/server/src/utilities/fileSystem/newApp.js b/packages/server/src/utilities/fileSystem/newApp.js new file mode 100644 index 0000000000..2f5c77912a --- /dev/null +++ b/packages/server/src/utilities/fileSystem/newApp.js @@ -0,0 +1,40 @@ +const packageJson = require("../../../package.json") +const { join } = require("path") +const { ObjectStoreBuckets } = require("../../constants") +const { streamUpload, downloadTarball } = require("./utilities") +const fs = require("fs") + +const BUCKET_NAME = ObjectStoreBuckets.APPS + +// can't really test this due to the downloading nature of it, wouldn't be a great test case +/* istanbul ignore next */ +exports.downloadLibraries = async appId => { + const LIBRARIES = ["standard-components"] + + const paths = {} + // Need to download tarballs directly from NPM as our users may not have node on their machine + for (let lib of LIBRARIES) { + // download tarball + const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` + const path = join(appId, "node_modules", "@budibase", lib) + paths[`@budibase/${lib}`] = await downloadTarball( + registryUrl, + BUCKET_NAME, + path + ) + } + return paths +} + +exports.newAppPublicPath = async appId => { + const path = join(appId, "public") + const sourcepath = require.resolve("@budibase/client") + const destPath = join(path, "budibase-client.js") + + await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath)) + await streamUpload( + BUCKET_NAME, + destPath + ".map", + fs.createReadStream(sourcepath + ".map") + ) +} diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileSystem/processor.js similarity index 62% rename from packages/server/src/utilities/fileProcessor.js rename to packages/server/src/utilities/fileSystem/processor.js index 15132b2d49..3778b50168 100644 --- a/packages/server/src/utilities/fileProcessor.js +++ b/packages/server/src/utilities/fileSystem/processor.js @@ -1,25 +1,20 @@ -const fs = require("fs") const jimp = require("jimp") -const fsPromises = fs.promises const FORMATS = { IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], } function processImage(file) { + // this will overwrite the temp file return jimp.read(file.path).then(img => { - return img.resize(300, jimp.AUTO).write(file.outputPath) + return img.resize(300, jimp.AUTO).write(file.path) }) } async function process(file) { if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) { await processImage(file) - return file } - - // No processing required - await fsPromises.copyFile(file.path, file.outputPath) return file } diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js new file mode 100644 index 0000000000..b586cfcfe5 --- /dev/null +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -0,0 +1,243 @@ +const sanitize = require("sanitize-s3-objectkey") +const AWS = require("aws-sdk") +const stream = require("stream") +const fetch = require("node-fetch") +const tar = require("tar-fs") +const zlib = require("zlib") +const { promisify } = require("util") +const { join } = require("path") +const fs = require("fs") +const { budibaseTempDir } = require("../budibaseDir") +const env = require("../../environment") +const { ObjectStoreBuckets } = require("../../constants") +const uuid = require("uuid/v4") + +const streamPipeline = promisify(stream.pipeline) +// use this as a temporary store of buckets that are being created +const STATE = { + bucketCreationPromises: {}, +} + +const CONTENT_TYPE_MAP = { + html: "text/html", + css: "text/css", + js: "application/javascript", +} +const STRING_CONTENT_TYPES = [ + CONTENT_TYPE_MAP.html, + CONTENT_TYPE_MAP.css, + CONTENT_TYPE_MAP.js, +] + +function publicPolicy(bucketName) { + return { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { + AWS: ["*"], + }, + Action: "s3:GetObject", + Resource: [`arn:aws:s3:::${bucketName}/*`], + }, + ], + } +} + +const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS] + +/** + * Gets a connection to the object store using the S3 SDK. + * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. + * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. + * @constructor + */ +exports.ObjectStore = bucket => { + if (env.SELF_HOSTED) { + AWS.config.update({ + accessKeyId: env.MINIO_ACCESS_KEY, + secretAccessKey: env.MINIO_SECRET_KEY, + }) + } + const config = { + s3ForcePathStyle: true, + signatureVersion: "v4", + params: { + Bucket: bucket, + }, + } + if (env.MINIO_URL) { + config.endpoint = env.MINIO_URL + } + return new AWS.S3(config) +} + +/** + * Given an object store and a bucket name this will make sure the bucket exists, + * if it does not exist then it will create it. + */ +exports.makeSureBucketExists = async (client, bucketName) => { + try { + await client + .headBucket({ + Bucket: bucketName, + }) + .promise() + } catch (err) { + const promises = STATE.bucketCreationPromises + if (promises[bucketName]) { + await promises[bucketName] + } else if (err.statusCode === 404) { + // bucket doesn't exist create it + promises[bucketName] = client + .createBucket({ + Bucket: bucketName, + }) + .promise() + await promises[bucketName] + delete promises[bucketName] + // public buckets are quite hidden in the system, make sure + // no bucket is set accidentally + if (PUBLIC_BUCKETS.includes(bucketName)) { + await client + .putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify(publicPolicy(bucketName)), + }) + .promise() + } + } else { + throw err + } + } +} + +/** + * Uploads the contents of a file given the required parameters, useful when + * temp files in use (for example file uploaded as an attachment). + */ +exports.upload = async ({ bucket, filename, path, type, metadata }) => { + const extension = [...filename.split(".")].pop() + const fileBytes = fs.readFileSync(path) + + const objectStore = exports.ObjectStore(bucket) + await exports.makeSureBucketExists(objectStore, bucket) + + const config = { + // windows file paths need to be converted to forward slashes for s3 + Key: sanitize(filename).replace(/\\/g, "/"), + Body: fileBytes, + ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()], + } + if (metadata) { + config.Metadata = metadata + } + return objectStore.upload(config).promise() +} + +/** + * Similar to the upload function but can be used to send a file stream + * through to the object store. + */ +exports.streamUpload = async (bucket, filename, stream) => { + const objectStore = exports.ObjectStore(bucket) + await exports.makeSureBucketExists(objectStore, bucket) + + const params = { + Bucket: bucket, + Key: sanitize(filename).replace(/\\/g, "/"), + Body: stream, + } + return objectStore.upload(params).promise() +} + +/** + * retrieves the contents of a file from the object store, if it is a known content type it + * will be converted, otherwise it will be returned as a buffer stream. + */ +exports.retrieve = async (bucket, filepath) => { + const objectStore = exports.ObjectStore(bucket) + const params = { + Bucket: bucket, + Key: sanitize(filepath).replace(/\\/g, "/"), + } + const response = await objectStore.getObject(params).promise() + // currently these are all strings + if (STRING_CONTENT_TYPES.includes(response.ContentType)) { + return response.Body.toString("utf8") + } else { + return response.Body + } +} + +/** + * Same as retrieval function but puts to a temporary file. + */ +exports.retrieveToTmp = async (bucket, filepath) => { + const data = await exports.retrieve(bucket, filepath) + const outputPath = join(budibaseTempDir(), uuid()) + fs.writeFileSync(outputPath, data) + return outputPath +} + +exports.deleteFolder = async (bucket, folder) => { + const client = exports.ObjectStore(bucket) + const listParams = { + Bucket: bucket, + Prefix: folder, + } + + let response = await client.listObjects(listParams).promise() + if (response.Contents.length === 0) { + return + } + const deleteParams = { + Bucket: bucket, + Delete: { + Objects: [], + }, + } + + response.Contents.forEach(content => { + deleteParams.Delete.Objects.push({ Key: content.Key }) + }) + + response = await client.deleteObjects(deleteParams).promise() + // can only empty 1000 items at once + if (response.Deleted.length === 1000) { + return exports.deleteFolder(bucket, folder) + } +} + +exports.uploadDirectory = async (bucket, localPath, bucketPath) => { + let uploads = [] + const files = fs.readdirSync(localPath, { withFileTypes: true }) + for (let file of files) { + const path = join(bucketPath, file.name) + const local = join(localPath, file.name) + if (file.isDirectory()) { + uploads.push(exports.uploadDirectory(bucket, local, path)) + } else { + uploads.push( + exports.streamUpload(bucket, path, fs.createReadStream(local)) + ) + } + } + await Promise.all(uploads) +} + +exports.downloadTarball = async (url, bucket, path) => { + const response = await fetch(url) + if (!response.ok) { + throw new Error(`unexpected response ${response.statusText}`) + } + + const tmpPath = join(budibaseTempDir(), path) + await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) + if (!env.isTest()) { + await exports.uploadDirectory(bucket, tmpPath, path) + } + // return the temporary path incase there is a use for it + return tmpPath +} diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 7420884d44..ad92987434 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,6 +1,5 @@ const env = require("../environment") const { DocumentTypes, SEPARATOR } = require("../db/utils") -const fs = require("fs") const CouchDB = require("../db") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -13,14 +12,7 @@ function confirmAppId(possibleAppId) { exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms)) -exports.isDev = () => { - return ( - !env.CLOUD && - env.NODE_ENV !== "production" && - env.NODE_ENV !== "jest" && - env.NODE_ENV !== "cypress" - ) -} +exports.isDev = env.isDev /** * Given a request tries to find the appId, which can be located in various places @@ -28,10 +20,18 @@ exports.isDev = () => { * @returns {string|undefined} If an appId was found it will be returned. */ exports.getAppId = ctx => { - let appId = confirmAppId(ctx.headers["x-budibase-app-id"]) - if (!appId) { - appId = confirmAppId(env.CLOUD ? ctx.subdomains[1] : ctx.params.appId) + const options = [ctx.headers["x-budibase-app-id"], ctx.params.appId] + if (ctx.subdomains) { + options.push(ctx.subdomains[1]) } + let appId + for (let option of options) { + appId = confirmAppId(option) + if (appId) { + break + } + } + // look in body if can't find it in subdomain if (!appId && ctx.request.body && ctx.request.body.appId) { appId = confirmAppId(ctx.request.body.appId) @@ -51,7 +51,7 @@ exports.getAppId = ctx => { * @returns {string} The name of the token trying to find */ exports.getCookieName = (name = "builder") => { - let environment = env.CLOUD ? "cloud" : "local" + let environment = env.isProd() ? "cloud" : "local" return `budibase:${name}:${environment}` } @@ -89,24 +89,6 @@ exports.isClient = ctx => { return ctx.headers["x-budibase-type"] === "client" } -/** - * Recursively walk a directory tree and execute a callback on all files. - * @param {String} dirPath - Directory to traverse - * @param {Function} callback - callback to execute on files - */ -exports.walkDir = (dirPath, callback) => { - for (let filename of fs.readdirSync(dirPath)) { - const filePath = `${dirPath}/${filename}` - const stat = fs.lstatSync(filePath) - - if (stat.isFile()) { - callback(filePath) - } else { - exports.walkDir(filePath, callback) - } - } -} - exports.getLogoUrl = () => { return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" } @@ -124,3 +106,7 @@ exports.getAllApps = async () => { .map(({ value }) => value) } } + +exports.checkSlashesInUrl = url => { + return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2") +} diff --git a/packages/server/src/utilities/initialiseBudibase.js b/packages/server/src/utilities/initialiseBudibase.js deleted file mode 100644 index bdbd194eaa..0000000000 --- a/packages/server/src/utilities/initialiseBudibase.js +++ /dev/null @@ -1,37 +0,0 @@ -const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra") -const { join, resolve } = require("./centralPath") -const { processString } = require("@budibase/string-templates") -const uuid = require("uuid") - -module.exports = async opts => { - await ensureDir(opts.dir) - await setCouchDbUrl(opts) - - // need an env file - await createDevEnvFile(opts) -} - -const setCouchDbUrl = async opts => { - if (!opts.couchDbUrl) { - const dataDir = join(opts.dir, ".data") - await ensureDir(dataDir) - opts.couchDbUrl = - dataDir + (dataDir.endsWith("/") || dataDir.endsWith("\\") ? "" : "/") - } -} - -const createDevEnvFile = async opts => { - const destConfigFile = join(opts.dir, "./.env") - let createConfig = !existsSync(destConfigFile) || opts.quiet - if (createConfig) { - const template = await readFile( - resolve(__dirname, "..", "..", ".env.template"), - { - encoding: "utf8", - } - ) - opts.cookieKey1 = opts.cookieKey1 || uuid.v4() - const config = await processString(template, opts) - await writeFile(destConfigFile, config, { flag: "w+" }) - } -} diff --git a/packages/server/src/utilities/rowProcessor.js b/packages/server/src/utilities/rowProcessor.js index eddd597459..97e2a2880c 100644 --- a/packages/server/src/utilities/rowProcessor.js +++ b/packages/server/src/utilities/rowProcessor.js @@ -180,7 +180,7 @@ exports.outputProcessing = async (appId, table, rows) => { rows ) // update the attachments URL depending on hosting - if (env.CLOUD && env.SELF_HOSTED) { + if (env.isProd() && env.SELF_HOSTED) { for (let [property, column] of Object.entries(table.schema)) { if (column.type === FieldTypes.ATTACHMENT) { for (let row of outputRows) { diff --git a/packages/server/src/utilities/templates.js b/packages/server/src/utilities/templates.js deleted file mode 100644 index c3d89477df..0000000000 --- a/packages/server/src/utilities/templates.js +++ /dev/null @@ -1,81 +0,0 @@ -const fs = require("fs-extra") -const { join } = require("./centralPath") -const os = require("os") -const fetch = require("node-fetch") -const stream = require("stream") -const tar = require("tar-fs") -const zlib = require("zlib") -const { promisify } = require("util") -const streamPipeline = promisify(stream.pipeline) -const { budibaseAppsDir } = require("./budibaseDir") -const env = require("../environment") -const CouchDB = require("../db") - -const DEFAULT_TEMPLATES_BUCKET = - "prod-budi-templates.s3-eu-west-1.amazonaws.com" - -exports.getLocalTemplates = function() { - const templatesDir = join(os.homedir(), ".budibase", "templates", "app") - const templateObj = { app: {} } - fs.ensureDirSync(templatesDir) - const templateNames = fs.readdirSync(templatesDir) - for (let name of templateNames) { - templateObj.app[name] = { - name, - category: "local", - description: "local template", - type: "app", - key: `app/${name}`, - } - } - return templateObj -} - -// can't really test this, downloading is just not something we should do in a behavioural test -/* istanbul ignore next */ -exports.downloadTemplate = async function(type, name) { - const dirName = join(budibaseAppsDir(), "templates", type, name) - if (env.LOCAL_TEMPLATES) { - return dirName - } - const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` - const response = await fetch(templateUrl) - - if (!response.ok) { - throw new Error( - `Error downloading template ${type}:${name}: ${response.statusText}` - ) - } - - // stream the response, unzip and extract - await streamPipeline( - response.body, - zlib.Unzip(), - tar.extract(join(budibaseAppsDir(), "templates", type)) - ) - - return dirName -} - -async function performDump({ dir, appId, name = "dump.txt" }) { - const writeStream = fs.createWriteStream(join(dir, name)) - // perform couch dump - const instanceDb = new CouchDB(appId) - await instanceDb.dump(writeStream, {}) -} - -exports.performDump = performDump - -exports.exportTemplateFromApp = async function({ templateName, appId }) { - // Copy frontend files - const templatesDir = join( - budibaseAppsDir(), - "templates", - "app", - templateName, - "db" - ) - fs.ensureDirSync(templatesDir) - await performDump({ dir: templatesDir, appId }) - return templatesDir -} diff --git a/packages/server/src/utilities/tests/csvParser.spec.js b/packages/server/src/utilities/tests/csvParser.spec.js index 48e275fdd1..76ea9a7eb3 100644 --- a/packages/server/src/utilities/tests/csvParser.spec.js +++ b/packages/server/src/utilities/tests/csvParser.spec.js @@ -1,4 +1,4 @@ -const fs = require("fs") +const { readFileSync } = require("../fileSystem") const csvParser = require("../csvParser") const CSV_PATH = __dirname + "/test.csv" @@ -33,7 +33,7 @@ const SCHEMAS = { } describe("CSV Parser", () => { - const csvString = fs.readFileSync(CSV_PATH, "utf8") + const csvString = readFileSync(CSV_PATH, "utf8") describe("parsing", () => { it("returns status and types for a valid CSV transformation", async () => { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index d809d9e673..d042d290d5 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -50,7 +50,7 @@ exports.Properties = { } exports.getAPIKey = async appId => { - if (env.SELF_HOSTED) { + if (!env.USE_QUOTAS) { return { apiKey: null } } return apiKeyTable.get({ primary: appId }) @@ -65,8 +65,7 @@ exports.getAPIKey = async appId => { * also been reset after this call. */ exports.update = async (apiKey, property, usage) => { - // don't try validate in builder - if (!env.CLOUD || env.SELF_HOSTED) { + if (!env.USE_QUOTAS) { return } try { diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 11aa95bf90..919dccf1a1 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2239,13 +2239,6 @@ crc@^3.4.4: dependencies: buffer "^5.1.0" -cross-env@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" - integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== - dependencies: - cross-spawn "^7.0.1" - cross-spawn@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41" @@ -2265,15 +2258,6 @@ cross-spawn@^6.0.0, cross-spawn@^6.0.5: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^7.0.1: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== - dependencies: - path-key "^3.1.0" - shebang-command "^2.0.0" - which "^2.0.1" - crypto-random-string@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" @@ -2576,6 +2560,11 @@ dmg-builder@22.9.1: js-yaml "^3.14.0" sanitize-filename "^1.6.3" +docker-compose@^0.23.6: + version "0.23.6" + resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.6.tgz#bd21e17d599f17fcf7a4b5d607cff0358a9c378b" + integrity sha512-y3Q8MkwG862rNqkvEQG59/7Fi2/fzs3NYDCvqUAAD+z0WGs2qcJ9hRcn34hWgWv9ouPkFqe3Vwca0h+4bIIRWw== + doctrine@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" @@ -6421,11 +6410,6 @@ path-key@^2.0.0, path-key@^2.0.1: resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= -path-key@^3.1.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" - integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== - path-parse@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" @@ -7553,23 +7537,11 @@ shebang-command@^1.2.0: dependencies: shebang-regex "^1.0.0" -shebang-command@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" - integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== - dependencies: - shebang-regex "^3.0.0" - shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= -shebang-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" - integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== - shell-env@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/shell-env/-/shell-env-0.3.0.tgz#2250339022989165bda4eb7bf383afeaaa92dc34" @@ -8719,13 +8691,6 @@ which@^1.2.9, which@^1.3.0: dependencies: isexe "^2.0.0" -which@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" - integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== - dependencies: - isexe "^2.0.0" - widest-line@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca" diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte index 509205f8f1..aeaebcc0fa 100644 --- a/packages/standard-components/src/Search.svelte +++ b/packages/standard-components/src/Search.svelte @@ -25,10 +25,11 @@ let tableDefinition let schema - // pagination - let page = 0 + let nextBookmark = null + let bookmark = null + let lastBookmark = null - $: fetchData(table, page) + $: fetchData(table, bookmark) // omit empty strings $: parsedSearch = Object.keys(search).reduce( (acc, next) => @@ -38,33 +39,43 @@ $: actions = [ { type: ActionTypes.RefreshDatasource, - callback: () => fetchData(table, page), + callback: () => fetchData(table, bookmark), metadata: { datasource: { type: "table", tableId: table } }, }, ] - async function fetchData(table, page) { + async function fetchData(table, mark) { if (table) { const tableDef = await API.fetchTableDefinition(table) schema = tableDef.schema - rows = await API.searchTableData({ + const output = await API.searchTableData({ tableId: table, search: parsedSearch, pagination: { pageSize, - page, + bookmark: mark, }, }) + rows = output.rows + nextBookmark = output.bookmark } loaded = true } function nextPage() { - page += 1 + lastBookmark = bookmark + bookmark = nextBookmark } function previousPage() { - page -= 1 + nextBookmark = bookmark + if (lastBookmark !== bookmark) { + bookmark = lastBookmark + } else { + // special case for going back to beginning + bookmark = null + lastBookmark = null + } } @@ -99,15 +110,15 @@ secondary on:click={() => { search = {} - page = 0 + bookmark = null }}> Reset @@ -129,10 +140,10 @@ {/if} {/if} diff --git a/packages/worker/package.json b/packages/worker/package.json index 1670fd729e..7d88654a88 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -33,6 +33,5 @@ "pouchdb": "^7.2.2", "pouchdb-all-dbs": "^1.0.2", "server-destroy": "^1.0.1" - }, - "gitHead": "1b95326b20d1352d36305910259228b96a683dc7" + } } diff --git a/packages/worker/src/db/index.js b/packages/worker/src/db/index.js index 24bbb55092..83f030c846 100644 --- a/packages/worker/src/db/index.js +++ b/packages/worker/src/db/index.js @@ -1,12 +1,9 @@ const PouchDB = require("pouchdb") const allDbs = require("pouchdb-all-dbs") const env = require("../environment") -const { join } = require("path") -const { homedir } = require("os") // level option is purely for testing (development) -const COUCH_DB_URL = - env.COUCH_DB_URL || `leveldb://${join(homedir(), ".budibase")}/.data/` +const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/" const Pouch = PouchDB.defaults({ prefix: COUCH_DB_URL,