From bc00afcf36e24f02d8ec54d0eb0c3e1a137320e3 Mon Sep 17 00:00:00 2001 From: Michael Shanks Date: Sun, 22 Dec 2019 07:12:21 +0000 Subject: [PATCH] #9 All ids - replace with folder list items --- packages/core/dist/budibase-core.cjs.js | 40 ++-- packages/core/dist/budibase-core.esm.mjs | 40 ++-- packages/core/dist/budibase-core.umd.js | 40 ++-- .../core/src/appInitialise/inProcIndexer.js | 0 packages/core/src/appInitialise/index.js | 16 +- .../core/src/appInitialise/initialiseData.js | 5 +- packages/core/src/collectionApi/delete.js | 48 +++-- packages/core/src/collectionApi/initialise.js | 4 +- packages/core/src/common/index.js | 16 +- packages/core/src/indexApi/aggregates.js | 10 +- packages/core/src/indexApi/buildIndex.js | 16 +- packages/core/src/indexApi/delete.js | 14 +- packages/core/src/indexApi/getIndexDir.js | 20 ++ packages/core/src/indexApi/listItems.js | 16 +- packages/core/src/indexing/allIds.js | 203 ++++++++++++------ packages/core/src/indexing/apply.js | 28 ++- packages/core/src/indexing/initialiseIndex.js | 8 +- packages/core/src/indexing/read.js | 33 --- packages/core/src/indexing/relevant.js | 32 +-- packages/core/src/indexing/sharding.js | 48 ++--- packages/core/src/indexing/sweeper.js | 0 packages/core/src/recordApi/delete.js | 25 ++- packages/core/src/recordApi/downloadFile.js | 4 +- packages/core/src/recordApi/getContext.js | 4 +- packages/core/src/recordApi/load.js | 4 +- packages/core/src/recordApi/recordInfo.js | 64 ++++-- packages/core/src/recordApi/save.js | 22 +- packages/core/src/recordApi/uploadFile.js | 36 +--- packages/core/src/recordApi/validate.js | 4 +- packages/core/src/templateApi/createNodes.js | 2 +- packages/core/src/templateApi/hierarchy.js | 19 +- packages/core/src/transactions/execute.js | 54 ++--- packages/core/test/collectionApi.spec.js | 3 +- .../core/test/indexApi.buildIndex.spec.js | 6 +- .../core/test/indexing.concurrency.spec.js | 18 +- .../test/indexing.getRelevantIndexes.spec.js | 41 ++-- packages/core/test/indexing.schema.spec.js | 2 +- packages/core/test/initialiseData.spec.js | 11 - packages/core/test/memory.js | 57 ++++- packages/core/test/recordApi.files.spec.js | 3 - .../core/test/recordApi.getRecordInfo.spec.js | 124 +++++++++++ packages/core/test/recordApi.save.spec.js | 65 +++--- packages/core/test/specHelpers.js | 8 +- .../templateApi.constructHeirarchy.spec.js | 2 +- 44 files changed, 764 insertions(+), 451 deletions(-) delete mode 100644 packages/core/src/appInitialise/inProcIndexer.js create mode 100644 packages/core/src/indexApi/getIndexDir.js delete mode 100644 packages/core/src/indexing/sweeper.js create mode 100644 packages/core/test/recordApi.getRecordInfo.spec.js diff --git a/packages/core/dist/budibase-core.cjs.js b/packages/core/dist/budibase-core.cjs.js index c8823f20ec..9eeb38417f 100644 --- a/packages/core/dist/budibase-core.cjs.js +++ b/packages/core/dist/budibase-core.cjs.js @@ -726,7 +726,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [ fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)), ]); -const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ +const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [ getFlattenedHierarchy, fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)), ]); @@ -764,7 +764,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [ ]); const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { - const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); + const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey); return isNothing(nodeByKey) ? getNode(appHierarchy, keyOrNodeKey) : nodeByKey; @@ -777,7 +777,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { : nodeByKey; }; -const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); +const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key)); const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ splitKey, @@ -876,7 +876,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe var hierarchy = { getLastPartInKey, getNodesInPath, - getExactNodeForPath, + getExactNodeForKey, hasMatchingAncestor, getNode, getNodeByKeyOrNodeKey, @@ -1612,7 +1612,7 @@ const load = app => async key => { const _load = async (app, key, keyStack = []) => { key = safeKey(key); - const recordNode = getExactNodeForPath(app.hierarchy)(key); + const recordNode = getExactNodeForKey(app.hierarchy)(key); const storedData = await app.datastore.loadJson( getRecordFileName(key), ); @@ -1758,7 +1758,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => { }; const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); const startShardName = !startRecord ? null @@ -4403,7 +4403,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => { )); indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } @@ -4435,7 +4435,7 @@ const getContext = app => recordKey => { const _getContext = (app, recordKey) => { recordKey = safeKey(recordKey); - const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); + const recordNode = getExactNodeForKey(app.hierarchy)(recordKey); const cachedReferenceIndexes = {}; @@ -4543,7 +4543,7 @@ const validate = app => async (record, context) => { ? _getContext(app, record.key) : context; - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const fieldParseFails = validateAllFieldParse(record, recordNode); // non parsing would cause further issues - exit here @@ -4603,7 +4603,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => { const initialiseChildCollections = async (app, recordKey) => { const childCollectionRecords = $(recordKey, [ - getExactNodeForPath(app.hierarchy), + getExactNodeForKey(app.hierarchy), n => n.children, fp.filter(isCollectionRecord), ]); @@ -4974,7 +4974,7 @@ const _save = async (app, record, context, skipValidation = false) => { } if (recordClone.isNew) { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); if(!recordNode) throw new Error("Cannot find node for " + record.key); @@ -5023,7 +5023,7 @@ const _save = async (app, record, context, skipValidation = false) => { }; const initialiseAncestorIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); for (const index of recordNode.indexes) { const indexKey = joinKey(record.key, index.name); @@ -5032,7 +5032,7 @@ const initialiseAncestorIndexes = async (app, record) => { }; const initialiseReverseReferenceIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [ @@ -5131,7 +5131,7 @@ const deleteRecords = async (app, key) => { const _deleteIndex = async (app, indexKey, includeFolder) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } @@ -5176,7 +5176,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => { // called deleteRecord because delete is a keyword const _deleteRecord = async (app, key, disableCleanup) => { key = safeKey(key); - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); const record = await _load(app, key); await transactionForDeleteRecord(app, record); @@ -5203,7 +5203,7 @@ const _deleteRecord = async (app, key, disableCleanup) => { }; const deleteIndexes = async (app, key) => { - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); /* const reverseIndexKeys = $(app.hierarchy, [ getFlattenedHierarchy, map(n => n.fields), @@ -5310,7 +5310,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) => }; const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const incorrectFileFields = $(recordNode.fields, [ fp.filter(f => f.type === 'file' @@ -5557,7 +5557,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } @@ -7431,7 +7431,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => { }; const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ - getExactNodeForPath(appHierarchy), + getExactNodeForKey(appHierarchy), n => n.fields, fp.filter(f => f.type === 'reference' && isSomething(record[f.name]) @@ -7841,7 +7841,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { } if (isReferenceIndex(indexNode)) { - const recordNode = getExactNodeForPath(hierarchy)(t.record.key); + const recordNode = getExactNodeForKey(hierarchy)(t.record.key); const refFields = $(recordNode.fields, [ fp.filter(fieldReversesReferenceToIndex(indexNode)), ]); diff --git a/packages/core/dist/budibase-core.esm.mjs b/packages/core/dist/budibase-core.esm.mjs index 40e5e57d20..724e4f771a 100644 --- a/packages/core/dist/budibase-core.esm.mjs +++ b/packages/core/dist/budibase-core.esm.mjs @@ -719,7 +719,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [ filter(n => new RegExp(`${n.pathRegx()}`).test(key)), ]); -const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ +const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [ getFlattenedHierarchy, find(n => new RegExp(`${n.pathRegx()}$`).test(key)), ]); @@ -757,7 +757,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [ ]); const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { - const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); + const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey); return isNothing(nodeByKey) ? getNode(appHierarchy, keyOrNodeKey) : nodeByKey; @@ -770,7 +770,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { : nodeByKey; }; -const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); +const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key)); const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ splitKey, @@ -869,7 +869,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe var hierarchy = { getLastPartInKey, getNodesInPath, - getExactNodeForPath, + getExactNodeForKey, hasMatchingAncestor, getNode, getNodeByKeyOrNodeKey, @@ -1605,7 +1605,7 @@ const load = app => async key => { const _load = async (app, key, keyStack = []) => { key = safeKey(key); - const recordNode = getExactNodeForPath(app.hierarchy)(key); + const recordNode = getExactNodeForKey(app.hierarchy)(key); const storedData = await app.datastore.loadJson( getRecordFileName(key), ); @@ -1751,7 +1751,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => { }; const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); const startShardName = !startRecord ? null @@ -4396,7 +4396,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => { )); indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } @@ -4428,7 +4428,7 @@ const getContext = app => recordKey => { const _getContext = (app, recordKey) => { recordKey = safeKey(recordKey); - const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); + const recordNode = getExactNodeForKey(app.hierarchy)(recordKey); const cachedReferenceIndexes = {}; @@ -4536,7 +4536,7 @@ const validate = app => async (record, context) => { ? _getContext(app, record.key) : context; - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const fieldParseFails = validateAllFieldParse(record, recordNode); // non parsing would cause further issues - exit here @@ -4596,7 +4596,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => { const initialiseChildCollections = async (app, recordKey) => { const childCollectionRecords = $(recordKey, [ - getExactNodeForPath(app.hierarchy), + getExactNodeForKey(app.hierarchy), n => n.children, filter(isCollectionRecord), ]); @@ -4967,7 +4967,7 @@ const _save = async (app, record, context, skipValidation = false) => { } if (recordClone.isNew) { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); if(!recordNode) throw new Error("Cannot find node for " + record.key); @@ -5016,7 +5016,7 @@ const _save = async (app, record, context, skipValidation = false) => { }; const initialiseAncestorIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); for (const index of recordNode.indexes) { const indexKey = joinKey(record.key, index.name); @@ -5025,7 +5025,7 @@ const initialiseAncestorIndexes = async (app, record) => { }; const initialiseReverseReferenceIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ map(f => $(f.typeOptions.reverseIndexNodeKeys, [ @@ -5124,7 +5124,7 @@ const deleteRecords = async (app, key) => { const _deleteIndex = async (app, indexKey, includeFolder) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } @@ -5169,7 +5169,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => { // called deleteRecord because delete is a keyword const _deleteRecord = async (app, key, disableCleanup) => { key = safeKey(key); - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); const record = await _load(app, key); await transactionForDeleteRecord(app, record); @@ -5196,7 +5196,7 @@ const _deleteRecord = async (app, key, disableCleanup) => { }; const deleteIndexes = async (app, key) => { - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); /* const reverseIndexKeys = $(app.hierarchy, [ getFlattenedHierarchy, map(n => n.fields), @@ -5303,7 +5303,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) => }; const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const incorrectFileFields = $(recordNode.fields, [ filter(f => f.type === 'file' @@ -5550,7 +5550,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } @@ -7424,7 +7424,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => { }; const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ - getExactNodeForPath(appHierarchy), + getExactNodeForKey(appHierarchy), n => n.fields, filter(f => f.type === 'reference' && isSomething(record[f.name]) @@ -7834,7 +7834,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { } if (isReferenceIndex(indexNode)) { - const recordNode = getExactNodeForPath(hierarchy)(t.record.key); + const recordNode = getExactNodeForKey(hierarchy)(t.record.key); const refFields = $(recordNode.fields, [ filter(fieldReversesReferenceToIndex(indexNode)), ]); diff --git a/packages/core/dist/budibase-core.umd.js b/packages/core/dist/budibase-core.umd.js index cf76715980..2f3d09d485 100644 --- a/packages/core/dist/budibase-core.umd.js +++ b/packages/core/dist/budibase-core.umd.js @@ -721,7 +721,7 @@ fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)), ]); - const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ + const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [ getFlattenedHierarchy, fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)), ]); @@ -759,7 +759,7 @@ ]); const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { - const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); + const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey); return isNothing(nodeByKey) ? getNode(appHierarchy, keyOrNodeKey) : nodeByKey; @@ -772,7 +772,7 @@ : nodeByKey; }; - const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); + const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key)); const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ splitKey, @@ -871,7 +871,7 @@ var hierarchy = { getLastPartInKey, getNodesInPath, - getExactNodeForPath, + getExactNodeForKey, hasMatchingAncestor, getNode, getNodeByKeyOrNodeKey, @@ -1607,7 +1607,7 @@ const _load = async (app, key, keyStack = []) => { key = safeKey(key); - const recordNode = getExactNodeForPath(app.hierarchy)(key); + const recordNode = getExactNodeForKey(app.hierarchy)(key); const storedData = await app.datastore.loadJson( getRecordFileName(key), ); @@ -1753,7 +1753,7 @@ }; const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); const startShardName = !startRecord ? null @@ -4398,7 +4398,7 @@ )); indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } @@ -4430,7 +4430,7 @@ const _getContext = (app, recordKey) => { recordKey = safeKey(recordKey); - const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); + const recordNode = getExactNodeForKey(app.hierarchy)(recordKey); const cachedReferenceIndexes = {}; @@ -4538,7 +4538,7 @@ ? _getContext(app, record.key) : context; - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const fieldParseFails = validateAllFieldParse(record, recordNode); // non parsing would cause further issues - exit here @@ -4598,7 +4598,7 @@ const initialiseChildCollections = async (app, recordKey) => { const childCollectionRecords = $(recordKey, [ - getExactNodeForPath(app.hierarchy), + getExactNodeForKey(app.hierarchy), n => n.children, fp.filter(isCollectionRecord), ]); @@ -4969,7 +4969,7 @@ } if (recordClone.isNew) { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); if(!recordNode) throw new Error("Cannot find node for " + record.key); @@ -5018,7 +5018,7 @@ }; const initialiseAncestorIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); for (const index of recordNode.indexes) { const indexKey = joinKey(record.key, index.name); @@ -5027,7 +5027,7 @@ }; const initialiseReverseReferenceIndexes = async (app, record) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [ @@ -5126,7 +5126,7 @@ const _deleteIndex = async (app, indexKey, includeFolder) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } @@ -5171,7 +5171,7 @@ // called deleteRecord because delete is a keyword const _deleteRecord = async (app, key, disableCleanup) => { key = safeKey(key); - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); const record = await _load(app, key); await transactionForDeleteRecord(app, record); @@ -5198,7 +5198,7 @@ }; const deleteIndexes = async (app, key) => { - const node = getExactNodeForPath(app.hierarchy)(key); + const node = getExactNodeForKey(app.hierarchy)(key); /* const reverseIndexKeys = $(app.hierarchy, [ getFlattenedHierarchy, map(n => n.fields), @@ -5305,7 +5305,7 @@ }; const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const incorrectFileFields = $(recordNode.fields, [ fp.filter(f => f.type === 'file' @@ -5552,7 +5552,7 @@ const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } @@ -7426,7 +7426,7 @@ }; const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ - getExactNodeForPath(appHierarchy), + getExactNodeForKey(appHierarchy), n => n.fields, fp.filter(f => f.type === 'reference' && isSomething(record[f.name]) @@ -7836,7 +7836,7 @@ } if (isReferenceIndex(indexNode)) { - const recordNode = getExactNodeForPath(hierarchy)(t.record.key); + const recordNode = getExactNodeForKey(hierarchy)(t.record.key); const refFields = $(recordNode.fields, [ fp.filter(fieldReversesReferenceToIndex(indexNode)), ]); diff --git a/packages/core/src/appInitialise/inProcIndexer.js b/packages/core/src/appInitialise/inProcIndexer.js deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/core/src/appInitialise/index.js b/packages/core/src/appInitialise/index.js index 9ffbe122ef..6996895b2b 100644 --- a/packages/core/src/appInitialise/index.js +++ b/packages/core/src/appInitialise/index.js @@ -1,23 +1,27 @@ import { retry } from '../common/index'; import { NotFoundError } from '../common/errors'; -const createJson = originalCreateFile => async (key, obj, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj)); +const createJson = originalCreateFile => async (key, obj, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj)); -const createNewFile = originalCreateFile => async (path, content, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, path, content); +const createNewFile = originalCreateFile => async (path, content, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, path, content); -const loadJson = datastore => async (key, retries = 5, delay = 500) => { +const loadJson = datastore => async (key, retries = 3, delay = 100) => { try { return await retry(JSON.parse, retries, delay, await datastore.loadFile(key)); } catch (err) { - throw new NotFoundError(err.message); + const newErr = new NotFoundError(err.message); + newErr.stack = err.stack; + throw(newErr); } } -const updateJson = datastore => async (key, obj, retries = 5, delay = 500) => { +const updateJson = datastore => async (key, obj, retries = 3, delay = 100) => { try { return await retry(datastore.updateFile, retries, delay, key, JSON.stringify(obj)); } catch (err) { - throw new NotFoundError(err.message); + const newErr = new NotFoundError(err.message); + newErr.stack = err.stack; + throw(newErr); } } diff --git a/packages/core/src/appInitialise/initialiseData.js b/packages/core/src/appInitialise/initialiseData.js index 20ce5eabf5..62e9d07dfe 100644 --- a/packages/core/src/appInitialise/initialiseData.js +++ b/packages/core/src/appInitialise/initialiseData.js @@ -35,7 +35,9 @@ const initialiseRootIndexes = async (datastore, hierarchy) => { ]); for (const index of globalIndexes) { - if (!await datastore.exists(index.nodeKey())) { await initialiseIndex(datastore, '', index); } + if (!await datastore.exists(index.nodeKey())) { + await initialiseIndex(datastore, '', index); + } } }; @@ -52,6 +54,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => { ]); for (let record of singleRecords) { + await datastore.createFolder(record.nodeKey()); const result = _getNew(record, ""); await _save(app,result); } diff --git a/packages/core/src/collectionApi/delete.js b/packages/core/src/collectionApi/delete.js index ac3f697de7..b11412d59f 100644 --- a/packages/core/src/collectionApi/delete.js +++ b/packages/core/src/collectionApi/delete.js @@ -1,13 +1,11 @@ -import { includes } from 'lodash/fp'; -import { getNodeForCollectionPath } from '../templateApi/hierarchy'; import { safeKey, apiWrapper, events, joinKey, } from '../common'; import { _deleteRecord } from '../recordApi/delete'; -import { getAllIdsIterator, getAllIdsShardKey, folderStructureArray } from '../indexing/allIds'; +import { getAllIdsIterator } from '../indexing/allIds'; import { permission } from '../authApi/permissions'; -import { getRecordInfo } from "../recordApi/recordInfo"; +import { getCollectionDir } from "../recordApi/recordInfo"; export const deleteCollection = (app, disableCleanup = false) => async key => apiWrapper( app, @@ -17,28 +15,38 @@ export const deleteCollection = (app, disableCleanup = false) => async key => ap _deleteCollection, app, key, disableCleanup, ); +/* + const recordNode = getCollectionNode(app.hierarchy, key); + +*/ export const _deleteCollection = async (app, key, disableCleanup) => { - const recordInfo = getRecordInfo(key); - await app.datastore.deleteFolder(recordInfo) + key = safeKey(key); + const collectionDir = getCollectionDir(app.hierarchy, key); + await deleteRecords(app, key); + await deleteCollectionFolder(app, collectionDir); if (!disableCleanup) { await app.cleanupTransactions(); } }; -const deleteCollectionFolder = async (app, key) => await app.datastore.deleteFolder(key); +const deleteCollectionFolder = async (app, dir) => + await app.datastore.deleteFolder(dir); +const deleteRecords = async (app, key) => { + + const iterate = await getAllIdsIterator(app)(key); -const deleteShardFolders = async (app, node, key) => { + let ids = await iterate(); + while (!ids.done) { + if (ids.result.collectionKey === key) { + for (const id of ids.result.ids) { + await _deleteRecord( + app, + joinKey(key, id), + true, + ); + } + } - await app.datastore.deleteFolder( - joinKey( - key, 'allids', - node.nodeId, - ), - ); - - await app.datastore.deleteFolder( - joinKey(key, 'allids'), - ); + ids = await iterate(); + } }; - - diff --git a/packages/core/src/collectionApi/initialise.js b/packages/core/src/collectionApi/initialise.js index 2f1f19cfa3..2384d171d4 100644 --- a/packages/core/src/collectionApi/initialise.js +++ b/packages/core/src/collectionApi/initialise.js @@ -3,13 +3,13 @@ import { getFlattenedHierarchy, isCollectionRecord, isRoot, - getExactNodeForPath, } from '../templateApi/hierarchy'; import { $, allTrue, joinKey } from '../common'; const ensureCollectionIsInitialised = async (datastore, node, dir) => { if (!await datastore.exists(dir)) { await datastore.createFolder(dir); + await datastore.createFolder(joinKey(dir, node.nodeId)); } }; @@ -29,7 +29,7 @@ export const initialiseRootCollections = async (datastore, hierarchy) => { await ensureCollectionIsInitialised( datastore, col, - col.collectionPathRegx(), + col.collectionPathRegx() ); } }; diff --git a/packages/core/src/common/index.js b/packages/core/src/common/index.js index 50538599a5..0a28bb37e2 100644 --- a/packages/core/src/common/index.js +++ b/packages/core/src/common/index.js @@ -4,13 +4,12 @@ import { tail, findIndex, startsWith, dropRight, flow, takeRight, trim, replace - } from 'lodash'; import { some, reduce, isEmpty, isArray, join, isString, isInteger, isDate, toNumber, isUndefined, isNaN, isNull, constant, - split, includes + split, includes, filter } from 'lodash/fp'; import { events, eventsList } from './events'; import { apiWrapper } from './apiWrapper'; @@ -32,7 +31,13 @@ export const safeKey = key => replace(`${keySep}${trimKeySep(key)}`, `${keySep}$ export const joinKey = (...strs) => { const paramsOrArray = strs.length === 1 & isArray(strs[0]) ? strs[0] : strs; - return safeKey(join(keySep)(paramsOrArray)); + return $(paramsOrArray, [ + filter(s => !isUndefined(s) + && !isNull(s) + && s.toString().length > 0), + join(keySep), + safeKey + ]); }; export const splitKey = $$(trimKeySep, splitByKeySep); export const getDirFomKey = $$(splitKey, dropRight, p => joinKey(...p)); @@ -183,6 +188,10 @@ export const toNumberOrNull = s => (isNull(s) ? null export const isArrayOfString = opts => isArray(opts) && all(isString)(opts); +export const pushAll = (target, items) => { + for(let i of items) target.push(i); +} + export const pause = async duration => new Promise(res => setTimeout(res, duration)); export const retry = async (fn, retries, delay, ...args) => { @@ -267,4 +276,5 @@ export default { insensitiveEquals, pause, retry, + pushAll }; diff --git a/packages/core/src/indexApi/aggregates.js b/packages/core/src/indexApi/aggregates.js index 2020a4dda8..4c77ce0973 100644 --- a/packages/core/src/indexApi/aggregates.js +++ b/packages/core/src/indexApi/aggregates.js @@ -10,12 +10,13 @@ import { getShardKeysInRange, } from '../indexing/sharding'; import { - getExactNodeForPath, isIndex, + getExactNodeForKey, isIndex, isShardedIndex, } from '../templateApi/hierarchy'; import { CONTINUE_READING_RECORDS } from '../indexing/serializer'; import { permission } from '../authApi/permissions'; import { BadRequestError } from '../common/errors'; +import { getIndexDir } from "./getIndexDir"; export const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndParams = null) => apiWrapper( app, @@ -27,13 +28,14 @@ export const aggregates = app => async (indexKey, rangeStartParams = null, range const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); + const indexDir = getIndexDir(app.hierarchy, indexKey); if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } if (isShardedIndex(indexNode)) { const shardKeys = await getShardKeysInRange( - app, indexKey, rangeStartParams, rangeEndParams, + app, indexNode, indexDir, rangeStartParams, rangeEndParams, ); let aggregateResult = null; for (const k of shardKeys) { @@ -53,7 +55,7 @@ const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { app.hierarchy, app.datastore, indexNode, - getUnshardedIndexDataKey(indexKey), + getUnshardedIndexDataKey(indexDir), ); }; diff --git a/packages/core/src/indexApi/buildIndex.js b/packages/core/src/indexApi/buildIndex.js index eb5534f2b1..2bf18a9948 100644 --- a/packages/core/src/indexApi/buildIndex.js +++ b/packages/core/src/indexApi/buildIndex.js @@ -1,16 +1,16 @@ import { - find, filter, + filter, includes, some, } from 'lodash/fp'; import { getAllIdsIterator } from '../indexing/allIds'; import { getFlattenedHierarchy, getRecordNodeById, - getCollectionNodeByKeyOrNodeKey, getNode, isIndex, - isRecord, isDecendant, getAllowedRecordNodesForIndex, + getNode, isIndex, + isRecord, getAllowedRecordNodesForIndex, fieldReversesReferenceToIndex, } from '../templateApi/hierarchy'; import { - joinKey, apiWrapper, events, $, allTrue, + joinKey, apiWrapper, events, $ } from '../common'; import { createBuildIndexFolder, @@ -82,9 +82,11 @@ const buildReverseReferenceIndex = async (app, indexNode) => { } }; +/* const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [ map(n => n.parent()), ]); +*/ const buildHeirarchalIndex = async (app, indexNode) => { let recordCount = 0; @@ -127,10 +129,11 @@ const buildHeirarchalIndex = async (app, indexNode) => { return recordCount; }; -const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children); +// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children); const recordNodeApplies = indexNode => recordNode => includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds); +/* const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [ getFlattenedHierarchy, filter( @@ -141,7 +144,9 @@ const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarc ), ), ]); +*/ + /* const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey, indexNode, indexKey, currentIndexedData, currentIndexedDataKey, recordCount = 0) => { @@ -194,5 +199,6 @@ const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey, return recordCount; }; +*/ export default buildIndex; diff --git a/packages/core/src/indexApi/delete.js b/packages/core/src/indexApi/delete.js index afcb809e87..eef4476d06 100644 --- a/packages/core/src/indexApi/delete.js +++ b/packages/core/src/indexApi/delete.js @@ -4,21 +4,23 @@ import { } from '../common'; import { isIndex, isShardedIndex, - getExactNodeForPath, + getExactNodeForKey, } from '../templateApi/hierarchy'; import { getAllShardKeys, getShardMapKey, getUnshardedIndexDataKey, } from '../indexing/sharding'; +import { getIndexDir } from "./getIndexDir"; export const _deleteIndex = async (app, indexKey, includeFolder) => { indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); + const indexDir = getIndexDir(app.hierarchy, indexKey); if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } if (isShardedIndex(indexNode)) { - const shardKeys = await getAllShardKeys(app, indexKey); + const shardKeys = await getAllShardKeys(app, indexNode, indexDir); for (const k of shardKeys) { await tryAwaitOrIgnore( app.datastore.deleteFile(k), @@ -26,20 +28,20 @@ export const _deleteIndex = async (app, indexKey, includeFolder) => { } tryAwaitOrIgnore( await app.datastore.deleteFile( - getShardMapKey(indexKey), + getShardMapKey(indexDir), ), ); } else { await tryAwaitOrIgnore( app.datastore.deleteFile( - getUnshardedIndexDataKey(indexKey), + getUnshardedIndexDataKey(indexDir), ), ); } if (includeFolder) { tryAwaitOrIgnore( - await app.datastore.deleteFolder(indexKey), + await app.datastore.deleteFolder(indexDir), ); } }; diff --git a/packages/core/src/indexApi/getIndexDir.js b/packages/core/src/indexApi/getIndexDir.js new file mode 100644 index 0000000000..91357d0a67 --- /dev/null +++ b/packages/core/src/indexApi/getIndexDir.js @@ -0,0 +1,20 @@ +import { getRecordInfo } from "../recordApi/recordInfo"; +import { + getParentKey, getLastPartInKey +} from "../templateApi/hierarchy"; +import { keySep } from "../common"; + +export const getIndexDir = (hierarchy, indexKey) => { + + const parentKey = getParentKey(indexKey); + + if(parentKey === "") return indexKey; + if(parentKey === keySep) return indexKey; + + const recordInfo = getRecordInfo( + hierarchy, + parentKey); + + return recordInfo.child( + getLastPartInKey(indexKey)); +} \ No newline at end of file diff --git a/packages/core/src/indexApi/listItems.js b/packages/core/src/indexApi/listItems.js index 8d21239533..c53bbfe827 100644 --- a/packages/core/src/indexApi/listItems.js +++ b/packages/core/src/indexApi/listItems.js @@ -9,10 +9,11 @@ import { getShardKeysInRange, } from '../indexing/sharding'; import { - getExactNodeForPath, isIndex, + getExactNodeForKey, isIndex, isShardedIndex, } from '../templateApi/hierarchy'; import { permission } from '../authApi/permissions'; +import { getIndexDir } from "./getIndexDir"; export const listItems = app => async (indexKey, options) => { indexKey = safeKey(indexKey); @@ -33,29 +34,30 @@ const _listItems = async (app, indexKey, options = defaultOptions) => { merge(defaultOptions), ]); - const getItems = async key => (isNonEmptyString(searchPhrase) + const getItems = async indexedDataKey => (isNonEmptyString(searchPhrase) ? await searchIndex( app.hierarchy, app.datastore, indexNode, - key, + indexedDataKey, searchPhrase, ) : await readIndex( app.hierarchy, app.datastore, indexNode, - key, + indexedDataKey, )); indexKey = safeKey(indexKey); - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); + const indexNode = getExactNodeForKey(app.hierarchy)(indexKey); + const indexDir = getIndexDir(app.hierarchy, indexKey); if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } if (isShardedIndex(indexNode)) { const shardKeys = await getShardKeysInRange( - app, indexKey, rangeStartParams, rangeEndParams, + app, indexNode, indexDir, rangeStartParams, rangeEndParams, ); const items = []; for (const k of shardKeys) { @@ -64,6 +66,6 @@ const _listItems = async (app, indexKey, options = defaultOptions) => { return flatten(items); } return await getItems( - getUnshardedIndexDataKey(indexKey), + getUnshardedIndexDataKey(indexDir), ); }; diff --git a/packages/core/src/indexing/allIds.js b/packages/core/src/indexing/allIds.js index 400c7d4782..639a9600dc 100644 --- a/packages/core/src/indexing/allIds.js +++ b/packages/core/src/indexing/allIds.js @@ -1,13 +1,14 @@ import { - join, flatten, orderBy, - filter + flatten, orderBy, + filter, isUndefined } from 'lodash/fp'; -import { +import hierarchy, { getFlattenedHierarchy, getCollectionNodeByKeyOrNodeKey, isCollectionRecord, isAncestor, } from '../templateApi/hierarchy'; import { joinKey, safeKey, $ } from '../common'; +import { getCollectionDir } from "../recordApi/recordInfo"; export const RECORDS_PER_FOLDER = 1000; export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-'; @@ -23,19 +24,39 @@ export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ * - [64, 64, 10] = all records fit into 64 * 64 * 10 folder * (there are 64 possible chars in allIsChars) */ -export const folderStructureArray = (recordNode, currentArray=[], currentFolderPosition=0) => { +export const folderStructureArray = (recordNode) => { + + const totalFolders = Math.ceil(recordNode.estimatedRecordCount / 1000); + const folderArray = []; + let levelCount = 1; + while(64**levelCount < totalFolders) { + levelCount += 1; + folderArray.push(64); + } + + const parentFactor = (64**folderArray.length); + if(parentFactor < totalFolders) { + folderArray.push( + Math.ceil(totalFolders / parentFactor) + ); + } + + return folderArray; + + /* const maxRecords = currentFolderPosition === 0 ? RECORDS_PER_FOLDER : currentFolderPosition * 64 * RECORDS_PER_FOLDER; if(maxRecords < recordNode.estimatedRecordCount) { return folderStructureArray( + recordNode, [...currentArray, 64], currentFolderPosition + 1); } else { - const childFolderCount = Math.ceil(maxRecords / recordNode.estimatedRecordCount); + const childFolderCount = Math.ceil(recordNode.estimatedRecordCount / maxRecords ); return [...currentArray, childFolderCount] - } + }*/ } @@ -51,88 +72,150 @@ export const getAllIdsIterator = app => async (collection_Key_or_NodeKey) => { const folderStructure = folderStructureArray(recordNode) let currentFolderContents = []; - let currentFolderIndexes = []; - let currentSubPath = []; + let currentPosition = []; + const collectionDir = getCollectionDir(app.hierarchy, collectionKey); const basePath = joinKey( - collectionKey, recordNode.nodeId.toString()); + collectionDir, recordNode.nodeId.toString()); - let folderLevel = 0; - const levels = folderStructure.length; + + // "folderStructure" determines the top, sharding folders + // we need to add one, for the collection root folder, which + // always exists + const levels = folderStructure.length + 1; const topLevel = levels -1; - const lastPathHasContent = () => - folderLevel === 0 - || currentFolderContents[folderLevel - 1].length > 0; + + /* populate initial directory structure in form: + [ + {path: "/a", contents: ["b", "c", "d"]}, + {path: "/a/b", contents: ["e","f","g"]}, + {path: "/a/b/e", contents: ["1-abcd","2-cdef","3-efgh"]}, + ] + // stores contents on each parent level + // top level has ID folders + */ + const firstFolder = async () => { - while (folderLevel < folderStructure.length && lastPathHasContent()) { - if(folderLevel < topLevel) { - const contentsThisLevel = - await app.datastore.getFolderContents( - join(basePath, ...currentSubPath)); + let folderLevel = 0; - currentFolderContents.push(contentsThisLevel); - currentFolderIndexes.push(0); - currentSubPath.push(currentFolderContents[0]) - } else { - // placesholders only for the top level (which will be populated by nextFolder()) - currentFolderContents.push([]) - currentFolderIndexes.push(-1); - currentSubPath.push(""); - } + const lastPathHasContent = () => + folderLevel === 0 + || currentFolderContents[folderLevel - 1].contents.length > 0; - folderLevel+=1; - } - + while (folderLevel <= topLevel && lastPathHasContent()) { - const nextFolder = async (lev=-1) => { - lev = (lev === -1) ? topLevel : lev; - if(currentFolderIndexes[lev] !== currentFolderContents[lev].length - 1){ - - const folderIndexThisLevel = currentFolderIndexes[lev] + 1; - currentFolderIndexes[lev] = folderIndexThisLevel; - currentSubPath[lev] = currentFolderContents[folderIndexThisLevel] - - if(lev < topLevel) { - let loopLev = lev + 1; - while(loopLev <= topLevel) { - currentFolderContents[loopLev] = - await app.datastore.getFolderContents(join(basePath, ...currentSubPath)); - loopLev+=1; - } + let thisPath = basePath; + for(let lev = 0; lev < currentPosition.length; lev++) { + thisPath = joinKey( + thisPath, currentFolderContents[lev].contents[0]); } - return false; // not complete + const contentsThisLevel = + await app.datastore.getFolderContents(thisPath); + currentFolderContents.push({ + contents:contentsThisLevel, + path: thisPath + }); - } else { - if(lev === 0) return true; // complete - return await nextFolder(lev - 1); + // should start as something like [0,0] + if(folderLevel < topLevel) + currentPosition.push(0); + + folderLevel+=1; } + + return (currentPosition.length === levels - 1); + } + + const isOnLastFolder = level => { + + const result = currentPosition[level] === currentFolderContents[level].contents.length - 1; + return result; } + + const getNextFolder = async (lev=undefined) => { + lev = isUndefined(lev) ? topLevel : lev; + const parentLev = lev - 1; + + if(parentLev < 0) return false; + + if(isOnLastFolder(parentLev)) { + return await getNextFolder(parentLev); + } + + const newPosition = currentPosition[parentLev] + 1; + currentPosition[parentLev] = newPosition; + + const nextFolder = joinKey( + currentFolderContents[parentLev].path, + currentFolderContents[parentLev].contents[newPosition]); + currentFolderContents[lev].contents = await app.datastore.getFolderContents( + nextFolder + ); + currentFolderContents[lev].path = nextFolder; + + if(lev !== topLevel) { + + // we just advanced a parent folder, so now need to + // do the same to the next levels + let loopLevel = lev + 1; + while(loopLevel <= topLevel) { + const loopParentLevel = loopLevel-1; + + currentPosition[loopParentLevel] = 0; + const nextLoopFolder = joinKey( + currentFolderContents[loopParentLevel].path, + currentFolderContents[loopParentLevel].contents[0]); + currentFolderContents[loopLevel].contents = await app.datastore.getFolderContents( + nextLoopFolder + ); + currentFolderContents[loopLevel].path = nextLoopFolder; + loopLevel+=1; + } + } + + // true ==has more ids... (just loaded more) + return true; + } + + + const idsCurrentFolder = () => + currentFolderContents[currentFolderContents.length - 1].contents; const fininshedResult = ({ done: true, result: { ids: [], collectionKey } }); + let hasStarted = false; + let hasMore = true; const getIdsFromCurrentfolder = async () => { - if(currentFolderIndexes.length < folderStructure) + if(!hasMore) { return fininshedResult; + } - const hasMore = await nextFolder(); + if(!hasStarted) { + hasMore = await firstFolder(); + hasStarted = true; + return ({ + result: { + ids: idsCurrentFolder(), + collectionKey + }, + done: false + }) + } - if(!hasMore) return fininshedResult; + hasMore = await getNextFolder(); - const result = ({ + return ({ result: { - ids: await app.datastore.getFolderContents( - joinKey(basePath, currentSubPath)), + ids: hasMore ? idsCurrentFolder() : [], collectionKey }, - done:false - }) - - return result; + done: !hasMore + }); } return getIdsFromCurrentfolder; diff --git a/packages/core/src/indexing/apply.js b/packages/core/src/indexing/apply.js index 1626bd6b2f..e282107484 100644 --- a/packages/core/src/indexing/apply.js +++ b/packages/core/src/indexing/apply.js @@ -1,13 +1,13 @@ import { ensureShardNameIsInShardMap } from './sharding'; import { getIndexWriter } from './serializer'; -import { isShardedIndex } from '../templateApi/hierarchy'; +import { isShardedIndex, getParentKey } from '../templateApi/hierarchy'; import {promiseWriteableStream} from "./promiseWritableStream"; import {promiseReadableStream} from "./promiseReadableStream"; -export const applyToShard = async (hierarchy, store, indexKey, +export const applyToShard = async (hierarchy, store, indexDir, indexNode, indexShardKey, recordsToWrite, keysToRemove) => { const createIfNotExists = recordsToWrite.length > 0; - const writer = await getWriter(hierarchy, store, indexKey, indexShardKey, indexNode, createIfNotExists); + const writer = await getWriter(hierarchy, store, indexDir, indexShardKey, indexNode, createIfNotExists); if (writer === SHARD_DELETED) return; await writer.updateIndex(recordsToWrite, keysToRemove); @@ -15,13 +15,17 @@ export const applyToShard = async (hierarchy, store, indexKey, }; const SHARD_DELETED = 'SHARD_DELETED'; -const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode, createIfNotExists) => { +const getWriter = async (hierarchy, store, indexDir, indexedDataKey, indexNode, createIfNotExists) => { let readableStream = null; if (isShardedIndex(indexNode)) { - await ensureShardNameIsInShardMap(store, indexKey, indexedDataKey); + await ensureShardNameIsInShardMap(store, indexDir, indexedDataKey); if(!await store.exists(indexedDataKey)) { - await store.createFile(indexedDataKey, ""); + if (await store.exists(getParentKey(indexedDataKey))) { + await store.createFile(indexedDataKey, ""); + } else { + return SHARD_DELETED; + } } } @@ -37,7 +41,11 @@ const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode, throw e; } else { if (createIfNotExists) { - await store.createFile(indexedDataKey, ''); + if(await store.exists(getParentKey(indexedDataKey))) { + await store.createFile(indexedDataKey, ''); + } else { + return SHARD_DELETED; + } } else { return SHARD_DELETED; } @@ -65,6 +73,12 @@ const swapTempFileIn = async (store, indexedDataKey, isRetry = false) => { await store.deleteFile(indexedDataKey); } catch (e) { // ignore failure, incase it has not been created yet + + // if parent folder does not exist, assume that this index + // should not be there + if(!await store.exists(getParentKey(indexedDataKey))) { + return; + } } try { await store.renameFile(tempFile, indexedDataKey); diff --git a/packages/core/src/indexing/initialiseIndex.js b/packages/core/src/indexing/initialiseIndex.js index b8b2cb54ba..3f3265c5be 100644 --- a/packages/core/src/indexing/initialiseIndex.js +++ b/packages/core/src/indexing/initialiseIndex.js @@ -3,19 +3,19 @@ import { joinKey } from '../common'; import { getShardMapKey, getUnshardedIndexDataKey, createIndexFile } from './sharding'; export const initialiseIndex = async (datastore, dir, index) => { - const indexKey = joinKey(dir, index.name); + const indexDir = joinKey(dir, index.name); - await datastore.createFolder(indexKey); + await datastore.createFolder(indexDir); if (isShardedIndex(index)) { await datastore.createFile( - getShardMapKey(indexKey), + getShardMapKey(indexDir), '[]', ); } else { await createIndexFile( datastore, - getUnshardedIndexDataKey(indexKey), + getUnshardedIndexDataKey(indexDir), index, ); } diff --git a/packages/core/src/indexing/read.js b/packages/core/src/indexing/read.js index ffb2c5da8e..1b512b349d 100644 --- a/packages/core/src/indexing/read.js +++ b/packages/core/src/indexing/read.js @@ -1,12 +1,4 @@ import lunr from 'lunr'; -import { - getHashCode, - joinKey -} from '../common'; -import { - getActualKeyOfParent, - isGlobalIndex, -} from '../templateApi/hierarchy'; import {promiseReadableStream} from "./promiseReadableStream"; import { createIndexFile } from './sharding'; import { generateSchema } from './indexSchemaCreator'; @@ -50,31 +42,6 @@ export const searchIndex = async (hierarchy, datastore, index, indexedDataKey, s return await doRead(hierarchy, datastore, index, indexedDataKey); }; -export const getIndexedDataKey_fromIndexKey = (indexKey) => - `${indexKey}${indexKey.endsWith('.csv') ? '' : '.csv'}`; - -export const uniqueIndexName = index => `idx_${ - getHashCode(`${index.filter}${index.map}`) -}.csv`; - -export const getIndexedDataKey = (decendantKey, indexNode) => { - if (isGlobalIndex(indexNode)) { return `${indexNode.nodeKey()}.csv`; } - - const indexedDataParentKey = getActualKeyOfParent( - indexNode.parent().nodeKey(), - decendantKey, - ); - - const indexName = indexNode.name - ? `${indexNode.name}.csv` - : uniqueIndexName(indexNode); - - return joinKey( - indexedDataParentKey, - indexName, - ); -}; - export const iterateIndex = (onGetItem, getFinalResult) => async (hierarchy, datastore, index, indexedDataKey) => { try { const readableStream = promiseReadableStream( diff --git a/packages/core/src/indexing/relevant.js b/packages/core/src/indexing/relevant.js index 58f03493c0..706dfb80a9 100644 --- a/packages/core/src/indexing/relevant.js +++ b/packages/core/src/indexing/relevant.js @@ -9,21 +9,23 @@ import { } from '../common'; import { getFlattenedHierarchy, getNode, getRecordNodeId, - getExactNodeForPath, recordNodeIdIsAllowed, + getExactNodeForKey, recordNodeIdIsAllowed, isRecord, isGlobalIndex, } from '../templateApi/hierarchy'; import { indexTypes } from '../templateApi/indexes'; +import { getIndexDir } from "../indexApi/getIndexDir"; +import { getRecordInfo} from "../recordApi/recordInfo"; -export const getRelevantAncestorIndexes = (appHierarchy, record) => { +export const getRelevantAncestorIndexes = (hierarchy, record) => { const key = record.key; const keyParts = splitKey(key); const nodeId = getRecordNodeId(key); - const flatHierarchy = orderBy(getFlattenedHierarchy(appHierarchy), + const flatHierarchy = orderBy(getFlattenedHierarchy(hierarchy), [node => node.pathRegx().length], ['desc']); - const makeindexNodeAndKey_ForAncestorIndex = (indexNode, indexKey) => makeIndexNodeAndKey(indexNode, joinKey(indexKey, indexNode.name)); + const makeindexNodeAndDir_ForAncestorIndex = (indexNode, parentRecordDir) => makeIndexNodeAndDir(indexNode, joinKey(parentRecordDir, indexNode.name)); const traverseAncestorIndexesInPath = () => reduce((acc, part) => { const currentIndexKey = joinKey(acc.lastIndexKey, part); @@ -42,8 +44,10 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => { || includes(nodeId)(i.allowedRecordNodeIds))), ]); + const currentRecordDir = getRecordInfo(hierarchy, currentIndexKey).dir; + each(v => acc.nodesAndKeys.push( - makeindexNodeAndKey_ForAncestorIndex(v, currentIndexKey), + makeindexNodeAndDir_ForAncestorIndex(v, currentRecordDir), ))(indexes); return acc; @@ -51,31 +55,35 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => { const rootIndexes = $(flatHierarchy, [ filter(n => isGlobalIndex(n) && recordNodeIdIsAllowed(n)(nodeId)), - map(i => makeIndexNodeAndKey(i, i.nodeKey())), + map(i => makeIndexNodeAndDir( + i, + getIndexDir(hierarchy, i.nodeKey()))), ]); return union(traverseAncestorIndexesInPath())(rootIndexes); }; -export const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ - getExactNodeForPath(appHierarchy), +export const getRelevantReverseReferenceIndexes = (hierarchy, record) => $(record.key, [ + getExactNodeForKey(hierarchy), n => n.fields, filter(f => f.type === 'reference' && isSomething(record[f.name]) && isNonEmptyString(record[f.name].key)), map(f => $(f.typeOptions.reverseIndexNodeKeys, [ map(n => ({ - recordNode: getNode(appHierarchy, n), + recordNode: getNode(hierarchy, n), field: f, })), ])), flatten, - map(n => makeIndexNodeAndKey( + map(n => makeIndexNodeAndDir( n.recordNode, - joinKey(record[n.field.name].key, n.recordNode.name), + joinKey( + getRecordInfo(hierarchy, record[n.field.name].key).dir, + n.recordNode.name), )), ]); -const makeIndexNodeAndKey = (indexNode, indexKey) => ({ indexNode, indexKey }); +const makeIndexNodeAndDir = (indexNode, indexDir) => ({ indexNode, indexDir }); export default getRelevantAncestorIndexes; diff --git a/packages/core/src/indexing/sharding.js b/packages/core/src/indexing/sharding.js index 57a73d2b8c..6cc95ea38b 100644 --- a/packages/core/src/indexing/sharding.js +++ b/packages/core/src/indexing/sharding.js @@ -5,13 +5,14 @@ import { import { getActualKeyOfParent, isGlobalIndex, getParentKey, isShardedIndex, - getExactNodeForPath, + getExactNodeForKey, } from '../templateApi/hierarchy'; import { joinKey, isNonEmptyString, splitKey, $, } from '../common'; -export const getIndexedDataKey = (indexNode, indexKey, record) => { +export const getIndexedDataKey = (indexNode, indexDir, record) => { + const getShardName = (indexNode, record) => { const shardNameFunc = compileCode(indexNode.getShardName); try { @@ -27,18 +28,16 @@ export const getIndexedDataKey = (indexNode, indexKey, record) => { ? `${getShardName(indexNode, record)}.csv` : 'index.csv'; - return joinKey(indexKey, shardName); + return joinKey(indexDir, shardName); }; -export const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { - const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); - +export const getShardKeysInRange = async (app, indexNode, indexDir, startRecord = null, endRecord = null) => { const startShardName = !startRecord ? null : shardNameFromKey( getIndexedDataKey( indexNode, - indexKey, + indexDir, startRecord, ), ); @@ -48,29 +47,29 @@ export const getShardKeysInRange = async (app, indexKey, startRecord = null, end : shardNameFromKey( getIndexedDataKey( indexNode, - indexKey, + indexDir, endRecord, ), ); - return $(await getShardMap(app.datastore, indexKey), [ + return $(await getShardMap(app.datastore, indexDir), [ filter(k => (startRecord === null || k >= startShardName) && (endRecord === null || k <= endShardName)), - map(k => joinKey(indexKey, `${k}.csv`)), + map(k => joinKey(indexDir, `${k}.csv`)), ]); }; -export const ensureShardNameIsInShardMap = async (store, indexKey, indexedDataKey) => { - const map = await getShardMap(store, indexKey); +export const ensureShardNameIsInShardMap = async (store, indexDir, indexedDataKey) => { + const map = await getShardMap(store, indexDir); const shardName = shardNameFromKey(indexedDataKey); if (!includes(shardName)(map)) { map.push(shardName); - await writeShardMap(store, indexKey, map); + await writeShardMap(store, indexDir, map); } }; -export const getShardMap = async (datastore, indexKey) => { - const shardMapKey = getShardMapKey(indexKey); +export const getShardMap = async (datastore, indexDir) => { + const shardMapKey = getShardMapKey(indexDir); try { return await datastore.loadJson(shardMapKey); } catch (_) { @@ -79,27 +78,26 @@ export const getShardMap = async (datastore, indexKey) => { } }; -export const writeShardMap = async (datastore, indexKey, shardMap) => await datastore.updateJson( - getShardMapKey(indexKey), +export const writeShardMap = async (datastore, indexDir, shardMap) => await datastore.updateJson( + getShardMapKey(indexDir), shardMap, ); -export const getAllShardKeys = async (app, indexKey) => await getShardKeysInRange(app, indexKey); +export const getAllShardKeys = async (app, indexNode, indexDir) => + await getShardKeysInRange(app, indexNode, indexDir); -export const getShardMapKey = indexKey => joinKey(indexKey, 'shardMap.json'); +export const getShardMapKey = indexDir => joinKey(indexDir, 'shardMap.json'); -export const getUnshardedIndexDataKey = indexKey => joinKey(indexKey, 'index.csv'); - -export const getIndexFolderKey = indexKey => indexKey; +export const getUnshardedIndexDataKey = indexDir => joinKey(indexDir, 'index.csv'); export const createIndexFile = async (datastore, indexedDataKey, index) => { if (isShardedIndex(index)) { - const indexKey = getParentKey(indexedDataKey); - const shardMap = await getShardMap(datastore, indexKey); + const indexDir = getParentKey(indexedDataKey); + const shardMap = await getShardMap(datastore, indexDir); shardMap.push( shardNameFromKey(indexedDataKey), ); - await writeShardMap(datastore, indexKey, shardMap); + await writeShardMap(datastore, indexDir, shardMap); } await datastore.createFile(indexedDataKey, ''); }; diff --git a/packages/core/src/indexing/sweeper.js b/packages/core/src/indexing/sweeper.js deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/core/src/recordApi/delete.js b/packages/core/src/recordApi/delete.js index 51fe614a9f..d51993cc81 100644 --- a/packages/core/src/recordApi/delete.js +++ b/packages/core/src/recordApi/delete.js @@ -1,11 +1,15 @@ import { safeKey, apiWrapper, - events, + events, joinKey, } from '../common'; -import { _loadFromInfo } from './load'; +import { _load } from './load'; +import { _deleteCollection } from '../collectionApi/delete'; +import { + getExactNodeForKey +} from '../templateApi/hierarchy'; import { transactionForDeleteRecord } from '../transactions/create'; import { permission } from '../authApi/permissions'; -import { getRecordInfo } from "./recordInfo"; +import { getRecordInfo } from './recordInfo'; export const deleteRecord = (app, disableCleanup = false) => async key => { key = safeKey(key); @@ -20,17 +24,22 @@ export const deleteRecord = (app, disableCleanup = false) => async key => { // called deleteRecord because delete is a keyword export const _deleteRecord = async (app, key, disableCleanup) => { - const recordInfo = getRecordInfo(app, key); + const recordInfo = getRecordInfo(app.hierarchy, key); key = recordInfo.key; + const node = getExactNodeForKey(app.hierarchy)(key); - const record = await _loadFromInfo(app, recordInfo); + const record = await _load(app, key); await transactionForDeleteRecord(app, record); + for (const collectionRecord of node.children) { + const collectionKey = joinKey( + key, collectionRecord.collectionName, + ); + await _deleteCollection(app, collectionKey, true); + } + await app.datastore.deleteFolder(recordInfo.dir); if (!disableCleanup) { await app.cleanupTransactions(); } }; - - - diff --git a/packages/core/src/recordApi/downloadFile.js b/packages/core/src/recordApi/downloadFile.js index d032097cca..9c05b9fce3 100644 --- a/packages/core/src/recordApi/downloadFile.js +++ b/packages/core/src/recordApi/downloadFile.js @@ -2,6 +2,7 @@ import { apiWrapper, events, isNothing } from '../common'; import { permission } from '../authApi/permissions'; import { safeGetFullFilePath } from './uploadFile'; import { BadRequestError } from '../common/errors'; +import { getRecordInfo } from "./recordInfo"; export const downloadFile = app => async (recordKey, relativePath) => apiWrapper( app, @@ -16,9 +17,10 @@ const _downloadFile = async (app, recordKey, relativePath) => { if (isNothing(recordKey)) { throw new BadRequestError('Record Key not supplied'); } if (isNothing(relativePath)) { throw new BadRequestError('file path not supplied'); } + const {dir} = getRecordInfo(app.hierarchy, recordKey); return await app.datastore.readableFileStream( safeGetFullFilePath( - recordKey, relativePath, + dir, relativePath, ), ); }; diff --git a/packages/core/src/recordApi/getContext.js b/packages/core/src/recordApi/getContext.js index cb9db43d99..a0903eff94 100644 --- a/packages/core/src/recordApi/getContext.js +++ b/packages/core/src/recordApi/getContext.js @@ -1,6 +1,6 @@ import { map, isString, has, some } from 'lodash/fp'; import { - getExactNodeForPath, + getExactNodeForKey, findField, getNode, isGlobalIndex, } from '../templateApi/hierarchy'; import { listItems } from '../indexApi/listItems'; @@ -23,7 +23,7 @@ export const getContext = app => recordKey => { export const _getContext = (app, recordKey) => { recordKey = safeKey(recordKey); - const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); + const recordNode = getExactNodeForKey(app.hierarchy)(recordKey); const cachedReferenceIndexes = {}; diff --git a/packages/core/src/recordApi/load.js b/packages/core/src/recordApi/load.js index 8738cac8c0..65b504a497 100644 --- a/packages/core/src/recordApi/load.js +++ b/packages/core/src/recordApi/load.js @@ -2,7 +2,7 @@ import { keyBy, mapValues, filter, map, includes, last, } from 'lodash/fp'; -import { getExactNodeForPath, getNode } from '../templateApi/hierarchy'; +import { getExactNodeForKey, getNode } from '../templateApi/hierarchy'; import { safeParseField } from '../types'; import { $, splitKey, safeKey, isNonEmptyString, @@ -72,7 +72,7 @@ export const _loadFromInfo = async (app, recordInfo, keyStack = []) => { export const _load = async (app, key, keyStack = []) => _loadFromInfo( app, - getRecordInfo(app, key), + getRecordInfo(app.hierarchy, key), keyStack); diff --git a/packages/core/src/recordApi/recordInfo.js b/packages/core/src/recordApi/recordInfo.js index a68ee17790..8d7b3ba283 100644 --- a/packages/core/src/recordApi/recordInfo.js +++ b/packages/core/src/recordApi/recordInfo.js @@ -1,18 +1,19 @@ import { - getExactNodeForPath, getActualKeyOfParent, isRoot + getExactNodeForKey, getActualKeyOfParent, + isRoot, isSingleRecord, getNodeForCollectionPath } from '../templateApi/hierarchy'; import { -map, reduce +reduce, find, filter, take } from 'lodash/fp'; import { -$, getDirFomKey, getFileFromKey, joinKey, safeKey +$, getFileFromKey, joinKey, safeKey, keySep } from '../common'; import { folderStructureArray, allIdChars } from "../indexing/allIds"; -export const getRecordInfo = (app, key) => { - const recordNode = getExactNodeForPath(app.hierarchy)(key); +export const getRecordInfo = (hierarchy, key) => { + const recordNode = getExactNodeForKey(hierarchy)(key); const pathInfo = getRecordDirectory(recordNode, key); const dir = joinKey(pathInfo.base, ...pathInfo.subdirs); @@ -25,6 +26,13 @@ export const getRecordInfo = (app, key) => { }; } +export const getCollectionDir = (hierarchy, collectionKey) => { + const recordNode = getNodeForCollectionPath(hierarchy)(collectionKey); + const dummyRecordKey = joinKey(collectionKey, "1-abcd"); + const pathInfo = getRecordDirectory(recordNode, dummyRecordKey); + return pathInfo.base; +} + const recordJson = (dir) => joinKey(dir, "record.json") @@ -34,31 +42,53 @@ const files = (dir) => const getRecordDirectory = (recordNode, key) => { const id = getFileFromKey(key); - const traverseParentKeys = (n, keys=[]) => { - if(isRoot(n)) return keys; - const k = getActualKeyOfParent(n, key); + const traverseParentKeys = (n, parents=[]) => { + if(isRoot(n)) return parents; + const k = getActualKeyOfParent(n.nodeKey(), key); + const thisNodeDir = { + node:n, + relativeDir: joinKey( + recordRelativeDirectory(n, getFileFromKey(k))) + }; + return traverseParentKeys( + n.parent(), + [thisNodeDir, ...parents]); } + const parentDirs = $(recordNode.parent(), [ + traverseParentKeys, + reduce((key, item) => { + return joinKey(key, item.node.collectionName, item.relativeDir) + }, keySep) + ]); + + const subdirs = isSingleRecord(recordNode) + ? [] + : recordRelativeDirectory(recordNode, id); + const base = isSingleRecord(recordNode) + ? joinKey(parentDirs, recordNode.name) + : joinKey(parentDirs, recordNode.collectionName); + return ({ - base:getDirFomKey(key), - subdirs: [recordNode.nodeId.toString(), ...subfolders] + subdirs, base }); } const recordRelativeDirectory = (recordNode, id) => { const folderStructure = folderStructureArray(recordNode); - + const strippedId = id.substring(recordNode.nodeId.toString().length + 1); const subfolders = $(folderStructure, [ reduce((result, currentCount) => { - result.folders.push( - folderForChar(id[result.level], currentCount) - ); - return {level:result.level+1, folders:result.folders} + result.folders.push( + folderForChar(strippedId[result.level], currentCount) + ); + return {level:result.level+1, folders:result.folders}; }, {level:0, folders:[]}), - map(f => f.folders), + f => f.folders, + filter(f => !!f) ]); - return [recordNode.nodeId.toString(), ...subfolders] + return [recordNode.nodeId.toString(), ...subfolders, id] } const folderForChar = (char, folderCount) => diff --git a/packages/core/src/recordApi/save.js b/packages/core/src/recordApi/save.js index 346ad51236..05eff7a950 100644 --- a/packages/core/src/recordApi/save.js +++ b/packages/core/src/recordApi/save.js @@ -4,13 +4,13 @@ import { } from 'lodash/fp'; import { initialiseChildCollections } from '../collectionApi/initialise'; import { validate } from './validate'; -import { _loadFromInfo, getRecordFileName } from './load'; +import { _loadFromInfo } from './load'; import { apiWrapper, events, $, joinKey, } from '../common'; import { - getFlattenedHierarchy, getExactNodeForPath, - isRecord, getNode, fieldReversesReferenceToNode, + getFlattenedHierarchy, isRecord, getNode, + fieldReversesReferenceToNode, } from '../templateApi/hierarchy'; import { transactionForCreateRecord, @@ -42,7 +42,7 @@ export const _save = async (app, record, context, skipValidation = false) => { } } - const recordInfo = getRecordInfo(app.record.key); + const recordInfo = getRecordInfo(app.hierarchy, record.key); const { recordNode, pathInfo, recordJson, files, @@ -92,7 +92,9 @@ export const _save = async (app, record, context, skipValidation = false) => { const initialiseAncestorIndexes = async (app, recordInfo) => { for (const index of recordInfo.recordNode.indexes) { const indexKey = recordInfo.child(index.name); - if (!await app.datastore.exists(indexKey)) { await initialiseIndex(app.datastore, recordInfo.dir, index); } + if (!await app.datastore.exists(indexKey)) { + await initialiseIndex(app.datastore, recordInfo.dir, index); + } } }; @@ -125,7 +127,7 @@ const fieldsThatReferenceThisRecord = (app, recordNode) => $(app.hierarchy, [ const createRecordFolderPath = async (datastore, pathInfo) => { - const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=[]) => { + const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=undefined) => { // iterate backwards through directory hierachy // until we get to a folder that exists, then create the rest @@ -138,12 +140,16 @@ const createRecordFolderPath = async (datastore, pathInfo) => { if(await datastore.exists(thisFolder)) { let creationFolder = thisFolder; - for(let nextDir of dirsThatNeedCreated) { + for(let nextDir of (dirsThatNeedCreated || []) ) { creationFolder = joinKey(creationFolder, nextDir); await datastore.createFolder(creationFolder); } - } else if(dirsThatNeedCreated.length > 0) { + } else if(!dirsThatNeedCreated || dirsThatNeedCreated.length > 0) { + + dirsThatNeedCreated = !dirsThatNeedCreated + ? [] + :dirsThatNeedCreated; await recursiveCreateFolder( take(subdirs.length - 1)(subdirs), diff --git a/packages/core/src/recordApi/uploadFile.js b/packages/core/src/recordApi/uploadFile.js index 4a4a91cec8..f92f5d80e6 100644 --- a/packages/core/src/recordApi/uploadFile.js +++ b/packages/core/src/recordApi/uploadFile.js @@ -3,15 +3,16 @@ import { map, some, } from 'lodash/fp'; import { generate } from 'shortid'; -import { _load } from './load'; +import { _loadFromInfo } from './load'; import { apiWrapper, events, splitKey, $, joinKey, isNothing, tryAwaitOrIgnore, } from '../common'; -import { getExactNodeForPath } from '../templateApi/hierarchy'; +import { getExactNodeForKey } from '../templateApi/hierarchy'; import { permission } from '../authApi/permissions'; import { isLegalFilename } from '../types/file'; import { BadRequestError, ForbiddenError } from '../common/errors'; +import { getRecordInfo } from "./recordInfo"; export const uploadFile = app => async (recordKey, readableStream, relativeFilePath) => apiWrapper( app, @@ -26,10 +27,11 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) => if (isNothing(relativeFilePath)) { throw new BadRequestError('file path not supplied'); } if (!isLegalFilename(relativeFilePath)) { throw new BadRequestError('Illegal filename'); } - const record = await _load(app, recordKey); + const recordInfo = getRecordInfo(app.hierarchy, recordKey); + const record = await _loadFromInfo(app, recordInfo); const fullFilePath = safeGetFullFilePath( - recordKey, relativeFilePath, + recordInfo.dir, relativeFilePath, ); const tempFilePath = `${fullFilePath}_${generate()}.temp`; @@ -54,30 +56,10 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) => .then(() => tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath)) .then(() => app.datastore.renameFile(tempFilePath, fullFilePath)); - /* - readableStream.pipe(outputStream); - - await new Promise(fulfill => outputStream.on('finish', fulfill)); - - const isExpectedFileSize = checkFileSizeAgainstFields( - app, - record, relativeFilePath, - await app.datastore.getFileSize(tempFilePath), - ); - - if (!isExpectedFileSize) { - throw new Error( - `Fields for ${relativeFilePath} do not have expected size`); - } - - await tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath); - - await app.datastore.renameFile(tempFilePath, fullFilePath); - */ }; const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const incorrectFileFields = $(recordNode.fields, [ filter(f => f.type === 'file' @@ -107,7 +89,7 @@ const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) return true; }; -export const safeGetFullFilePath = (recordKey, relativeFilePath) => { +export const safeGetFullFilePath = (recordDir, relativeFilePath) => { const naughtyUser = () => { throw new ForbiddenError('naughty naughty'); }; if (relativeFilePath.startsWith('..')) naughtyUser(); @@ -116,7 +98,7 @@ export const safeGetFullFilePath = (recordKey, relativeFilePath) => { if (includes('..')(pathParts)) naughtyUser(); - const recordKeyParts = splitKey(recordKey); + const recordKeyParts = splitKey(recordDir); const fullPathParts = [ ...recordKeyParts, diff --git a/packages/core/src/recordApi/validate.js b/packages/core/src/recordApi/validate.js index 1e760c42df..d0694ed054 100644 --- a/packages/core/src/recordApi/validate.js +++ b/packages/core/src/recordApi/validate.js @@ -4,7 +4,7 @@ import { } from 'lodash/fp'; import { compileExpression } from '@nx-js/compiler-util'; import _ from 'lodash'; -import { getExactNodeForPath } from '../templateApi/hierarchy'; +import { getExactNodeForKey } from '../templateApi/hierarchy'; import { validateFieldParse, validateTypeConstraints } from '../types'; import { $, isNothing, isNonEmptyString } from '../common'; import { _getContext } from './getContext'; @@ -63,7 +63,7 @@ export const validate = app => async (record, context) => { ? _getContext(app, record.key) : context; - const recordNode = getExactNodeForPath(app.hierarchy)(record.key); + const recordNode = getExactNodeForKey(app.hierarchy)(record.key); const fieldParseFails = validateAllFieldParse(record, recordNode); // non parsing would cause further issues - exit here diff --git a/packages/core/src/templateApi/createNodes.js b/packages/core/src/templateApi/createNodes.js index aac0cf2e34..1a4fcb39de 100644 --- a/packages/core/src/templateApi/createNodes.js +++ b/packages/core/src/templateApi/createNodes.js @@ -171,7 +171,7 @@ const _getNewRecordTemplate = (parent, name, createDefaultIndex, isSingle) => { validationRules: [], nodeId: getNodeId(parent), indexes: [], - allidsShardFactor: isRecord(parent) ? 1 : 64, + estimatedRecordCount: isRecord(parent) ? 500 : 1000000, collectionName: '', isSingle, }); diff --git a/packages/core/src/templateApi/hierarchy.js b/packages/core/src/templateApi/hierarchy.js index dec1bf90c7..34c47679a3 100644 --- a/packages/core/src/templateApi/hierarchy.js +++ b/packages/core/src/templateApi/hierarchy.js @@ -49,7 +49,7 @@ export const getNodesInPath = appHierarchy => key => $(appHierarchy, [ filter(n => new RegExp(`${n.pathRegx()}`).test(key)), ]); -export const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ +export const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [ getFlattenedHierarchy, find(n => new RegExp(`${n.pathRegx()}$`).test(key)), ]); @@ -87,7 +87,7 @@ export const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [ ]); export const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { - const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); + const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey); return isNothing(nodeByKey) ? getNode(appHierarchy, keyOrNodeKey) : nodeByKey; @@ -100,13 +100,14 @@ export const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { : nodeByKey; }; -export const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); +export const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key)); -export const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ - splitKey, - take(splitKey(parentNodeKey).length), - ks => joinKey(...ks), -]); +export const getActualKeyOfParent = (parentNodeKey, actualChildKey) => + $(actualChildKey, [ + splitKey, + take(splitKey(parentNodeKey).length), + ks => joinKey(...ks), + ]); export const getParentKey = (key) => { return $(key, [ @@ -199,7 +200,7 @@ export const fieldReversesReferenceToIndex = indexNode => field => field.type == export default { getLastPartInKey, getNodesInPath, - getExactNodeForPath, + getExactNodeForKey, hasMatchingAncestor, getNode, getNodeByKeyOrNodeKey, diff --git a/packages/core/src/transactions/execute.js b/packages/core/src/transactions/execute.js index a71a3c6172..110228e9c2 100644 --- a/packages/core/src/transactions/execute.js +++ b/packages/core/src/transactions/execute.js @@ -24,8 +24,10 @@ import { applyToShard } from '../indexing/apply'; import { getActualKeyOfParent, isGlobalIndex, fieldReversesReferenceToIndex, isReferenceIndex, - getExactNodeForPath, + getExactNodeForKey, } from '../templateApi/hierarchy'; +import { getRecordInfo } from "../recordApi/recordInfo"; +import { getIndexDir } from '../indexApi/getIndexDir'; export const executeTransactions = app => async (transactions) => { const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions); @@ -33,7 +35,7 @@ export const executeTransactions = app => async (transactions) => { for (const shard of keys(recordsByShard)) { await applyToShard( app.hierarchy, app.datastore, - recordsByShard[shard].indexKey, + recordsByShard[shard].indexDir, recordsByShard[shard].indexNode, shard, recordsByShard[shard].writes, @@ -77,8 +79,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => { transByShard[t.indexShardKey] = { writes: [], removes: [], - indexKey: t.indexKey, - indexNodeKey: t.indexNodeKey, + indexDir: t.indexDir, + indexNodeKey: t.indexNode.nodeKey(), indexNode: t.indexNode, }; } @@ -109,10 +111,10 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => { return ({ mappedRecord, indexNode: indexNodeAndPath.indexNode, - indexKey: indexNodeAndPath.indexKey, + indexDir: indexNodeAndPath.indexDir, indexShardKey: getIndexedDataKey( indexNodeAndPath.indexNode, - indexNodeAndPath.indexKey, + indexNodeAndPath.indexDir, mappedRecord.result, ), }); @@ -219,54 +221,56 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { if (!isNonEmptyArray(buildTransactions)) return []; const indexNode = transactions.indexNode; - const getIndexKeys = (t) => { + const getIndexDirs = (t) => { if (isGlobalIndex(indexNode)) { return [indexNode.nodeKey()]; } if (isReferenceIndex(indexNode)) { - const recordNode = getExactNodeForPath(hierarchy)(t.record.key); + const recordNode = getExactNodeForKey(hierarchy)(t.record.key); const refFields = $(recordNode.fields, [ filter(fieldReversesReferenceToIndex(indexNode)), ]); - const indexKeys = []; + const indexDirs = []; for (const refField of refFields) { const refValue = t.record[refField.name]; if (isSomething(refValue) && isNonEmptyString(refValue.key)) { - const indexKey = joinKey( - refValue.key, + const indexDir = joinKey( + getRecordInfo(hierarchy, refValue.key).dir, indexNode.name, ); - if (!includes(indexKey)(indexKeys)) { indexKeys.push(indexKey); } + if (!includes(indexDir)(indexDirs)) { indexDirs.push(indexDir); } } } - return indexKeys; + return indexDirs; } - return [joinKey( + const indexKey = joinKey( getActualKeyOfParent( indexNode.parent().nodeKey(), t.record.key, ), indexNode.name, - )]; + ); + + return [getIndexDir(hierarchy, indexKey)]; }; return $(buildTransactions, [ map((t) => { const mappedRecord = evaluate(t.record)(indexNode); if (!mappedRecord.passedFilter) return null; - const indexKeys = getIndexKeys(t); - return $(indexKeys, [ - map(indexKey => ({ + const indexDirs = getIndexDirs(t); + return $(indexDirs, [ + map(indexDir => ({ mappedRecord, indexNode, - indexKey, + indexDir, indexShardKey: getIndexedDataKey( indexNode, - indexKey, + indexDir, mappedRecord.result, ), })), @@ -286,10 +290,10 @@ const get_Create_Delete_TransactionsByShard = pred => (hierarchy, transactions) return ({ mappedRecord, indexNode: n.indexNode, - indexKey: n.indexKey, + indexDir: n.indexDir, indexShardKey: getIndexedDataKey( n.indexNode, - n.indexKey, + n.indexDir, mappedRecord.result, ), }); @@ -327,17 +331,17 @@ const diffReverseRefForUpdate = (appHierarchy, oldRecord, newRecord) => { ); const unReferenced = differenceBy( - i => i.indexKey, + i => i.indexDir, oldIndexes, newIndexes, ); const newlyReferenced = differenceBy( - i => i.indexKey, + i => i.indexDir, newIndexes, oldIndexes, ); const notChanged = intersectionBy( - i => i.indexKey, + i => i.indexDir, newIndexes, oldIndexes, ); diff --git a/packages/core/test/collectionApi.spec.js b/packages/core/test/collectionApi.spec.js index 2a22a60e72..963821201c 100644 --- a/packages/core/test/collectionApi.spec.js +++ b/packages/core/test/collectionApi.spec.js @@ -1,5 +1,4 @@ -import {setupApphierarchy, basicAppHierarchyCreator_WithFields, - basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers"; +import {setupApphierarchy, basicAppHierarchyCreator_WithFields} from "./specHelpers"; import {includes, union} from "lodash"; import {joinKey} from "../src/common"; diff --git a/packages/core/test/indexApi.buildIndex.spec.js b/packages/core/test/indexApi.buildIndex.spec.js index 621a6d1adb..16f3f3f3d8 100644 --- a/packages/core/test/indexApi.buildIndex.spec.js +++ b/packages/core/test/indexApi.buildIndex.spec.js @@ -4,7 +4,7 @@ import { joinKey } from "../src/common"; import {some} from "lodash"; import {_deleteIndex} from "../src/indexApi/delete"; import {permission} from "../src/authApi/permissions"; -import { getExactNodeForPath } from "../src/templateApi/hierarchy"; +import { getExactNodeForKey } from "../src/templateApi/hierarchy"; describe("buildIndex > Global index", () => { @@ -213,7 +213,7 @@ describe("buildIndex > nested collection", () => { const indexKey = joinKey(customer.key, "invoice_index"); await _deleteIndex(app, indexKey, false); - const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey); + const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey); await indexApi.buildIndex(indexNode.nodeKey()); const indexItems = await indexApi.listItems(indexKey); @@ -269,7 +269,7 @@ describe("buildIndex > nested collection", () => { const indexKey = joinKey(customer.key, "invoice_index"); await _deleteIndex(app, indexKey, false); - const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey); + const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey); await indexApi.buildIndex( indexNode.nodeKey()); const indexItems = await indexApi.listItems(indexKey); diff --git a/packages/core/test/indexing.concurrency.spec.js b/packages/core/test/indexing.concurrency.spec.js index f3d73bdf2c..cbfaa92390 100644 --- a/packages/core/test/indexing.concurrency.spec.js +++ b/packages/core/test/indexing.concurrency.spec.js @@ -5,7 +5,7 @@ import {getLockFileContent} from "../src/common/lock"; import {some, isArray} from "lodash"; import {cleanup} from "../src/transactions/cleanup"; import {LOCK_FILE_KEY} from "../src/transactions/transactionsCommon"; - +import { getRecordInfo } from "../src/recordApi/recordInfo"; describe("cleanup transactions", () => { @@ -96,7 +96,7 @@ describe("cleanup transactions", () => { it("should not reindex when transactionId does not match that of the record", async () => { - const {recordApi, app, + const {recordApi, app, indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); const record = recordApi.getNew("/customers", "customer"); record.surname = "Ledog"; @@ -109,8 +109,10 @@ describe("cleanup transactions", () => { await recordApi.save(savedRecord); savedRecord.transactionId = "something else"; + + const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key); await recordApi._storeHandle.updateJson( - joinKey(savedRecord.key, "record.json"), + recordInfo.child("record.json"), savedRecord); await cleanup(app); @@ -123,7 +125,7 @@ describe("cleanup transactions", () => { it("should not reindex when transactionId does not match that of the record, and has multiple transactions", async () => { - const {recordApi, app, + const {recordApi, app, indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); const record = recordApi.getNew("/customers", "customer"); record.surname = "Ledog"; @@ -139,8 +141,10 @@ describe("cleanup transactions", () => { await recordApi.save(savedRecord); savedRecord.transactionId = "something else"; + + const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key); await recordApi._storeHandle.updateJson( - joinKey(savedRecord.key, "record.json"), + recordInfo.child("record.json"), savedRecord); await cleanup(app); @@ -228,7 +232,7 @@ describe("cleanup transactions", () => { indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); const record = recordApi.getNew("/customers", "customer"); record.surname = "Ledog"; - const savedRecord = await recordApi.save(record); + await recordApi.save(record); const currentTime = await app.getEpochTime(); await recordApi._storeHandle.createFile( LOCK_FILE_KEY, @@ -252,7 +256,7 @@ describe("cleanup transactions", () => { indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); const record = recordApi.getNew("/customers", "customer"); record.surname = "Ledog"; - const savedRecord = await recordApi.save(record); + await recordApi.save(record); await recordApi._storeHandle.createFile( LOCK_FILE_KEY, getLockFileContent(30000, (new Date(1990,1,1,0,0,0,0).getTime())) diff --git a/packages/core/test/indexing.getRelevantIndexes.spec.js b/packages/core/test/indexing.getRelevantIndexes.spec.js index 5d6714bdb0..b5bbabdc49 100644 --- a/packages/core/test/indexing.getRelevantIndexes.spec.js +++ b/packages/core/test/indexing.getRelevantIndexes.spec.js @@ -1,11 +1,14 @@ -import {getMemoryTemplateApi, - basicAppHierarchyCreator_WithFields, - setupApphierarchy, - basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers"; -import {getRelevantReverseReferenceIndexes, - getRelevantAncestorIndexes} from "../src/indexing/relevant"; +import { +setupApphierarchy, +basicAppHierarchyCreator_WithFields_AndIndexes +} from "./specHelpers"; +import { +getRelevantReverseReferenceIndexes, +getRelevantAncestorIndexes +} from "../src/indexing/relevant"; import {some} from "lodash"; import {joinKey} from "../src/common"; +import { getRecordInfo } from "../src/recordApi/recordInfo"; describe("getRelevantIndexes", () => { @@ -45,7 +48,7 @@ describe("getRelevantIndexes", () => { expect(indexes.length).toBe(4); const indexExists = key => - some(indexes, c => c.indexKey === key); + some(indexes, c => c.indexDir === key); expect(indexExists("/customer_index")).toBeTruthy(); expect(indexExists("/deceased")).toBeTruthy(); @@ -64,7 +67,7 @@ describe("getRelevantIndexes", () => { appHierarchy.root, invoice); const indexExists = key => - some(indexes, c => c.indexKey === key); + some(indexes, c => c.indexDir === key); expect(indexExists("/customersBySurname")).toBeFalsy(); }); @@ -82,7 +85,7 @@ describe("getRelevantIndexes", () => { expect(indexes.length).toBe(4); const indexExists = key => - some(indexes, c => c.indexKey === key); + some(indexes, c => c.indexDir === key); expect(indexExists("/customersBySurname")).toBeTruthy(); }); @@ -96,14 +99,14 @@ describe("getRelevantIndexes", () => { const indexes = getRelevantAncestorIndexes( appHierarchy.root, invoice); - + const {dir} = getRecordInfo(appHierarchy.root, `/customers/${nodeid}-1234`); expect(indexes.length).toBe(4); - expect(some(indexes, i => i.indexKey === `/customer_invoices`)).toBeTruthy(); - expect(some(indexes, i => i.indexKey === `/customers/${nodeid}-1234/invoice_index`)).toBeTruthy(); + expect(some(indexes, i => i.indexDir === `/customer_invoices`)).toBeTruthy(); + expect(some(indexes, i => i.indexDir === `${dir}/invoice_index`)).toBeTruthy(); }); it("should get reverseReferenceIndex accross hierarchy branches", async () => { - const {appHierarchy, + const {appHierarchy, recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes); const partner = recordApi.getNew("/partners", "partner"); @@ -118,8 +121,9 @@ describe("getRelevantIndexes", () => { const indexes = getRelevantReverseReferenceIndexes( appHierarchy.root, customer); expect(indexes.length).toBe(1); - expect(indexes[0].indexKey) - .toBe(joinKey(partner.key, appHierarchy.partnerCustomersReverseIndex.name)); + const partnerdir = getRecordInfo(appHierarchy.root, partner.key).dir; + expect(indexes[0].indexDir) + .toBe(joinKey(partnerdir, appHierarchy.partnerCustomersReverseIndex.name)); }); @@ -136,8 +140,11 @@ describe("getRelevantIndexes", () => { const indexes = getRelevantReverseReferenceIndexes( appHierarchy.root, referredToCustomer); + + const referredByCustomerDir = getRecordInfo(appHierarchy.root, referredByCustomer.key).dir; + expect(indexes.length).toBe(1); - expect(indexes[0].indexKey) - .toBe(joinKey(referredByCustomer.key, appHierarchy.referredToCustomersReverseIndex.name)); + expect(indexes[0].indexDir) + .toBe(joinKey(referredByCustomerDir, appHierarchy.referredToCustomersReverseIndex.name)); }); }); \ No newline at end of file diff --git a/packages/core/test/indexing.schema.spec.js b/packages/core/test/indexing.schema.spec.js index 25bc0ee757..7b3b7d92e1 100644 --- a/packages/core/test/indexing.schema.spec.js +++ b/packages/core/test/indexing.schema.spec.js @@ -1,5 +1,5 @@ import {generateSchema} from "../src/indexing/indexSchemaCreator"; -import {setupApphierarchy, findCollectionDefaultIndex} from "./specHelpers"; +import {setupApphierarchy} from "./specHelpers"; import {find} from "lodash"; import {indexTypes} from "../src/templateApi/indexes"; diff --git a/packages/core/test/initialiseData.spec.js b/packages/core/test/initialiseData.spec.js index edf38e9092..bafc9e3a10 100644 --- a/packages/core/test/initialiseData.spec.js +++ b/packages/core/test/initialiseData.spec.js @@ -21,17 +21,6 @@ describe("initialiseData", () => { expect(await datastore.exists(`/customers`)).toBeTruthy(); }); - - it("should create allids folders", async () => { - const {appDef, datastore, h} = getApplicationDefinition(); - await initialiseData(datastore, appDef); - - const allIdsTypeFolder = "/customers/allids/" + h.customerRecord.nodeId; - const allIdsFolder = "/customers/allids"; - expect(await datastore.exists(allIdsTypeFolder)).toBeTruthy(); - expect(await datastore.exists(allIdsFolder)).toBeTruthy(); - }); - it("should create transactions folder", async () => { const {appDef, datastore} = getApplicationDefinition(); await initialiseData(datastore, appDef); diff --git a/packages/core/test/memory.js b/packages/core/test/memory.js index 11da8a5fd3..25506470bb 100644 --- a/packages/core/test/memory.js +++ b/packages/core/test/memory.js @@ -2,11 +2,16 @@ import {isUndefined, has} from "lodash"; import {take} from "lodash/fp"; import {Readable, Writable} from "readable-stream"; import { Buffer } from "safe-buffer"; -import {splitKey, joinKey, $} from "../src/common"; +import {splitKey, joinKey, $, keySep, getFileFromKey} from "../src/common"; import {getLastPartInKey} from "../src/templateApi/hierarchy"; const folderMarker = "OH-YES-ITSA-FOLDER-"; -const isFolder = val => val.includes(folderMarker); +const isFolder = val => { + if(isUndefined(val)) { + throw new Error("Passed undefined value for folder"); + } + return val.includes(folderMarker); +} const getParentFolderKey = key => $(key, [ @@ -16,7 +21,9 @@ const getParentFolderKey = key => ]); const getParentFolder = (data,key) => { + if(key === keySep) return null; const parentKey = getParentFolderKey(key); + if(parentKey === keySep) return null; if(data[parentKey] === undefined) throw new Error("Parent folder for " + key + " does not exist (" + parentKey + ")"); return JSON.parse(data[parentKey]); @@ -39,12 +46,18 @@ export const createFile = data => async (path, content) => { }; export const updateFile = data => async (path, content) => { // putting this check in to force use of create - if(!await exists(data)(path)) throw new Error("cannot update " + path + " - does not exist"); + if(!await exists(data)(path)) { + throw new Error("cannot update " + path + " - does not exist"); + } data[path] = content; } export const writableFileStream = data => async (path) => { //if(!await exists(data)(path)) throw new Error("cannot write stream to " + path + " - does not exist"); + if(!getParentFolder(data, path)) { + throw new Error("Parent folder for " + path + " does not exist"); + } + const stream = Writable(); stream._write = (chunk, encoding, done) => { data[path] = data[path] === undefined @@ -52,6 +65,9 @@ export const writableFileStream = data => async (path) => { data[path] = [...data[path], ...chunk]; done(); }; + + addItemToParentFolder(data, path); + return stream; }; @@ -77,11 +93,19 @@ export const renameFile = data => async (oldKey, newKey) => { if(await exists(data)(newKey)) throw new Error("cannot rename path: " + newKey + " ... already exists"); data[newKey] = data[oldKey]; delete data[oldKey]; + + const parent = getParentFolder(data, newKey); + const oldFileName = getFileFromKey(oldKey); + const newFileName = getFileFromKey(newKey); + parent.items = [...parent.items.filter(i => i !== oldFileName), newFileName]; + data[getParentFolderKey(newKey)] = JSON.stringify(parent); }; export const loadFile = data => async (path) => { const result = data[path]; - if(isUndefined(result)) throw new Error("Load failed - path " + path + " does not exist"); + if(isUndefined(result)) { + throw new Error("Load failed - path " + path + " does not exist"); + } return result; }; export const exists = data => async (path) => has(data, path); @@ -95,7 +119,8 @@ export const deleteFile = data => async (path) => { delete data[path]; } export const createFolder = data => async (path) => { - if(await exists(data)(path)) throw new Error("Cannot create folder, path " + path + " already exists"); + if(await exists(data)(path)) + throw new Error("Cannot create folder, path " + path + " already exists"); addItemToParentFolder(data, path); data[path] = JSON.stringify({folderMarker, items:[]}); } @@ -103,14 +128,30 @@ export const deleteFolder = data => async (path) => { if(!await exists(data)(path)) throw new Error("Cannot delete folder, path " + path + " does not exist"); if(!isFolder(data[path])) throw new Error("DeleteFolder: Path " + path + " is not a folder"); + + for(let item of JSON.parse(data[path]).items) { + const fullItemPath = `${path}/${item}`; + if(isFolder(data[fullItemPath])) { + await deleteFolder(data)(fullItemPath); + } else { + await deleteFile(data)(fullItemPath); + } + } + + const parent = getParentFolder(data, path); + if(parent) { + parent.items = parent.items.filter(f => f !== getLastPartInKey(path)); + data[getParentFolderKey(path)] = JSON.stringify(parent); + } + delete data[path]; } -export const getFolderContents = data => async (folderPath) => { - if(!isFolder(data[folderPath])) - throw new Error("Not a folder: " + folderPath); +export const getFolderContents = data => async (folderPath) => { if(!await exists(data)(folderPath)) throw new Error("Folder does not exist: " + folderPath); + if(!isFolder(data[folderPath])) + throw new Error("Not a folder: " + folderPath); return JSON.parse(data[folderPath]).items; }; diff --git a/packages/core/test/recordApi.files.spec.js b/packages/core/test/recordApi.files.spec.js index 2aefa12b62..d5ccc93625 100644 --- a/packages/core/test/recordApi.files.spec.js +++ b/packages/core/test/recordApi.files.spec.js @@ -1,8 +1,5 @@ import {setupApphierarchy, basicAppHierarchyCreator_WithFields} from "./specHelpers"; -import {keys, filter} from "lodash/fp"; -import {$} from "../src/common"; -import {permission} from "../src/authApi/permissions"; import {Readable} from "readable-stream"; diff --git a/packages/core/test/recordApi.getRecordInfo.spec.js b/packages/core/test/recordApi.getRecordInfo.spec.js new file mode 100644 index 0000000000..b14aa92971 --- /dev/null +++ b/packages/core/test/recordApi.getRecordInfo.spec.js @@ -0,0 +1,124 @@ +import { folderStructureArray } from "../src/indexing/allIds"; +import { getRecordInfo } from "../src/recordApi/recordInfo"; +import {setupApphierarchy} from "./specHelpers"; + +describe("getRecordInfo", () => { + + it("dir should not be sharded when record count = 1000", async () => { + const {root} = (await setup({parentCount: 1000})).appHierarchy; + const {dir} = getRecordInfo(root, "/parents/1-abcd"); + expect(dir).toBe("/parents/1/1-abcd"); + }); + + it("dir should be sharded when record count = 1001", async () => { + const {root} = (await setup({parentCount: 1001})).appHierarchy; + const {dir} = getRecordInfo(root, "/parents/1-abcd"); + expect(dir).toBe("/parents/1/0123456789abcdefghijklmnopqrstuv/1-abcd"); + }); + + it("dir should be sharded to one char per folder when record count = 63,000 (64*1000)", async () => { + const {root} = (await setup({parentCount: 64000})).appHierarchy; + const {dir} = getRecordInfo(root, "/parents/1-abcd"); + expect(dir).toBe("/parents/1/a/1-abcd"); + }); + + it("dir should be sharded to one char per folder, on 2 levels when record count = 4096000 (64*64*1000)", async () => { + const {root} = (await setup({parentCount: 4096000})).appHierarchy; + const {dir} = getRecordInfo(root, "/parents/1-abcd"); + expect(dir).toBe("/parents/1/a/b/1-abcd"); + }); + + it("child dir should not be sharded when record count = 1000", async () => { + const {root, child} = (await setup({parentCount: 4096000, childCount: 1000})).appHierarchy; + const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg`); + expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/${child.nodeId}-defg`); + }); + + it("grandchild dir should not be sharded when record count = 1000", async () => { + const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000})).appHierarchy; + const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`); + expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/${grandchild.nodeId}-hijk`); + }); + + it("grandchild dir should be sharded when record count = 4096000", async () => { + const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy; + const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`); + expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`); + }); + + it("child levels can be sharded, with parent not", async () => { + const {root, child, grandchild} = (await setup({parentCount: 1000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy; + const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`); + expect(dir).toBe(`/parents/1/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`); + }); + + +}); + + +describe("folderStructureArray", () => { + + const recordNode = (count) => ({estimatedRecordCount: count}); + + it("should return [] when folder count < 1000", () => { + const result = folderStructureArray(recordNode(999)); + expect(result).toEqual([]); + }); + + it("should return [4] when folder count between 3000 - 4000", () => { + const result = folderStructureArray(recordNode(3456)); + expect(result).toEqual([4]); + }) + + it("should return [64, 2] when folder count between 64000 - 65000", () => { + const result = folderStructureArray(recordNode(64001)); + expect(result).toEqual([64, 2]); + }) + + it("should return [64, 64] when folder = 4095999", () => { + const result = folderStructureArray(recordNode(4095999)); + expect(result).toEqual([64, 64]); + }); + + it("should return [64, 64] when folder = 4096000", () => { + const result = folderStructureArray(recordNode(4096000)); + expect(result).toEqual([64, 64]); + }); + + it("should return [64, 64, 2] when folder = 4096001", () => { + const result = folderStructureArray(recordNode(4096001)); + expect(result).toEqual([64, 64, 2]); + }); + +}); + +const setup = ({parentCount, childCount, grandChildCount}) => + setupApphierarchy((templateApi) => { + + const root = templateApi.getNewRootLevel(); + + const addField = (recordNode) => { + const field = templateApi.getNewField("string"); + field.name = "test"; + templateApi.addField(recordNode, field); + return field; + }; + + const parent = templateApi.getNewRecordTemplate(root, "parent"); + parent.estimatedRecordCount = parentCount || 1000; + parent.collectionName = "parents"; + addField(parent); + const child = templateApi.getNewRecordTemplate(parent, "child"); + child.estimatedRecordCount = childCount || 1000; + child.collectionName = "children"; + addField(child); + const grandchild = templateApi.getNewRecordTemplate(child, "grandchild"); + grandchild.estimatedRecordCount = grandChildCount || 1000; + grandchild.collectionName = "grandchildren"; + addField(grandchild); + + return ({ + parent, child, grandchild, root + }); + }); + diff --git a/packages/core/test/recordApi.save.spec.js b/packages/core/test/recordApi.save.spec.js index a12f8f753a..d50a05976b 100644 --- a/packages/core/test/recordApi.save.spec.js +++ b/packages/core/test/recordApi.save.spec.js @@ -2,7 +2,7 @@ import {setupApphierarchy, basicAppHierarchyCreator_WithFields, stubEventHandler} from "./specHelpers"; import {events, isNonEmptyString} from "../src/common"; import {permission} from "../src/authApi/permissions"; - +import { getRecordInfo } from "../src/recordApi/recordInfo"; describe('recordApi > save then load', () => { @@ -202,48 +202,41 @@ describe("save", () => { }); it("should create folder and index for subcollection", async () => { - const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); - const record = recordApi.getNew("/customers", "customer"); - record.surname = "Ledog"; - - await recordApi.save(record); - expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index/index.csv`)).toBeTruthy() - expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index`)).toBeTruthy() - expect(await recordApi._storeHandle.exists(`${record.key}/invoices`)).toBeTruthy() - }); - - it("should create index folder and shardMap for sharded reverse reference index", async () => { - const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); - const record = recordApi.getNew("/customers", "customer"); - record.surname = "Ledog"; - - await recordApi.save(record); - expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers/shardMap.json`)).toBeTruthy(); - expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers`)).toBeTruthy(); - }); - - it("should create folder for record", async () => { - const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); - const record = recordApi.getNew("/customers", "customer"); - record.surname = "Ledog"; - - await recordApi.save(record); - expect(await recordApi._storeHandle.exists(`${record.key}`)).toBeTruthy(); - expect(await recordApi._storeHandle.exists(`${record.key}/record.json`)).toBeTruthy(); - }); - - it("should create allids file", async () => { const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); const record = recordApi.getNew("/customers", "customer"); record.surname = "Ledog"; await recordApi.save(record); - - const allIdsPath = `/customers/allids/${appHierarchy.customerRecord.nodeId}/${record.id[2]}`; - expect(await recordApi._storeHandle.exists(allIdsPath)).toBeTruthy(); - + const recordDir = getRecordInfo(appHierarchy.root, record.key).dir; + expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index/index.csv`)).toBeTruthy() + expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index`)).toBeTruthy() + expect(await recordApi._storeHandle.exists(`${recordDir}/invoices`)).toBeTruthy() }); + it("should create index folder and shardMap for sharded reverse reference index", async () => { + const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); + const record = recordApi.getNew("/customers", "customer"); + record.surname = "Ledog"; + + await recordApi.save(record); + const recordDir = getRecordInfo(appHierarchy.root, record.key).dir; + expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers/shardMap.json`)).toBeTruthy(); + expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers`)).toBeTruthy(); + }); + + it("should create folder for record", async () => { + const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); + const record = recordApi.getNew("/customers", "customer"); + record.surname = "Ledog"; + + await recordApi.save(record); + const recordDir = getRecordInfo(appHierarchy.root, record.key).dir; + + expect(await recordApi._storeHandle.exists(`${recordDir}`)).toBeTruthy(); + expect(await recordApi._storeHandle.exists(`${recordDir}/record.json`)).toBeTruthy(); + }); + + it("create should throw error, user user does not have permission", async () => { const {recordApi, app, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); const record = recordApi.getNew("/customers", "customer"); diff --git a/packages/core/test/specHelpers.js b/packages/core/test/specHelpers.js index fc7bd0900f..2a0d6449d6 100644 --- a/packages/core/test/specHelpers.js +++ b/packages/core/test/specHelpers.js @@ -22,12 +22,10 @@ import {permission} from "../src/authApi/permissions"; import {generateFullPermissions} from "../src/authApi/generateFullPermissions" import {initialiseData} from "../src/appInitialise/initialiseData"; -const exp = module.exports; - export const testFileArea = (testNameArea) => path.join("test", "fs_test_area", testNameArea); -export const testConfigFolder = (testAreaName) => path.join(exp.testFileArea(testAreaName), configFolder); -export const testFieldDefinitionsPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), fieldDefinitions); -export const testTemplatesPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), templateDefinitions); +export const testConfigFolder = (testAreaName) => path.join(testFileArea(testAreaName), configFolder); +export const testFieldDefinitionsPath = (testAreaName) => path.join(testFileArea(testAreaName), fieldDefinitions); +export const testTemplatesPath = (testAreaName) => path.join(testFileArea(testAreaName), templateDefinitions); export const getMemoryStore = () => setupDatastore(memory({})); export const getMemoryTemplateApi = () => { diff --git a/packages/core/test/templateApi.constructHeirarchy.spec.js b/packages/core/test/templateApi.constructHeirarchy.spec.js index 41f4f8f0be..79e91914ac 100644 --- a/packages/core/test/templateApi.constructHeirarchy.spec.js +++ b/packages/core/test/templateApi.constructHeirarchy.spec.js @@ -27,7 +27,7 @@ describe("hierarchy node creation", () => { expect(record.indexes).toEqual([]); expect(record.parent()).toBe(root); expect(record.collectionName).toBe(""); - expect(record.allidsShardFactor).toBe(64); + expect(record.estimatedRecordCount).toBe(1000000); expect(record.isSingle).toBe(false); record.collectionName = "records";