MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#9 All ids - replace with folder list items

This commit is contained in:
Michael Shanks 2019-12-22 07:12:21 +00:00
parent fa683385b6
commit bc00afcf36
44 changed files with 764 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
packages/core/dist/budibase-core.cjs.js vendored
View File

@ -726,7 +726,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [
fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)), fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]); ]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)), fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]); ]);
@ -764,7 +764,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]); ]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey) return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey) ? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey; : nodeByKey;
@ -777,7 +777,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey; : nodeByKey;
}; };
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey, splitKey,
@ -876,7 +876,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe
var hierarchy = { var hierarchy = {
getLastPartInKey, getLastPartInKey,
getNodesInPath, getNodesInPath,
getExactNodeForPath, getExactNodeForKey,
hasMatchingAncestor, hasMatchingAncestor,
getNode, getNode,
getNodeByKeyOrNodeKey, getNodeByKeyOrNodeKey,
@ -1612,7 +1612,7 @@ const load = app => async key => {
const _load = async (app, key, keyStack = []) => { const _load = async (app, key, keyStack = []) => {
key = safeKey(key); key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key); const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson( const storedData = await app.datastore.loadJson(
getRecordFileName(key), getRecordFileName(key),
); );
@ -1758,7 +1758,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => {
}; };
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord const startShardName = !startRecord
? null ? null
@ -4403,7 +4403,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
)); ));
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4435,7 +4435,7 @@ const getContext = app => recordKey => {
const _getContext = (app, recordKey) => { const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey); recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {}; const cachedReferenceIndexes = {};
@ -4543,7 +4543,7 @@ const validate = app => async (record, context) => {
? _getContext(app, record.key) ? _getContext(app, record.key)
: context; : context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode); const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here // non parsing would cause further issues - exit here
@ -4603,7 +4603,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => {
const initialiseChildCollections = async (app, recordKey) => { const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [ const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy), getExactNodeForKey(app.hierarchy),
n => n.children, n => n.children,
fp.filter(isCollectionRecord), fp.filter(isCollectionRecord),
]); ]);
@ -4974,7 +4974,7 @@ const _save = async (app, record, context, skipValidation = false) => {
} }
if (recordClone.isNew) { if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode) if(!recordNode)
throw new Error("Cannot find node for " + record.key); throw new Error("Cannot find node for " + record.key);
@ -5023,7 +5023,7 @@ const _save = async (app, record, context, skipValidation = false) => {
}; };
const initialiseAncestorIndexes = async (app, record) => { const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) { for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name); const indexKey = joinKey(record.key, index.name);
@ -5032,7 +5032,7 @@ const initialiseAncestorIndexes = async (app, record) => {
}; };
const initialiseReverseReferenceIndexes = async (app, record) => { const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [ fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5131,7 +5131,7 @@ const deleteRecords = async (app, key) => {
const _deleteIndex = async (app, indexKey, includeFolder) => { const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5176,7 +5176,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword // called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => { const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key); key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key); const record = await _load(app, key);
await transactionForDeleteRecord(app, record); await transactionForDeleteRecord(app, record);
@ -5203,7 +5203,7 @@ const _deleteRecord = async (app, key, disableCleanup) => {
}; };
const deleteIndexes = async (app, key) => { const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [ /* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
map(n => n.fields), map(n => n.fields),
@ -5310,7 +5310,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
}; };
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [ const incorrectFileFields = $(recordNode.fields, [
fp.filter(f => f.type === 'file' fp.filter(f => f.type === 'file'
@ -5557,7 +5557,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7431,7 +7431,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => {
}; };
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy), getExactNodeForKey(appHierarchy),
n => n.fields, n => n.fields,
fp.filter(f => f.type === 'reference' fp.filter(f => f.type === 'reference'
&& isSomething(record[f.name]) && isSomething(record[f.name])
@ -7841,7 +7841,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
} }
if (isReferenceIndex(indexNode)) { if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key); const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [ const refFields = $(recordNode.fields, [
fp.filter(fieldReversesReferenceToIndex(indexNode)), fp.filter(fieldReversesReferenceToIndex(indexNode)),
]); ]);

40
packages/core/dist/budibase-core.esm.mjs vendored
View File

@ -719,7 +719,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [
filter(n => new RegExp(`${n.pathRegx()}`).test(key)), filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]); ]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
find(n => new RegExp(`${n.pathRegx()}$`).test(key)), find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]); ]);
@ -757,7 +757,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]); ]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey) return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey) ? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey; : nodeByKey;
@ -770,7 +770,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey; : nodeByKey;
}; };
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey, splitKey,
@ -869,7 +869,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe
var hierarchy = { var hierarchy = {
getLastPartInKey, getLastPartInKey,
getNodesInPath, getNodesInPath,
getExactNodeForPath, getExactNodeForKey,
hasMatchingAncestor, hasMatchingAncestor,
getNode, getNode,
getNodeByKeyOrNodeKey, getNodeByKeyOrNodeKey,
@ -1605,7 +1605,7 @@ const load = app => async key => {
const _load = async (app, key, keyStack = []) => { const _load = async (app, key, keyStack = []) => {
key = safeKey(key); key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key); const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson( const storedData = await app.datastore.loadJson(
getRecordFileName(key), getRecordFileName(key),
); );
@ -1751,7 +1751,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => {
}; };
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord const startShardName = !startRecord
? null ? null
@ -4396,7 +4396,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
)); ));
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4428,7 +4428,7 @@ const getContext = app => recordKey => {
const _getContext = (app, recordKey) => { const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey); recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {}; const cachedReferenceIndexes = {};
@ -4536,7 +4536,7 @@ const validate = app => async (record, context) => {
? _getContext(app, record.key) ? _getContext(app, record.key)
: context; : context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode); const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here // non parsing would cause further issues - exit here
@ -4596,7 +4596,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => {
const initialiseChildCollections = async (app, recordKey) => { const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [ const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy), getExactNodeForKey(app.hierarchy),
n => n.children, n => n.children,
filter(isCollectionRecord), filter(isCollectionRecord),
]); ]);
@ -4967,7 +4967,7 @@ const _save = async (app, record, context, skipValidation = false) => {
} }
if (recordClone.isNew) { if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode) if(!recordNode)
throw new Error("Cannot find node for " + record.key); throw new Error("Cannot find node for " + record.key);
@ -5016,7 +5016,7 @@ const _save = async (app, record, context, skipValidation = false) => {
}; };
const initialiseAncestorIndexes = async (app, record) => { const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) { for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name); const indexKey = joinKey(record.key, index.name);
@ -5025,7 +5025,7 @@ const initialiseAncestorIndexes = async (app, record) => {
}; };
const initialiseReverseReferenceIndexes = async (app, record) => { const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
map(f => $(f.typeOptions.reverseIndexNodeKeys, [ map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5124,7 +5124,7 @@ const deleteRecords = async (app, key) => {
const _deleteIndex = async (app, indexKey, includeFolder) => { const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5169,7 +5169,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword // called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => { const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key); key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key); const record = await _load(app, key);
await transactionForDeleteRecord(app, record); await transactionForDeleteRecord(app, record);
@ -5196,7 +5196,7 @@ const _deleteRecord = async (app, key, disableCleanup) => {
}; };
const deleteIndexes = async (app, key) => { const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [ /* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
map(n => n.fields), map(n => n.fields),
@ -5303,7 +5303,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
}; };
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [ const incorrectFileFields = $(recordNode.fields, [
filter(f => f.type === 'file' filter(f => f.type === 'file'
@ -5550,7 +5550,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7424,7 +7424,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => {
}; };
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy), getExactNodeForKey(appHierarchy),
n => n.fields, n => n.fields,
filter(f => f.type === 'reference' filter(f => f.type === 'reference'
&& isSomething(record[f.name]) && isSomething(record[f.name])
@ -7834,7 +7834,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
} }
if (isReferenceIndex(indexNode)) { if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key); const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [ const refFields = $(recordNode.fields, [
filter(fieldReversesReferenceToIndex(indexNode)), filter(fieldReversesReferenceToIndex(indexNode)),
]); ]);

40
packages/core/dist/budibase-core.umd.js vendored
View File

@ -721,7 +721,7 @@
fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)), fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]); ]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)), fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]); ]);
@ -759,7 +759,7 @@
]); ]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey) return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey) ? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey; : nodeByKey;
@ -772,7 +772,7 @@
: nodeByKey; : nodeByKey;
}; };
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey, splitKey,
@ -871,7 +871,7 @@
var hierarchy = { var hierarchy = {
getLastPartInKey, getLastPartInKey,
getNodesInPath, getNodesInPath,
getExactNodeForPath, getExactNodeForKey,
hasMatchingAncestor, hasMatchingAncestor,
getNode, getNode,
getNodeByKeyOrNodeKey, getNodeByKeyOrNodeKey,
@ -1607,7 +1607,7 @@
const _load = async (app, key, keyStack = []) => { const _load = async (app, key, keyStack = []) => {
key = safeKey(key); key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key); const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson( const storedData = await app.datastore.loadJson(
getRecordFileName(key), getRecordFileName(key),
); );
@ -1753,7 +1753,7 @@
}; };
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord const startShardName = !startRecord
? null ? null
@ -4398,7 +4398,7 @@
)); ));
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4430,7 +4430,7 @@
const _getContext = (app, recordKey) => { const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey); recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {}; const cachedReferenceIndexes = {};
@ -4538,7 +4538,7 @@
? _getContext(app, record.key) ? _getContext(app, record.key)
: context; : context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode); const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here // non parsing would cause further issues - exit here
@ -4598,7 +4598,7 @@
const initialiseChildCollections = async (app, recordKey) => { const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [ const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy), getExactNodeForKey(app.hierarchy),
n => n.children, n => n.children,
fp.filter(isCollectionRecord), fp.filter(isCollectionRecord),
]); ]);
@ -4969,7 +4969,7 @@
} }
if (recordClone.isNew) { if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode) if(!recordNode)
throw new Error("Cannot find node for " + record.key); throw new Error("Cannot find node for " + record.key);
@ -5018,7 +5018,7 @@
}; };
const initialiseAncestorIndexes = async (app, record) => { const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) { for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name); const indexKey = joinKey(record.key, index.name);
@ -5027,7 +5027,7 @@
}; };
const initialiseReverseReferenceIndexes = async (app, record) => { const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [ const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [ fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5126,7 +5126,7 @@
const _deleteIndex = async (app, indexKey, includeFolder) => { const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5171,7 +5171,7 @@
// called deleteRecord because delete is a keyword // called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => { const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key); key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key); const record = await _load(app, key);
await transactionForDeleteRecord(app, record); await transactionForDeleteRecord(app, record);
@ -5198,7 +5198,7 @@
}; };
const deleteIndexes = async (app, key) => { const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key); const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [ /* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
map(n => n.fields), map(n => n.fields),
@ -5305,7 +5305,7 @@
}; };
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [ const incorrectFileFields = $(recordNode.fields, [
fp.filter(f => f.type === 'file' fp.filter(f => f.type === 'file'
@ -5552,7 +5552,7 @@
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7426,7 +7426,7 @@
}; };
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy), getExactNodeForKey(appHierarchy),
n => n.fields, n => n.fields,
fp.filter(f => f.type === 'reference' fp.filter(f => f.type === 'reference'
&& isSomething(record[f.name]) && isSomething(record[f.name])
@ -7836,7 +7836,7 @@
} }
if (isReferenceIndex(indexNode)) { if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key); const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [ const refFields = $(recordNode.fields, [
fp.filter(fieldReversesReferenceToIndex(indexNode)), fp.filter(fieldReversesReferenceToIndex(indexNode)),
]); ]);

0
packages/core/src/appInitialise/inProcIndexer.js
View File

16
packages/core/src/appInitialise/index.js
View File

@ -1,23 +1,27 @@
import { retry } from '../common/index'; import { retry } from '../common/index';
import { NotFoundError } from '../common/errors'; import { NotFoundError } from '../common/errors';
const createJson = originalCreateFile => async (key, obj, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj)); const createJson = originalCreateFile => async (key, obj, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj));
const createNewFile = originalCreateFile => async (path, content, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, path, content); const createNewFile = originalCreateFile => async (path, content, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, path, content);
const loadJson = datastore => async (key, retries = 5, delay = 500) => { const loadJson = datastore => async (key, retries = 3, delay = 100) => {
try { try {
return await retry(JSON.parse, retries, delay, await datastore.loadFile(key)); return await retry(JSON.parse, retries, delay, await datastore.loadFile(key));
} catch (err) { } catch (err) {
throw new NotFoundError(err.message); const newErr = new NotFoundError(err.message);
newErr.stack = err.stack;
throw(newErr);
} }
} }
const updateJson = datastore => async (key, obj, retries = 5, delay = 500) => { const updateJson = datastore => async (key, obj, retries = 3, delay = 100) => {
try { try {
return await retry(datastore.updateFile, retries, delay, key, JSON.stringify(obj)); return await retry(datastore.updateFile, retries, delay, key, JSON.stringify(obj));
} catch (err) { } catch (err) {
throw new NotFoundError(err.message); const newErr = new NotFoundError(err.message);
newErr.stack = err.stack;
throw(newErr);
} }
} }

5
packages/core/src/appInitialise/initialiseData.js
View File

@ -35,7 +35,9 @@ const initialiseRootIndexes = async (datastore, hierarchy) => {
]); ]);
for (const index of globalIndexes) { for (const index of globalIndexes) {
if (!await datastore.exists(index.nodeKey())) { await initialiseIndex(datastore, '', index); } if (!await datastore.exists(index.nodeKey())) {
await initialiseIndex(datastore, '', index);
}
} }
}; };
@ -52,6 +54,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => {
]); ]);
for (let record of singleRecords) { for (let record of singleRecords) {
await datastore.createFolder(record.nodeKey());
const result = _getNew(record, ""); const result = _getNew(record, "");
await _save(app,result); await _save(app,result);
} }

48
packages/core/src/collectionApi/delete.js
View File

@ -1,13 +1,11 @@
import { includes } from 'lodash/fp';
import { getNodeForCollectionPath } from '../templateApi/hierarchy';
import { import {
safeKey, apiWrapper, safeKey, apiWrapper,
events, joinKey, events, joinKey,
} from '../common'; } from '../common';
import { _deleteRecord } from '../recordApi/delete'; import { _deleteRecord } from '../recordApi/delete';
import { getAllIdsIterator, getAllIdsShardKey, folderStructureArray } from '../indexing/allIds'; import { getAllIdsIterator } from '../indexing/allIds';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { getRecordInfo } from "../recordApi/recordInfo"; import { getCollectionDir } from "../recordApi/recordInfo";
export const deleteCollection = (app, disableCleanup = false) => async key => apiWrapper( export const deleteCollection = (app, disableCleanup = false) => async key => apiWrapper(
app, app,
@ -17,28 +15,38 @@ export const deleteCollection = (app, disableCleanup = false) => async key => ap
_deleteCollection, app, key, disableCleanup, _deleteCollection, app, key, disableCleanup,
); );
/*
const recordNode = getCollectionNode(app.hierarchy, key);
*/
export const _deleteCollection = async (app, key, disableCleanup) => { export const _deleteCollection = async (app, key, disableCleanup) => {
const recordInfo = getRecordInfo(key); key = safeKey(key);
await app.datastore.deleteFolder(recordInfo) const collectionDir = getCollectionDir(app.hierarchy, key);
await deleteRecords(app, key);
await deleteCollectionFolder(app, collectionDir);
if (!disableCleanup) { await app.cleanupTransactions(); } if (!disableCleanup) { await app.cleanupTransactions(); }
}; };
const deleteCollectionFolder = async (app, key) => await app.datastore.deleteFolder(key); const deleteCollectionFolder = async (app, dir) =>
await app.datastore.deleteFolder(dir);
const deleteRecords = async (app, key) => {
const iterate = await getAllIdsIterator(app)(key);
const deleteShardFolders = async (app, node, key) => { let ids = await iterate();
while (!ids.done) {
if (ids.result.collectionKey === key) {
for (const id of ids.result.ids) {
await _deleteRecord(
app,
joinKey(key, id),
true,
);
}
}
await app.datastore.deleteFolder( ids = await iterate();
joinKey( }
key, 'allids',
node.nodeId,
),
);
await app.datastore.deleteFolder(
joinKey(key, 'allids'),
);
}; };

4
packages/core/src/collectionApi/initialise.js
View File

@ -3,13 +3,13 @@ import {
getFlattenedHierarchy, getFlattenedHierarchy,
isCollectionRecord, isCollectionRecord,
isRoot, isRoot,
getExactNodeForPath,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { $, allTrue, joinKey } from '../common'; import { $, allTrue, joinKey } from '../common';
const ensureCollectionIsInitialised = async (datastore, node, dir) => { const ensureCollectionIsInitialised = async (datastore, node, dir) => {
if (!await datastore.exists(dir)) { if (!await datastore.exists(dir)) {
await datastore.createFolder(dir); await datastore.createFolder(dir);
await datastore.createFolder(joinKey(dir, node.nodeId));
} }
}; };
@ -29,7 +29,7 @@ export const initialiseRootCollections = async (datastore, hierarchy) => {
await ensureCollectionIsInitialised( await ensureCollectionIsInitialised(
datastore, datastore,
col, col,
col.collectionPathRegx(), col.collectionPathRegx()
); );
} }
}; };

16
packages/core/src/common/index.js
View File

@ -4,13 +4,12 @@ import {
tail, findIndex, startsWith, tail, findIndex, startsWith,
dropRight, flow, takeRight, trim, dropRight, flow, takeRight, trim,
replace replace
} from 'lodash'; } from 'lodash';
import { import {
some, reduce, isEmpty, isArray, join, some, reduce, isEmpty, isArray, join,
isString, isInteger, isDate, toNumber, isString, isInteger, isDate, toNumber,
isUndefined, isNaN, isNull, constant, isUndefined, isNaN, isNull, constant,
split, includes split, includes, filter
} from 'lodash/fp'; } from 'lodash/fp';
import { events, eventsList } from './events'; import { events, eventsList } from './events';
import { apiWrapper } from './apiWrapper'; import { apiWrapper } from './apiWrapper';
@ -32,7 +31,13 @@ export const safeKey = key => replace(`${keySep}${trimKeySep(key)}`, `${keySep}$
export const joinKey = (...strs) => { export const joinKey = (...strs) => {
const paramsOrArray = strs.length === 1 & isArray(strs[0]) const paramsOrArray = strs.length === 1 & isArray(strs[0])
? strs[0] : strs; ? strs[0] : strs;
return safeKey(join(keySep)(paramsOrArray)); return $(paramsOrArray, [
filter(s => !isUndefined(s)
&& !isNull(s)
&& s.toString().length > 0),
join(keySep),
safeKey
]);
}; };
export const splitKey = $$(trimKeySep, splitByKeySep); export const splitKey = $$(trimKeySep, splitByKeySep);
export const getDirFomKey = $$(splitKey, dropRight, p => joinKey(...p)); export const getDirFomKey = $$(splitKey, dropRight, p => joinKey(...p));
@ -183,6 +188,10 @@ export const toNumberOrNull = s => (isNull(s) ? null
export const isArrayOfString = opts => isArray(opts) && all(isString)(opts); export const isArrayOfString = opts => isArray(opts) && all(isString)(opts);
export const pushAll = (target, items) => {
for(let i of items) target.push(i);
}
export const pause = async duration => new Promise(res => setTimeout(res, duration)); export const pause = async duration => new Promise(res => setTimeout(res, duration));
export const retry = async (fn, retries, delay, ...args) => { export const retry = async (fn, retries, delay, ...args) => {
@ -267,4 +276,5 @@ export default {
insensitiveEquals, insensitiveEquals,
pause, pause,
retry, retry,
pushAll
}; };

10
packages/core/src/indexApi/aggregates.js
View File

@ -10,12 +10,13 @@ import {
getShardKeysInRange, getShardKeysInRange,
} from '../indexing/sharding'; } from '../indexing/sharding';
import { import {
getExactNodeForPath, isIndex, getExactNodeForKey, isIndex,
isShardedIndex, isShardedIndex,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { CONTINUE_READING_RECORDS } from '../indexing/serializer'; import { CONTINUE_READING_RECORDS } from '../indexing/serializer';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { BadRequestError } from '../common/errors'; import { BadRequestError } from '../common/errors';
import { getIndexDir } from "./getIndexDir";
export const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndParams = null) => apiWrapper( export const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndParams = null) => apiWrapper(
app, app,
@ -27,13 +28,14 @@ export const aggregates = app => async (indexKey, rangeStartParams = null, range
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => { const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
if (isShardedIndex(indexNode)) { if (isShardedIndex(indexNode)) {
const shardKeys = await getShardKeysInRange( const shardKeys = await getShardKeysInRange(
app, indexKey, rangeStartParams, rangeEndParams, app, indexNode, indexDir, rangeStartParams, rangeEndParams,
); );
let aggregateResult = null; let aggregateResult = null;
for (const k of shardKeys) { for (const k of shardKeys) {
@ -53,7 +55,7 @@ const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
app.hierarchy, app.hierarchy,
app.datastore, app.datastore,
indexNode, indexNode,
getUnshardedIndexDataKey(indexKey), getUnshardedIndexDataKey(indexDir),
); );
}; };

16
packages/core/src/indexApi/buildIndex.js
View File

@ -1,16 +1,16 @@
import { import {
find, filter, filter,
includes, some, includes, some,
} from 'lodash/fp'; } from 'lodash/fp';
import { getAllIdsIterator } from '../indexing/allIds'; import { getAllIdsIterator } from '../indexing/allIds';
import { import {
getFlattenedHierarchy, getRecordNodeById, getFlattenedHierarchy, getRecordNodeById,
getCollectionNodeByKeyOrNodeKey, getNode, isIndex, getNode, isIndex,
isRecord, isDecendant, getAllowedRecordNodesForIndex, isRecord, getAllowedRecordNodesForIndex,
fieldReversesReferenceToIndex, fieldReversesReferenceToIndex,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { import {
joinKey, apiWrapper, events, $, allTrue, joinKey, apiWrapper, events, $
} from '../common'; } from '../common';
import { import {
createBuildIndexFolder, createBuildIndexFolder,
@ -82,9 +82,11 @@ const buildReverseReferenceIndex = async (app, indexNode) => {
} }
}; };
/*
const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [ const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [
map(n => n.parent()), map(n => n.parent()),
]); ]);
*/
const buildHeirarchalIndex = async (app, indexNode) => { const buildHeirarchalIndex = async (app, indexNode) => {
let recordCount = 0; let recordCount = 0;
@ -127,10 +129,11 @@ const buildHeirarchalIndex = async (app, indexNode) => {
return recordCount; return recordCount;
}; };
const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children); // const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
const recordNodeApplies = indexNode => recordNode => includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds); const recordNodeApplies = indexNode => recordNode => includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds);
/*
const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [ const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
filter( filter(
@ -141,7 +144,9 @@ const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarc
), ),
), ),
]); ]);
*/
/*
const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey, const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
indexNode, indexKey, currentIndexedData, indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount = 0) => { currentIndexedDataKey, recordCount = 0) => {
@ -194,5 +199,6 @@ const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
return recordCount; return recordCount;
}; };
*/
export default buildIndex; export default buildIndex;

14
packages/core/src/indexApi/delete.js
View File

@ -4,21 +4,23 @@ import {
} from '../common'; } from '../common';
import { import {
isIndex, isShardedIndex, isIndex, isShardedIndex,
getExactNodeForPath, getExactNodeForKey,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { import {
getAllShardKeys, getShardMapKey, getAllShardKeys, getShardMapKey,
getUnshardedIndexDataKey, getUnshardedIndexDataKey,
} from '../indexing/sharding'; } from '../indexing/sharding';
import { getIndexDir } from "./getIndexDir";
export const _deleteIndex = async (app, indexKey, includeFolder) => { export const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
if (isShardedIndex(indexNode)) { if (isShardedIndex(indexNode)) {
const shardKeys = await getAllShardKeys(app, indexKey); const shardKeys = await getAllShardKeys(app, indexNode, indexDir);
for (const k of shardKeys) { for (const k of shardKeys) {
await tryAwaitOrIgnore( await tryAwaitOrIgnore(
app.datastore.deleteFile(k), app.datastore.deleteFile(k),
@ -26,20 +28,20 @@ export const _deleteIndex = async (app, indexKey, includeFolder) => {
} }
tryAwaitOrIgnore( tryAwaitOrIgnore(
await app.datastore.deleteFile( await app.datastore.deleteFile(
getShardMapKey(indexKey), getShardMapKey(indexDir),
), ),
); );
} else { } else {
await tryAwaitOrIgnore( await tryAwaitOrIgnore(
app.datastore.deleteFile( app.datastore.deleteFile(
getUnshardedIndexDataKey(indexKey), getUnshardedIndexDataKey(indexDir),
), ),
); );
} }
if (includeFolder) { if (includeFolder) {
tryAwaitOrIgnore( tryAwaitOrIgnore(
await app.datastore.deleteFolder(indexKey), await app.datastore.deleteFolder(indexDir),
); );
} }
}; };

20
packages/core/src/indexApi/getIndexDir.js Normal file
View File

@ -0,0 +1,20 @@
import { getRecordInfo } from "../recordApi/recordInfo";
import {
getParentKey, getLastPartInKey
} from "../templateApi/hierarchy";
import { keySep } from "../common";
export const getIndexDir = (hierarchy, indexKey) => {
const parentKey = getParentKey(indexKey);
if(parentKey === "") return indexKey;
if(parentKey === keySep) return indexKey;
const recordInfo = getRecordInfo(
hierarchy,
parentKey);
return recordInfo.child(
getLastPartInKey(indexKey));
}

16
packages/core/src/indexApi/listItems.js
View File

@ -9,10 +9,11 @@ import {
getShardKeysInRange, getShardKeysInRange,
} from '../indexing/sharding'; } from '../indexing/sharding';
import { import {
getExactNodeForPath, isIndex, getExactNodeForKey, isIndex,
isShardedIndex, isShardedIndex,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { getIndexDir } from "./getIndexDir";
export const listItems = app => async (indexKey, options) => { export const listItems = app => async (indexKey, options) => {
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
@ -33,29 +34,30 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
merge(defaultOptions), merge(defaultOptions),
]); ]);
const getItems = async key => (isNonEmptyString(searchPhrase) const getItems = async indexedDataKey => (isNonEmptyString(searchPhrase)
? await searchIndex( ? await searchIndex(
app.hierarchy, app.hierarchy,
app.datastore, app.datastore,
indexNode, indexNode,
key, indexedDataKey,
searchPhrase, searchPhrase,
) )
: await readIndex( : await readIndex(
app.hierarchy, app.hierarchy,
app.datastore, app.datastore,
indexNode, indexNode,
key, indexedDataKey,
)); ));
indexKey = safeKey(indexKey); indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey); const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); } if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
if (isShardedIndex(indexNode)) { if (isShardedIndex(indexNode)) {
const shardKeys = await getShardKeysInRange( const shardKeys = await getShardKeysInRange(
app, indexKey, rangeStartParams, rangeEndParams, app, indexNode, indexDir, rangeStartParams, rangeEndParams,
); );
const items = []; const items = [];
for (const k of shardKeys) { for (const k of shardKeys) {
@ -64,6 +66,6 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
return flatten(items); return flatten(items);
} }
return await getItems( return await getItems(
getUnshardedIndexDataKey(indexKey), getUnshardedIndexDataKey(indexDir),
); );
}; };

203
packages/core/src/indexing/allIds.js
View File

@ -1,13 +1,14 @@
import { import {
join, flatten, orderBy, flatten, orderBy,
filter filter, isUndefined
} from 'lodash/fp'; } from 'lodash/fp';
import { import hierarchy, {
getFlattenedHierarchy, getFlattenedHierarchy,
getCollectionNodeByKeyOrNodeKey, getCollectionNodeByKeyOrNodeKey,
isCollectionRecord, isAncestor, isCollectionRecord, isAncestor,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { joinKey, safeKey, $ } from '../common'; import { joinKey, safeKey, $ } from '../common';
import { getCollectionDir } from "../recordApi/recordInfo";
export const RECORDS_PER_FOLDER = 1000; export const RECORDS_PER_FOLDER = 1000;
export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-'; export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-';
@ -23,19 +24,39 @@ export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ
* - [64, 64, 10] = all records fit into 64 * 64 * 10 folder * - [64, 64, 10] = all records fit into 64 * 64 * 10 folder
* (there are 64 possible chars in allIsChars) * (there are 64 possible chars in allIsChars)
*/ */
export const folderStructureArray = (recordNode, currentArray=[], currentFolderPosition=0) => { export const folderStructureArray = (recordNode) => {
const totalFolders = Math.ceil(recordNode.estimatedRecordCount / 1000);
const folderArray = [];
let levelCount = 1;
while(64**levelCount < totalFolders) {
levelCount += 1;
folderArray.push(64);
}
const parentFactor = (64**folderArray.length);
if(parentFactor < totalFolders) {
folderArray.push(
Math.ceil(totalFolders / parentFactor)
);
}
return folderArray;
/*
const maxRecords = currentFolderPosition === 0 const maxRecords = currentFolderPosition === 0
? RECORDS_PER_FOLDER ? RECORDS_PER_FOLDER
: currentFolderPosition * 64 * RECORDS_PER_FOLDER; : currentFolderPosition * 64 * RECORDS_PER_FOLDER;
if(maxRecords < recordNode.estimatedRecordCount) { if(maxRecords < recordNode.estimatedRecordCount) {
return folderStructureArray( return folderStructureArray(
recordNode,
[...currentArray, 64], [...currentArray, 64],
currentFolderPosition + 1); currentFolderPosition + 1);
} else { } else {
const childFolderCount = Math.ceil(maxRecords / recordNode.estimatedRecordCount); const childFolderCount = Math.ceil(recordNode.estimatedRecordCount / maxRecords );
return [...currentArray, childFolderCount] return [...currentArray, childFolderCount]
} }*/
} }
@ -51,88 +72,150 @@ export const getAllIdsIterator = app => async (collection_Key_or_NodeKey) => {
const folderStructure = folderStructureArray(recordNode) const folderStructure = folderStructureArray(recordNode)
let currentFolderContents = []; let currentFolderContents = [];
let currentFolderIndexes = []; let currentPosition = [];
let currentSubPath = [];
const collectionDir = getCollectionDir(app.hierarchy, collectionKey);
const basePath = joinKey( const basePath = joinKey(
collectionKey, recordNode.nodeId.toString()); collectionDir, recordNode.nodeId.toString());
let folderLevel = 0;
const levels = folderStructure.length; // "folderStructure" determines the top, sharding folders
// we need to add one, for the collection root folder, which
// always exists
const levels = folderStructure.length + 1;
const topLevel = levels -1; const topLevel = levels -1;
const lastPathHasContent = () =>
folderLevel === 0 /* populate initial directory structure in form:
|| currentFolderContents[folderLevel - 1].length > 0; [
{path: "/a", contents: ["b", "c", "d"]},
{path: "/a/b", contents: ["e","f","g"]},
{path: "/a/b/e", contents: ["1-abcd","2-cdef","3-efgh"]},
]
// stores contents on each parent level
// top level has ID folders
*/
const firstFolder = async () => {
while (folderLevel < folderStructure.length && lastPathHasContent()) { let folderLevel = 0;
if(folderLevel < topLevel) {
const contentsThisLevel =
await app.datastore.getFolderContents(
join(basePath, ...currentSubPath));
currentFolderContents.push(contentsThisLevel); const lastPathHasContent = () =>
currentFolderIndexes.push(0); folderLevel === 0
currentSubPath.push(currentFolderContents[0]) || currentFolderContents[folderLevel - 1].contents.length > 0;
} else {
// placesholders only for the top level (which will be populated by nextFolder())
currentFolderContents.push([])
currentFolderIndexes.push(-1);
currentSubPath.push("");
}
folderLevel+=1;
}
while (folderLevel <= topLevel && lastPathHasContent()) {
const nextFolder = async (lev=-1) => { let thisPath = basePath;
lev = (lev === -1) ? topLevel : lev; for(let lev = 0; lev < currentPosition.length; lev++) {
if(currentFolderIndexes[lev] !== currentFolderContents[lev].length - 1){ thisPath = joinKey(
thisPath, currentFolderContents[lev].contents[0]);
const folderIndexThisLevel = currentFolderIndexes[lev] + 1;
currentFolderIndexes[lev] = folderIndexThisLevel;
currentSubPath[lev] = currentFolderContents[folderIndexThisLevel]
if(lev < topLevel) {
let loopLev = lev + 1;
while(loopLev <= topLevel) {
currentFolderContents[loopLev] =
await app.datastore.getFolderContents(join(basePath, ...currentSubPath));
loopLev+=1;
}
} }
return false; // not complete const contentsThisLevel =
await app.datastore.getFolderContents(thisPath);
currentFolderContents.push({
contents:contentsThisLevel,
path: thisPath
});
} else { // should start as something like [0,0]
if(lev === 0) return true; // complete if(folderLevel < topLevel)
return await nextFolder(lev - 1); currentPosition.push(0);
folderLevel+=1;
} }
return (currentPosition.length === levels - 1);
}
const isOnLastFolder = level => {
const result = currentPosition[level] === currentFolderContents[level].contents.length - 1;
return result;
} }
const getNextFolder = async (lev=undefined) => {
lev = isUndefined(lev) ? topLevel : lev;
const parentLev = lev - 1;
if(parentLev < 0) return false;
if(isOnLastFolder(parentLev)) {
return await getNextFolder(parentLev);
}
const newPosition = currentPosition[parentLev] + 1;
currentPosition[parentLev] = newPosition;
const nextFolder = joinKey(
currentFolderContents[parentLev].path,
currentFolderContents[parentLev].contents[newPosition]);
currentFolderContents[lev].contents = await app.datastore.getFolderContents(
nextFolder
);
currentFolderContents[lev].path = nextFolder;
if(lev !== topLevel) {
// we just advanced a parent folder, so now need to
// do the same to the next levels
let loopLevel = lev + 1;
while(loopLevel <= topLevel) {
const loopParentLevel = loopLevel-1;
currentPosition[loopParentLevel] = 0;
const nextLoopFolder = joinKey(
currentFolderContents[loopParentLevel].path,
currentFolderContents[loopParentLevel].contents[0]);
currentFolderContents[loopLevel].contents = await app.datastore.getFolderContents(
nextLoopFolder
);
currentFolderContents[loopLevel].path = nextLoopFolder;
loopLevel+=1;
}
}
// true ==has more ids... (just loaded more)
return true;
}
const idsCurrentFolder = () =>
currentFolderContents[currentFolderContents.length - 1].contents;
const fininshedResult = ({ done: true, result: { ids: [], collectionKey } }); const fininshedResult = ({ done: true, result: { ids: [], collectionKey } });
let hasStarted = false;
let hasMore = true;
const getIdsFromCurrentfolder = async () => { const getIdsFromCurrentfolder = async () => {
if(currentFolderIndexes.length < folderStructure) if(!hasMore) {
return fininshedResult; return fininshedResult;
}
const hasMore = await nextFolder(); if(!hasStarted) {
hasMore = await firstFolder();
hasStarted = true;
return ({
result: {
ids: idsCurrentFolder(),
collectionKey
},
done: false
})
}
if(!hasMore) return fininshedResult; hasMore = await getNextFolder();
const result = ({ return ({
result: { result: {
ids: await app.datastore.getFolderContents( ids: hasMore ? idsCurrentFolder() : [],
joinKey(basePath, currentSubPath)),
collectionKey collectionKey
}, },
done:false done: !hasMore
}) });
return result;
} }
return getIdsFromCurrentfolder; return getIdsFromCurrentfolder;

28
packages/core/src/indexing/apply.js
View File

@ -1,13 +1,13 @@
import { ensureShardNameIsInShardMap } from './sharding'; import { ensureShardNameIsInShardMap } from './sharding';
import { getIndexWriter } from './serializer'; import { getIndexWriter } from './serializer';
import { isShardedIndex } from '../templateApi/hierarchy'; import { isShardedIndex, getParentKey } from '../templateApi/hierarchy';
import {promiseWriteableStream} from "./promiseWritableStream"; import {promiseWriteableStream} from "./promiseWritableStream";
import {promiseReadableStream} from "./promiseReadableStream"; import {promiseReadableStream} from "./promiseReadableStream";
export const applyToShard = async (hierarchy, store, indexKey, export const applyToShard = async (hierarchy, store, indexDir,
indexNode, indexShardKey, recordsToWrite, keysToRemove) => { indexNode, indexShardKey, recordsToWrite, keysToRemove) => {
const createIfNotExists = recordsToWrite.length > 0; const createIfNotExists = recordsToWrite.length > 0;
const writer = await getWriter(hierarchy, store, indexKey, indexShardKey, indexNode, createIfNotExists); const writer = await getWriter(hierarchy, store, indexDir, indexShardKey, indexNode, createIfNotExists);
if (writer === SHARD_DELETED) return; if (writer === SHARD_DELETED) return;
await writer.updateIndex(recordsToWrite, keysToRemove); await writer.updateIndex(recordsToWrite, keysToRemove);
@ -15,13 +15,17 @@ export const applyToShard = async (hierarchy, store, indexKey,
}; };
const SHARD_DELETED = 'SHARD_DELETED'; const SHARD_DELETED = 'SHARD_DELETED';
const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode, createIfNotExists) => { const getWriter = async (hierarchy, store, indexDir, indexedDataKey, indexNode, createIfNotExists) => {
let readableStream = null; let readableStream = null;
if (isShardedIndex(indexNode)) { if (isShardedIndex(indexNode)) {
await ensureShardNameIsInShardMap(store, indexKey, indexedDataKey); await ensureShardNameIsInShardMap(store, indexDir, indexedDataKey);
if(!await store.exists(indexedDataKey)) { if(!await store.exists(indexedDataKey)) {
await store.createFile(indexedDataKey, ""); if (await store.exists(getParentKey(indexedDataKey))) {
await store.createFile(indexedDataKey, "");
} else {
return SHARD_DELETED;
}
} }
} }
@ -37,7 +41,11 @@ const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode,
throw e; throw e;
} else { } else {
if (createIfNotExists) { if (createIfNotExists) {
await store.createFile(indexedDataKey, ''); if(await store.exists(getParentKey(indexedDataKey))) {
await store.createFile(indexedDataKey, '');
} else {
return SHARD_DELETED;
}
} else { } else {
return SHARD_DELETED; return SHARD_DELETED;
} }
@ -65,6 +73,12 @@ const swapTempFileIn = async (store, indexedDataKey, isRetry = false) => {
await store.deleteFile(indexedDataKey); await store.deleteFile(indexedDataKey);
} catch (e) { } catch (e) {
// ignore failure, incase it has not been created yet // ignore failure, incase it has not been created yet
// if parent folder does not exist, assume that this index
// should not be there
if(!await store.exists(getParentKey(indexedDataKey))) {
return;
}
} }
try { try {
await store.renameFile(tempFile, indexedDataKey); await store.renameFile(tempFile, indexedDataKey);

8
packages/core/src/indexing/initialiseIndex.js
View File

@ -3,19 +3,19 @@ import { joinKey } from '../common';
import { getShardMapKey, getUnshardedIndexDataKey, createIndexFile } from './sharding'; import { getShardMapKey, getUnshardedIndexDataKey, createIndexFile } from './sharding';
export const initialiseIndex = async (datastore, dir, index) => { export const initialiseIndex = async (datastore, dir, index) => {
const indexKey = joinKey(dir, index.name); const indexDir = joinKey(dir, index.name);
await datastore.createFolder(indexKey); await datastore.createFolder(indexDir);
if (isShardedIndex(index)) { if (isShardedIndex(index)) {
await datastore.createFile( await datastore.createFile(
getShardMapKey(indexKey), getShardMapKey(indexDir),
'[]', '[]',
); );
} else { } else {
await createIndexFile( await createIndexFile(
datastore, datastore,
getUnshardedIndexDataKey(indexKey), getUnshardedIndexDataKey(indexDir),
index, index,
); );
} }

33
packages/core/src/indexing/read.js
View File

@ -1,12 +1,4 @@
import lunr from 'lunr'; import lunr from 'lunr';
import {
getHashCode,
joinKey
} from '../common';
import {
getActualKeyOfParent,
isGlobalIndex,
} from '../templateApi/hierarchy';
import {promiseReadableStream} from "./promiseReadableStream"; import {promiseReadableStream} from "./promiseReadableStream";
import { createIndexFile } from './sharding'; import { createIndexFile } from './sharding';
import { generateSchema } from './indexSchemaCreator'; import { generateSchema } from './indexSchemaCreator';
@ -50,31 +42,6 @@ export const searchIndex = async (hierarchy, datastore, index, indexedDataKey, s
return await doRead(hierarchy, datastore, index, indexedDataKey); return await doRead(hierarchy, datastore, index, indexedDataKey);
}; };
export const getIndexedDataKey_fromIndexKey = (indexKey) =>
`${indexKey}${indexKey.endsWith('.csv') ? '' : '.csv'}`;
export const uniqueIndexName = index => `idx_${
getHashCode(`${index.filter}${index.map}`)
}.csv`;
export const getIndexedDataKey = (decendantKey, indexNode) => {
if (isGlobalIndex(indexNode)) { return `${indexNode.nodeKey()}.csv`; }
const indexedDataParentKey = getActualKeyOfParent(
indexNode.parent().nodeKey(),
decendantKey,
);
const indexName = indexNode.name
? `${indexNode.name}.csv`
: uniqueIndexName(indexNode);
return joinKey(
indexedDataParentKey,
indexName,
);
};
export const iterateIndex = (onGetItem, getFinalResult) => async (hierarchy, datastore, index, indexedDataKey) => { export const iterateIndex = (onGetItem, getFinalResult) => async (hierarchy, datastore, index, indexedDataKey) => {
try { try {
const readableStream = promiseReadableStream( const readableStream = promiseReadableStream(

32
packages/core/src/indexing/relevant.js
View File

@ -9,21 +9,23 @@ import {
} from '../common'; } from '../common';
import { import {
getFlattenedHierarchy, getNode, getRecordNodeId, getFlattenedHierarchy, getNode, getRecordNodeId,
getExactNodeForPath, recordNodeIdIsAllowed, getExactNodeForKey, recordNodeIdIsAllowed,
isRecord, isGlobalIndex, isRecord, isGlobalIndex,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { indexTypes } from '../templateApi/indexes'; import { indexTypes } from '../templateApi/indexes';
import { getIndexDir } from "../indexApi/getIndexDir";
import { getRecordInfo} from "../recordApi/recordInfo";
export const getRelevantAncestorIndexes = (appHierarchy, record) => { export const getRelevantAncestorIndexes = (hierarchy, record) => {
const key = record.key; const key = record.key;
const keyParts = splitKey(key); const keyParts = splitKey(key);
const nodeId = getRecordNodeId(key); const nodeId = getRecordNodeId(key);
const flatHierarchy = orderBy(getFlattenedHierarchy(appHierarchy), const flatHierarchy = orderBy(getFlattenedHierarchy(hierarchy),
[node => node.pathRegx().length], [node => node.pathRegx().length],
['desc']); ['desc']);
const makeindexNodeAndKey_ForAncestorIndex = (indexNode, indexKey) => makeIndexNodeAndKey(indexNode, joinKey(indexKey, indexNode.name)); const makeindexNodeAndDir_ForAncestorIndex = (indexNode, parentRecordDir) => makeIndexNodeAndDir(indexNode, joinKey(parentRecordDir, indexNode.name));
const traverseAncestorIndexesInPath = () => reduce((acc, part) => { const traverseAncestorIndexesInPath = () => reduce((acc, part) => {
const currentIndexKey = joinKey(acc.lastIndexKey, part); const currentIndexKey = joinKey(acc.lastIndexKey, part);
@ -42,8 +44,10 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => {
|| includes(nodeId)(i.allowedRecordNodeIds))), || includes(nodeId)(i.allowedRecordNodeIds))),
]); ]);
const currentRecordDir = getRecordInfo(hierarchy, currentIndexKey).dir;
each(v => acc.nodesAndKeys.push( each(v => acc.nodesAndKeys.push(
makeindexNodeAndKey_ForAncestorIndex(v, currentIndexKey), makeindexNodeAndDir_ForAncestorIndex(v, currentRecordDir),
))(indexes); ))(indexes);
return acc; return acc;
@ -51,31 +55,35 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => {
const rootIndexes = $(flatHierarchy, [ const rootIndexes = $(flatHierarchy, [
filter(n => isGlobalIndex(n) && recordNodeIdIsAllowed(n)(nodeId)), filter(n => isGlobalIndex(n) && recordNodeIdIsAllowed(n)(nodeId)),
map(i => makeIndexNodeAndKey(i, i.nodeKey())), map(i => makeIndexNodeAndDir(
i,
getIndexDir(hierarchy, i.nodeKey()))),
]); ]);
return union(traverseAncestorIndexesInPath())(rootIndexes); return union(traverseAncestorIndexesInPath())(rootIndexes);
}; };
export const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [ export const getRelevantReverseReferenceIndexes = (hierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy), getExactNodeForKey(hierarchy),
n => n.fields, n => n.fields,
filter(f => f.type === 'reference' filter(f => f.type === 'reference'
&& isSomething(record[f.name]) && isSomething(record[f.name])
&& isNonEmptyString(record[f.name].key)), && isNonEmptyString(record[f.name].key)),
map(f => $(f.typeOptions.reverseIndexNodeKeys, [ map(f => $(f.typeOptions.reverseIndexNodeKeys, [
map(n => ({ map(n => ({
recordNode: getNode(appHierarchy, n), recordNode: getNode(hierarchy, n),
field: f, field: f,
})), })),
])), ])),
flatten, flatten,
map(n => makeIndexNodeAndKey( map(n => makeIndexNodeAndDir(
n.recordNode, n.recordNode,
joinKey(record[n.field.name].key, n.recordNode.name), joinKey(
getRecordInfo(hierarchy, record[n.field.name].key).dir,
n.recordNode.name),
)), )),
]); ]);
const makeIndexNodeAndKey = (indexNode, indexKey) => ({ indexNode, indexKey }); const makeIndexNodeAndDir = (indexNode, indexDir) => ({ indexNode, indexDir });
export default getRelevantAncestorIndexes; export default getRelevantAncestorIndexes;

48
packages/core/src/indexing/sharding.js
View File

@ -5,13 +5,14 @@ import {
import { import {
getActualKeyOfParent, isGlobalIndex, getActualKeyOfParent, isGlobalIndex,
getParentKey, isShardedIndex, getParentKey, isShardedIndex,
getExactNodeForPath, getExactNodeForKey,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { import {
joinKey, isNonEmptyString, splitKey, $, joinKey, isNonEmptyString, splitKey, $,
} from '../common'; } from '../common';
export const getIndexedDataKey = (indexNode, indexKey, record) => { export const getIndexedDataKey = (indexNode, indexDir, record) => {
const getShardName = (indexNode, record) => { const getShardName = (indexNode, record) => {
const shardNameFunc = compileCode(indexNode.getShardName); const shardNameFunc = compileCode(indexNode.getShardName);
try { try {
@ -27,18 +28,16 @@ export const getIndexedDataKey = (indexNode, indexKey, record) => {
? `${getShardName(indexNode, record)}.csv` ? `${getShardName(indexNode, record)}.csv`
: 'index.csv'; : 'index.csv';
return joinKey(indexKey, shardName); return joinKey(indexDir, shardName);
}; };
export const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => { export const getShardKeysInRange = async (app, indexNode, indexDir, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const startShardName = !startRecord const startShardName = !startRecord
? null ? null
: shardNameFromKey( : shardNameFromKey(
getIndexedDataKey( getIndexedDataKey(
indexNode, indexNode,
indexKey, indexDir,
startRecord, startRecord,
), ),
); );
@ -48,29 +47,29 @@ export const getShardKeysInRange = async (app, indexKey, startRecord = null, end
: shardNameFromKey( : shardNameFromKey(
getIndexedDataKey( getIndexedDataKey(
indexNode, indexNode,
indexKey, indexDir,
endRecord, endRecord,
), ),
); );
return $(await getShardMap(app.datastore, indexKey), [ return $(await getShardMap(app.datastore, indexDir), [
filter(k => (startRecord === null || k >= startShardName) filter(k => (startRecord === null || k >= startShardName)
&& (endRecord === null || k <= endShardName)), && (endRecord === null || k <= endShardName)),
map(k => joinKey(indexKey, `${k}.csv`)), map(k => joinKey(indexDir, `${k}.csv`)),
]); ]);
}; };
export const ensureShardNameIsInShardMap = async (store, indexKey, indexedDataKey) => { export const ensureShardNameIsInShardMap = async (store, indexDir, indexedDataKey) => {
const map = await getShardMap(store, indexKey); const map = await getShardMap(store, indexDir);
const shardName = shardNameFromKey(indexedDataKey); const shardName = shardNameFromKey(indexedDataKey);
if (!includes(shardName)(map)) { if (!includes(shardName)(map)) {
map.push(shardName); map.push(shardName);
await writeShardMap(store, indexKey, map); await writeShardMap(store, indexDir, map);
} }
}; };
export const getShardMap = async (datastore, indexKey) => { export const getShardMap = async (datastore, indexDir) => {
const shardMapKey = getShardMapKey(indexKey); const shardMapKey = getShardMapKey(indexDir);
try { try {
return await datastore.loadJson(shardMapKey); return await datastore.loadJson(shardMapKey);
} catch (_) { } catch (_) {
@ -79,27 +78,26 @@ export const getShardMap = async (datastore, indexKey) => {
} }
}; };
export const writeShardMap = async (datastore, indexKey, shardMap) => await datastore.updateJson( export const writeShardMap = async (datastore, indexDir, shardMap) => await datastore.updateJson(
getShardMapKey(indexKey), getShardMapKey(indexDir),
shardMap, shardMap,
); );
export const getAllShardKeys = async (app, indexKey) => await getShardKeysInRange(app, indexKey); export const getAllShardKeys = async (app, indexNode, indexDir) =>
await getShardKeysInRange(app, indexNode, indexDir);
export const getShardMapKey = indexKey => joinKey(indexKey, 'shardMap.json'); export const getShardMapKey = indexDir => joinKey(indexDir, 'shardMap.json');
export const getUnshardedIndexDataKey = indexKey => joinKey(indexKey, 'index.csv'); export const getUnshardedIndexDataKey = indexDir => joinKey(indexDir, 'index.csv');
export const getIndexFolderKey = indexKey => indexKey;
export const createIndexFile = async (datastore, indexedDataKey, index) => { export const createIndexFile = async (datastore, indexedDataKey, index) => {
if (isShardedIndex(index)) { if (isShardedIndex(index)) {
const indexKey = getParentKey(indexedDataKey); const indexDir = getParentKey(indexedDataKey);
const shardMap = await getShardMap(datastore, indexKey); const shardMap = await getShardMap(datastore, indexDir);
shardMap.push( shardMap.push(
shardNameFromKey(indexedDataKey), shardNameFromKey(indexedDataKey),
); );
await writeShardMap(datastore, indexKey, shardMap); await writeShardMap(datastore, indexDir, shardMap);
} }
await datastore.createFile(indexedDataKey, ''); await datastore.createFile(indexedDataKey, '');
}; };

0
packages/core/src/indexing/sweeper.js
View File

25
packages/core/src/recordApi/delete.js
View File

@ -1,11 +1,15 @@
import { import {
safeKey, apiWrapper, safeKey, apiWrapper,
events, events, joinKey,
} from '../common'; } from '../common';
import { _loadFromInfo } from './load'; import { _load } from './load';
import { _deleteCollection } from '../collectionApi/delete';
import {
getExactNodeForKey
} from '../templateApi/hierarchy';
import { transactionForDeleteRecord } from '../transactions/create'; import { transactionForDeleteRecord } from '../transactions/create';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { getRecordInfo } from "./recordInfo"; import { getRecordInfo } from './recordInfo';
export const deleteRecord = (app, disableCleanup = false) => async key => { export const deleteRecord = (app, disableCleanup = false) => async key => {
key = safeKey(key); key = safeKey(key);
@ -20,17 +24,22 @@ export const deleteRecord = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword // called deleteRecord because delete is a keyword
export const _deleteRecord = async (app, key, disableCleanup) => { export const _deleteRecord = async (app, key, disableCleanup) => {
const recordInfo = getRecordInfo(app, key); const recordInfo = getRecordInfo(app.hierarchy, key);
key = recordInfo.key; key = recordInfo.key;
const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _loadFromInfo(app, recordInfo); const record = await _load(app, key);
await transactionForDeleteRecord(app, record); await transactionForDeleteRecord(app, record);
for (const collectionRecord of node.children) {
const collectionKey = joinKey(
key, collectionRecord.collectionName,
);
await _deleteCollection(app, collectionKey, true);
}
await app.datastore.deleteFolder(recordInfo.dir); await app.datastore.deleteFolder(recordInfo.dir);
if (!disableCleanup) { await app.cleanupTransactions(); } if (!disableCleanup) { await app.cleanupTransactions(); }
}; };

4
packages/core/src/recordApi/downloadFile.js
View File

@ -2,6 +2,7 @@ import { apiWrapper, events, isNothing } from '../common';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { safeGetFullFilePath } from './uploadFile'; import { safeGetFullFilePath } from './uploadFile';
import { BadRequestError } from '../common/errors'; import { BadRequestError } from '../common/errors';
import { getRecordInfo } from "./recordInfo";
export const downloadFile = app => async (recordKey, relativePath) => apiWrapper( export const downloadFile = app => async (recordKey, relativePath) => apiWrapper(
app, app,
@ -16,9 +17,10 @@ const _downloadFile = async (app, recordKey, relativePath) => {
if (isNothing(recordKey)) { throw new BadRequestError('Record Key not supplied'); } if (isNothing(recordKey)) { throw new BadRequestError('Record Key not supplied'); }
if (isNothing(relativePath)) { throw new BadRequestError('file path not supplied'); } if (isNothing(relativePath)) { throw new BadRequestError('file path not supplied'); }
const {dir} = getRecordInfo(app.hierarchy, recordKey);
return await app.datastore.readableFileStream( return await app.datastore.readableFileStream(
safeGetFullFilePath( safeGetFullFilePath(
recordKey, relativePath, dir, relativePath,
), ),
); );
}; };

4
packages/core/src/recordApi/getContext.js
View File

@ -1,6 +1,6 @@
import { map, isString, has, some } from 'lodash/fp'; import { map, isString, has, some } from 'lodash/fp';
import { import {
getExactNodeForPath, getExactNodeForKey,
findField, getNode, isGlobalIndex, findField, getNode, isGlobalIndex,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { listItems } from '../indexApi/listItems'; import { listItems } from '../indexApi/listItems';
@ -23,7 +23,7 @@ export const getContext = app => recordKey => {
export const _getContext = (app, recordKey) => { export const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey); recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey); const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {}; const cachedReferenceIndexes = {};

4
packages/core/src/recordApi/load.js
View File

@ -2,7 +2,7 @@ import {
keyBy, mapValues, filter, keyBy, mapValues, filter,
map, includes, last, map, includes, last,
} from 'lodash/fp'; } from 'lodash/fp';
import { getExactNodeForPath, getNode } from '../templateApi/hierarchy'; import { getExactNodeForKey, getNode } from '../templateApi/hierarchy';
import { safeParseField } from '../types'; import { safeParseField } from '../types';
import { import {
$, splitKey, safeKey, isNonEmptyString, $, splitKey, safeKey, isNonEmptyString,
@ -72,7 +72,7 @@ export const _loadFromInfo = async (app, recordInfo, keyStack = []) => {
export const _load = async (app, key, keyStack = []) => export const _load = async (app, key, keyStack = []) =>
_loadFromInfo( _loadFromInfo(
app, app,
getRecordInfo(app, key), getRecordInfo(app.hierarchy, key),
keyStack); keyStack);

64
packages/core/src/recordApi/recordInfo.js
View File

@ -1,18 +1,19 @@
import { import {
getExactNodeForPath, getActualKeyOfParent, isRoot getExactNodeForKey, getActualKeyOfParent,
isRoot, isSingleRecord, getNodeForCollectionPath
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { import {
map, reduce reduce, find, filter, take
} from 'lodash/fp'; } from 'lodash/fp';
import { import {
$, getDirFomKey, getFileFromKey, joinKey, safeKey $, getFileFromKey, joinKey, safeKey, keySep
} from '../common'; } from '../common';
import { import {
folderStructureArray, allIdChars folderStructureArray, allIdChars
} from "../indexing/allIds"; } from "../indexing/allIds";
export const getRecordInfo = (app, key) => { export const getRecordInfo = (hierarchy, key) => {
const recordNode = getExactNodeForPath(app.hierarchy)(key); const recordNode = getExactNodeForKey(hierarchy)(key);
const pathInfo = getRecordDirectory(recordNode, key); const pathInfo = getRecordDirectory(recordNode, key);
const dir = joinKey(pathInfo.base, ...pathInfo.subdirs); const dir = joinKey(pathInfo.base, ...pathInfo.subdirs);
@ -25,6 +26,13 @@ export const getRecordInfo = (app, key) => {
}; };
} }
export const getCollectionDir = (hierarchy, collectionKey) => {
const recordNode = getNodeForCollectionPath(hierarchy)(collectionKey);
const dummyRecordKey = joinKey(collectionKey, "1-abcd");
const pathInfo = getRecordDirectory(recordNode, dummyRecordKey);
return pathInfo.base;
}
const recordJson = (dir) => const recordJson = (dir) =>
joinKey(dir, "record.json") joinKey(dir, "record.json")
@ -34,31 +42,53 @@ const files = (dir) =>
const getRecordDirectory = (recordNode, key) => { const getRecordDirectory = (recordNode, key) => {
const id = getFileFromKey(key); const id = getFileFromKey(key);
const traverseParentKeys = (n, keys=[]) => { const traverseParentKeys = (n, parents=[]) => {
if(isRoot(n)) return keys; if(isRoot(n)) return parents;
const k = getActualKeyOfParent(n, key); const k = getActualKeyOfParent(n.nodeKey(), key);
const thisNodeDir = {
node:n,
relativeDir: joinKey(
recordRelativeDirectory(n, getFileFromKey(k)))
};
return traverseParentKeys(
n.parent(),
[thisNodeDir, ...parents]);
} }
const parentDirs = $(recordNode.parent(), [
traverseParentKeys,
reduce((key, item) => {
return joinKey(key, item.node.collectionName, item.relativeDir)
}, keySep)
]);
const subdirs = isSingleRecord(recordNode)
? []
: recordRelativeDirectory(recordNode, id);
const base = isSingleRecord(recordNode)
? joinKey(parentDirs, recordNode.name)
: joinKey(parentDirs, recordNode.collectionName);
return ({ return ({
base:getDirFomKey(key), subdirs, base
subdirs: [recordNode.nodeId.toString(), ...subfolders]
}); });
} }
const recordRelativeDirectory = (recordNode, id) => { const recordRelativeDirectory = (recordNode, id) => {
const folderStructure = folderStructureArray(recordNode); const folderStructure = folderStructureArray(recordNode);
const strippedId = id.substring(recordNode.nodeId.toString().length + 1);
const subfolders = $(folderStructure, [ const subfolders = $(folderStructure, [
reduce((result, currentCount) => { reduce((result, currentCount) => {
result.folders.push( result.folders.push(
folderForChar(id[result.level], currentCount) folderForChar(strippedId[result.level], currentCount)
); );
return {level:result.level+1, folders:result.folders} return {level:result.level+1, folders:result.folders};
}, {level:0, folders:[]}), }, {level:0, folders:[]}),
map(f => f.folders), f => f.folders,
filter(f => !!f)
]); ]);
return [recordNode.nodeId.toString(), ...subfolders] return [recordNode.nodeId.toString(), ...subfolders, id]
} }
const folderForChar = (char, folderCount) => const folderForChar = (char, folderCount) =>

22
packages/core/src/recordApi/save.js
View File

@ -4,13 +4,13 @@ import {
} from 'lodash/fp'; } from 'lodash/fp';
import { initialiseChildCollections } from '../collectionApi/initialise'; import { initialiseChildCollections } from '../collectionApi/initialise';
import { validate } from './validate'; import { validate } from './validate';
import { _loadFromInfo, getRecordFileName } from './load'; import { _loadFromInfo } from './load';
import { import {
apiWrapper, events, $, joinKey, apiWrapper, events, $, joinKey,
} from '../common'; } from '../common';
import { import {
getFlattenedHierarchy, getExactNodeForPath, getFlattenedHierarchy, isRecord, getNode,
isRecord, getNode, fieldReversesReferenceToNode, fieldReversesReferenceToNode,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { import {
transactionForCreateRecord, transactionForCreateRecord,
@ -42,7 +42,7 @@ export const _save = async (app, record, context, skipValidation = false) => {
} }
} }
const recordInfo = getRecordInfo(app.record.key); const recordInfo = getRecordInfo(app.hierarchy, record.key);
const { const {
recordNode, pathInfo, recordNode, pathInfo,
recordJson, files, recordJson, files,
@ -92,7 +92,9 @@ export const _save = async (app, record, context, skipValidation = false) => {
const initialiseAncestorIndexes = async (app, recordInfo) => { const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) { for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name); const indexKey = recordInfo.child(index.name);
if (!await app.datastore.exists(indexKey)) { await initialiseIndex(app.datastore, recordInfo.dir, index); } if (!await app.datastore.exists(indexKey)) {
await initialiseIndex(app.datastore, recordInfo.dir, index);
}
} }
}; };
@ -125,7 +127,7 @@ const fieldsThatReferenceThisRecord = (app, recordNode) => $(app.hierarchy, [
const createRecordFolderPath = async (datastore, pathInfo) => { const createRecordFolderPath = async (datastore, pathInfo) => {
const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=[]) => { const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=undefined) => {
// iterate backwards through directory hierachy // iterate backwards through directory hierachy
// until we get to a folder that exists, then create the rest // until we get to a folder that exists, then create the rest
@ -138,12 +140,16 @@ const createRecordFolderPath = async (datastore, pathInfo) => {
if(await datastore.exists(thisFolder)) { if(await datastore.exists(thisFolder)) {
let creationFolder = thisFolder; let creationFolder = thisFolder;
for(let nextDir of dirsThatNeedCreated) { for(let nextDir of (dirsThatNeedCreated || []) ) {
creationFolder = joinKey(creationFolder, nextDir); creationFolder = joinKey(creationFolder, nextDir);
await datastore.createFolder(creationFolder); await datastore.createFolder(creationFolder);
} }
} else if(dirsThatNeedCreated.length > 0) { } else if(!dirsThatNeedCreated || dirsThatNeedCreated.length > 0) {
dirsThatNeedCreated = !dirsThatNeedCreated
? []
:dirsThatNeedCreated;
await recursiveCreateFolder( await recursiveCreateFolder(
take(subdirs.length - 1)(subdirs), take(subdirs.length - 1)(subdirs),

36
packages/core/src/recordApi/uploadFile.js
View File

@ -3,15 +3,16 @@ import {
map, some, map, some,
} from 'lodash/fp'; } from 'lodash/fp';
import { generate } from 'shortid'; import { generate } from 'shortid';
import { _load } from './load'; import { _loadFromInfo } from './load';
import { import {
apiWrapper, events, splitKey, apiWrapper, events, splitKey,
$, joinKey, isNothing, tryAwaitOrIgnore, $, joinKey, isNothing, tryAwaitOrIgnore,
} from '../common'; } from '../common';
import { getExactNodeForPath } from '../templateApi/hierarchy'; import { getExactNodeForKey } from '../templateApi/hierarchy';
import { permission } from '../authApi/permissions'; import { permission } from '../authApi/permissions';
import { isLegalFilename } from '../types/file'; import { isLegalFilename } from '../types/file';
import { BadRequestError, ForbiddenError } from '../common/errors'; import { BadRequestError, ForbiddenError } from '../common/errors';
import { getRecordInfo } from "./recordInfo";
export const uploadFile = app => async (recordKey, readableStream, relativeFilePath) => apiWrapper( export const uploadFile = app => async (recordKey, readableStream, relativeFilePath) => apiWrapper(
app, app,
@ -26,10 +27,11 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
if (isNothing(relativeFilePath)) { throw new BadRequestError('file path not supplied'); } if (isNothing(relativeFilePath)) { throw new BadRequestError('file path not supplied'); }
if (!isLegalFilename(relativeFilePath)) { throw new BadRequestError('Illegal filename'); } if (!isLegalFilename(relativeFilePath)) { throw new BadRequestError('Illegal filename'); }
const record = await _load(app, recordKey); const recordInfo = getRecordInfo(app.hierarchy, recordKey);
const record = await _loadFromInfo(app, recordInfo);
const fullFilePath = safeGetFullFilePath( const fullFilePath = safeGetFullFilePath(
recordKey, relativeFilePath, recordInfo.dir, relativeFilePath,
); );
const tempFilePath = `${fullFilePath}_${generate()}.temp`; const tempFilePath = `${fullFilePath}_${generate()}.temp`;
@ -54,30 +56,10 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
.then(() => tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath)) .then(() => tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath))
.then(() => app.datastore.renameFile(tempFilePath, fullFilePath)); .then(() => app.datastore.renameFile(tempFilePath, fullFilePath));
/*
readableStream.pipe(outputStream);
await new Promise(fulfill => outputStream.on('finish', fulfill));
const isExpectedFileSize = checkFileSizeAgainstFields(
app,
record, relativeFilePath,
await app.datastore.getFileSize(tempFilePath),
);
if (!isExpectedFileSize) {
throw new Error(
`Fields for ${relativeFilePath} do not have expected size`);
}
await tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath);
await app.datastore.renameFile(tempFilePath, fullFilePath);
*/
}; };
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => { const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [ const incorrectFileFields = $(recordNode.fields, [
filter(f => f.type === 'file' filter(f => f.type === 'file'
@ -107,7 +89,7 @@ const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize)
return true; return true;
}; };
export const safeGetFullFilePath = (recordKey, relativeFilePath) => { export const safeGetFullFilePath = (recordDir, relativeFilePath) => {
const naughtyUser = () => { throw new ForbiddenError('naughty naughty'); }; const naughtyUser = () => { throw new ForbiddenError('naughty naughty'); };
if (relativeFilePath.startsWith('..')) naughtyUser(); if (relativeFilePath.startsWith('..')) naughtyUser();
@ -116,7 +98,7 @@ export const safeGetFullFilePath = (recordKey, relativeFilePath) => {
if (includes('..')(pathParts)) naughtyUser(); if (includes('..')(pathParts)) naughtyUser();
const recordKeyParts = splitKey(recordKey); const recordKeyParts = splitKey(recordDir);
const fullPathParts = [ const fullPathParts = [
...recordKeyParts, ...recordKeyParts,

4
packages/core/src/recordApi/validate.js
View File

@ -4,7 +4,7 @@ import {
} from 'lodash/fp'; } from 'lodash/fp';
import { compileExpression } from '@nx-js/compiler-util'; import { compileExpression } from '@nx-js/compiler-util';
import _ from 'lodash'; import _ from 'lodash';
import { getExactNodeForPath } from '../templateApi/hierarchy'; import { getExactNodeForKey } from '../templateApi/hierarchy';
import { validateFieldParse, validateTypeConstraints } from '../types'; import { validateFieldParse, validateTypeConstraints } from '../types';
import { $, isNothing, isNonEmptyString } from '../common'; import { $, isNothing, isNonEmptyString } from '../common';
import { _getContext } from './getContext'; import { _getContext } from './getContext';
@ -63,7 +63,7 @@ export const validate = app => async (record, context) => {
? _getContext(app, record.key) ? _getContext(app, record.key)
: context; : context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key); const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode); const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here // non parsing would cause further issues - exit here

2
packages/core/src/templateApi/createNodes.js
View File

@ -171,7 +171,7 @@ const _getNewRecordTemplate = (parent, name, createDefaultIndex, isSingle) => {
validationRules: [], validationRules: [],
nodeId: getNodeId(parent), nodeId: getNodeId(parent),
indexes: [], indexes: [],
allidsShardFactor: isRecord(parent) ? 1 : 64, estimatedRecordCount: isRecord(parent) ? 500 : 1000000,
collectionName: '', collectionName: '',
isSingle, isSingle,
}); });

19
packages/core/src/templateApi/hierarchy.js
View File

@ -49,7 +49,7 @@ export const getNodesInPath = appHierarchy => key => $(appHierarchy, [
filter(n => new RegExp(`${n.pathRegx()}`).test(key)), filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]); ]);
export const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [ export const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy, getFlattenedHierarchy,
find(n => new RegExp(`${n.pathRegx()}$`).test(key)), find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]); ]);
@ -87,7 +87,7 @@ export const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]); ]);
export const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => { export const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey); const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey) return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey) ? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey; : nodeByKey;
@ -100,13 +100,14 @@ export const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey; : nodeByKey;
}; };
export const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key)); export const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
export const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [ export const getActualKeyOfParent = (parentNodeKey, actualChildKey) =>
splitKey, $(actualChildKey, [
take(splitKey(parentNodeKey).length), splitKey,
ks => joinKey(...ks), take(splitKey(parentNodeKey).length),
]); ks => joinKey(...ks),
]);
export const getParentKey = (key) => { export const getParentKey = (key) => {
return $(key, [ return $(key, [
@ -199,7 +200,7 @@ export const fieldReversesReferenceToIndex = indexNode => field => field.type ==
export default { export default {
getLastPartInKey, getLastPartInKey,
getNodesInPath, getNodesInPath,
getExactNodeForPath, getExactNodeForKey,
hasMatchingAncestor, hasMatchingAncestor,
getNode, getNode,
getNodeByKeyOrNodeKey, getNodeByKeyOrNodeKey,

54
packages/core/src/transactions/execute.js
View File

@ -24,8 +24,10 @@ import { applyToShard } from '../indexing/apply';
import { import {
getActualKeyOfParent, getActualKeyOfParent,
isGlobalIndex, fieldReversesReferenceToIndex, isReferenceIndex, isGlobalIndex, fieldReversesReferenceToIndex, isReferenceIndex,
getExactNodeForPath, getExactNodeForKey,
} from '../templateApi/hierarchy'; } from '../templateApi/hierarchy';
import { getRecordInfo } from "../recordApi/recordInfo";
import { getIndexDir } from '../indexApi/getIndexDir';
export const executeTransactions = app => async (transactions) => { export const executeTransactions = app => async (transactions) => {
const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions); const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions);
@ -33,7 +35,7 @@ export const executeTransactions = app => async (transactions) => {
for (const shard of keys(recordsByShard)) { for (const shard of keys(recordsByShard)) {
await applyToShard( await applyToShard(
app.hierarchy, app.datastore, app.hierarchy, app.datastore,
recordsByShard[shard].indexKey, recordsByShard[shard].indexDir,
recordsByShard[shard].indexNode, recordsByShard[shard].indexNode,
shard, shard,
recordsByShard[shard].writes, recordsByShard[shard].writes,
@ -77,8 +79,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
transByShard[t.indexShardKey] = { transByShard[t.indexShardKey] = {
writes: [], writes: [],
removes: [], removes: [],
indexKey: t.indexKey, indexDir: t.indexDir,
indexNodeKey: t.indexNodeKey, indexNodeKey: t.indexNode.nodeKey(),
indexNode: t.indexNode, indexNode: t.indexNode,
}; };
} }
@ -109,10 +111,10 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => {
return ({ return ({
mappedRecord, mappedRecord,
indexNode: indexNodeAndPath.indexNode, indexNode: indexNodeAndPath.indexNode,
indexKey: indexNodeAndPath.indexKey, indexDir: indexNodeAndPath.indexDir,
indexShardKey: getIndexedDataKey( indexShardKey: getIndexedDataKey(
indexNodeAndPath.indexNode, indexNodeAndPath.indexNode,
indexNodeAndPath.indexKey, indexNodeAndPath.indexDir,
mappedRecord.result, mappedRecord.result,
), ),
}); });
@ -219,54 +221,56 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
if (!isNonEmptyArray(buildTransactions)) return []; if (!isNonEmptyArray(buildTransactions)) return [];
const indexNode = transactions.indexNode; const indexNode = transactions.indexNode;
const getIndexKeys = (t) => { const getIndexDirs = (t) => {
if (isGlobalIndex(indexNode)) { if (isGlobalIndex(indexNode)) {
return [indexNode.nodeKey()]; return [indexNode.nodeKey()];
} }
if (isReferenceIndex(indexNode)) { if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key); const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [ const refFields = $(recordNode.fields, [
filter(fieldReversesReferenceToIndex(indexNode)), filter(fieldReversesReferenceToIndex(indexNode)),
]); ]);
const indexKeys = []; const indexDirs = [];
for (const refField of refFields) { for (const refField of refFields) {
const refValue = t.record[refField.name]; const refValue = t.record[refField.name];
if (isSomething(refValue) if (isSomething(refValue)
&& isNonEmptyString(refValue.key)) { && isNonEmptyString(refValue.key)) {
const indexKey = joinKey( const indexDir = joinKey(
refValue.key, getRecordInfo(hierarchy, refValue.key).dir,
indexNode.name, indexNode.name,
); );
if (!includes(indexKey)(indexKeys)) { indexKeys.push(indexKey); } if (!includes(indexDir)(indexDirs)) { indexDirs.push(indexDir); }
} }
} }
return indexKeys; return indexDirs;
} }
return [joinKey( const indexKey = joinKey(
getActualKeyOfParent( getActualKeyOfParent(
indexNode.parent().nodeKey(), indexNode.parent().nodeKey(),
t.record.key, t.record.key,
), ),
indexNode.name, indexNode.name,
)]; );
return [getIndexDir(hierarchy, indexKey)];
}; };
return $(buildTransactions, [ return $(buildTransactions, [
map((t) => { map((t) => {
const mappedRecord = evaluate(t.record)(indexNode); const mappedRecord = evaluate(t.record)(indexNode);
if (!mappedRecord.passedFilter) return null; if (!mappedRecord.passedFilter) return null;
const indexKeys = getIndexKeys(t); const indexDirs = getIndexDirs(t);
return $(indexKeys, [ return $(indexDirs, [
map(indexKey => ({ map(indexDir => ({
mappedRecord, mappedRecord,
indexNode, indexNode,
indexKey, indexDir,
indexShardKey: getIndexedDataKey( indexShardKey: getIndexedDataKey(
indexNode, indexNode,
indexKey, indexDir,
mappedRecord.result, mappedRecord.result,
), ),
})), })),
@ -286,10 +290,10 @@ const get_Create_Delete_TransactionsByShard = pred => (hierarchy, transactions)
return ({ return ({
mappedRecord, mappedRecord,
indexNode: n.indexNode, indexNode: n.indexNode,
indexKey: n.indexKey, indexDir: n.indexDir,
indexShardKey: getIndexedDataKey( indexShardKey: getIndexedDataKey(
n.indexNode, n.indexNode,
n.indexKey, n.indexDir,
mappedRecord.result, mappedRecord.result,
), ),
}); });
@ -327,17 +331,17 @@ const diffReverseRefForUpdate = (appHierarchy, oldRecord, newRecord) => {
); );
const unReferenced = differenceBy( const unReferenced = differenceBy(
i => i.indexKey, i => i.indexDir,
oldIndexes, newIndexes, oldIndexes, newIndexes,
); );
const newlyReferenced = differenceBy( const newlyReferenced = differenceBy(
i => i.indexKey, i => i.indexDir,
newIndexes, oldIndexes, newIndexes, oldIndexes,
); );
const notChanged = intersectionBy( const notChanged = intersectionBy(
i => i.indexKey, i => i.indexDir,
newIndexes, oldIndexes, newIndexes, oldIndexes,
); );

3
packages/core/test/collectionApi.spec.js
View File

@ -1,5 +1,4 @@
import {setupApphierarchy, basicAppHierarchyCreator_WithFields, import {setupApphierarchy, basicAppHierarchyCreator_WithFields} from "./specHelpers";
basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers";
import {includes, union} from "lodash"; import {includes, union} from "lodash";
import {joinKey} from "../src/common"; import {joinKey} from "../src/common";

6
packages/core/test/indexApi.buildIndex.spec.js
View File

@ -4,7 +4,7 @@ import { joinKey } from "../src/common";
import {some} from "lodash"; import {some} from "lodash";
import {_deleteIndex} from "../src/indexApi/delete"; import {_deleteIndex} from "../src/indexApi/delete";
import {permission} from "../src/authApi/permissions"; import {permission} from "../src/authApi/permissions";
import { getExactNodeForPath } from "../src/templateApi/hierarchy"; import { getExactNodeForKey } from "../src/templateApi/hierarchy";
describe("buildIndex > Global index", () => { describe("buildIndex > Global index", () => {
@ -213,7 +213,7 @@ describe("buildIndex > nested collection", () => {
const indexKey = joinKey(customer.key, "invoice_index"); const indexKey = joinKey(customer.key, "invoice_index");
await _deleteIndex(app, indexKey, false); await _deleteIndex(app, indexKey, false);
const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey); const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey);
await indexApi.buildIndex(indexNode.nodeKey()); await indexApi.buildIndex(indexNode.nodeKey());
const indexItems = await indexApi.listItems(indexKey); const indexItems = await indexApi.listItems(indexKey);
@ -269,7 +269,7 @@ describe("buildIndex > nested collection", () => {
const indexKey = joinKey(customer.key, "invoice_index"); const indexKey = joinKey(customer.key, "invoice_index");
await _deleteIndex(app, indexKey, false); await _deleteIndex(app, indexKey, false);
const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey); const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey);
await indexApi.buildIndex( await indexApi.buildIndex(
indexNode.nodeKey()); indexNode.nodeKey());
const indexItems = await indexApi.listItems(indexKey); const indexItems = await indexApi.listItems(indexKey);

18
packages/core/test/indexing.concurrency.spec.js
View File

@ -5,7 +5,7 @@ import {getLockFileContent} from "../src/common/lock";
import {some, isArray} from "lodash"; import {some, isArray} from "lodash";
import {cleanup} from "../src/transactions/cleanup"; import {cleanup} from "../src/transactions/cleanup";
import {LOCK_FILE_KEY} from "../src/transactions/transactionsCommon"; import {LOCK_FILE_KEY} from "../src/transactions/transactionsCommon";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe("cleanup transactions", () => { describe("cleanup transactions", () => {
@ -96,7 +96,7 @@ describe("cleanup transactions", () => {
it("should not reindex when transactionId does not match that of the record", async () => { it("should not reindex when transactionId does not match that of the record", async () => {
const {recordApi, app, const {recordApi, app,
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog"; record.surname = "Ledog";
@ -109,8 +109,10 @@ describe("cleanup transactions", () => {
await recordApi.save(savedRecord); await recordApi.save(savedRecord);
savedRecord.transactionId = "something else"; savedRecord.transactionId = "something else";
const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key);
await recordApi._storeHandle.updateJson( await recordApi._storeHandle.updateJson(
joinKey(savedRecord.key, "record.json"), recordInfo.child("record.json"),
savedRecord); savedRecord);
await cleanup(app); await cleanup(app);
@ -123,7 +125,7 @@ describe("cleanup transactions", () => {
it("should not reindex when transactionId does not match that of the record, and has multiple transactions", async () => { it("should not reindex when transactionId does not match that of the record, and has multiple transactions", async () => {
const {recordApi, app, const {recordApi, app,
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog"; record.surname = "Ledog";
@ -139,8 +141,10 @@ describe("cleanup transactions", () => {
await recordApi.save(savedRecord); await recordApi.save(savedRecord);
savedRecord.transactionId = "something else"; savedRecord.transactionId = "something else";
const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key);
await recordApi._storeHandle.updateJson( await recordApi._storeHandle.updateJson(
joinKey(savedRecord.key, "record.json"), recordInfo.child("record.json"),
savedRecord); savedRecord);
await cleanup(app); await cleanup(app);
@ -228,7 +232,7 @@ describe("cleanup transactions", () => {
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog"; record.surname = "Ledog";
const savedRecord = await recordApi.save(record); await recordApi.save(record);
const currentTime = await app.getEpochTime(); const currentTime = await app.getEpochTime();
await recordApi._storeHandle.createFile( await recordApi._storeHandle.createFile(
LOCK_FILE_KEY, LOCK_FILE_KEY,
@ -252,7 +256,7 @@ describe("cleanup transactions", () => {
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true); indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog"; record.surname = "Ledog";
const savedRecord = await recordApi.save(record); await recordApi.save(record);
await recordApi._storeHandle.createFile( await recordApi._storeHandle.createFile(
LOCK_FILE_KEY, LOCK_FILE_KEY,
getLockFileContent(30000, (new Date(1990,1,1,0,0,0,0).getTime())) getLockFileContent(30000, (new Date(1990,1,1,0,0,0,0).getTime()))

41
packages/core/test/indexing.getRelevantIndexes.spec.js
View File

@ -1,11 +1,14 @@
import {getMemoryTemplateApi, import {
basicAppHierarchyCreator_WithFields, setupApphierarchy,
setupApphierarchy, basicAppHierarchyCreator_WithFields_AndIndexes
basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers"; } from "./specHelpers";
import {getRelevantReverseReferenceIndexes, import {
getRelevantAncestorIndexes} from "../src/indexing/relevant"; getRelevantReverseReferenceIndexes,
getRelevantAncestorIndexes
} from "../src/indexing/relevant";
import {some} from "lodash"; import {some} from "lodash";
import {joinKey} from "../src/common"; import {joinKey} from "../src/common";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe("getRelevantIndexes", () => { describe("getRelevantIndexes", () => {
@ -45,7 +48,7 @@ describe("getRelevantIndexes", () => {
expect(indexes.length).toBe(4); expect(indexes.length).toBe(4);
const indexExists = key => const indexExists = key =>
some(indexes, c => c.indexKey === key); some(indexes, c => c.indexDir === key);
expect(indexExists("/customer_index")).toBeTruthy(); expect(indexExists("/customer_index")).toBeTruthy();
expect(indexExists("/deceased")).toBeTruthy(); expect(indexExists("/deceased")).toBeTruthy();
@ -64,7 +67,7 @@ describe("getRelevantIndexes", () => {
appHierarchy.root, invoice); appHierarchy.root, invoice);
const indexExists = key => const indexExists = key =>
some(indexes, c => c.indexKey === key); some(indexes, c => c.indexDir === key);
expect(indexExists("/customersBySurname")).toBeFalsy(); expect(indexExists("/customersBySurname")).toBeFalsy();
}); });
@ -82,7 +85,7 @@ describe("getRelevantIndexes", () => {
expect(indexes.length).toBe(4); expect(indexes.length).toBe(4);
const indexExists = key => const indexExists = key =>
some(indexes, c => c.indexKey === key); some(indexes, c => c.indexDir === key);
expect(indexExists("/customersBySurname")).toBeTruthy(); expect(indexExists("/customersBySurname")).toBeTruthy();
}); });
@ -96,14 +99,14 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantAncestorIndexes( const indexes = getRelevantAncestorIndexes(
appHierarchy.root, invoice); appHierarchy.root, invoice);
const {dir} = getRecordInfo(appHierarchy.root, `/customers/${nodeid}-1234`);
expect(indexes.length).toBe(4); expect(indexes.length).toBe(4);
expect(some(indexes, i => i.indexKey === `/customer_invoices`)).toBeTruthy(); expect(some(indexes, i => i.indexDir === `/customer_invoices`)).toBeTruthy();
expect(some(indexes, i => i.indexKey === `/customers/${nodeid}-1234/invoice_index`)).toBeTruthy(); expect(some(indexes, i => i.indexDir === `${dir}/invoice_index`)).toBeTruthy();
}); });
it("should get reverseReferenceIndex accross hierarchy branches", async () => { it("should get reverseReferenceIndex accross hierarchy branches", async () => {
const {appHierarchy, const {appHierarchy,
recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes); recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes);
const partner = recordApi.getNew("/partners", "partner"); const partner = recordApi.getNew("/partners", "partner");
@ -118,8 +121,9 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantReverseReferenceIndexes( const indexes = getRelevantReverseReferenceIndexes(
appHierarchy.root, customer); appHierarchy.root, customer);
expect(indexes.length).toBe(1); expect(indexes.length).toBe(1);
expect(indexes[0].indexKey) const partnerdir = getRecordInfo(appHierarchy.root, partner.key).dir;
.toBe(joinKey(partner.key, appHierarchy.partnerCustomersReverseIndex.name)); expect(indexes[0].indexDir)
.toBe(joinKey(partnerdir, appHierarchy.partnerCustomersReverseIndex.name));
}); });
@ -136,8 +140,11 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantReverseReferenceIndexes( const indexes = getRelevantReverseReferenceIndexes(
appHierarchy.root, referredToCustomer); appHierarchy.root, referredToCustomer);
const referredByCustomerDir = getRecordInfo(appHierarchy.root, referredByCustomer.key).dir;
expect(indexes.length).toBe(1); expect(indexes.length).toBe(1);
expect(indexes[0].indexKey) expect(indexes[0].indexDir)
.toBe(joinKey(referredByCustomer.key, appHierarchy.referredToCustomersReverseIndex.name)); .toBe(joinKey(referredByCustomerDir, appHierarchy.referredToCustomersReverseIndex.name));
}); });
}); });

2
packages/core/test/indexing.schema.spec.js
View File

@ -1,5 +1,5 @@
import {generateSchema} from "../src/indexing/indexSchemaCreator"; import {generateSchema} from "../src/indexing/indexSchemaCreator";
import {setupApphierarchy, findCollectionDefaultIndex} from "./specHelpers"; import {setupApphierarchy} from "./specHelpers";
import {find} from "lodash"; import {find} from "lodash";
import {indexTypes} from "../src/templateApi/indexes"; import {indexTypes} from "../src/templateApi/indexes";

11
packages/core/test/initialiseData.spec.js
View File

@ -21,17 +21,6 @@ describe("initialiseData", () => {
expect(await datastore.exists(`/customers`)).toBeTruthy(); expect(await datastore.exists(`/customers`)).toBeTruthy();
}); });
it("should create allids folders", async () => {
const {appDef, datastore, h} = getApplicationDefinition();
await initialiseData(datastore, appDef);
const allIdsTypeFolder = "/customers/allids/" + h.customerRecord.nodeId;
const allIdsFolder = "/customers/allids";
expect(await datastore.exists(allIdsTypeFolder)).toBeTruthy();
expect(await datastore.exists(allIdsFolder)).toBeTruthy();
});
it("should create transactions folder", async () => { it("should create transactions folder", async () => {
const {appDef, datastore} = getApplicationDefinition(); const {appDef, datastore} = getApplicationDefinition();
await initialiseData(datastore, appDef); await initialiseData(datastore, appDef);

57
packages/core/test/memory.js
View File

@ -2,11 +2,16 @@ import {isUndefined, has} from "lodash";
import {take} from "lodash/fp"; import {take} from "lodash/fp";
import {Readable, Writable} from "readable-stream"; import {Readable, Writable} from "readable-stream";
import { Buffer } from "safe-buffer"; import { Buffer } from "safe-buffer";
import {splitKey, joinKey, $} from "../src/common"; import {splitKey, joinKey, $, keySep, getFileFromKey} from "../src/common";
import {getLastPartInKey} from "../src/templateApi/hierarchy"; import {getLastPartInKey} from "../src/templateApi/hierarchy";
const folderMarker = "OH-YES-ITSA-FOLDER-"; const folderMarker = "OH-YES-ITSA-FOLDER-";
const isFolder = val => val.includes(folderMarker); const isFolder = val => {
if(isUndefined(val)) {
throw new Error("Passed undefined value for folder");
}
return val.includes(folderMarker);
}
const getParentFolderKey = key => const getParentFolderKey = key =>
$(key, [ $(key, [
@ -16,7 +21,9 @@ const getParentFolderKey = key =>
]); ]);
const getParentFolder = (data,key) => { const getParentFolder = (data,key) => {
if(key === keySep) return null;
const parentKey = getParentFolderKey(key); const parentKey = getParentFolderKey(key);
if(parentKey === keySep) return null;
if(data[parentKey] === undefined) if(data[parentKey] === undefined)
throw new Error("Parent folder for " + key + " does not exist (" + parentKey + ")"); throw new Error("Parent folder for " + key + " does not exist (" + parentKey + ")");
return JSON.parse(data[parentKey]); return JSON.parse(data[parentKey]);
@ -39,12 +46,18 @@ export const createFile = data => async (path, content) => {
}; };
export const updateFile = data => async (path, content) => { export const updateFile = data => async (path, content) => {
// putting this check in to force use of create // putting this check in to force use of create
if(!await exists(data)(path)) throw new Error("cannot update " + path + " - does not exist"); if(!await exists(data)(path)) {
throw new Error("cannot update " + path + " - does not exist");
}
data[path] = content; data[path] = content;
} }
export const writableFileStream = data => async (path) => { export const writableFileStream = data => async (path) => {
//if(!await exists(data)(path)) throw new Error("cannot write stream to " + path + " - does not exist"); //if(!await exists(data)(path)) throw new Error("cannot write stream to " + path + " - does not exist");
if(!getParentFolder(data, path)) {
throw new Error("Parent folder for " + path + " does not exist");
}
const stream = Writable(); const stream = Writable();
stream._write = (chunk, encoding, done) => { stream._write = (chunk, encoding, done) => {
data[path] = data[path] === undefined data[path] = data[path] === undefined
@ -52,6 +65,9 @@ export const writableFileStream = data => async (path) => {
data[path] = [...data[path], ...chunk]; data[path] = [...data[path], ...chunk];
done(); done();
}; };
addItemToParentFolder(data, path);
return stream; return stream;
}; };
@ -77,11 +93,19 @@ export const renameFile = data => async (oldKey, newKey) => {
if(await exists(data)(newKey)) throw new Error("cannot rename path: " + newKey + " ... already exists"); if(await exists(data)(newKey)) throw new Error("cannot rename path: " + newKey + " ... already exists");
data[newKey] = data[oldKey]; data[newKey] = data[oldKey];
delete data[oldKey]; delete data[oldKey];
const parent = getParentFolder(data, newKey);
const oldFileName = getFileFromKey(oldKey);
const newFileName = getFileFromKey(newKey);
parent.items = [...parent.items.filter(i => i !== oldFileName), newFileName];
data[getParentFolderKey(newKey)] = JSON.stringify(parent);
}; };
export const loadFile = data => async (path) => { export const loadFile = data => async (path) => {
const result = data[path]; const result = data[path];
if(isUndefined(result)) throw new Error("Load failed - path " + path + " does not exist"); if(isUndefined(result)) {
throw new Error("Load failed - path " + path + " does not exist");
}
return result; return result;
}; };
export const exists = data => async (path) => has(data, path); export const exists = data => async (path) => has(data, path);
@ -95,7 +119,8 @@ export const deleteFile = data => async (path) => {
delete data[path]; delete data[path];
} }
export const createFolder = data => async (path) => { export const createFolder = data => async (path) => {
if(await exists(data)(path)) throw new Error("Cannot create folder, path " + path + " already exists"); if(await exists(data)(path))
throw new Error("Cannot create folder, path " + path + " already exists");
addItemToParentFolder(data, path); addItemToParentFolder(data, path);
data[path] = JSON.stringify({folderMarker, items:[]}); data[path] = JSON.stringify({folderMarker, items:[]});
} }
@ -103,14 +128,30 @@ export const deleteFolder = data => async (path) => {
if(!await exists(data)(path)) throw new Error("Cannot delete folder, path " + path + " does not exist"); if(!await exists(data)(path)) throw new Error("Cannot delete folder, path " + path + " does not exist");
if(!isFolder(data[path])) if(!isFolder(data[path]))
throw new Error("DeleteFolder: Path " + path + " is not a folder"); throw new Error("DeleteFolder: Path " + path + " is not a folder");
for(let item of JSON.parse(data[path]).items) {
const fullItemPath = `${path}/${item}`;
if(isFolder(data[fullItemPath])) {
await deleteFolder(data)(fullItemPath);
} else {
await deleteFile(data)(fullItemPath);
}
}
const parent = getParentFolder(data, path);
if(parent) {
parent.items = parent.items.filter(f => f !== getLastPartInKey(path));
data[getParentFolderKey(path)] = JSON.stringify(parent);
}
delete data[path]; delete data[path];
} }
export const getFolderContents = data => async (folderPath) => { export const getFolderContents = data => async (folderPath) => {
if(!isFolder(data[folderPath]))
throw new Error("Not a folder: " + folderPath);
if(!await exists(data)(folderPath)) if(!await exists(data)(folderPath))
throw new Error("Folder does not exist: " + folderPath); throw new Error("Folder does not exist: " + folderPath);
if(!isFolder(data[folderPath]))
throw new Error("Not a folder: " + folderPath);
return JSON.parse(data[folderPath]).items; return JSON.parse(data[folderPath]).items;
}; };

3
packages/core/test/recordApi.files.spec.js
View File

@ -1,8 +1,5 @@
import {setupApphierarchy, import {setupApphierarchy,
basicAppHierarchyCreator_WithFields} from "./specHelpers"; basicAppHierarchyCreator_WithFields} from "./specHelpers";
import {keys, filter} from "lodash/fp";
import {$} from "../src/common";
import {permission} from "../src/authApi/permissions";
import {Readable} from "readable-stream"; import {Readable} from "readable-stream";

124
packages/core/test/recordApi.getRecordInfo.spec.js Normal file
View File

@ -0,0 +1,124 @@
import { folderStructureArray } from "../src/indexing/allIds";
import { getRecordInfo } from "../src/recordApi/recordInfo";
import {setupApphierarchy} from "./specHelpers";
describe("getRecordInfo", () => {
it("dir should not be sharded when record count = 1000", async () => {
const {root} = (await setup({parentCount: 1000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/1-abcd");
});
it("dir should be sharded when record count = 1001", async () => {
const {root} = (await setup({parentCount: 1001})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/0123456789abcdefghijklmnopqrstuv/1-abcd");
});
it("dir should be sharded to one char per folder when record count = 63,000 (64*1000)", async () => {
const {root} = (await setup({parentCount: 64000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/a/1-abcd");
});
it("dir should be sharded to one char per folder, on 2 levels when record count = 4096000 (64*64*1000)", async () => {
const {root} = (await setup({parentCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/a/b/1-abcd");
});
it("child dir should not be sharded when record count = 1000", async () => {
const {root, child} = (await setup({parentCount: 4096000, childCount: 1000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/${child.nodeId}-defg`);
});
it("grandchild dir should not be sharded when record count = 1000", async () => {
const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/${grandchild.nodeId}-hijk`);
});
it("grandchild dir should be sharded when record count = 4096000", async () => {
const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`);
});
it("child levels can be sharded, with parent not", async () => {
const {root, child, grandchild} = (await setup({parentCount: 1000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`);
});
});
describe("folderStructureArray", () => {
const recordNode = (count) => ({estimatedRecordCount: count});
it("should return [] when folder count < 1000", () => {
const result = folderStructureArray(recordNode(999));
expect(result).toEqual([]);
});
it("should return [4] when folder count between 3000 - 4000", () => {
const result = folderStructureArray(recordNode(3456));
expect(result).toEqual([4]);
})
it("should return [64, 2] when folder count between 64000 - 65000", () => {
const result = folderStructureArray(recordNode(64001));
expect(result).toEqual([64, 2]);
})
it("should return [64, 64] when folder = 4095999", () => {
const result = folderStructureArray(recordNode(4095999));
expect(result).toEqual([64, 64]);
});
it("should return [64, 64] when folder = 4096000", () => {
const result = folderStructureArray(recordNode(4096000));
expect(result).toEqual([64, 64]);
});
it("should return [64, 64, 2] when folder = 4096001", () => {
const result = folderStructureArray(recordNode(4096001));
expect(result).toEqual([64, 64, 2]);
});
});
const setup = ({parentCount, childCount, grandChildCount}) =>
setupApphierarchy((templateApi) => {
const root = templateApi.getNewRootLevel();
const addField = (recordNode) => {
const field = templateApi.getNewField("string");
field.name = "test";
templateApi.addField(recordNode, field);
return field;
};
const parent = templateApi.getNewRecordTemplate(root, "parent");
parent.estimatedRecordCount = parentCount || 1000;
parent.collectionName = "parents";
addField(parent);
const child = templateApi.getNewRecordTemplate(parent, "child");
child.estimatedRecordCount = childCount || 1000;
child.collectionName = "children";
addField(child);
const grandchild = templateApi.getNewRecordTemplate(child, "grandchild");
grandchild.estimatedRecordCount = grandChildCount || 1000;
grandchild.collectionName = "grandchildren";
addField(grandchild);
return ({
parent, child, grandchild, root
});
});

65
packages/core/test/recordApi.save.spec.js
View File

@ -2,7 +2,7 @@ import {setupApphierarchy, basicAppHierarchyCreator_WithFields,
stubEventHandler} from "./specHelpers"; stubEventHandler} from "./specHelpers";
import {events, isNonEmptyString} from "../src/common"; import {events, isNonEmptyString} from "../src/common";
import {permission} from "../src/authApi/permissions"; import {permission} from "../src/authApi/permissions";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe('recordApi > save then load', () => { describe('recordApi > save then load', () => {
@ -202,48 +202,41 @@ describe("save", () => {
}); });
it("should create folder and index for subcollection", async () => { it("should create folder and index for subcollection", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index/index.csv`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${record.key}/invoices`)).toBeTruthy()
});
it("should create index folder and shardMap for sharded reverse reference index", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers/shardMap.json`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers`)).toBeTruthy();
});
it("should create folder for record", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${record.key}/record.json`)).toBeTruthy();
});
it("should create allids file", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog"; record.surname = "Ledog";
await recordApi.save(record); await recordApi.save(record);
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
const allIdsPath = `/customers/allids/${appHierarchy.customerRecord.nodeId}/${record.id[2]}`; expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index/index.csv`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(allIdsPath)).toBeTruthy(); expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${recordDir}/invoices`)).toBeTruthy()
}); });
it("should create index folder and shardMap for sharded reverse reference index", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers/shardMap.json`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers`)).toBeTruthy();
});
it("should create folder for record", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
expect(await recordApi._storeHandle.exists(`${recordDir}`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${recordDir}/record.json`)).toBeTruthy();
});
it("create should throw error, user user does not have permission", async () => { it("create should throw error, user user does not have permission", async () => {
const {recordApi, app, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields); const {recordApi, app, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer"); const record = recordApi.getNew("/customers", "customer");

8
packages/core/test/specHelpers.js
View File

@ -22,12 +22,10 @@ import {permission} from "../src/authApi/permissions";
import {generateFullPermissions} from "../src/authApi/generateFullPermissions" import {generateFullPermissions} from "../src/authApi/generateFullPermissions"
import {initialiseData} from "../src/appInitialise/initialiseData"; import {initialiseData} from "../src/appInitialise/initialiseData";
const exp = module.exports;
export const testFileArea = (testNameArea) => path.join("test", "fs_test_area", testNameArea); export const testFileArea = (testNameArea) => path.join("test", "fs_test_area", testNameArea);
export const testConfigFolder = (testAreaName) => path.join(exp.testFileArea(testAreaName), configFolder); export const testConfigFolder = (testAreaName) => path.join(testFileArea(testAreaName), configFolder);
export const testFieldDefinitionsPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), fieldDefinitions); export const testFieldDefinitionsPath = (testAreaName) => path.join(testFileArea(testAreaName), fieldDefinitions);
export const testTemplatesPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), templateDefinitions); export const testTemplatesPath = (testAreaName) => path.join(testFileArea(testAreaName), templateDefinitions);
export const getMemoryStore = () => setupDatastore(memory({})); export const getMemoryStore = () => setupDatastore(memory({}));
export const getMemoryTemplateApi = () => { export const getMemoryTemplateApi = () => {

2
packages/core/test/templateApi.constructHeirarchy.spec.js
View File

@ -27,7 +27,7 @@ describe("hierarchy node creation", () => {
expect(record.indexes).toEqual([]); expect(record.indexes).toEqual([]);
expect(record.parent()).toBe(root); expect(record.parent()).toBe(root);
expect(record.collectionName).toBe(""); expect(record.collectionName).toBe("");
expect(record.allidsShardFactor).toBe(64); expect(record.estimatedRecordCount).toBe(1000000);
expect(record.isSingle).toBe(false); expect(record.isSingle).toBe(false);
record.collectionName = "records"; record.collectionName = "records";