MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#8 Diff Hierarchy & upgrade data

This commit is contained in:
Michael Shanks 2020-03-20 13:39:38 +00:00
parent ad62a9c7d8
commit c5b1cc8a9f
28 changed files with 962 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/core/src/appInitialise/cloneApp.js Normal file
View File

@ -0,0 +1,8 @@
import { setCleanupFunc } from "../transactions/setCleanupFunc"
export const cloneApp = (app, mergeWith) => {
const newApp = { ...app }
Object.assign(newApp, mergeWith)
setCleanupFunc(newApp)
return newApp
}

8
packages/core/src/appInitialise/initialiseData.js
View File

@ -21,18 +21,25 @@ export const initialiseData = async (
applicationDefinition,
accessLevels
) => {
if (!await datastore.exists(configFolder))
await datastore.createFolder(configFolder)
if (!await datastore.exists(appDefinitionFile))
await datastore.createJson(appDefinitionFile, applicationDefinition)
await initialiseRootCollections(datastore, applicationDefinition.hierarchy)
await initialiseRootIndexes(datastore, applicationDefinition.hierarchy)
if (!await datastore.exists(TRANSACTIONS_FOLDER))
await datastore.createFolder(TRANSACTIONS_FOLDER)
if (!await datastore.exists(AUTH_FOLDER))
await datastore.createFolder(AUTH_FOLDER)
if (!await datastore.exists(USERS_LIST_FILE))
await datastore.createJson(USERS_LIST_FILE, [])
if (!await datastore.exists(ACCESS_LEVELS_FILE))
await datastore.createJson(
ACCESS_LEVELS_FILE,
accessLevels ? accessLevels : { version: 0, levels: [] }
@ -64,6 +71,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => {
const singleRecords = $(flathierarchy, [filter(isSingleRecord)])
for (let record of singleRecords) {
if (await datastore.exists(record.nodeKey())) continue
await datastore.createFolder(record.nodeKey())
const result = _getNew(record, "")
await _save(app, result)

6
packages/core/src/index.js
View File

@ -7,7 +7,7 @@ import getActionsApi from "./actionsApi"
import { setupDatastore, createEventAggregator } from "./appInitialise"
import { initialiseActions } from "./actionsApi/initialise"
import { isSomething, crypto } from "./common"
import { cleanup } from "./transactions/cleanup"
import { setCleanupFunc } from "./transactions/setCleanupFunc"
import { generateFullPermissions } from "./authApi/generateFullPermissions"
import { getApplicationDefinition } from "./templateApi/getApplicationDefinition"
import common from "./common"
@ -40,9 +40,7 @@ export const getAppApis = async (
const templateApi = getTemplateApi(app)
app.cleanupTransactions = isSomething(cleanupTransactions)
? cleanupTransactions
: async () => await cleanup(app)
setCleanupFunc(app, cleanupTransactions)
app.getEpochTime = isSomething(getEpochTime)
? getEpochTime

81
packages/core/src/indexApi/buildIndex.js
View File

@ -6,8 +6,10 @@ import {
getNode,
isIndex,
isRecord,
getActualKeyOfParent,
getAllowedRecordNodesForIndex,
fieldReversesReferenceToIndex,
isTopLevelIndex,
} from "../templateApi/hierarchy"
import { joinKey, apiWrapper, events, $ } from "../common"
import {
@ -16,6 +18,8 @@ import {
} from "../transactions/create"
import { permission } from "../authApi/permissions"
import { BadRequestError } from "../common/errors"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { getRecordInfo } from "../recordApi/recordInfo"
/** rebuilds an index
* @param {object} app - the application container
@ -32,7 +36,7 @@ export const buildIndex = app => async indexNodeKey =>
indexNodeKey
)
const _buildIndex = async (app, indexNodeKey) => {
export const _buildIndex = async (app, indexNodeKey) => {
const indexNode = getNode(app.hierarchy, indexNodeKey)
await createBuildIndexFolder(app.datastore, indexNodeKey)
@ -89,12 +93,6 @@ const buildReverseReferenceIndex = async (app, indexNode) => {
}
}
/*
const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [
map(n => n.parent()),
]);
*/
const buildHeirarchalIndex = async (app, indexNode) => {
let recordCount = 0
@ -139,77 +137,8 @@ const buildHeirarchalIndex = async (app, indexNode) => {
return recordCount
}
// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
const recordNodeApplies = indexNode => recordNode =>
includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds)
/*
const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [
getFlattenedHierarchy,
filter(
allTrue(
isRecord,
isDecendant(ancestorNode),
recordNodeApplies(indexNode),
),
),
]);
*/
/*
const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount = 0) => {
const collectionNode = getCollectionNodeByKeyOrNodeKey(
app.hierarchy,
collection_Key_or_NodeKey,
);
const allIdsIterator = await getAllIdsIterator(app)(collection_Key_or_NodeKey);
const createTransactionsForIds = async (collectionKey, allIds) => {
for (const recordId of allIds) {
const recordKey = joinKey(collectionKey, recordId);
const recordNode = chooseChildRecordNodeByKey(
collectionNode,
recordId,
);
if (recordNodeApplies(indexNode)(recordNode)) {
await transactionForBuildIndex(
app, indexNode.nodeKey(),
recordKey, recordCount,
);
recordCount++;
}
if (hasApplicableDecendant(app.hierarchy, recordNode, indexNode)) {
for (const childCollectionNode of recordNode.children) {
recordCount = await applyAllDecendantRecords(
app,
joinKey(recordKey, childCollectionNode.collectionName),
indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount,
);
}
}
}
};
let allIds = await allIdsIterator();
while (allIds.done === false) {
await createTransactionsForIds(
allIds.result.collectionKey,
allIds.result.ids,
);
allIds = await allIdsIterator();
}
return recordCount;
};
*/
export default buildIndex

2
packages/core/src/indexApi/listItems.js
View File

@ -33,7 +33,7 @@ const defaultOptions = {
searchPhrase: null,
}
const _listItems = async (app, indexKey, options = defaultOptions) => {
export const _listItems = async (app, indexKey, options = defaultOptions) => {
const { searchPhrase, rangeStartParams, rangeEndParams } = $({}, [
merge(options),
merge(defaultOptions),

3
packages/core/src/indexing/allIds.js
View File

@ -2,6 +2,7 @@ import { flatten, orderBy, filter, isUndefined } from "lodash/fp"
import {
getFlattenedHierarchy,
getCollectionNodeByKeyOrNodeKey,
getNodeByKeyOrNodeKey,
isCollectionRecord,
isAncestor,
} from "../templateApi/hierarchy"
@ -60,7 +61,7 @@ export const getAllIdsIterator = app => async collection_Key_or_NodeKey => {
const recordNode = getCollectionNodeByKeyOrNodeKey(
app.hierarchy,
collection_Key_or_NodeKey
)
) || getNodeByKeyOrNodeKey(app.hierarchy, collection_Key_or_NodeKey)
const getAllIdsIteratorForCollectionKey = async (
recordNode,

12
packages/core/src/indexing/initialiseIndex.js
View File

@ -9,11 +9,19 @@ import {
export const initialiseIndex = async (datastore, dir, index) => {
const indexDir = joinKey(dir, index.name)
let newDir = false
if (!await datastore.exists(indexDir)) {
await datastore.createFolder(indexDir)
newDir = true
}
if (isShardedIndex(index)) {
await datastore.createFile(getShardMapKey(indexDir), "[]")
const shardFile = getShardMapKey(indexDir)
if (newDir || !await datastore.exists(shardFile))
await datastore.createFile(shardFile, "[]")
} else {
await createIndexFile(datastore, getUnshardedIndexDataKey(indexDir), index)
const indexFile = getUnshardedIndexDataKey(indexDir)
if (newDir || !await datastore.exists(indexFile))
await createIndexFile(datastore, indexFile, index)
}
}

12
packages/core/src/indexing/read.js
View File

@ -3,6 +3,9 @@ import { promiseReadableStream } from "./promiseReadableStream"
import { createIndexFile } from "./sharding"
import { generateSchema } from "./indexSchemaCreator"
import { getIndexReader, CONTINUE_READING_RECORDS } from "./serializer"
import { getAllowedRecordNodesForIndex, getRecordNodeId } from "../templateApi/hierarchy"
import { $ } from "../common"
import { filter, includes, find } from "lodash/fp"
export const readIndex = async (
hierarchy,
@ -11,8 +14,10 @@ export const readIndex = async (
indexedDataKey
) => {
const records = []
const getType = typeLoader(index, hierarchy)
const doRead = iterateIndex(
async item => {
item.type = getType(item.key)
records.push(item)
return CONTINUE_READING_RECORDS
},
@ -31,8 +36,10 @@ export const searchIndex = async (
) => {
const records = []
const schema = generateSchema(hierarchy, index)
const getType = typeLoader(index, hierarchy)
const doRead = iterateIndex(
async item => {
item.type = getType(item.key)
const idx = lunr(function() {
this.ref("key")
for (const field of schema) {
@ -76,3 +83,8 @@ export const iterateIndex = (onGetItem, getFinalResult) => async (
return []
}
}
const typeLoader = (index, hierarchy) => {
const allowedNodes = getAllowedRecordNodesForIndex(hierarchy, index)
return key => find(n => getRecordNodeId(key) === n.nodeId)(allowedNodes).name
}

81
packages/core/src/recordApi/initialiseChildren.js Normal file
View File

@ -0,0 +1,81 @@
import { isString, flatten, map, filter } from "lodash/fp"
import { initialiseChildCollections } from "../collectionApi/initialise"
import { _loadFromInfo } from "./load"
import { $ } from "../common"
import {
getFlattenedHierarchy,
isRecord,
getNode,
isTopLevelRecord,
fieldReversesReferenceToNode,
} from "../templateApi/hierarchy"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { getRecordInfo } from "./recordInfo"
export const initialiseChildren = async (app, recordInfoOrKey) => {
const recordInfo = isString(recordInfoOrKey)
? getRecordInfo(app.hierarchy, recordInfoOrKey)
: recordInfoOrKey
await initialiseReverseReferenceIndexes(app, recordInfo)
await initialiseAncestorIndexes(app, recordInfo)
await initialiseChildCollections(app, recordInfo)
}
export const initialiseChildrenForNode = async (app, recordNode) => {
if (isTopLevelRecord(recordNode)) {
await initialiseChildren(
app, recordNode.nodeKey())
return
}
const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const initialisingRecordKey = joinKey(
result.collectionKey, id)
await initialiseChildren(app, initialisingRecordKey)
}
iterateResult = await iterate()
}
}
const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name)
if (!(await app.datastore.exists(indexKey))) {
await initialiseIndex(app.datastore, recordInfo.dir, index)
}
}
}
const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
const indexNodes = $(
fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
[
map(f =>
$(f.typeOptions.reverseIndexNodeKeys, [
map(n => getNode(app.hierarchy, n)),
])
),
flatten,
]
)
for (const indexNode of indexNodes) {
await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
}
}
const fieldsThatReferenceThisRecord = (app, recordNode) =>
$(app.hierarchy, [
getFlattenedHierarchy,
filter(isRecord),
map(n => n.fields),
flatten,
filter(fieldReversesReferenceToNode(recordNode)),
])

42
packages/core/src/recordApi/save.js
View File

@ -1,5 +1,4 @@
import { cloneDeep, take, takeRight, flatten, map, filter } from "lodash/fp"
import { initialiseChildCollections } from "../collectionApi/initialise"
import { validate } from "./validate"
import { _loadFromInfo } from "./load"
import { apiWrapper, events, $, joinKey } from "../common"
@ -17,6 +16,7 @@ import { permission } from "../authApi/permissions"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { BadRequestError } from "../common/errors"
import { getRecordInfo } from "./recordInfo"
import { initialiseChildren } from "./initialiseChildren"
export const save = app => async (record, context) =>
apiWrapper(
@ -59,9 +59,7 @@ export const _save = async (app, record, context, skipValidation = false) => {
await createRecordFolderPath(app.datastore, pathInfo)
await app.datastore.createFolder(files)
await app.datastore.createJson(recordJson, recordClone)
await initialiseReverseReferenceIndexes(app, recordInfo)
await initialiseAncestorIndexes(app, recordInfo)
await initialiseChildCollections(app, recordInfo)
await initialiseChildren(app, recordInfo)
await app.publish(events.recordApi.save.onRecordCreated, {
record: recordClone,
})
@ -87,42 +85,6 @@ export const _save = async (app, record, context, skipValidation = false) => {
return returnedClone
}
const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name)
if (!(await app.datastore.exists(indexKey))) {
await initialiseIndex(app.datastore, recordInfo.dir, index)
}
}
}
const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
const indexNodes = $(
fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
[
map(f =>
$(f.typeOptions.reverseIndexNodeKeys, [
map(n => getNode(app.hierarchy, n)),
])
),
flatten,
]
)
for (const indexNode of indexNodes) {
await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
}
}
const fieldsThatReferenceThisRecord = (app, recordNode) =>
$(app.hierarchy, [
getFlattenedHierarchy,
filter(isRecord),
map(n => n.fields),
flatten,
filter(fieldReversesReferenceToNode(recordNode)),
])
const createRecordFolderPath = async (datastore, pathInfo) => {
const recursiveCreateFolder = async (
subdirs,

34
packages/core/src/templateApi/deleteAllIndexFilesForNode.js Normal file
View File

@ -0,0 +1,34 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getRecordInfo } from "../recordApi/recordInfo"
import { isTopLevelIndex, getParentKey, getLastPartInKey } from "./hierarchy"
import { safeKey, joinKey } from "../common"
export const deleteAllIndexFilesForNode = async (app, indexNode) => {
if (isTopLevelIndex(indexNode)) {
await app.datastore.deleteFolder(indexNode.nodeKey())
return
}
const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const deletingIndexKey = joinKey(
result.collectionKey, id, indexNode.name)
await deleteIndexFolder(app, deletingIndexKey)
}
iterateResult = await iterate()
}
}
const deleteIndexFolder = async (app, indexKey) => {
indexKey = safeKey(indexKey)
const indexName = getLastPartInKey(indexKey)
const parentRecordKey = getParentKey(indexKey)
const recordInfo = getRecordInfo(app.hierarchy, parentRecordKey)
await app.datastore.deleteFolder(
joinKey(recordInfo.dir, indexName))
}

32
packages/core/src/templateApi/deleteAllRecordsForNode.js Normal file
View File

@ -0,0 +1,32 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getCollectionDir } from "../recordApi/recordInfo"
import { isTopLevelRecord, getCollectionKey } from "./hierarchy"
import { safeKey, joinKey } from "../common"
export const deleteAllRecordsForNode = async (app, recordNode) => {
if (isTopLevelRecord(recordNode)) {
await deleteRecordCollection(
app, recordNode.collectionName)
return
}
const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const deletingCollectionKey = joinKey(
result.collectionKey, id, recordNode.collectionName)
await deleteRecordCollection(app, deletingCollectionKey)
}
iterateResult = await iterate()
}
}
const deleteRecordCollection = async (app, collectionKey) => {
collectionKey = safeKey(collectionKey)
await app.datastore.deleteFolder(
getCollectionDir(app.hierarchy, collectionKey))
}

7
packages/core/src/templateApi/diffHierarchy.js
View File

@ -1,6 +1,6 @@
import { getFlattenedHierarchy, isRecord, isIndex, isAncestor } from "./hierarchy"
import { $, none } from "../common"
import { map, filter, some, find } from "lodash/fp"
import { map, filter, some, find, difference } from "lodash/fp"
export const HierarchyChangeTypes = {
recordCreated: "Record Created",
@ -10,7 +10,7 @@ export const HierarchyChangeTypes = {
recordEstimatedRecordTypeChanged: "Record's Estimated Record Count Changed",
indexCreated: "Index Created",
indexDeleted: "Index Deleted",
indexChanged: "index Changed",
indexChanged: "Index Changed",
}
export const diffHierarchy = (oldHierarchy, newHierarchy) => {
@ -123,7 +123,7 @@ const findDeletedIndexes = (oldHierarchyFlat, newHierarchyFlat, deletedRecords)
const findUpdatedIndexes = (oldHierarchyFlat, newHierarchyFlat) =>
$(oldHierarchyFlat, [
filter(isRecord),
filter(isIndex),
filter(nodeExistsIn(newHierarchyFlat)),
filter(nodeChanged(newHierarchyFlat, indexHasChanged)),
map(n => changeItem(
@ -150,6 +150,7 @@ const indexHasChanged = (_new, old) =>
_new.map !== old.map
|| _new.filter !== old.filter
|| _new.getShardName !== old.getShardName
|| difference(_new.allowedRecordNodeIds)(old.allowedRecordNodeIds).length > 0
const isFieldSame = f1 => f2 =>
f1.name === f2.name && f1.type === f2.type

24
packages/core/src/templateApi/hierarchy.js
View File

@ -191,6 +191,22 @@ export const getAllowedRecordNodesForIndex = (appHierarchy, indexNode) => {
}
}
export const getDependantIndexes = (hierarchy, recordNode) => {
const allIndexes = $(hierarchy, [ getFlattenedHierarchy, filter(isIndex)])
const allowedAncestors = $(allIndexes, [
filter(isAncestorIndex),
filter(i => recordNodeIsAllowed(i)(recordNode)),
])
const allowedReference = $(allIndexes, [
filter(isReferenceIndex),
filter(i => some(fieldReversesReferenceToIndex(i))(recordNode.fields))
])
return [...allowedAncestors, ...allowedReference]
}
export const getNodeFromNodeKeyHash = hierarchy => hash =>
$(hierarchy, [
getFlattenedHierarchy,
@ -206,13 +222,19 @@ export const isaggregateGroup = node =>
export const isShardedIndex = node =>
isIndex(node) && isNonEmptyString(node.getShardName)
export const isRoot = node => isSomething(node) && node.isRoot()
export const findRoot = node => isRoot(node) ? node : findRoot(node.parent())
export const isDecendantOfARecord = hasMatchingAncestor(isRecord)
export const isGlobalIndex = node => isIndex(node) && isRoot(node.parent())
export const isReferenceIndex = node =>
isIndex(node) && node.indexType === indexTypes.reference
export const isAncestorIndex = node =>
isIndex(node) && node.indexType === indexTypes.ancestor
export const isTopLevelRecord = node => isRoot(node.parent()) && isRecord(node)
export const isTopLevelIndex = node => isRoot(node.parent()) && isIndex(node)
export const getCollectionKey = recordKey => $(recordKey, [
splitKey,
parts => joinKey(parts.slice(0, parts.length - 1))
])
export const fieldReversesReferenceToNode = node => field =>
field.type === "reference" &&
intersection(field.typeOptions.reverseIndexNodeKeys)(

2
packages/core/src/templateApi/index.js
View File

@ -28,6 +28,7 @@ import { saveApplicationHierarchy } from "./saveApplicationHierarchy"
import { saveActionsAndTriggers } from "./saveActionsAndTriggers"
import { all } from "../types"
import { getBehaviourSources } from "./getBehaviourSources"
import { upgradeData } from "./upgradeData"
const api = app => ({
getApplicationDefinition: getApplicationDefinition(app.datastore),
@ -57,6 +58,7 @@ const api = app => ({
validateNode,
validateAll,
validateTriggers,
upgradeData: upgradeData(app)
})
export const getTemplateApi = app => api(app)

27
packages/core/src/templateApi/initialiseNewIndex.js Normal file
View File

@ -0,0 +1,27 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getRecordInfo } from "../recordApi/recordInfo"
import { isTopLevelIndex } from "./hierarchy"
import { joinKey } from "../common"
import { initialiseIndex } from "../indexing/initialiseIndex"
export const initialiseNewIndex = async (app, indexNode) => {
if (isTopLevelIndex(indexNode)) {
await initialiseIndex(app.datastore, "/", indexNode)
return
}
const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const recordKey = joinKey(result.collectionKey, id)
await initialiseIndex(
app.datastore,
getRecordInfo(app.hierarchy, recordKey).dir,
indexNode)
}
iterateResult = await iterate()
}
}

207
packages/core/src/templateApi/upgradeData.js
View File

@ -1,17 +1,194 @@
/*
const changeActions = {
rebuildIndex: indexNodeKey => ({
type: "rebuildIndex",
indexNodeKey,
}),
reshardRecords: recordNodeKey => ({
type: "reshardRecords",
recordNodeKey,
}),
deleteRecords: recordNodeKey => ({
type: "reshardRecords",
recordNodeKey,
}),
renameRecord
import { diffHierarchy, HierarchyChangeTypes } from "./diffHierarchy"
import { $, switchCase } from "../common"
import {
differenceBy,
isEqual,
some,
map,
filter,
uniqBy,
flatten
} from "lodash/fp"
import {
findRoot,
getDependantIndexes,
isTopLevelRecord,
isAncestorIndex
} from "./hierarchy"
import { generateSchema } from "../indexing/indexSchemaCreator"
import { _buildIndex } from "../indexApi/buildIndex"
import { constructHierarchy } from "./createNodes"
import { deleteAllRecordsForNode } from "./deleteAllRecordsForNode"
import { deleteAllIndexFilesForNode } from "./deleteAllIndexFilesForNode"
import { cloneApp } from "../appInitialise/cloneApp"
import { initialiseData } from "../appInitialise/initialiseData"
import { initialiseChildrenForNode } from "../recordApi/initialiseChildren"
import { initialiseNewIndex } from "./initialiseNewIndex"
import { saveApplicationHierarchy } from "../templateApi/saveApplicationHierarchy"
export const upgradeData = app => async newHierarchy => {
const diff = diffHierarchy(app.hierarchy, newHierarchy)
const changeActions = gatherChangeActions(diff)
if (changeActions.length === 0) return
newHierarchy = constructHierarchy(newHierarchy)
const newApp = newHierarchy && cloneApp(app, {
hierarchy: newHierarchy
})
await doUpgrade(app, newApp, changeActions)
await saveApplicationHierarchy(newApp)(newHierarchy)
}
const gatherChangeActions = (diff) =>
$(diff, [
map(actionForChange),
flatten,
uniqBy(a => a.compareKey)
])
const doUpgrade = async (oldApp, newApp, changeActions) => {
for(let action of changeActions) {
await action.run(oldApp, newApp, action.diff)
}
}
const actionForChange = diff =>
switchCase(
[isChangeType(HierarchyChangeTypes.recordCreated), recordCreatedAction],
[isChangeType(HierarchyChangeTypes.recordDeleted), deleteRecordsAction],
[
isChangeType(HierarchyChangeTypes.recordFieldsChanged),
rebuildAffectedIndexesAction
],
[isChangeType(HierarchyChangeTypes.recordRenamed), renameRecordAction],
[
isChangeType(HierarchyChangeTypes.recordEstimatedRecordTypeChanged),
reshardRecordsAction
],
[isChangeType(HierarchyChangeTypes.indexCreated), newIndexAction],
[isChangeType(HierarchyChangeTypes.indexDeleted), deleteIndexAction],
[isChangeType(HierarchyChangeTypes.indexChanged), rebuildIndexAction],
)(diff)
const isChangeType = changeType => change =>
change.type === changeType
const action = (diff, compareKey, run) => ({
diff,
compareKey,
run,
})
const reshardRecordsAction = diff =>
[action(diff, `reshardRecords-${diff.oldNode.nodeKey()}`, runReshardRecords)]
const rebuildIndexAction = diff =>
[action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
const newIndexAction = diff => {
if (isAncestorIndex(diff.newNode)) {
return [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
} else {
return [action(diff, `newIndex-${diff.newNode.nodeKey()}`, runNewIndex)]
}
}
const deleteIndexAction = diff =>
[action(diff, `deleteIndex-${diff.oldNode.nodeKey()}`, runDeleteIndex)]
const deleteRecordsAction = diff =>
[action(diff, `deleteRecords-${diff.oldNode.nodeKey()}`, runDeleteRecords)]
const renameRecordAction = diff =>
[action(diff, `renameRecords-${diff.oldNode.nodeKey()}`, runRenameRecord)]
const recordCreatedAction = diff => {
if (isTopLevelRecord(diff.newNode)) {
return [action(diff, `initialiseRoot`, runInitialiseRoot)]
}
return [action(diff, `initialiseChildRecord-${diff.newNode.nodeKey()}`, runInitialiseChildRecord)]
}
const rebuildAffectedIndexesAction = diff =>{
const newHierarchy = findRoot(diff.newNode)
const oldHierarchy = findRoot(diff.oldNode)
const indexes = getDependantIndexes(newHierarchy, diff.newNode)
const changedFields = (() => {
const addedFields = differenceBy(f => f.name)
(diff.oldNode.fields)
(diff.newNode.fields)
const removedFields = differenceBy(f => f.name)
(diff.newNode.fields)
(diff.oldNode.fields)
return map(f => f.name)([...addedFields, ...removedFields])
})()
const isIndexAffected = i => {
if (!isEqual(
generateSchema(oldHierarchy, i),
generateSchema(newHierarchy, i))) return true
if (some(f => indexes.filter.indexOf(`record.${f}`) > -1)(changedFields))
return true
if (some(f => indexes.getShardName.indexOf(`record.${f}`) > -1)(changedFields))
return true
return false
}
return $(indexes, [
filter(isIndexAffected),
map(i => action({ newNode:i }, `rebuildIndex-${i.nodeKey()}`, runRebuildIndex))
])
}
const runReshardRecords = async change => {
throw new Error("Resharding of records is not supported yet")
}
const runRebuildIndex = async (_, newApp, diff) => {
await _buildIndex(newApp, diff.newNode.nodeKey())
}
const runDeleteIndex = async (oldApp, _, diff) => {
await deleteAllIndexFilesForNode(oldApp, diff.oldNode)
}
const runDeleteRecords = async (oldApp, _, diff) => {
await deleteAllRecordsForNode(oldApp, diff.oldNode)
}
const runNewIndex = async (_, newApp, diff) => {
await initialiseNewIndex(newApp, diff.newNode)
}
const runRenameRecord = change => {
/*
Going to disllow this in the builder. once a collection key is set... its done
*/
}
const runInitialiseRoot = async (_, newApp) => {
await initialiseData(newApp.datastore, newApp)
}
const runInitialiseChildRecord = async (_, newApp, diff) => {
await initialiseChildrenForNode(newApp.datastore, diff.newNode)
}

15
packages/core/src/transactions/cleanup.js
View File

@ -14,8 +14,10 @@ export const cleanup = async app => {
const lock = await getTransactionLock(app)
if (isNolock(lock)) return
try {
const _cleanupBatch = async () => {
let processed = 0
const transactions = await retrieve(app)
let i = 1
if (transactions.length > 0) {
await executeTransactions(app)(transactions)
@ -34,10 +36,21 @@ export const cleanup = async app => {
])
await Promise.all(deleteFiles)
processed = transactions.length
}
return processed
}
try {
let count = -1
while (count !== 0) {
count = await _cleanupBatch()
}
} finally {
await releaseLock(app, lock)
}
}
const getTransactionLock = async app =>

29
packages/core/src/transactions/execute.js
View File

@ -1,6 +1,7 @@
import {
filter,
map,
reduce,
isUndefined,
includes,
flatten,
@ -10,6 +11,7 @@ import {
keys,
differenceBy,
difference,
some,
} from "lodash/fp"
import { union } from "lodash"
import {
@ -38,14 +40,22 @@ import {
fieldReversesReferenceToIndex,
isReferenceIndex,
getExactNodeForKey,
getParentKey
} from "../templateApi/hierarchy"
import { getRecordInfo } from "../recordApi/recordInfo"
import { getIndexDir } from "../indexApi/getIndexDir"
import { initialiseIndex } from "../indexing/initialiseIndex"
export const executeTransactions = app => async transactions => {
const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions)
for (const shard of keys(recordsByShard)) {
if (recordsByShard[shard].isRebuild)
await initialiseIndex(
app.datastore,
getParentKey(recordsByShard[shard].indexDir),
recordsByShard[shard].indexNode
)
await applyToShard(
app.hierarchy,
app.datastore,
@ -66,9 +76,9 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
const indexBuild = getBuildIndexTransactionsByShard(hierarchy, transactions)
const toRemove = [...deletes, ...updates.toRemove]
const toRemove = [...deletes, ...updates.toRemove, ...indexBuild.toRemove]
const toWrite = [...created, ...updates.toWrite, ...indexBuild]
const toWrite = [...created, ...updates.toWrite, ...indexBuild.toWrite]
const transByShard = {}
@ -77,6 +87,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
transByShard[t.indexShardKey] = {
writes: [],
removes: [],
isRebuild: some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toWrite)
|| some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toRemove),
indexDir: t.indexDir,
indexNodeKey: t.indexNode.nodeKey(),
indexNode: t.indexNode,
@ -207,7 +219,7 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => {
const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
const buildTransactions = $(transactions, [filter(isBuildIndex)])
if (!isNonEmptyArray(buildTransactions)) return []
if (!isNonEmptyArray(buildTransactions)) return { toWrite:[], toRemove:[] }
const indexNode = transactions.indexNode
const getIndexDirs = t => {
@ -248,7 +260,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
return $(buildTransactions, [
map(t => {
const mappedRecord = evaluate(t.record)(indexNode)
if (!mappedRecord.passedFilter) return null
mappedRecord.result = mappedRecord.result || t.record
const indexDirs = getIndexDirs(t)
return $(indexDirs, [
map(indexDir => ({
@ -262,9 +274,16 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
),
})),
])
}),
flatten,
filter(isSomething),
reduce((obj, res) => {
if (res.mappedRecord.passedFilter)
obj.toWrite.push(res)
else
obj.toRemove.push(res)
return obj
}, { toWrite: [], toRemove: [] })
])
}

41
packages/core/src/transactions/retrieve.js
View File

@ -22,32 +22,41 @@ export const retrieve = async app => {
TRANSACTIONS_FOLDER
)
let transactions = []
if (some(isBuildIndexFolder)(transactionFiles)) {
const buildIndexFolder = find(isBuildIndexFolder)(transactionFiles)
transactions = await retrieveBuildIndexTransactions(
const buildIndexFolders = filter(isBuildIndexFolder)(transactionFiles)
let currentFolderIndex = 0
while (currentFolderIndex < buildIndexFolders.length) {
const buildIndexFolder = buildIndexFolders[currentFolderIndex]
const transactions = await retrieveBuildIndexTransactions(
app,
joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)
)
if(transactions.length === 0) {
await app.datastore.deleteFolder(
joinKey(TRANSACTIONS_FOLDER, buildIndexFolder))
} else {
return transactions
}
currentFolderIndex += 1
}
if (transactions.length > 0) return transactions
return []
}
return await retrieveStandardTransactions(app, transactionFiles)
}
const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
const childFolders = await app.datastore.getFolderContents(buildIndexFolder)
if (childFolders.length === 0) {
// cleanup
await app.datastore.deleteFolder(buildIndexFolder)
const childFolderCount = childFolders.length
if (childFolderCount === 0) {
return []
}
const getTransactionFiles = async (childFolderIndex = 0) => {
if (childFolderIndex >= childFolders.length) return []
if (childFolderIndex >= childFolders.length) {
return { childFolderKey: "", files: [] }
}
const childFolderKey = joinKey(
buildIndexFolder,
@ -55,17 +64,19 @@ const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
)
const files = await app.datastore.getFolderContents(childFolderKey)
if (files.length === 0) {
if (files.length > 0) {
return { childFolderKey, files }
}
await app.datastore.deleteFolder(childFolderKey)
return await getTransactionFiles(childFolderIndex + 1)
}
return { childFolderKey, files }
}
const transactionFiles = await getTransactionFiles()
if (transactionFiles.files.length === 0) return []
if (transactionFiles.files.length === 0) {
return []
}
const transactions = $(transactionFiles.files, [map(parseTransactionId)])

14
packages/core/src/transactions/setCleanupFunc.js Normal file
View File

@ -0,0 +1,14 @@
import { cleanup } from "./cleanup"
export const setCleanupFunc = (app, cleanupTransactions) => {
if (cleanupTransactions) {
app.cleanupTransactions = cleanupTransactions
return
}
if (!app.cleanupTransactions || app.cleanupTransactions.isDefault) {
const newCleanup = async () => cleanup(app)
newCleanup.isDefault = true
app.cleanupTransactions = newCleanup
}
}

4
packages/core/src/transactions/transactionsCommon.js
View File

@ -1,18 +1,20 @@
import { joinKey, keySep, getHashCode } from "../common"
import { getLastPartInKey } from "../templateApi/hierarchy"
import { includes } from "lodash/fp"
export const TRANSACTIONS_FOLDER = `${keySep}.transactions`
export const LOCK_FILENAME = "lock"
export const LOCK_FILE_KEY = joinKey(TRANSACTIONS_FOLDER, LOCK_FILENAME)
export const idSep = "$"
const isOfType = typ => trans => trans.transactionType === typ
const isOfType = (...typ) => trans => includes(trans.transactionType)(typ)
export const CREATE_RECORD_TRANSACTION = "create"
export const UPDATE_RECORD_TRANSACTION = "update"
export const DELETE_RECORD_TRANSACTION = "delete"
export const BUILD_INDEX_TRANSACTION = "build"
export const isUpdate_Or_Rebuild = isOfType(UPDATE_RECORD_TRANSACTION, BUILD_INDEX_TRANSACTION)
export const isUpdate = isOfType(UPDATE_RECORD_TRANSACTION)
export const isDelete = isOfType(DELETE_RECORD_TRANSACTION)
export const isCreate = isOfType(CREATE_RECORD_TRANSACTION)

14
packages/core/test/specHelpers.js
View File

@ -24,7 +24,7 @@ import { filter, find } from "lodash/fp"
import { createBehaviourSources } from "../src/actionsApi/buildBehaviourSource"
import { createAction, createTrigger } from "../src/templateApi/createActions"
import { initialiseActions } from "../src/actionsApi/initialise"
import { cleanup } from "../src/transactions/cleanup"
import { setCleanupFunc } from "../src/transactions/setCleanupFunc"
import { permission } from "../src/authApi/permissions"
import { generateFullPermissions } from "../src/authApi/generateFullPermissions"
import { initialiseData } from "../src/appInitialise/initialiseData"
@ -39,9 +39,9 @@ export const testTemplatesPath = testAreaName =>
path.join(testFileArea(testAreaName), templateDefinitions)
export const getMemoryStore = () => setupDatastore(memory({}))
export const getMemoryTemplateApi = () => {
export const getMemoryTemplateApi = (store) => {
const app = {
datastore: getMemoryStore(),
datastore: store || getMemoryStore(),
publish: () => {},
getEpochTime: async () => new Date().getTime(),
user: { name: "", permissions: [permission.writeTemplates.get()] },
@ -78,8 +78,9 @@ export const appFromTempalteApi = async (
const fullPermissions = generateFullPermissions(app)
app.user.permissions = fullPermissions
if (disableCleanupTransactions) app.cleanupTransactions = async () => {}
else app.cleanupTransactions = async () => await cleanup(app)
if (disableCleanupTransactions) setCleanupFunc(app, async () => {})
else setCleanupFunc(app)
return app
}
@ -100,8 +101,9 @@ export const getRecordApiFromTemplateApi = async (
disableCleanupTransactions = false
) => {
const app = await appFromTempalteApi(templateApi, disableCleanupTransactions)
const recordapi = getRecordApi()
const recordapi = getRecordApi(app)
recordapi._storeHandle = app.datastore
return recordapi
}
export const getCollectionApiFromTemplateApi = async (

162
packages/core/test/templateApi.diffHierarchy.spec.js
View File

@ -1,6 +1,5 @@
import { getMemoryTemplateApi } from "./specHelpers"
import { setup } from "./upgradeDataSetup"
import { diffHierarchy, HierarchyChangeTypes } from "../src/templateApi/diffHierarchy"
import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
describe("diffHierarchy", () => {
@ -13,7 +12,7 @@ describe("diffHierarchy", () => {
it("should detect root record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
@ -25,7 +24,7 @@ describe("diffHierarchy", () => {
it("should only detect root record, when newly created root record has children ", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
newSetup.templateApi.getNewRecordTemplate(opportunity, "invoice", true)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
@ -38,7 +37,7 @@ describe("diffHierarchy", () => {
it("should detect child record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.contact, "opportunity", false)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
@ -49,8 +48,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record deleted", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -61,8 +60,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record deleted", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -73,8 +72,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record renamed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.collectionKey = "CONTACTS"
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -85,8 +84,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record renamed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.collectionKey = "CONTACTS"
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -97,8 +96,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record field removed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.fields = newSetup.contact.fields.filter(f => f.name !== "name")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -109,8 +108,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record field removed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.fields = newSetup.deal.fields.filter(f => f.name !== "name")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -121,8 +120,8 @@ describe("diffHierarchy", () => {
})
it("should detect record field added", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
const notesField = newSetup.templateApi.getNewField("string")
notesField.name = "notes"
newSetup.templateApi.addField(newSetup.contact, notesField)
@ -136,8 +135,8 @@ describe("diffHierarchy", () => {
})
it("should detect 1 record field added and 1 removed (total no. fields unchanged)", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
const notesField = newSetup.templateApi.getNewField("string")
notesField.name = "notes"
newSetup.templateApi.addField(newSetup.contact, notesField)
@ -151,8 +150,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record estimated record count changed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.estimatedRecordCount = 987
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -163,8 +162,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record estimated record count changed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.estimatedRecordCount = 987
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -174,44 +173,97 @@ describe("diffHierarchy", () => {
}])
})
it("should detect root record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
it("should detect root index created", async () => {
const oldHierarchy = (await setup()).root
const newSetup = await setup()
const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
newNode: opportunity,
newNode: all_deals,
oldNode: null,
type: HierarchyChangeTypes.recordCreated
type: HierarchyChangeTypes.indexCreated
}])
})
it("should detect child index created", async () => {
const oldHierarchy = (await setup()).root
const newSetup = await setup()
const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
newNode: all_deals,
oldNode: null,
type: HierarchyChangeTypes.indexCreated
}])
})
const setup = async () => {
const { templateApi } = await getMemoryTemplateApi()
const root = templateApi.getNewRootLevel()
const contact = templateApi.getNewRecordTemplate(root, "contact", true)
it("should detect root index deleted", async () => {
const oldSetup = await setup()
const newSetup = await setup()
newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: null,
oldNode: oldSetup.root.indexes.find(i => i.name === "contact_index"),
type: HierarchyChangeTypes.indexDeleted
}])
})
const nameField = templateApi.getNewField("string")
nameField.name = "name"
const statusField = templateApi.getNewField("string")
statusField.name = "status"
it("should detect child index deleted", async () => {
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: null,
oldNode: oldSetup.contact.indexes.find(i => i.name === "deal_index"),
type: HierarchyChangeTypes.indexDeleted
}])
})
templateApi.addField(contact, nameField)
templateApi.addField(contact, statusField)
const lead = templateApi.getNewRecordTemplate(root, "lead", true)
const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
templateApi.addField(deal, {...nameField})
templateApi.addField(deal, {...statusField})
getFlattenedHierarchy(root)
return {
root, contact, lead, deal, templateApi,
all_contacts: root.indexes[0],
all_leads: root.indexes[1],
deals_for_contacts: contact.indexes[0]
}
const testIndexChanged = (parent, makechange) => async () => {
const oldSetup = await setup()
const newSetup = await setup()
makechange(newSetup)
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: newSetup[parent].indexes[0],
oldNode: oldSetup[parent].indexes[0],
type: HierarchyChangeTypes.indexChanged
}])
}
it("should detect root index map changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].map = "new"
}))
it("should detect root index filter changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].filter = "new"
}))
it("should detect root index shardName changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].getShardName = "new"
}))
it("should detect root index allowedRecordIds changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].allowedRecordNodeIds.push(3)
}))
it("should detect child index allowedRecordIds changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].allowedRecordNodeIds.push(3)
}))
it("should detect child index map changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].map = "new"
}))
it("should detect child index filter changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].filter = "new"
}))
it("should detect child index shardName changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].getShardName = "new"
}))
})

244
packages/core/test/templateApi.upgradeData.spec.js Normal file
View File

@ -0,0 +1,244 @@
import {
getRecordApiFromTemplateApi,
getIndexApiFromTemplateApi,
} from "./specHelpers"
import { upgradeData } from "../src/templateApi/upgradeData"
import { setup } from "./upgradeDataSetup"
import { $, splitKey } from "../src/common"
import { keys, filter } from "lodash/fp"
import { _listItems } from "../src/indexApi/listItems"
import { _save } from "../src/recordApi/save"
describe("upgradeData", () => {
it("should delete all records and child records, when root record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => splitKey(k)[0] === "contacts"),
])
expect(remainingKeys.length).toBe(0)
})
it("should not delete other root record types, when root record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => splitKey(k)[0] === "leads"),
])
expect(remainingKeys.length > 0).toBe(true)
})
it("should delete all child records, when child record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
const startingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => k.includes("/deals/")),
])
expect(startingKeys.length > 0).toBe(true)
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => k.includes("/deals/")),
])
expect(remainingKeys.length).toBe(0)
})
it("should build a new root index", async () => {
const { oldSetup, newSetup } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
newIndex.name = "more_contacts"
newIndex.allowedRecordNodeIds = [newSetup.contact.nodeId]
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, "/more_contacts")
expect(itemsInNewIndex.length).toBe(2)
})
it("should update a root index", async () => {
const { oldSetup, newSetup } = await configure()
const contact_index = indexByName(newSetup.root, "contact_index")
contact_index.filter = "record.name === 'bobby'"
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, "/contact_index")
expect(itemsInNewIndex.length).toBe(1)
})
it("should delete a root index", async () => {
const { oldSetup, newSetup } = await configure()
// no exception
await _listItems(newSetup.app, "/contact_index")
newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
await upgradeData(oldSetup.app)(newSetup.root)
let er
try {
await _listItems(newSetup.app, "/contact_index")
} catch (e) {
er = e
}
expect(er).toBeDefined()
})
it("should build a new child index", async () => {
const { oldSetup, newSetup, records } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
newIndex.name = "more_deals"
newIndex.allowedRecordNodeIds = [newSetup.deal.nodeId]
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, `${records.contact1.key}/more_deals`)
expect(itemsInNewIndex.length).toBe(2)
})
it("should update a child index", async () => {
const { oldSetup, newSetup, records } = await configure()
const deal_index = indexByName(newSetup.contact, "deal_index")
deal_index.filter = "record.status === 'new'"
let itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
expect(itemsInIndex.length).toBe(2)
await upgradeData(oldSetup.app)(newSetup.root)
itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
expect(itemsInIndex.length).toBe(1)
})
it("should delete a child index", async () => {
const { oldSetup, newSetup, records } = await configure()
// no exception
await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
await upgradeData(oldSetup.app)(newSetup.root)
let er
try {
await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
} catch (e) {
er = e
}
expect(er).toBeDefined()
})
it("should build a new reference index", async () => {
const { oldSetup, newSetup, records, recordApi } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.lead)
newIndex.name = "contact_leads"
newIndex.allowedRecordNodeIds = [newSetup.lead.nodeId]
newIndex.indexType = "reference"
const leadField = newSetup.templateApi.getNewField("string")
leadField.name = "lead"
leadField.type = "reference"
leadField.typeOptions = {
reverseIndexNodeKeys: [ newIndex.nodeKey() ],
indexNodeKey: "/lead_index",
displayValue: "name"
}
newSetup.templateApi.addField(newSetup.contact, leadField)
await upgradeData(oldSetup.app)(newSetup.root)
const indexKey = `${records.lead1.key}/contact_leads`
let itemsInNewIndex = await _listItems(newSetup.app, indexKey)
expect(itemsInNewIndex.length).toBe(0)
records.contact1.lead = records.lead1
records.contact1.isNew = false
await _save(newSetup.app, records.contact1)
itemsInNewIndex = await _listItems(newSetup.app, indexKey)
expect(itemsInNewIndex.length).toBe(1)
})
})
const configure = async () => {
const oldSetup = await setup()
const recordApi = await getRecordApiFromTemplateApi(oldSetup.templateApi)
const indexApi = await getIndexApiFromTemplateApi(oldSetup.templateApi)
const newSetup = await setup(oldSetup.store)
const records = await createSomeRecords(recordApi)
return { oldSetup, newSetup, recordApi, records, indexApi }
}
const createSomeRecords = async recordApi => {
const contact1 = recordApi.getNew("/contacts", "contact")
contact1.name = "bobby"
const contact2 = recordApi.getNew("/contacts", "contact")
contact2.name = "poppy"
await recordApi.save(contact1)
await recordApi.save(contact2)
const deal1 = recordApi.getNew(`${contact1.key}/deals`, "deal")
deal1.name = "big mad deal"
deal1.status = "new"
const deal2 = recordApi.getNew(`${contact1.key}/deals`, "deal")
deal2.name = "smaller deal"
deal2.status = "old"
const deal3 = recordApi.getNew(`${contact2.key}/deals`, "deal")
deal3.name = "ok deal"
deal3.status = "new"
await recordApi.save(deal1)
await recordApi.save(deal2)
await recordApi.save(deal3)
const lead1 = recordApi.getNew("/leads", "lead")
lead1.name = "big new lead"
await recordApi.save(lead1)
return {
contact1, contact2, deal1, deal2, deal3, lead1,
}
}
const indexByName = (parent, name) => parent.indexes.find(i => i.name === name)

47
packages/core/test/upgradeDataSetup.js Normal file
View File

@ -0,0 +1,47 @@
import { getMemoryTemplateApi, appFromTempalteApi } from "./specHelpers"
import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
import { initialiseData } from "../src/appInitialise/initialiseData"
export const setup = async (store) => {
const { templateApi } = await getMemoryTemplateApi(store)
const root = templateApi.getNewRootLevel()
const contact = templateApi.getNewRecordTemplate(root, "contact", true)
contact.collectionName = "contacts"
const nameField = templateApi.getNewField("string")
nameField.name = "name"
const statusField = templateApi.getNewField("string")
statusField.name = "status"
templateApi.addField(contact, nameField)
templateApi.addField(contact, statusField)
const lead = templateApi.getNewRecordTemplate(root, "lead", true)
lead.collectionName = "leads"
const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
deal.collectionName = "deals"
templateApi.addField(deal, {...nameField})
templateApi.addField(deal, {...statusField})
templateApi.addField(lead, {...nameField})
getFlattenedHierarchy(root)
if (!store)
await initialiseData(templateApi._storeHandle, {
hierarchy: root,
actions: [],
triggers: [],
})
const app = await appFromTempalteApi(templateApi)
app.hierarchy = root
return {
root, contact, lead, app,
deal, templateApi, store: templateApi._storeHandle,
all_contacts: root.indexes[0],
all_leads: root.indexes[1],
deals_for_contacts: contact.indexes[0],
}
}

6
packages/server/middleware/routeHandlers/upgradeData.js Normal file
View File

@ -0,0 +1,6 @@
const StatusCodes = require("../../utilities/statusCodes")
module.exports = async ctx => {
await ctx.instance.templateApi.upgradeData(ctx.request.body.newHierarchy)
ctx.response.status = StatusCodes.OK
}

4
packages/server/middleware/routers.js
View File

@ -238,6 +238,10 @@ module.exports = (config, app) => {
ctx.response.status = StatusCodes.UNAUTHORIZED
}
})
.post(
"/_builder/instance/:appname/:instanceid/api/upgradeData",
routeHandlers.upgradeData
)
.post("/:appname/api/changeMyPassword", routeHandlers.changeMyPassword)
.post(
"/_builder/instance/:appname/:instanceid/api/changeMyPassword",