MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #160 from mjashanks/master 

Diff hierarchy & upgradeData
This commit is contained in:
Michael Shanks 2020-03-20 14:03:09 +00:00 committed by GitHub
parent 12f31c71d4 0399eec4e6
commit 5c24504b42
30 changed files with 965 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/builder/src/database/ActionsHeader.svelte
View File

@ -58,6 +58,7 @@
padding: 1.5rem;
width: 100%;
align-items: right;
box-sizing: border-box;
}
.actions-modal-body {

8
packages/core/src/appInitialise/cloneApp.js Normal file
View File

@ -0,0 +1,8 @@
import { setCleanupFunc } from "../transactions/setCleanupFunc"
export const cloneApp = (app, mergeWith) => {
const newApp = { ...app }
Object.assign(newApp, mergeWith)
setCleanupFunc(newApp)
return newApp
}

26
packages/core/src/appInitialise/initialiseData.js
View File

@ -21,22 +21,29 @@ export const initialiseData = async (
applicationDefinition,
accessLevels
) => {
await datastore.createFolder(configFolder)
await datastore.createJson(appDefinitionFile, applicationDefinition)
if (!await datastore.exists(configFolder))
await datastore.createFolder(configFolder)
if (!await datastore.exists(appDefinitionFile))
await datastore.createJson(appDefinitionFile, applicationDefinition)
await initialiseRootCollections(datastore, applicationDefinition.hierarchy)
await initialiseRootIndexes(datastore, applicationDefinition.hierarchy)
await datastore.createFolder(TRANSACTIONS_FOLDER)
if (!await datastore.exists(TRANSACTIONS_FOLDER))
await datastore.createFolder(TRANSACTIONS_FOLDER)
await datastore.createFolder(AUTH_FOLDER)
if (!await datastore.exists(AUTH_FOLDER))
await datastore.createFolder(AUTH_FOLDER)
await datastore.createJson(USERS_LIST_FILE, [])
if (!await datastore.exists(USERS_LIST_FILE))
await datastore.createJson(USERS_LIST_FILE, [])
await datastore.createJson(
ACCESS_LEVELS_FILE,
accessLevels ? accessLevels : { version: 0, levels: [] }
)
if (!await datastore.exists(ACCESS_LEVELS_FILE))
await datastore.createJson(
ACCESS_LEVELS_FILE,
accessLevels ? accessLevels : { version: 0, levels: [] }
)
await initialiseRootSingleRecords(datastore, applicationDefinition.hierarchy)
}
@ -64,6 +71,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => {
const singleRecords = $(flathierarchy, [filter(isSingleRecord)])
for (let record of singleRecords) {
if (await datastore.exists(record.nodeKey())) continue
await datastore.createFolder(record.nodeKey())
const result = _getNew(record, "")
await _save(app, result)

6
packages/core/src/index.js
View File

@ -7,7 +7,7 @@ import getActionsApi from "./actionsApi"
import { setupDatastore, createEventAggregator } from "./appInitialise"
import { initialiseActions } from "./actionsApi/initialise"
import { isSomething, crypto } from "./common"
import { cleanup } from "./transactions/cleanup"
import { setCleanupFunc } from "./transactions/setCleanupFunc"
import { generateFullPermissions } from "./authApi/generateFullPermissions"
import { getApplicationDefinition } from "./templateApi/getApplicationDefinition"
import common from "./common"
@ -40,9 +40,7 @@ export const getAppApis = async (
const templateApi = getTemplateApi(app)
app.cleanupTransactions = isSomething(cleanupTransactions)
? cleanupTransactions
: async () => await cleanup(app)
setCleanupFunc(app, cleanupTransactions)
app.getEpochTime = isSomething(getEpochTime)
? getEpochTime

83
packages/core/src/indexApi/buildIndex.js
View File

@ -6,8 +6,10 @@ import {
getNode,
isIndex,
isRecord,
getActualKeyOfParent,
getAllowedRecordNodesForIndex,
fieldReversesReferenceToIndex,
isTopLevelIndex,
} from "../templateApi/hierarchy"
import { joinKey, apiWrapper, events, $ } from "../common"
import {
@ -16,6 +18,8 @@ import {
} from "../transactions/create"
import { permission } from "../authApi/permissions"
import { BadRequestError } from "../common/errors"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { getRecordInfo } from "../recordApi/recordInfo"
/** rebuilds an index
* @param {object} app - the application container
@ -32,7 +36,7 @@ export const buildIndex = app => async indexNodeKey =>
indexNodeKey
)
const _buildIndex = async (app, indexNodeKey) => {
export const _buildIndex = async (app, indexNodeKey) => {
const indexNode = getNode(app.hierarchy, indexNodeKey)
await createBuildIndexFolder(app.datastore, indexNodeKey)
@ -89,12 +93,6 @@ const buildReverseReferenceIndex = async (app, indexNode) => {
}
}
/*
const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [
map(n => n.parent()),
]);
*/
const buildHeirarchalIndex = async (app, indexNode) => {
let recordCount = 0
@ -127,7 +125,7 @@ const buildHeirarchalIndex = async (app, indexNode) => {
)
let allIds = await allIdsIterator()
while (allIds.done === false) {
while (allIds.done === false) {
await createTransactionsForIds(
allIds.result.collectionKey,
allIds.result.ids
@ -139,77 +137,8 @@ const buildHeirarchalIndex = async (app, indexNode) => {
return recordCount
}
// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
const recordNodeApplies = indexNode => recordNode =>
includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds)
/*
const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [
getFlattenedHierarchy,
filter(
allTrue(
isRecord,
isDecendant(ancestorNode),
recordNodeApplies(indexNode),
),
),
]);
*/
/*
const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount = 0) => {
const collectionNode = getCollectionNodeByKeyOrNodeKey(
app.hierarchy,
collection_Key_or_NodeKey,
);
const allIdsIterator = await getAllIdsIterator(app)(collection_Key_or_NodeKey);
const createTransactionsForIds = async (collectionKey, allIds) => {
for (const recordId of allIds) {
const recordKey = joinKey(collectionKey, recordId);
const recordNode = chooseChildRecordNodeByKey(
collectionNode,
recordId,
);
if (recordNodeApplies(indexNode)(recordNode)) {
await transactionForBuildIndex(
app, indexNode.nodeKey(),
recordKey, recordCount,
);
recordCount++;
}
if (hasApplicableDecendant(app.hierarchy, recordNode, indexNode)) {
for (const childCollectionNode of recordNode.children) {
recordCount = await applyAllDecendantRecords(
app,
joinKey(recordKey, childCollectionNode.collectionName),
indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount,
);
}
}
}
};
let allIds = await allIdsIterator();
while (allIds.done === false) {
await createTransactionsForIds(
allIds.result.collectionKey,
allIds.result.ids,
);
allIds = await allIdsIterator();
}
return recordCount;
};
*/
export default buildIndex

2
packages/core/src/indexApi/listItems.js
View File

@ -33,7 +33,7 @@ const defaultOptions = {
searchPhrase: null,
}
const _listItems = async (app, indexKey, options = defaultOptions) => {
export const _listItems = async (app, indexKey, options = defaultOptions) => {
const { searchPhrase, rangeStartParams, rangeEndParams } = $({}, [
merge(options),
merge(defaultOptions),

3
packages/core/src/indexing/allIds.js
View File

@ -2,6 +2,7 @@ import { flatten, orderBy, filter, isUndefined } from "lodash/fp"
import {
getFlattenedHierarchy,
getCollectionNodeByKeyOrNodeKey,
getNodeByKeyOrNodeKey,
isCollectionRecord,
isAncestor,
} from "../templateApi/hierarchy"
@ -60,7 +61,7 @@ export const getAllIdsIterator = app => async collection_Key_or_NodeKey => {
const recordNode = getCollectionNodeByKeyOrNodeKey(
app.hierarchy,
collection_Key_or_NodeKey
)
) || getNodeByKeyOrNodeKey(app.hierarchy, collection_Key_or_NodeKey)
const getAllIdsIteratorForCollectionKey = async (
recordNode,

14
packages/core/src/indexing/initialiseIndex.js
View File

@ -9,11 +9,19 @@ import {
export const initialiseIndex = async (datastore, dir, index) => {
const indexDir = joinKey(dir, index.name)
await datastore.createFolder(indexDir)
let newDir = false
if (!await datastore.exists(indexDir)) {
await datastore.createFolder(indexDir)
newDir = true
}
if (isShardedIndex(index)) {
await datastore.createFile(getShardMapKey(indexDir), "[]")
const shardFile = getShardMapKey(indexDir)
if (newDir || !await datastore.exists(shardFile))
await datastore.createFile(shardFile, "[]")
} else {
await createIndexFile(datastore, getUnshardedIndexDataKey(indexDir), index)
const indexFile = getUnshardedIndexDataKey(indexDir)
if (newDir || !await datastore.exists(indexFile))
await createIndexFile(datastore, indexFile, index)
}
}

12
packages/core/src/indexing/read.js
View File

@ -3,6 +3,9 @@ import { promiseReadableStream } from "./promiseReadableStream"
import { createIndexFile } from "./sharding"
import { generateSchema } from "./indexSchemaCreator"
import { getIndexReader, CONTINUE_READING_RECORDS } from "./serializer"
import { getAllowedRecordNodesForIndex, getRecordNodeId } from "../templateApi/hierarchy"
import { $ } from "../common"
import { filter, includes, find } from "lodash/fp"
export const readIndex = async (
hierarchy,
@ -11,8 +14,10 @@ export const readIndex = async (
indexedDataKey
) => {
const records = []
const getType = typeLoader(index, hierarchy)
const doRead = iterateIndex(
async item => {
item.type = getType(item.key)
records.push(item)
return CONTINUE_READING_RECORDS
},
@ -31,8 +36,10 @@ export const searchIndex = async (
) => {
const records = []
const schema = generateSchema(hierarchy, index)
const getType = typeLoader(index, hierarchy)
const doRead = iterateIndex(
async item => {
item.type = getType(item.key)
const idx = lunr(function() {
this.ref("key")
for (const field of schema) {
@ -76,3 +83,8 @@ export const iterateIndex = (onGetItem, getFinalResult) => async (
return []
}
}
const typeLoader = (index, hierarchy) => {
const allowedNodes = getAllowedRecordNodesForIndex(hierarchy, index)
return key => find(n => getRecordNodeId(key) === n.nodeId)(allowedNodes).name
}

81
packages/core/src/recordApi/initialiseChildren.js Normal file
View File

@ -0,0 +1,81 @@
import { isString, flatten, map, filter } from "lodash/fp"
import { initialiseChildCollections } from "../collectionApi/initialise"
import { _loadFromInfo } from "./load"
import { $ } from "../common"
import {
getFlattenedHierarchy,
isRecord,
getNode,
isTopLevelRecord,
fieldReversesReferenceToNode,
} from "../templateApi/hierarchy"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { getRecordInfo } from "./recordInfo"
export const initialiseChildren = async (app, recordInfoOrKey) => {
const recordInfo = isString(recordInfoOrKey)
? getRecordInfo(app.hierarchy, recordInfoOrKey)
: recordInfoOrKey
await initialiseReverseReferenceIndexes(app, recordInfo)
await initialiseAncestorIndexes(app, recordInfo)
await initialiseChildCollections(app, recordInfo)
}
export const initialiseChildrenForNode = async (app, recordNode) => {
if (isTopLevelRecord(recordNode)) {
await initialiseChildren(
app, recordNode.nodeKey())
return
}
const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const initialisingRecordKey = joinKey(
result.collectionKey, id)
await initialiseChildren(app, initialisingRecordKey)
}
iterateResult = await iterate()
}
}
const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name)
if (!(await app.datastore.exists(indexKey))) {
await initialiseIndex(app.datastore, recordInfo.dir, index)
}
}
}
const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
const indexNodes = $(
fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
[
map(f =>
$(f.typeOptions.reverseIndexNodeKeys, [
map(n => getNode(app.hierarchy, n)),
])
),
flatten,
]
)
for (const indexNode of indexNodes) {
await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
}
}
const fieldsThatReferenceThisRecord = (app, recordNode) =>
$(app.hierarchy, [
getFlattenedHierarchy,
filter(isRecord),
map(n => n.fields),
flatten,
filter(fieldReversesReferenceToNode(recordNode)),
])

42
packages/core/src/recordApi/save.js
View File

@ -1,5 +1,4 @@
import { cloneDeep, take, takeRight, flatten, map, filter } from "lodash/fp"
import { initialiseChildCollections } from "../collectionApi/initialise"
import { validate } from "./validate"
import { _loadFromInfo } from "./load"
import { apiWrapper, events, $, joinKey } from "../common"
@ -17,6 +16,7 @@ import { permission } from "../authApi/permissions"
import { initialiseIndex } from "../indexing/initialiseIndex"
import { BadRequestError } from "../common/errors"
import { getRecordInfo } from "./recordInfo"
import { initialiseChildren } from "./initialiseChildren"
export const save = app => async (record, context) =>
apiWrapper(
@ -59,9 +59,7 @@ export const _save = async (app, record, context, skipValidation = false) => {
await createRecordFolderPath(app.datastore, pathInfo)
await app.datastore.createFolder(files)
await app.datastore.createJson(recordJson, recordClone)
await initialiseReverseReferenceIndexes(app, recordInfo)
await initialiseAncestorIndexes(app, recordInfo)
await initialiseChildCollections(app, recordInfo)
await initialiseChildren(app, recordInfo)
await app.publish(events.recordApi.save.onRecordCreated, {
record: recordClone,
})
@ -87,42 +85,6 @@ export const _save = async (app, record, context, skipValidation = false) => {
return returnedClone
}
const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name)
if (!(await app.datastore.exists(indexKey))) {
await initialiseIndex(app.datastore, recordInfo.dir, index)
}
}
}
const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
const indexNodes = $(
fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
[
map(f =>
$(f.typeOptions.reverseIndexNodeKeys, [
map(n => getNode(app.hierarchy, n)),
])
),
flatten,
]
)
for (const indexNode of indexNodes) {
await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
}
}
const fieldsThatReferenceThisRecord = (app, recordNode) =>
$(app.hierarchy, [
getFlattenedHierarchy,
filter(isRecord),
map(n => n.fields),
flatten,
filter(fieldReversesReferenceToNode(recordNode)),
])
const createRecordFolderPath = async (datastore, pathInfo) => {
const recursiveCreateFolder = async (
subdirs,

34
packages/core/src/templateApi/deleteAllIndexFilesForNode.js Normal file
View File

@ -0,0 +1,34 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getRecordInfo } from "../recordApi/recordInfo"
import { isTopLevelIndex, getParentKey, getLastPartInKey } from "./hierarchy"
import { safeKey, joinKey } from "../common"
export const deleteAllIndexFilesForNode = async (app, indexNode) => {
if (isTopLevelIndex(indexNode)) {
await app.datastore.deleteFolder(indexNode.nodeKey())
return
}
const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const deletingIndexKey = joinKey(
result.collectionKey, id, indexNode.name)
await deleteIndexFolder(app, deletingIndexKey)
}
iterateResult = await iterate()
}
}
const deleteIndexFolder = async (app, indexKey) => {
indexKey = safeKey(indexKey)
const indexName = getLastPartInKey(indexKey)
const parentRecordKey = getParentKey(indexKey)
const recordInfo = getRecordInfo(app.hierarchy, parentRecordKey)
await app.datastore.deleteFolder(
joinKey(recordInfo.dir, indexName))
}

32
packages/core/src/templateApi/deleteAllRecordsForNode.js Normal file
View File

@ -0,0 +1,32 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getCollectionDir } from "../recordApi/recordInfo"
import { isTopLevelRecord, getCollectionKey } from "./hierarchy"
import { safeKey, joinKey } from "../common"
export const deleteAllRecordsForNode = async (app, recordNode) => {
if (isTopLevelRecord(recordNode)) {
await deleteRecordCollection(
app, recordNode.collectionName)
return
}
const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const deletingCollectionKey = joinKey(
result.collectionKey, id, recordNode.collectionName)
await deleteRecordCollection(app, deletingCollectionKey)
}
iterateResult = await iterate()
}
}
const deleteRecordCollection = async (app, collectionKey) => {
collectionKey = safeKey(collectionKey)
await app.datastore.deleteFolder(
getCollectionDir(app.hierarchy, collectionKey))
}

7
packages/core/src/templateApi/diffHierarchy.js
View File

@ -1,6 +1,6 @@
import { getFlattenedHierarchy, isRecord, isIndex, isAncestor } from "./hierarchy"
import { $, none } from "../common"
import { map, filter, some, find } from "lodash/fp"
import { map, filter, some, find, difference } from "lodash/fp"
export const HierarchyChangeTypes = {
recordCreated: "Record Created",
@ -10,7 +10,7 @@ export const HierarchyChangeTypes = {
recordEstimatedRecordTypeChanged: "Record's Estimated Record Count Changed",
indexCreated: "Index Created",
indexDeleted: "Index Deleted",
indexChanged: "index Changed",
indexChanged: "Index Changed",
}
export const diffHierarchy = (oldHierarchy, newHierarchy) => {
@ -123,7 +123,7 @@ const findDeletedIndexes = (oldHierarchyFlat, newHierarchyFlat, deletedRecords)
const findUpdatedIndexes = (oldHierarchyFlat, newHierarchyFlat) =>
$(oldHierarchyFlat, [
filter(isRecord),
filter(isIndex),
filter(nodeExistsIn(newHierarchyFlat)),
filter(nodeChanged(newHierarchyFlat, indexHasChanged)),
map(n => changeItem(
@ -150,6 +150,7 @@ const indexHasChanged = (_new, old) =>
_new.map !== old.map
|| _new.filter !== old.filter
|| _new.getShardName !== old.getShardName
|| difference(_new.allowedRecordNodeIds)(old.allowedRecordNodeIds).length > 0
const isFieldSame = f1 => f2 =>
f1.name === f2.name && f1.type === f2.type

24
packages/core/src/templateApi/hierarchy.js
View File

@ -191,6 +191,22 @@ export const getAllowedRecordNodesForIndex = (appHierarchy, indexNode) => {
}
}
export const getDependantIndexes = (hierarchy, recordNode) => {
const allIndexes = $(hierarchy, [ getFlattenedHierarchy, filter(isIndex)])
const allowedAncestors = $(allIndexes, [
filter(isAncestorIndex),
filter(i => recordNodeIsAllowed(i)(recordNode)),
])
const allowedReference = $(allIndexes, [
filter(isReferenceIndex),
filter(i => some(fieldReversesReferenceToIndex(i))(recordNode.fields))
])
return [...allowedAncestors, ...allowedReference]
}
export const getNodeFromNodeKeyHash = hierarchy => hash =>
$(hierarchy, [
getFlattenedHierarchy,
@ -206,13 +222,19 @@ export const isaggregateGroup = node =>
export const isShardedIndex = node =>
isIndex(node) && isNonEmptyString(node.getShardName)
export const isRoot = node => isSomething(node) && node.isRoot()
export const findRoot = node => isRoot(node) ? node : findRoot(node.parent())
export const isDecendantOfARecord = hasMatchingAncestor(isRecord)
export const isGlobalIndex = node => isIndex(node) && isRoot(node.parent())
export const isReferenceIndex = node =>
isIndex(node) && node.indexType === indexTypes.reference
export const isAncestorIndex = node =>
isIndex(node) && node.indexType === indexTypes.ancestor
export const isTopLevelRecord = node => isRoot(node.parent()) && isRecord(node)
export const isTopLevelIndex = node => isRoot(node.parent()) && isIndex(node)
export const getCollectionKey = recordKey => $(recordKey, [
splitKey,
parts => joinKey(parts.slice(0, parts.length - 1))
])
export const fieldReversesReferenceToNode = node => field =>
field.type === "reference" &&
intersection(field.typeOptions.reverseIndexNodeKeys)(

2
packages/core/src/templateApi/index.js
View File

@ -28,6 +28,7 @@ import { saveApplicationHierarchy } from "./saveApplicationHierarchy"
import { saveActionsAndTriggers } from "./saveActionsAndTriggers"
import { all } from "../types"
import { getBehaviourSources } from "./getBehaviourSources"
import { upgradeData } from "./upgradeData"
const api = app => ({
getApplicationDefinition: getApplicationDefinition(app.datastore),
@ -57,6 +58,7 @@ const api = app => ({
validateNode,
validateAll,
validateTriggers,
upgradeData: upgradeData(app)
})
export const getTemplateApi = app => api(app)

27
packages/core/src/templateApi/initialiseNewIndex.js Normal file
View File

@ -0,0 +1,27 @@
import { getAllIdsIterator } from "../indexing/allIds"
import { getRecordInfo } from "../recordApi/recordInfo"
import { isTopLevelIndex } from "./hierarchy"
import { joinKey } from "../common"
import { initialiseIndex } from "../indexing/initialiseIndex"
export const initialiseNewIndex = async (app, indexNode) => {
if (isTopLevelIndex(indexNode)) {
await initialiseIndex(app.datastore, "/", indexNode)
return
}
const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
let iterateResult = await iterate()
while (!iterateResult.done) {
const { result } = iterateResult
for (const id of result.ids) {
const recordKey = joinKey(result.collectionKey, id)
await initialiseIndex(
app.datastore,
getRecordInfo(app.hierarchy, recordKey).dir,
indexNode)
}
iterateResult = await iterate()
}
}

209
packages/core/src/templateApi/upgradeData.js
View File

@ -1,17 +1,194 @@
/*
const changeActions = {
rebuildIndex: indexNodeKey => ({
type: "rebuildIndex",
indexNodeKey,
}),
reshardRecords: recordNodeKey => ({
type: "reshardRecords",
recordNodeKey,
}),
deleteRecords: recordNodeKey => ({
type: "reshardRecords",
recordNodeKey,
}),
renameRecord
import { diffHierarchy, HierarchyChangeTypes } from "./diffHierarchy"
import { $, switchCase } from "../common"
import {
differenceBy,
isEqual,
some,
map,
filter,
uniqBy,
flatten
} from "lodash/fp"
import {
findRoot,
getDependantIndexes,
isTopLevelRecord,
isAncestorIndex
} from "./hierarchy"
import { generateSchema } from "../indexing/indexSchemaCreator"
import { _buildIndex } from "../indexApi/buildIndex"
import { constructHierarchy } from "./createNodes"
import { deleteAllRecordsForNode } from "./deleteAllRecordsForNode"
import { deleteAllIndexFilesForNode } from "./deleteAllIndexFilesForNode"
import { cloneApp } from "../appInitialise/cloneApp"
import { initialiseData } from "../appInitialise/initialiseData"
import { initialiseChildrenForNode } from "../recordApi/initialiseChildren"
import { initialiseNewIndex } from "./initialiseNewIndex"
import { saveApplicationHierarchy } from "../templateApi/saveApplicationHierarchy"
export const upgradeData = app => async newHierarchy => {
const diff = diffHierarchy(app.hierarchy, newHierarchy)
const changeActions = gatherChangeActions(diff)
if (changeActions.length === 0) return
newHierarchy = constructHierarchy(newHierarchy)
const newApp = newHierarchy && cloneApp(app, {
hierarchy: newHierarchy
})
await doUpgrade(app, newApp, changeActions)
await saveApplicationHierarchy(newApp)(newHierarchy)
}
*/
const gatherChangeActions = (diff) =>
$(diff, [
map(actionForChange),
flatten,
uniqBy(a => a.compareKey)
])
const doUpgrade = async (oldApp, newApp, changeActions) => {
for(let action of changeActions) {
await action.run(oldApp, newApp, action.diff)
}
}
const actionForChange = diff =>
switchCase(
[isChangeType(HierarchyChangeTypes.recordCreated), recordCreatedAction],
[isChangeType(HierarchyChangeTypes.recordDeleted), deleteRecordsAction],
[
isChangeType(HierarchyChangeTypes.recordFieldsChanged),
rebuildAffectedIndexesAction
],
[isChangeType(HierarchyChangeTypes.recordRenamed), renameRecordAction],
[
isChangeType(HierarchyChangeTypes.recordEstimatedRecordTypeChanged),
reshardRecordsAction
],
[isChangeType(HierarchyChangeTypes.indexCreated), newIndexAction],
[isChangeType(HierarchyChangeTypes.indexDeleted), deleteIndexAction],
[isChangeType(HierarchyChangeTypes.indexChanged), rebuildIndexAction],
)(diff)
const isChangeType = changeType => change =>
change.type === changeType
const action = (diff, compareKey, run) => ({
diff,
compareKey,
run,
})
const reshardRecordsAction = diff =>
[action(diff, `reshardRecords-${diff.oldNode.nodeKey()}`, runReshardRecords)]
const rebuildIndexAction = diff =>
[action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
const newIndexAction = diff => {
if (isAncestorIndex(diff.newNode)) {
return [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
} else {
return [action(diff, `newIndex-${diff.newNode.nodeKey()}`, runNewIndex)]
}
}
const deleteIndexAction = diff =>
[action(diff, `deleteIndex-${diff.oldNode.nodeKey()}`, runDeleteIndex)]
const deleteRecordsAction = diff =>
[action(diff, `deleteRecords-${diff.oldNode.nodeKey()}`, runDeleteRecords)]
const renameRecordAction = diff =>
[action(diff, `renameRecords-${diff.oldNode.nodeKey()}`, runRenameRecord)]
const recordCreatedAction = diff => {
if (isTopLevelRecord(diff.newNode)) {
return [action(diff, `initialiseRoot`, runInitialiseRoot)]
}
return [action(diff, `initialiseChildRecord-${diff.newNode.nodeKey()}`, runInitialiseChildRecord)]
}
const rebuildAffectedIndexesAction = diff =>{
const newHierarchy = findRoot(diff.newNode)
const oldHierarchy = findRoot(diff.oldNode)
const indexes = getDependantIndexes(newHierarchy, diff.newNode)
const changedFields = (() => {
const addedFields = differenceBy(f => f.name)
(diff.oldNode.fields)
(diff.newNode.fields)
const removedFields = differenceBy(f => f.name)
(diff.newNode.fields)
(diff.oldNode.fields)
return map(f => f.name)([...addedFields, ...removedFields])
})()
const isIndexAffected = i => {
if (!isEqual(
generateSchema(oldHierarchy, i),
generateSchema(newHierarchy, i))) return true
if (some(f => indexes.filter.indexOf(`record.${f}`) > -1)(changedFields))
return true
if (some(f => indexes.getShardName.indexOf(`record.${f}`) > -1)(changedFields))
return true
return false
}
return $(indexes, [
filter(isIndexAffected),
map(i => action({ newNode:i }, `rebuildIndex-${i.nodeKey()}`, runRebuildIndex))
])
}
const runReshardRecords = async change => {
throw new Error("Resharding of records is not supported yet")
}
const runRebuildIndex = async (_, newApp, diff) => {
await _buildIndex(newApp, diff.newNode.nodeKey())
}
const runDeleteIndex = async (oldApp, _, diff) => {
await deleteAllIndexFilesForNode(oldApp, diff.oldNode)
}
const runDeleteRecords = async (oldApp, _, diff) => {
await deleteAllRecordsForNode(oldApp, diff.oldNode)
}
const runNewIndex = async (_, newApp, diff) => {
await initialiseNewIndex(newApp, diff.newNode)
}
const runRenameRecord = change => {
/*
Going to disllow this in the builder. once a collection key is set... its done
*/
}
const runInitialiseRoot = async (_, newApp) => {
await initialiseData(newApp.datastore, newApp)
}
const runInitialiseChildRecord = async (_, newApp, diff) => {
await initialiseChildrenForNode(newApp.datastore, diff.newNode)
}

15
packages/core/src/transactions/cleanup.js
View File

@ -14,8 +14,10 @@ export const cleanup = async app => {
const lock = await getTransactionLock(app)
if (isNolock(lock)) return
try {
const _cleanupBatch = async () => {
let processed = 0
const transactions = await retrieve(app)
let i = 1
if (transactions.length > 0) {
await executeTransactions(app)(transactions)
@ -34,10 +36,21 @@ export const cleanup = async app => {
])
await Promise.all(deleteFiles)
processed = transactions.length
}
return processed
}
try {
let count = -1
while (count !== 0) {
count = await _cleanupBatch()
}
} finally {
await releaseLock(app, lock)
}
}
const getTransactionLock = async app =>

31
packages/core/src/transactions/execute.js
View File

@ -1,6 +1,7 @@
import {
filter,
map,
reduce,
isUndefined,
includes,
flatten,
@ -10,6 +11,7 @@ import {
keys,
differenceBy,
difference,
some,
} from "lodash/fp"
import { union } from "lodash"
import {
@ -38,14 +40,22 @@ import {
fieldReversesReferenceToIndex,
isReferenceIndex,
getExactNodeForKey,
getParentKey
} from "../templateApi/hierarchy"
import { getRecordInfo } from "../recordApi/recordInfo"
import { getIndexDir } from "../indexApi/getIndexDir"
import { initialiseIndex } from "../indexing/initialiseIndex"
export const executeTransactions = app => async transactions => {
const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions)
for (const shard of keys(recordsByShard)) {
if (recordsByShard[shard].isRebuild)
await initialiseIndex(
app.datastore,
getParentKey(recordsByShard[shard].indexDir),
recordsByShard[shard].indexNode
)
await applyToShard(
app.hierarchy,
app.datastore,
@ -66,9 +76,9 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
const indexBuild = getBuildIndexTransactionsByShard(hierarchy, transactions)
const toRemove = [...deletes, ...updates.toRemove]
const toRemove = [...deletes, ...updates.toRemove, ...indexBuild.toRemove]
const toWrite = [...created, ...updates.toWrite, ...indexBuild]
const toWrite = [...created, ...updates.toWrite, ...indexBuild.toWrite]
const transByShard = {}
@ -77,6 +87,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
transByShard[t.indexShardKey] = {
writes: [],
removes: [],
isRebuild: some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toWrite)
|| some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toRemove),
indexDir: t.indexDir,
indexNodeKey: t.indexNode.nodeKey(),
indexNode: t.indexNode,
@ -207,7 +219,7 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => {
const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
const buildTransactions = $(transactions, [filter(isBuildIndex)])
if (!isNonEmptyArray(buildTransactions)) return []
if (!isNonEmptyArray(buildTransactions)) return { toWrite:[], toRemove:[] }
const indexNode = transactions.indexNode
const getIndexDirs = t => {
@ -247,8 +259,8 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
return $(buildTransactions, [
map(t => {
const mappedRecord = evaluate(t.record)(indexNode)
if (!mappedRecord.passedFilter) return null
const mappedRecord = evaluate(t.record)(indexNode)
mappedRecord.result = mappedRecord.result || t.record
const indexDirs = getIndexDirs(t)
return $(indexDirs, [
map(indexDir => ({
@ -262,9 +274,16 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
),
})),
])
}),
flatten,
filter(isSomething),
reduce((obj, res) => {
if (res.mappedRecord.passedFilter)
obj.toWrite.push(res)
else
obj.toRemove.push(res)
return obj
}, { toWrite: [], toRemove: [] })
])
}

49
packages/core/src/transactions/retrieve.js
View File

@ -22,32 +22,41 @@ export const retrieve = async app => {
TRANSACTIONS_FOLDER
)
let transactions = []
if (some(isBuildIndexFolder)(transactionFiles)) {
const buildIndexFolder = find(isBuildIndexFolder)(transactionFiles)
transactions = await retrieveBuildIndexTransactions(
app,
joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)
)
const buildIndexFolders = filter(isBuildIndexFolder)(transactionFiles)
let currentFolderIndex = 0
while (currentFolderIndex < buildIndexFolders.length) {
const buildIndexFolder = buildIndexFolders[currentFolderIndex]
const transactions = await retrieveBuildIndexTransactions(
app,
joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)
)
if(transactions.length === 0) {
await app.datastore.deleteFolder(
joinKey(TRANSACTIONS_FOLDER, buildIndexFolder))
} else {
return transactions
}
currentFolderIndex += 1
}
return []
}
if (transactions.length > 0) return transactions
return await retrieveStandardTransactions(app, transactionFiles)
}
const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
const childFolders = await app.datastore.getFolderContents(buildIndexFolder)
if (childFolders.length === 0) {
// cleanup
await app.datastore.deleteFolder(buildIndexFolder)
const childFolderCount = childFolders.length
if (childFolderCount === 0) {
return []
}
const getTransactionFiles = async (childFolderIndex = 0) => {
if (childFolderIndex >= childFolders.length) return []
if (childFolderIndex >= childFolders.length) {
return { childFolderKey: "", files: [] }
}
const childFolderKey = joinKey(
buildIndexFolder,
@ -55,17 +64,19 @@ const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
)
const files = await app.datastore.getFolderContents(childFolderKey)
if (files.length === 0) {
await app.datastore.deleteFolder(childFolderKey)
return await getTransactionFiles(childFolderIndex + 1)
if (files.length > 0) {
return { childFolderKey, files }
}
return { childFolderKey, files }
await app.datastore.deleteFolder(childFolderKey)
return await getTransactionFiles(childFolderIndex + 1)
}
const transactionFiles = await getTransactionFiles()
if (transactionFiles.files.length === 0) return []
if (transactionFiles.files.length === 0) {
return []
}
const transactions = $(transactionFiles.files, [map(parseTransactionId)])

14
packages/core/src/transactions/setCleanupFunc.js Normal file
View File

@ -0,0 +1,14 @@
import { cleanup } from "./cleanup"
export const setCleanupFunc = (app, cleanupTransactions) => {
if (cleanupTransactions) {
app.cleanupTransactions = cleanupTransactions
return
}
if (!app.cleanupTransactions || app.cleanupTransactions.isDefault) {
const newCleanup = async () => cleanup(app)
newCleanup.isDefault = true
app.cleanupTransactions = newCleanup
}
}

4
packages/core/src/transactions/transactionsCommon.js
View File

@ -1,18 +1,20 @@
import { joinKey, keySep, getHashCode } from "../common"
import { getLastPartInKey } from "../templateApi/hierarchy"
import { includes } from "lodash/fp"
export const TRANSACTIONS_FOLDER = `${keySep}.transactions`
export const LOCK_FILENAME = "lock"
export const LOCK_FILE_KEY = joinKey(TRANSACTIONS_FOLDER, LOCK_FILENAME)
export const idSep = "$"
const isOfType = typ => trans => trans.transactionType === typ
const isOfType = (...typ) => trans => includes(trans.transactionType)(typ)
export const CREATE_RECORD_TRANSACTION = "create"
export const UPDATE_RECORD_TRANSACTION = "update"
export const DELETE_RECORD_TRANSACTION = "delete"
export const BUILD_INDEX_TRANSACTION = "build"
export const isUpdate_Or_Rebuild = isOfType(UPDATE_RECORD_TRANSACTION, BUILD_INDEX_TRANSACTION)
export const isUpdate = isOfType(UPDATE_RECORD_TRANSACTION)
export const isDelete = isOfType(DELETE_RECORD_TRANSACTION)
export const isCreate = isOfType(CREATE_RECORD_TRANSACTION)

14
packages/core/test/specHelpers.js
View File

@ -24,7 +24,7 @@ import { filter, find } from "lodash/fp"
import { createBehaviourSources } from "../src/actionsApi/buildBehaviourSource"
import { createAction, createTrigger } from "../src/templateApi/createActions"
import { initialiseActions } from "../src/actionsApi/initialise"
import { cleanup } from "../src/transactions/cleanup"
import { setCleanupFunc } from "../src/transactions/setCleanupFunc"
import { permission } from "../src/authApi/permissions"
import { generateFullPermissions } from "../src/authApi/generateFullPermissions"
import { initialiseData } from "../src/appInitialise/initialiseData"
@ -39,9 +39,9 @@ export const testTemplatesPath = testAreaName =>
path.join(testFileArea(testAreaName), templateDefinitions)
export const getMemoryStore = () => setupDatastore(memory({}))
export const getMemoryTemplateApi = () => {
export const getMemoryTemplateApi = (store) => {
const app = {
datastore: getMemoryStore(),
datastore: store || getMemoryStore(),
publish: () => {},
getEpochTime: async () => new Date().getTime(),
user: { name: "", permissions: [permission.writeTemplates.get()] },
@ -78,8 +78,9 @@ export const appFromTempalteApi = async (
const fullPermissions = generateFullPermissions(app)
app.user.permissions = fullPermissions
if (disableCleanupTransactions) app.cleanupTransactions = async () => {}
else app.cleanupTransactions = async () => await cleanup(app)
if (disableCleanupTransactions) setCleanupFunc(app, async () => {})
else setCleanupFunc(app)
return app
}
@ -100,8 +101,9 @@ export const getRecordApiFromTemplateApi = async (
disableCleanupTransactions = false
) => {
const app = await appFromTempalteApi(templateApi, disableCleanupTransactions)
const recordapi = getRecordApi()
const recordapi = getRecordApi(app)
recordapi._storeHandle = app.datastore
return recordapi
}
export const getCollectionApiFromTemplateApi = async (

168
packages/core/test/templateApi.diffHierarchy.spec.js
View File

@ -1,6 +1,5 @@
import { getMemoryTemplateApi } from "./specHelpers"
import { setup } from "./upgradeDataSetup"
import { diffHierarchy, HierarchyChangeTypes } from "../src/templateApi/diffHierarchy"
import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
describe("diffHierarchy", () => {
@ -13,7 +12,7 @@ describe("diffHierarchy", () => {
it("should detect root record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
@ -25,7 +24,7 @@ describe("diffHierarchy", () => {
it("should only detect root record, when newly created root record has children ", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
newSetup.templateApi.getNewRecordTemplate(opportunity, "invoice", true)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
@ -38,7 +37,7 @@ describe("diffHierarchy", () => {
it("should detect child record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const newSetup = await setup()
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.contact, "opportunity", false)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
@ -49,8 +48,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record deleted", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -61,8 +60,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record deleted", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -73,8 +72,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record renamed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.collectionKey = "CONTACTS"
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -85,8 +84,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record renamed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.collectionKey = "CONTACTS"
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -97,8 +96,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record field removed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.fields = newSetup.contact.fields.filter(f => f.name !== "name")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -109,8 +108,8 @@ describe("diffHierarchy", () => {
})
it("should detect child record field removed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.fields = newSetup.deal.fields.filter(f => f.name !== "name")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -121,8 +120,8 @@ describe("diffHierarchy", () => {
})
it("should detect record field added", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
const notesField = newSetup.templateApi.getNewField("string")
notesField.name = "notes"
newSetup.templateApi.addField(newSetup.contact, notesField)
@ -136,8 +135,8 @@ describe("diffHierarchy", () => {
})
it("should detect 1 record field added and 1 removed (total no. fields unchanged)", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
const notesField = newSetup.templateApi.getNewField("string")
notesField.name = "notes"
newSetup.templateApi.addField(newSetup.contact, notesField)
@ -151,8 +150,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record estimated record count changed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.estimatedRecordCount = 987
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -163,8 +162,8 @@ describe("diffHierarchy", () => {
})
it("should detect root record estimated record count changed", async () => {
const oldSetup = (await setup());
const newSetup = (await setup());
const oldSetup = await setup()
const newSetup = await setup()
newSetup.deal.estimatedRecordCount = 987
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
@ -174,44 +173,97 @@ describe("diffHierarchy", () => {
}])
})
it("should detect root record created", async () => {
const oldHierarchy = (await setup()).root;
const newSetup = (await setup());
const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
it("should detect root index created", async () => {
const oldHierarchy = (await setup()).root
const newSetup = await setup()
const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
newNode: opportunity,
newNode: all_deals,
oldNode: null,
type: HierarchyChangeTypes.recordCreated
type: HierarchyChangeTypes.indexCreated
}])
})
it("should detect child index created", async () => {
const oldHierarchy = (await setup()).root
const newSetup = await setup()
const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
const diff = diffHierarchy(oldHierarchy, newSetup.root)
expect(diff).toEqual([{
newNode: all_deals,
oldNode: null,
type: HierarchyChangeTypes.indexCreated
}])
})
it("should detect root index deleted", async () => {
const oldSetup = await setup()
const newSetup = await setup()
newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: null,
oldNode: oldSetup.root.indexes.find(i => i.name === "contact_index"),
type: HierarchyChangeTypes.indexDeleted
}])
})
it("should detect child index deleted", async () => {
const oldSetup = await setup()
const newSetup = await setup()
newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: null,
oldNode: oldSetup.contact.indexes.find(i => i.name === "deal_index"),
type: HierarchyChangeTypes.indexDeleted
}])
})
const testIndexChanged = (parent, makechange) => async () => {
const oldSetup = await setup()
const newSetup = await setup()
makechange(newSetup)
const diff = diffHierarchy(oldSetup.root, newSetup.root)
expect(diff).toEqual([{
newNode: newSetup[parent].indexes[0],
oldNode: oldSetup[parent].indexes[0],
type: HierarchyChangeTypes.indexChanged
}])
}
it("should detect root index map changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].map = "new"
}))
it("should detect root index filter changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].filter = "new"
}))
it("should detect root index shardName changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].getShardName = "new"
}))
it("should detect root index allowedRecordIds changed", testIndexChanged("root", newSetup => {
newSetup.root.indexes[0].allowedRecordNodeIds.push(3)
}))
it("should detect child index allowedRecordIds changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].allowedRecordNodeIds.push(3)
}))
it("should detect child index map changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].map = "new"
}))
it("should detect child index filter changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].filter = "new"
}))
it("should detect child index shardName changed", testIndexChanged("contact", newSetup => {
newSetup.contact.indexes[0].getShardName = "new"
}))
})
const setup = async () => {
const { templateApi } = await getMemoryTemplateApi()
const root = templateApi.getNewRootLevel()
const contact = templateApi.getNewRecordTemplate(root, "contact", true)
const nameField = templateApi.getNewField("string")
nameField.name = "name"
const statusField = templateApi.getNewField("string")
statusField.name = "status"
templateApi.addField(contact, nameField)
templateApi.addField(contact, statusField)
const lead = templateApi.getNewRecordTemplate(root, "lead", true)
const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
templateApi.addField(deal, {...nameField})
templateApi.addField(deal, {...statusField})
getFlattenedHierarchy(root)
return {
root, contact, lead, deal, templateApi,
all_contacts: root.indexes[0],
all_leads: root.indexes[1],
deals_for_contacts: contact.indexes[0]
}
}

244
packages/core/test/templateApi.upgradeData.spec.js Normal file
View File

@ -0,0 +1,244 @@
import {
getRecordApiFromTemplateApi,
getIndexApiFromTemplateApi,
} from "./specHelpers"
import { upgradeData } from "../src/templateApi/upgradeData"
import { setup } from "./upgradeDataSetup"
import { $, splitKey } from "../src/common"
import { keys, filter } from "lodash/fp"
import { _listItems } from "../src/indexApi/listItems"
import { _save } from "../src/recordApi/save"
describe("upgradeData", () => {
it("should delete all records and child records, when root record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => splitKey(k)[0] === "contacts"),
])
expect(remainingKeys.length).toBe(0)
})
it("should not delete other root record types, when root record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => splitKey(k)[0] === "leads"),
])
expect(remainingKeys.length > 0).toBe(true)
})
it("should delete all child records, when child record node deleted", async () => {
const { oldSetup, newSetup, recordApi } = await configure()
newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
const startingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => k.includes("/deals/")),
])
expect(startingKeys.length > 0).toBe(true)
await upgradeData(oldSetup.app)(newSetup.root)
const remainingKeys = $(recordApi._storeHandle.data, [
keys,
filter(k => k.includes("/deals/")),
])
expect(remainingKeys.length).toBe(0)
})
it("should build a new root index", async () => {
const { oldSetup, newSetup } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
newIndex.name = "more_contacts"
newIndex.allowedRecordNodeIds = [newSetup.contact.nodeId]
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, "/more_contacts")
expect(itemsInNewIndex.length).toBe(2)
})
it("should update a root index", async () => {
const { oldSetup, newSetup } = await configure()
const contact_index = indexByName(newSetup.root, "contact_index")
contact_index.filter = "record.name === 'bobby'"
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, "/contact_index")
expect(itemsInNewIndex.length).toBe(1)
})
it("should delete a root index", async () => {
const { oldSetup, newSetup } = await configure()
// no exception
await _listItems(newSetup.app, "/contact_index")
newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
await upgradeData(oldSetup.app)(newSetup.root)
let er
try {
await _listItems(newSetup.app, "/contact_index")
} catch (e) {
er = e
}
expect(er).toBeDefined()
})
it("should build a new child index", async () => {
const { oldSetup, newSetup, records } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
newIndex.name = "more_deals"
newIndex.allowedRecordNodeIds = [newSetup.deal.nodeId]
await upgradeData(oldSetup.app)(newSetup.root)
const itemsInNewIndex = await _listItems(newSetup.app, `${records.contact1.key}/more_deals`)
expect(itemsInNewIndex.length).toBe(2)
})
it("should update a child index", async () => {
const { oldSetup, newSetup, records } = await configure()
const deal_index = indexByName(newSetup.contact, "deal_index")
deal_index.filter = "record.status === 'new'"
let itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
expect(itemsInIndex.length).toBe(2)
await upgradeData(oldSetup.app)(newSetup.root)
itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
expect(itemsInIndex.length).toBe(1)
})
it("should delete a child index", async () => {
const { oldSetup, newSetup, records } = await configure()
// no exception
await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
await upgradeData(oldSetup.app)(newSetup.root)
let er
try {
await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
} catch (e) {
er = e
}
expect(er).toBeDefined()
})
it("should build a new reference index", async () => {
const { oldSetup, newSetup, records, recordApi } = await configure()
const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.lead)
newIndex.name = "contact_leads"
newIndex.allowedRecordNodeIds = [newSetup.lead.nodeId]
newIndex.indexType = "reference"
const leadField = newSetup.templateApi.getNewField("string")
leadField.name = "lead"
leadField.type = "reference"
leadField.typeOptions = {
reverseIndexNodeKeys: [ newIndex.nodeKey() ],
indexNodeKey: "/lead_index",
displayValue: "name"
}
newSetup.templateApi.addField(newSetup.contact, leadField)
await upgradeData(oldSetup.app)(newSetup.root)
const indexKey = `${records.lead1.key}/contact_leads`
let itemsInNewIndex = await _listItems(newSetup.app, indexKey)
expect(itemsInNewIndex.length).toBe(0)
records.contact1.lead = records.lead1
records.contact1.isNew = false
await _save(newSetup.app, records.contact1)
itemsInNewIndex = await _listItems(newSetup.app, indexKey)
expect(itemsInNewIndex.length).toBe(1)
})
})
const configure = async () => {
const oldSetup = await setup()
const recordApi = await getRecordApiFromTemplateApi(oldSetup.templateApi)
const indexApi = await getIndexApiFromTemplateApi(oldSetup.templateApi)
const newSetup = await setup(oldSetup.store)
const records = await createSomeRecords(recordApi)
return { oldSetup, newSetup, recordApi, records, indexApi }
}
const createSomeRecords = async recordApi => {
const contact1 = recordApi.getNew("/contacts", "contact")
contact1.name = "bobby"
const contact2 = recordApi.getNew("/contacts", "contact")
contact2.name = "poppy"
await recordApi.save(contact1)
await recordApi.save(contact2)
const deal1 = recordApi.getNew(`${contact1.key}/deals`, "deal")
deal1.name = "big mad deal"
deal1.status = "new"
const deal2 = recordApi.getNew(`${contact1.key}/deals`, "deal")
deal2.name = "smaller deal"
deal2.status = "old"
const deal3 = recordApi.getNew(`${contact2.key}/deals`, "deal")
deal3.name = "ok deal"
deal3.status = "new"
await recordApi.save(deal1)
await recordApi.save(deal2)
await recordApi.save(deal3)
const lead1 = recordApi.getNew("/leads", "lead")
lead1.name = "big new lead"
await recordApi.save(lead1)
return {
contact1, contact2, deal1, deal2, deal3, lead1,
}
}
const indexByName = (parent, name) => parent.indexes.find(i => i.name === name)

47
packages/core/test/upgradeDataSetup.js Normal file
View File

@ -0,0 +1,47 @@
import { getMemoryTemplateApi, appFromTempalteApi } from "./specHelpers"
import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
import { initialiseData } from "../src/appInitialise/initialiseData"
export const setup = async (store) => {
const { templateApi } = await getMemoryTemplateApi(store)
const root = templateApi.getNewRootLevel()
const contact = templateApi.getNewRecordTemplate(root, "contact", true)
contact.collectionName = "contacts"
const nameField = templateApi.getNewField("string")
nameField.name = "name"
const statusField = templateApi.getNewField("string")
statusField.name = "status"
templateApi.addField(contact, nameField)
templateApi.addField(contact, statusField)
const lead = templateApi.getNewRecordTemplate(root, "lead", true)
lead.collectionName = "leads"
const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
deal.collectionName = "deals"
templateApi.addField(deal, {...nameField})
templateApi.addField(deal, {...statusField})
templateApi.addField(lead, {...nameField})
getFlattenedHierarchy(root)
if (!store)
await initialiseData(templateApi._storeHandle, {
hierarchy: root,
actions: [],
triggers: [],
})
const app = await appFromTempalteApi(templateApi)
app.hierarchy = root
return {
root, contact, lead, app,
deal, templateApi, store: templateApi._storeHandle,
all_contacts: root.indexes[0],
all_leads: root.indexes[1],
deals_for_contacts: contact.indexes[0],
}
}

2
packages/server/middleware/routeHandlers/index.js
View File

@ -18,6 +18,7 @@ const lookupField = require("./lookupField")
const getRecord = require("./getRecord")
const deleteRecord = require("./deleteRecord")
const saveAppHierarchy = require("./saveAppHierarchy")
const upgradeData = require("./saveAppHierarchy")
module.exports = {
authenticate,
@ -40,4 +41,5 @@ module.exports = {
getRecord,
deleteRecord,
saveAppHierarchy,
upgradeData,
}

6
packages/server/middleware/routeHandlers/upgradeData.js Normal file
View File

@ -0,0 +1,6 @@
const StatusCodes = require("../../utilities/statusCodes")
module.exports = async ctx => {
await ctx.instance.templateApi.upgradeData(ctx.request.body.newHierarchy)
ctx.response.status = StatusCodes.OK
}

4
packages/server/middleware/routers.js
View File

@ -238,6 +238,10 @@ module.exports = (config, app) => {
ctx.response.status = StatusCodes.UNAUTHORIZED
}
})
.post(
"/_builder/instance/:appname/:instanceid/api/upgradeData",
routeHandlers.upgradeData
)
.post("/:appname/api/changeMyPassword", routeHandlers.changeMyPassword)
.post(
"/_builder/instance/:appname/:instanceid/api/changeMyPassword",