Merge pull request #160 from mjashanks/master

Diff hierarchy & upgradeData

packages/builder/src/database/ActionsHeader.svelte

@@ -58,6 +58,7 @@
     padding: 1.5rem;
     width: 100%;
     align-items: right;
+    box-sizing: border-box;
   }
 
   .actions-modal-body {

packages/core/src/appInitialise/cloneApp.js

@@ -0,0 +1,8 @@
+import { setCleanupFunc } from "../transactions/setCleanupFunc"
+
+export const cloneApp = (app, mergeWith) => {
+  const newApp = { ...app }
+  Object.assign(newApp, mergeWith)
+  setCleanupFunc(newApp)
+  return newApp
+}

packages/core/src/appInitialise/initialiseData.js

@@ -21,22 +21,29 @@ export const initialiseData = async (
   applicationDefinition,
   accessLevels
 ) => {
-  await datastore.createFolder(configFolder)
+  if (!await datastore.exists(configFolder))
-  await datastore.createJson(appDefinitionFile, applicationDefinition)
+    await datastore.createFolder(configFolder)
+
+  if (!await datastore.exists(appDefinitionFile))
+    await datastore.createJson(appDefinitionFile, applicationDefinition)
 
   await initialiseRootCollections(datastore, applicationDefinition.hierarchy)
   await initialiseRootIndexes(datastore, applicationDefinition.hierarchy)
 
-  await datastore.createFolder(TRANSACTIONS_FOLDER)
+  if (!await datastore.exists(TRANSACTIONS_FOLDER))
+    await datastore.createFolder(TRANSACTIONS_FOLDER)
 
-  await datastore.createFolder(AUTH_FOLDER)
+  if (!await datastore.exists(AUTH_FOLDER))
+    await datastore.createFolder(AUTH_FOLDER)
 
-  await datastore.createJson(USERS_LIST_FILE, [])
+  if (!await datastore.exists(USERS_LIST_FILE))
+    await datastore.createJson(USERS_LIST_FILE, [])
 
-  await datastore.createJson(
+  if (!await datastore.exists(ACCESS_LEVELS_FILE))
-    ACCESS_LEVELS_FILE,
+    await datastore.createJson(
-    accessLevels ? accessLevels : { version: 0, levels: [] }
+      ACCESS_LEVELS_FILE,
-  )
+      accessLevels ? accessLevels : { version: 0, levels: [] }
+    )
 
   await initialiseRootSingleRecords(datastore, applicationDefinition.hierarchy)
 }
@@ -64,6 +71,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => {
   const singleRecords = $(flathierarchy, [filter(isSingleRecord)])
 
   for (let record of singleRecords) {
+    if (await datastore.exists(record.nodeKey())) continue
     await datastore.createFolder(record.nodeKey())
     const result = _getNew(record, "")
     await _save(app, result)

packages/core/src/index.js

@@ -7,7 +7,7 @@ import getActionsApi from "./actionsApi"
 import { setupDatastore, createEventAggregator } from "./appInitialise"
 import { initialiseActions } from "./actionsApi/initialise"
 import { isSomething, crypto } from "./common"
-import { cleanup } from "./transactions/cleanup"
+import { setCleanupFunc } from "./transactions/setCleanupFunc"
 import { generateFullPermissions } from "./authApi/generateFullPermissions"
 import { getApplicationDefinition } from "./templateApi/getApplicationDefinition"
 import common from "./common"
@@ -40,9 +40,7 @@ export const getAppApis = async (
 
   const templateApi = getTemplateApi(app)
 
-  app.cleanupTransactions = isSomething(cleanupTransactions)
+  setCleanupFunc(app, cleanupTransactions)
-    ? cleanupTransactions
-    : async () => await cleanup(app)
 
   app.getEpochTime = isSomething(getEpochTime)
     ? getEpochTime

packages/core/src/indexApi/buildIndex.js

@@ -6,8 +6,10 @@ import {
   getNode,
   isIndex,
   isRecord,
+  getActualKeyOfParent,
   getAllowedRecordNodesForIndex,
   fieldReversesReferenceToIndex,
+  isTopLevelIndex,
 } from "../templateApi/hierarchy"
 import { joinKey, apiWrapper, events, $ } from "../common"
 import {
@@ -16,6 +18,8 @@ import {
 } from "../transactions/create"
 import { permission } from "../authApi/permissions"
 import { BadRequestError } from "../common/errors"
+import { initialiseIndex } from "../indexing/initialiseIndex"
+import { getRecordInfo } from "../recordApi/recordInfo"
 
 /** rebuilds an index
  * @param {object} app - the application container
@@ -32,7 +36,7 @@ export const buildIndex = app => async indexNodeKey =>
     indexNodeKey
   )
 
-const _buildIndex = async (app, indexNodeKey) => {
+export const _buildIndex = async (app, indexNodeKey) => {
   const indexNode = getNode(app.hierarchy, indexNodeKey)
 
   await createBuildIndexFolder(app.datastore, indexNodeKey)
@@ -89,12 +93,6 @@ const buildReverseReferenceIndex = async (app, indexNode) => {
   }
 }
 
-/*
-const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [
-  map(n => n.parent()),
-]);
-*/
-
 const buildHeirarchalIndex = async (app, indexNode) => {
   let recordCount = 0
 
@@ -139,77 +137,8 @@ const buildHeirarchalIndex = async (app, indexNode) => {
   return recordCount
 }
 
-// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
-
 const recordNodeApplies = indexNode => recordNode =>
   includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds)
 
-/*
-const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [
-  getFlattenedHierarchy,
-  filter(
-    allTrue(
-      isRecord,
-      isDecendant(ancestorNode),
-      recordNodeApplies(indexNode),
-    ),
-  ),
-]);
-*/
-
-/*
-const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
-  indexNode, indexKey, currentIndexedData,
-  currentIndexedDataKey, recordCount = 0) => {
-  const collectionNode = getCollectionNodeByKeyOrNodeKey(
-    app.hierarchy,
-    collection_Key_or_NodeKey,
-  );
-
-  const allIdsIterator = await getAllIdsIterator(app)(collection_Key_or_NodeKey);
-
-
-  const createTransactionsForIds = async (collectionKey, allIds) => {
-    for (const recordId of allIds) {
-      const recordKey = joinKey(collectionKey, recordId);
-
-      const recordNode = chooseChildRecordNodeByKey(
-        collectionNode,
-        recordId,
-      );
-
-      if (recordNodeApplies(indexNode)(recordNode)) {
-        await transactionForBuildIndex(
-          app, indexNode.nodeKey(),
-          recordKey, recordCount,
-        );
-        recordCount++;
-      }
-
-      if (hasApplicableDecendant(app.hierarchy, recordNode, indexNode)) {
-        for (const childCollectionNode of recordNode.children) {
-          recordCount = await applyAllDecendantRecords(
-            app,
-            joinKey(recordKey, childCollectionNode.collectionName),
-            indexNode, indexKey, currentIndexedData,
-            currentIndexedDataKey, recordCount,
-          );
-        }
-      }
-    }
-  };
-
-  let allIds = await allIdsIterator();
-  while (allIds.done === false) {
-    await createTransactionsForIds(
-      allIds.result.collectionKey,
-      allIds.result.ids,
-    );
-    allIds = await allIdsIterator();
-  }
-
-  return recordCount;
-};
-*/
 
 export default buildIndex

packages/core/src/indexApi/listItems.js

@@ -33,7 +33,7 @@ const defaultOptions = {
   searchPhrase: null,
 }
 
-const _listItems = async (app, indexKey, options = defaultOptions) => {
+export const _listItems = async (app, indexKey, options = defaultOptions) => {
   const { searchPhrase, rangeStartParams, rangeEndParams } = $({}, [
     merge(options),
     merge(defaultOptions),

packages/core/src/indexing/allIds.js

@@ -2,6 +2,7 @@ import { flatten, orderBy, filter, isUndefined } from "lodash/fp"
 import {
   getFlattenedHierarchy,
   getCollectionNodeByKeyOrNodeKey,
+  getNodeByKeyOrNodeKey,
   isCollectionRecord,
   isAncestor,
 } from "../templateApi/hierarchy"
@@ -60,7 +61,7 @@ export const getAllIdsIterator = app => async collection_Key_or_NodeKey => {
   const recordNode = getCollectionNodeByKeyOrNodeKey(
     app.hierarchy,
     collection_Key_or_NodeKey
-  )
+  ) || getNodeByKeyOrNodeKey(app.hierarchy, collection_Key_or_NodeKey)
 
   const getAllIdsIteratorForCollectionKey = async (
     recordNode,

packages/core/src/indexing/initialiseIndex.js

@@ -9,11 +9,19 @@ import {
 export const initialiseIndex = async (datastore, dir, index) => {
   const indexDir = joinKey(dir, index.name)
 
-  await datastore.createFolder(indexDir)
+  let newDir = false
+  if (!await datastore.exists(indexDir)) {
+    await datastore.createFolder(indexDir)
+    newDir = true
+  }
 
   if (isShardedIndex(index)) {
-    await datastore.createFile(getShardMapKey(indexDir), "[]")
+    const shardFile = getShardMapKey(indexDir)
+    if (newDir || !await datastore.exists(shardFile))
+      await datastore.createFile(shardFile, "[]")
   } else {
-    await createIndexFile(datastore, getUnshardedIndexDataKey(indexDir), index)
+    const indexFile = getUnshardedIndexDataKey(indexDir)
+    if (newDir || !await datastore.exists(indexFile))
+      await createIndexFile(datastore, indexFile, index)
   }
 }

packages/core/src/indexing/read.js

@@ -3,6 +3,9 @@ import { promiseReadableStream } from "./promiseReadableStream"
 import { createIndexFile } from "./sharding"
 import { generateSchema } from "./indexSchemaCreator"
 import { getIndexReader, CONTINUE_READING_RECORDS } from "./serializer"
+import { getAllowedRecordNodesForIndex, getRecordNodeId } from "../templateApi/hierarchy"
+import { $ } from "../common"
+import { filter, includes, find } from "lodash/fp"
 
 export const readIndex = async (
   hierarchy,
@@ -11,8 +14,10 @@ export const readIndex = async (
   indexedDataKey
 ) => {
   const records = []
+  const getType = typeLoader(index, hierarchy)
   const doRead = iterateIndex(
     async item => {
+      item.type = getType(item.key)
       records.push(item)
       return CONTINUE_READING_RECORDS
     },
@@ -31,8 +36,10 @@ export const searchIndex = async (
 ) => {
   const records = []
   const schema = generateSchema(hierarchy, index)
+  const getType = typeLoader(index, hierarchy)
   const doRead = iterateIndex(
     async item => {
+      item.type = getType(item.key)
       const idx = lunr(function() {
         this.ref("key")
         for (const field of schema) {
@@ -76,3 +83,8 @@ export const iterateIndex = (onGetItem, getFinalResult) => async (
     return []
   }
 }
+
+const typeLoader = (index, hierarchy) => {
+  const allowedNodes = getAllowedRecordNodesForIndex(hierarchy, index)
+  return key => find(n => getRecordNodeId(key) === n.nodeId)(allowedNodes).name
+}

packages/core/src/recordApi/initialiseChildren.js

@@ -0,0 +1,81 @@
+import { isString, flatten, map, filter } from "lodash/fp"
+import { initialiseChildCollections } from "../collectionApi/initialise"
+import { _loadFromInfo } from "./load"
+import { $ } from "../common"
+import {
+  getFlattenedHierarchy,
+  isRecord,
+  getNode,
+  isTopLevelRecord,
+  fieldReversesReferenceToNode,
+} from "../templateApi/hierarchy"
+import { initialiseIndex } from "../indexing/initialiseIndex"
+import { getRecordInfo } from "./recordInfo"
+
+export const initialiseChildren = async (app, recordInfoOrKey) => {
+  const recordInfo = isString(recordInfoOrKey) 
+        ?  getRecordInfo(app.hierarchy, recordInfoOrKey)
+        : recordInfoOrKey
+  await initialiseReverseReferenceIndexes(app, recordInfo)
+  await initialiseAncestorIndexes(app, recordInfo)
+  await initialiseChildCollections(app, recordInfo)
+}
+
+export const initialiseChildrenForNode = async (app, recordNode) => {
+
+  if (isTopLevelRecord(recordNode)) {
+    await initialiseChildren(
+      app, recordNode.nodeKey())
+    return
+  }
+
+  const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
+  let iterateResult = await iterate()
+  while (!iterateResult.done) {
+    const { result } = iterateResult
+    for (const id of result.ids) {
+      const initialisingRecordKey = joinKey(
+        result.collectionKey, id)
+      await initialiseChildren(app, initialisingRecordKey)
+    }
+    iterateResult = await iterate()
+  }
+}
+
+const initialiseAncestorIndexes = async (app, recordInfo) => {
+  for (const index of recordInfo.recordNode.indexes) {
+    const indexKey = recordInfo.child(index.name)
+    if (!(await app.datastore.exists(indexKey))) {
+      await initialiseIndex(app.datastore, recordInfo.dir, index)
+    }
+  }
+}
+
+const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
+  const indexNodes = $(
+    fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
+    [
+      map(f =>
+        $(f.typeOptions.reverseIndexNodeKeys, [
+          map(n => getNode(app.hierarchy, n)),
+        ])
+      ),
+      flatten,
+    ]
+  )
+
+  for (const indexNode of indexNodes) {
+    await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
+  }
+}
+
+const fieldsThatReferenceThisRecord = (app, recordNode) =>
+  $(app.hierarchy, [
+    getFlattenedHierarchy,
+    filter(isRecord),
+    map(n => n.fields),
+    flatten,
+    filter(fieldReversesReferenceToNode(recordNode)),
+  ])
+
+

packages/core/src/recordApi/save.js

@@ -1,5 +1,4 @@
 import { cloneDeep, take, takeRight, flatten, map, filter } from "lodash/fp"
-import { initialiseChildCollections } from "../collectionApi/initialise"
 import { validate } from "./validate"
 import { _loadFromInfo } from "./load"
 import { apiWrapper, events, $, joinKey } from "../common"
@@ -17,6 +16,7 @@ import { permission } from "../authApi/permissions"
 import { initialiseIndex } from "../indexing/initialiseIndex"
 import { BadRequestError } from "../common/errors"
 import { getRecordInfo } from "./recordInfo"
+import { initialiseChildren } from "./initialiseChildren"
 
 export const save = app => async (record, context) =>
   apiWrapper(
@@ -59,9 +59,7 @@ export const _save = async (app, record, context, skipValidation = false) => {
     await createRecordFolderPath(app.datastore, pathInfo)
     await app.datastore.createFolder(files)
     await app.datastore.createJson(recordJson, recordClone)
-    await initialiseReverseReferenceIndexes(app, recordInfo)
+    await initialiseChildren(app, recordInfo)
-    await initialiseAncestorIndexes(app, recordInfo)
-    await initialiseChildCollections(app, recordInfo)
     await app.publish(events.recordApi.save.onRecordCreated, {
       record: recordClone,
     })
@@ -87,42 +85,6 @@ export const _save = async (app, record, context, skipValidation = false) => {
   return returnedClone
 }
 
-const initialiseAncestorIndexes = async (app, recordInfo) => {
-  for (const index of recordInfo.recordNode.indexes) {
-    const indexKey = recordInfo.child(index.name)
-    if (!(await app.datastore.exists(indexKey))) {
-      await initialiseIndex(app.datastore, recordInfo.dir, index)
-    }
-  }
-}
-
-const initialiseReverseReferenceIndexes = async (app, recordInfo) => {
-  const indexNodes = $(
-    fieldsThatReferenceThisRecord(app, recordInfo.recordNode),
-    [
-      map(f =>
-        $(f.typeOptions.reverseIndexNodeKeys, [
-          map(n => getNode(app.hierarchy, n)),
-        ])
-      ),
-      flatten,
-    ]
-  )
-
-  for (const indexNode of indexNodes) {
-    await initialiseIndex(app.datastore, recordInfo.dir, indexNode)
-  }
-}
-
-const fieldsThatReferenceThisRecord = (app, recordNode) =>
-  $(app.hierarchy, [
-    getFlattenedHierarchy,
-    filter(isRecord),
-    map(n => n.fields),
-    flatten,
-    filter(fieldReversesReferenceToNode(recordNode)),
-  ])
-
 const createRecordFolderPath = async (datastore, pathInfo) => {
   const recursiveCreateFolder = async (
     subdirs,

packages/core/src/templateApi/deleteAllIndexFilesForNode.js

@@ -0,0 +1,34 @@
+import { getAllIdsIterator } from "../indexing/allIds"
+import { getRecordInfo } from "../recordApi/recordInfo"
+import { isTopLevelIndex, getParentKey, getLastPartInKey } from "./hierarchy"
+import { safeKey, joinKey } from "../common"
+
+export const deleteAllIndexFilesForNode = async (app, indexNode) => {
+
+  if (isTopLevelIndex(indexNode)) {
+    await app.datastore.deleteFolder(indexNode.nodeKey())
+    return
+  }
+
+  const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
+  let iterateResult = await iterate()
+  while (!iterateResult.done) {
+    const { result } = iterateResult
+    for (const id of result.ids) {
+      const deletingIndexKey = joinKey(
+        result.collectionKey, id, indexNode.name)
+      await deleteIndexFolder(app, deletingIndexKey)
+    }
+    iterateResult = await iterate()
+  }
+
+}
+
+const deleteIndexFolder = async (app, indexKey) => {
+  indexKey = safeKey(indexKey)
+  const indexName = getLastPartInKey(indexKey)
+  const parentRecordKey = getParentKey(indexKey)
+  const recordInfo = getRecordInfo(app.hierarchy, parentRecordKey)
+  await app.datastore.deleteFolder(
+    joinKey(recordInfo.dir, indexName))
+}

packages/core/src/templateApi/deleteAllRecordsForNode.js

@@ -0,0 +1,32 @@
+import { getAllIdsIterator } from "../indexing/allIds"
+import { getCollectionDir } from "../recordApi/recordInfo"
+import { isTopLevelRecord, getCollectionKey } from "./hierarchy"
+import { safeKey, joinKey } from "../common"
+
+export const deleteAllRecordsForNode = async (app, recordNode) => {
+
+  if (isTopLevelRecord(recordNode)) {
+    await deleteRecordCollection(
+      app, recordNode.collectionName)
+    return
+  }
+
+  const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey())
+  let iterateResult = await iterate()
+  while (!iterateResult.done) {
+    const { result } = iterateResult
+    for (const id of result.ids) {
+      const deletingCollectionKey = joinKey(
+        result.collectionKey, id, recordNode.collectionName)
+      await deleteRecordCollection(app, deletingCollectionKey)
+    }
+    iterateResult = await iterate()
+  }
+
+}
+
+const deleteRecordCollection = async (app, collectionKey) => {
+  collectionKey = safeKey(collectionKey)
+  await app.datastore.deleteFolder(
+    getCollectionDir(app.hierarchy, collectionKey))
+}

packages/core/src/templateApi/diffHierarchy.js

@@ -1,6 +1,6 @@
 import { getFlattenedHierarchy, isRecord, isIndex, isAncestor } from "./hierarchy"
 import { $, none } from "../common"
-import { map, filter, some, find } from "lodash/fp"
+import { map, filter, some, find, difference } from "lodash/fp"
 
 export const HierarchyChangeTypes = {
   recordCreated: "Record Created",
@@ -10,7 +10,7 @@ export const HierarchyChangeTypes = {
   recordEstimatedRecordTypeChanged: "Record's Estimated Record Count Changed",
   indexCreated: "Index Created",
   indexDeleted: "Index Deleted",
-  indexChanged: "index Changed",
+  indexChanged: "Index Changed",
 }
 
 export const diffHierarchy = (oldHierarchy, newHierarchy) => {
@@ -123,7 +123,7 @@ const findDeletedIndexes = (oldHierarchyFlat, newHierarchyFlat, deletedRecords)
 
 const findUpdatedIndexes = (oldHierarchyFlat, newHierarchyFlat) => 
   $(oldHierarchyFlat, [
-    filter(isRecord),
+    filter(isIndex),
     filter(nodeExistsIn(newHierarchyFlat)),
     filter(nodeChanged(newHierarchyFlat, indexHasChanged)),
     map(n => changeItem(
@@ -150,6 +150,7 @@ const indexHasChanged = (_new, old) =>
   _new.map !== old.map 
   || _new.filter !== old.filter
   || _new.getShardName !== old.getShardName
+  || difference(_new.allowedRecordNodeIds)(old.allowedRecordNodeIds).length > 0
 
 const isFieldSame = f1 => f2 => 
   f1.name === f2.name && f1.type === f2.type

packages/core/src/templateApi/hierarchy.js

@@ -191,6 +191,22 @@ export const getAllowedRecordNodesForIndex = (appHierarchy, indexNode) => {
   }
 }
 
+export const getDependantIndexes = (hierarchy, recordNode) => {
+  const allIndexes = $(hierarchy, [ getFlattenedHierarchy, filter(isIndex)])
+  
+  const allowedAncestors = $(allIndexes, [
+    filter(isAncestorIndex),
+    filter(i => recordNodeIsAllowed(i)(recordNode)),
+  ])
+
+  const allowedReference = $(allIndexes, [
+    filter(isReferenceIndex),
+    filter(i => some(fieldReversesReferenceToIndex(i))(recordNode.fields))
+  ])
+
+  return [...allowedAncestors, ...allowedReference]
+}
+
 export const getNodeFromNodeKeyHash = hierarchy => hash =>
   $(hierarchy, [
     getFlattenedHierarchy,
@@ -206,13 +222,19 @@ export const isaggregateGroup = node =>
 export const isShardedIndex = node =>
   isIndex(node) && isNonEmptyString(node.getShardName)
 export const isRoot = node => isSomething(node) && node.isRoot()
+export const findRoot = node => isRoot(node) ? node : findRoot(node.parent())
 export const isDecendantOfARecord = hasMatchingAncestor(isRecord)
 export const isGlobalIndex = node => isIndex(node) && isRoot(node.parent())
 export const isReferenceIndex = node =>
   isIndex(node) && node.indexType === indexTypes.reference
 export const isAncestorIndex = node =>
   isIndex(node) && node.indexType === indexTypes.ancestor
-
+export const isTopLevelRecord = node => isRoot(node.parent()) && isRecord(node)
+export const isTopLevelIndex = node => isRoot(node.parent()) && isIndex(node)
+export const getCollectionKey = recordKey => $(recordKey, [
+  splitKey,
+  parts => joinKey(parts.slice(0, parts.length - 1))
+])
 export const fieldReversesReferenceToNode = node => field =>
   field.type === "reference" &&
   intersection(field.typeOptions.reverseIndexNodeKeys)(

packages/core/src/templateApi/index.js

@@ -28,6 +28,7 @@ import { saveApplicationHierarchy } from "./saveApplicationHierarchy"
 import { saveActionsAndTriggers } from "./saveActionsAndTriggers"
 import { all } from "../types"
 import { getBehaviourSources } from "./getBehaviourSources"
+import { upgradeData } from "./upgradeData"
 
 const api = app => ({
   getApplicationDefinition: getApplicationDefinition(app.datastore),
@@ -57,6 +58,7 @@ const api = app => ({
   validateNode,
   validateAll,
   validateTriggers,
+  upgradeData: upgradeData(app)
 })
 
 export const getTemplateApi = app => api(app)

packages/core/src/templateApi/initialiseNewIndex.js

@@ -0,0 +1,27 @@
+import { getAllIdsIterator } from "../indexing/allIds"
+import { getRecordInfo } from "../recordApi/recordInfo"
+import { isTopLevelIndex } from "./hierarchy"
+import { joinKey } from "../common"
+import { initialiseIndex } from "../indexing/initialiseIndex"
+
+export const initialiseNewIndex = async (app, indexNode) => {
+
+  if (isTopLevelIndex(indexNode)) {
+    await initialiseIndex(app.datastore, "/", indexNode)
+    return
+  }
+
+  const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey())
+  let iterateResult = await iterate()
+  while (!iterateResult.done) {
+    const { result } = iterateResult
+    for (const id of result.ids) {
+      const recordKey = joinKey(result.collectionKey, id)
+      await initialiseIndex(
+        app.datastore, 
+        getRecordInfo(app.hierarchy, recordKey).dir,
+        indexNode)
+    }
+    iterateResult = await iterate()
+  }
+}

packages/core/src/templateApi/upgradeData.js

@@ -1,17 +1,194 @@
- /*
+import { diffHierarchy, HierarchyChangeTypes } from "./diffHierarchy"
-const changeActions = {
+import { $, switchCase } from "../common"
-  rebuildIndex: indexNodeKey => ({
+import { 
-    type: "rebuildIndex",
+  differenceBy, 
-    indexNodeKey,
+  isEqual, 
-  }),
+  some, 
-  reshardRecords: recordNodeKey => ({
+  map, 
-    type: "reshardRecords",
+  filter,
-    recordNodeKey,
+  uniqBy,
-  }),
+  flatten
-  deleteRecords: recordNodeKey => ({
+} from "lodash/fp"
-    type: "reshardRecords",
+import { 
-    recordNodeKey,
+  findRoot, 
-  }),
+  getDependantIndexes, 
-  renameRecord
+  isTopLevelRecord, 
+  isAncestorIndex 
+} from "./hierarchy"
+import { generateSchema } from "../indexing/indexSchemaCreator"
+import { _buildIndex } from "../indexApi/buildIndex"
+import { constructHierarchy } from "./createNodes"
+import { deleteAllRecordsForNode } from "./deleteAllRecordsForNode"
+import { deleteAllIndexFilesForNode } from "./deleteAllIndexFilesForNode"
+import { cloneApp } from "../appInitialise/cloneApp"
+import { initialiseData } from "../appInitialise/initialiseData"
+import { initialiseChildrenForNode } from "../recordApi/initialiseChildren"
+import { initialiseNewIndex } from "./initialiseNewIndex"
+import { saveApplicationHierarchy } from "../templateApi/saveApplicationHierarchy"
+
+export const upgradeData = app => async newHierarchy => {
+  const diff = diffHierarchy(app.hierarchy, newHierarchy)
+  const changeActions = gatherChangeActions(diff)
+
+  if (changeActions.length === 0) return
+
+  newHierarchy = constructHierarchy(newHierarchy)
+  const newApp = newHierarchy && cloneApp(app, {
+     hierarchy: newHierarchy 
+  })
+  await doUpgrade(app, newApp, changeActions)
+  await saveApplicationHierarchy(newApp)(newHierarchy)
+}
+
+const gatherChangeActions = (diff) => 
+  $(diff, [
+    map(actionForChange),
+    flatten,
+    uniqBy(a => a.compareKey)
+  ])
+
+const doUpgrade = async (oldApp, newApp, changeActions) => {
+  for(let action of changeActions) {
+    await action.run(oldApp, newApp, action.diff)
+  }
+}
+
+const actionForChange = diff => 
+  switchCase(
+
+    [isChangeType(HierarchyChangeTypes.recordCreated), recordCreatedAction],
+
+    [isChangeType(HierarchyChangeTypes.recordDeleted), deleteRecordsAction],
+
+    [
+      isChangeType(HierarchyChangeTypes.recordFieldsChanged), 
+      rebuildAffectedIndexesAction
+    ],
+
+    [isChangeType(HierarchyChangeTypes.recordRenamed), renameRecordAction],
+
+    [
+      isChangeType(HierarchyChangeTypes.recordEstimatedRecordTypeChanged), 
+      reshardRecordsAction
+    ],
+
+    [isChangeType(HierarchyChangeTypes.indexCreated), newIndexAction],
+
+    [isChangeType(HierarchyChangeTypes.indexDeleted), deleteIndexAction],
+
+    [isChangeType(HierarchyChangeTypes.indexChanged), rebuildIndexAction],
+
+  )(diff)
+
+
+const isChangeType = changeType => change => 
+  change.type === changeType
+
+const action = (diff, compareKey, run) => ({
+  diff,
+  compareKey, 
+  run,
+})
+
+ 
+const reshardRecordsAction = diff =>
+  [action(diff, `reshardRecords-${diff.oldNode.nodeKey()}`, runReshardRecords)]
+
+const rebuildIndexAction = diff =>
+  [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
+
+const newIndexAction = diff => {
+  if (isAncestorIndex(diff.newNode)) {
+    return [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)]
+  } else {
+    return [action(diff, `newIndex-${diff.newNode.nodeKey()}`, runNewIndex)]
+  }
+}
+
+const deleteIndexAction = diff =>
+  [action(diff, `deleteIndex-${diff.oldNode.nodeKey()}`, runDeleteIndex)]
+
+const deleteRecordsAction = diff =>
+  [action(diff, `deleteRecords-${diff.oldNode.nodeKey()}`, runDeleteRecords)]
+
+const renameRecordAction = diff =>
+  [action(diff, `renameRecords-${diff.oldNode.nodeKey()}`, runRenameRecord)]
+
+const recordCreatedAction = diff => {
+  if (isTopLevelRecord(diff.newNode)) {
+    return [action(diff, `initialiseRoot`, runInitialiseRoot)]
+  }
+
+  return [action(diff, `initialiseChildRecord-${diff.newNode.nodeKey()}`, runInitialiseChildRecord)]
+}
+
+const rebuildAffectedIndexesAction = diff =>{
+  const newHierarchy = findRoot(diff.newNode)
+  const oldHierarchy = findRoot(diff.oldNode)
+  const indexes = getDependantIndexes(newHierarchy, diff.newNode)
+
+  const changedFields = (() => {
+    const addedFields = differenceBy(f => f.name)
+                          (diff.oldNode.fields) 
+                          (diff.newNode.fields)
+
+    const removedFields = differenceBy(f => f.name)
+                            (diff.newNode.fields) 
+                            (diff.oldNode.fields)
+    
+    return map(f => f.name)([...addedFields, ...removedFields])
+  })()
+
+  const isIndexAffected = i => {
+    if (!isEqual(
+      generateSchema(oldHierarchy, i),
+      generateSchema(newHierarchy, i))) return true
+    
+    if (some(f => indexes.filter.indexOf(`record.${f}`) > -1)(changedFields))
+      return true
+
+    if (some(f => indexes.getShardName.indexOf(`record.${f}`) > -1)(changedFields))
+      return true
+
+    return false
+  }
+
+  return $(indexes, [
+    filter(isIndexAffected),
+    map(i => action({ newNode:i }, `rebuildIndex-${i.nodeKey()}`, runRebuildIndex))
+  ])
+}
+ 
+const runReshardRecords = async change => {
+  throw new Error("Resharding of records is not supported yet")
+}
+
+const runRebuildIndex = async (_, newApp, diff) => {
+  await _buildIndex(newApp, diff.newNode.nodeKey())
+}
+
+const runDeleteIndex = async  (oldApp, _, diff) => {
+  await deleteAllIndexFilesForNode(oldApp, diff.oldNode)
+}
+
+const runDeleteRecords = async (oldApp, _, diff) => {
+  await deleteAllRecordsForNode(oldApp, diff.oldNode)
+}
+
+const runNewIndex = async (_, newApp, diff) => {
+  await initialiseNewIndex(newApp, diff.newNode)
+}
+
+const runRenameRecord = change => {
+  /*
+  Going to disllow this in the builder. once a collection key is set... its done
+  */
+}
+
+const runInitialiseRoot = async (_, newApp) => {
+  await initialiseData(newApp.datastore, newApp)
+}
+
+const runInitialiseChildRecord = async (_, newApp, diff) => {
+  await initialiseChildrenForNode(newApp.datastore, diff.newNode)
 } 
-*/

packages/core/src/transactions/cleanup.js

@@ -14,8 +14,10 @@ export const cleanup = async app => {
   const lock = await getTransactionLock(app)
   if (isNolock(lock)) return
 
-  try {
+  const _cleanupBatch = async () => {
+    let processed = 0
     const transactions = await retrieve(app)
+    let i = 1
     if (transactions.length > 0) {
       await executeTransactions(app)(transactions)
 
@@ -34,10 +36,21 @@ export const cleanup = async app => {
       ])
 
       await Promise.all(deleteFiles)
+
+      processed = transactions.length
+    }
+    return processed
+  }
+
+  try {
+    let count = -1
+    while (count !== 0) {
+      count = await _cleanupBatch()
     }
   } finally {
     await releaseLock(app, lock)
   }
+  
 }
 
 const getTransactionLock = async app =>

packages/core/src/transactions/execute.js

@@ -1,6 +1,7 @@
 import {
   filter,
   map,
+  reduce,
   isUndefined,
   includes,
   flatten,
@@ -10,6 +11,7 @@ import {
   keys,
   differenceBy,
   difference,
+  some,
 } from "lodash/fp"
 import { union } from "lodash"
 import {
@@ -38,14 +40,22 @@ import {
   fieldReversesReferenceToIndex,
   isReferenceIndex,
   getExactNodeForKey,
+  getParentKey
 } from "../templateApi/hierarchy"
 import { getRecordInfo } from "../recordApi/recordInfo"
 import { getIndexDir } from "../indexApi/getIndexDir"
+import { initialiseIndex } from "../indexing/initialiseIndex"
 
 export const executeTransactions = app => async transactions => {
   const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions)
 
   for (const shard of keys(recordsByShard)) {
+    if (recordsByShard[shard].isRebuild)
+      await initialiseIndex(
+        app.datastore, 
+        getParentKey(recordsByShard[shard].indexDir),
+        recordsByShard[shard].indexNode
+      )
     await applyToShard(
       app.hierarchy,
       app.datastore,
@@ -66,9 +76,9 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
 
   const indexBuild = getBuildIndexTransactionsByShard(hierarchy, transactions)
 
-  const toRemove = [...deletes, ...updates.toRemove]
+  const toRemove = [...deletes, ...updates.toRemove, ...indexBuild.toRemove]
 
-  const toWrite = [...created, ...updates.toWrite, ...indexBuild]
+  const toWrite = [...created, ...updates.toWrite, ...indexBuild.toWrite]
 
   const transByShard = {}
 
@@ -77,6 +87,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
       transByShard[t.indexShardKey] = {
         writes: [],
         removes: [],
+        isRebuild: some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toWrite)
+                   || some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toRemove),
         indexDir: t.indexDir,
         indexNodeKey: t.indexNode.nodeKey(),
         indexNode: t.indexNode,
@@ -207,7 +219,7 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => {
 
 const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
   const buildTransactions = $(transactions, [filter(isBuildIndex)])
-  if (!isNonEmptyArray(buildTransactions)) return []
+  if (!isNonEmptyArray(buildTransactions)) return { toWrite:[], toRemove:[] }
   const indexNode = transactions.indexNode
 
   const getIndexDirs = t => {
@@ -248,7 +260,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
   return $(buildTransactions, [
     map(t => {
       const mappedRecord = evaluate(t.record)(indexNode) 
-      if (!mappedRecord.passedFilter) return null
+      mappedRecord.result = mappedRecord.result || t.record
       const indexDirs = getIndexDirs(t)
       return $(indexDirs, [
         map(indexDir => ({
@@ -262,9 +274,16 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
           ),
         })),
       ])
+      
     }),
     flatten,
-    filter(isSomething),
+    reduce((obj, res) => {
+      if (res.mappedRecord.passedFilter)
+        obj.toWrite.push(res)
+      else 
+        obj.toRemove.push(res)
+      return obj
+    }, { toWrite: [], toRemove: [] })
   ])
 }
 

packages/core/src/transactions/retrieve.js

@@ -22,32 +22,41 @@ export const retrieve = async app => {
     TRANSACTIONS_FOLDER
   )
 
-  let transactions = []
-
   if (some(isBuildIndexFolder)(transactionFiles)) {
-    const buildIndexFolder = find(isBuildIndexFolder)(transactionFiles)
+    const buildIndexFolders = filter(isBuildIndexFolder)(transactionFiles)
+    let currentFolderIndex = 0
+    while (currentFolderIndex < buildIndexFolders.length) {
+      const buildIndexFolder = buildIndexFolders[currentFolderIndex]
+      const transactions = await retrieveBuildIndexTransactions(
+        app,
+        joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)
+      )
+      if(transactions.length === 0) {
+        await app.datastore.deleteFolder(
+          joinKey(TRANSACTIONS_FOLDER, buildIndexFolder))
+      } else {
+        return transactions
+      }
+      currentFolderIndex += 1
+    }
   
-    transactions = await retrieveBuildIndexTransactions(
+    return []
-      app,
-      joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)
-    )
   }
 
-  if (transactions.length > 0) return transactions
-
   return await retrieveStandardTransactions(app, transactionFiles)
 }
 
 const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
   const childFolders = await app.datastore.getFolderContents(buildIndexFolder)
-  if (childFolders.length === 0) {
+  const childFolderCount = childFolders.length
-    // cleanup
+  if (childFolderCount === 0) {
-    await app.datastore.deleteFolder(buildIndexFolder)
     return []
   }
 
   const getTransactionFiles = async (childFolderIndex = 0) => {
-    if (childFolderIndex >= childFolders.length) return []
+    if (childFolderIndex >= childFolders.length) {
+      return { childFolderKey: "", files: [] }
+    }
 
     const childFolderKey = joinKey(
       buildIndexFolder,
@@ -55,17 +64,19 @@ const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => {
     )
     const files = await app.datastore.getFolderContents(childFolderKey)
 
-    if (files.length === 0) {
+    if (files.length > 0) {
-      await app.datastore.deleteFolder(childFolderKey)
+      return { childFolderKey, files } 
-      return await getTransactionFiles(childFolderIndex + 1)
     }
 
-    return { childFolderKey, files }
+    await app.datastore.deleteFolder(childFolderKey)
+    return await getTransactionFiles(childFolderIndex + 1)
   }
 
   const transactionFiles = await getTransactionFiles()
 
-  if (transactionFiles.files.length === 0) return []
+  if (transactionFiles.files.length === 0) {
+    return []
+  }
 
   const transactions = $(transactionFiles.files, [map(parseTransactionId)])
 

packages/core/src/transactions/setCleanupFunc.js

@@ -0,0 +1,14 @@
+import { cleanup } from "./cleanup"
+
+export const setCleanupFunc = (app, cleanupTransactions) => {
+  if (cleanupTransactions) {
+    app.cleanupTransactions = cleanupTransactions
+    return
+  }
+
+  if (!app.cleanupTransactions || app.cleanupTransactions.isDefault) {
+    const newCleanup = async () => cleanup(app)
+    newCleanup.isDefault = true
+    app.cleanupTransactions = newCleanup
+  }
+}

packages/core/src/transactions/transactionsCommon.js

@@ -1,18 +1,20 @@
 import { joinKey, keySep, getHashCode } from "../common"
 import { getLastPartInKey } from "../templateApi/hierarchy"
+import { includes } from "lodash/fp"
 
 export const TRANSACTIONS_FOLDER = `${keySep}.transactions`
 export const LOCK_FILENAME = "lock"
 export const LOCK_FILE_KEY = joinKey(TRANSACTIONS_FOLDER, LOCK_FILENAME)
 export const idSep = "$"
 
-const isOfType = typ => trans => trans.transactionType === typ
+const isOfType = (...typ) => trans => includes(trans.transactionType)(typ)
 
 export const CREATE_RECORD_TRANSACTION = "create"
 export const UPDATE_RECORD_TRANSACTION = "update"
 export const DELETE_RECORD_TRANSACTION = "delete"
 export const BUILD_INDEX_TRANSACTION = "build"
 
+export const isUpdate_Or_Rebuild = isOfType(UPDATE_RECORD_TRANSACTION, BUILD_INDEX_TRANSACTION)
 export const isUpdate = isOfType(UPDATE_RECORD_TRANSACTION)
 export const isDelete = isOfType(DELETE_RECORD_TRANSACTION)
 export const isCreate = isOfType(CREATE_RECORD_TRANSACTION)

packages/core/test/specHelpers.js

@@ -24,7 +24,7 @@ import { filter, find } from "lodash/fp"
 import { createBehaviourSources } from "../src/actionsApi/buildBehaviourSource"
 import { createAction, createTrigger } from "../src/templateApi/createActions"
 import { initialiseActions } from "../src/actionsApi/initialise"
-import { cleanup } from "../src/transactions/cleanup"
+import { setCleanupFunc } from "../src/transactions/setCleanupFunc"
 import { permission } from "../src/authApi/permissions"
 import { generateFullPermissions } from "../src/authApi/generateFullPermissions"
 import { initialiseData } from "../src/appInitialise/initialiseData"
@@ -39,9 +39,9 @@ export const testTemplatesPath = testAreaName =>
   path.join(testFileArea(testAreaName), templateDefinitions)
 
 export const getMemoryStore = () => setupDatastore(memory({}))
-export const getMemoryTemplateApi = () => {
+export const getMemoryTemplateApi = (store) => {
   const app = {
-    datastore: getMemoryStore(),
+    datastore: store || getMemoryStore(),
     publish: () => {},
     getEpochTime: async () => new Date().getTime(),
     user: { name: "", permissions: [permission.writeTemplates.get()] },
@@ -78,8 +78,9 @@ export const appFromTempalteApi = async (
   const fullPermissions = generateFullPermissions(app)
   app.user.permissions = fullPermissions
 
-  if (disableCleanupTransactions) app.cleanupTransactions = async () => {}
+  if (disableCleanupTransactions) setCleanupFunc(app, async () => {})
-  else app.cleanupTransactions = async () => await cleanup(app)
+  else setCleanupFunc(app)
+
   return app
 }
 
@@ -100,8 +101,9 @@ export const getRecordApiFromTemplateApi = async (
   disableCleanupTransactions = false
 ) => {
   const app = await appFromTempalteApi(templateApi, disableCleanupTransactions)
-  const recordapi = getRecordApi()
+  const recordapi = getRecordApi(app)
   recordapi._storeHandle = app.datastore
+  return recordapi
 }
 
 export const getCollectionApiFromTemplateApi = async (

packages/core/test/templateApi.diffHierarchy.spec.js

@@ -1,6 +1,5 @@
-import { getMemoryTemplateApi } from "./specHelpers"
+import { setup } from "./upgradeDataSetup"
 import { diffHierarchy, HierarchyChangeTypes } from "../src/templateApi/diffHierarchy"
-import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
 
 describe("diffHierarchy", () => {
 
@@ -13,7 +12,7 @@ describe("diffHierarchy", () => {
 
   it("should detect root record created", async () => {
     const oldHierarchy = (await setup()).root;
-    const newSetup = (await setup());
+    const newSetup = await setup()
     const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
     const diff = diffHierarchy(oldHierarchy, newSetup.root)
     expect(diff).toEqual([{
@@ -25,7 +24,7 @@ describe("diffHierarchy", () => {
 
   it("should only detect root record, when newly created root record has children ", async () => {
     const oldHierarchy = (await setup()).root;
-    const newSetup = (await setup());
+    const newSetup = await setup()
     const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
     newSetup.templateApi.getNewRecordTemplate(opportunity, "invoice", true)
     const diff = diffHierarchy(oldHierarchy, newSetup.root)
@@ -38,7 +37,7 @@ describe("diffHierarchy", () => {
 
   it("should detect child record created", async () => {
     const oldHierarchy = (await setup()).root;
-    const newSetup = (await setup());
+    const newSetup = await setup()
     const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.contact, "opportunity", false)
     const diff = diffHierarchy(oldHierarchy, newSetup.root)
     expect(diff).toEqual([{
@@ -49,8 +48,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect root record deleted", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -61,8 +60,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect child record deleted", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -73,8 +72,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect root record renamed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.contact.collectionKey = "CONTACTS"
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -85,8 +84,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect child record renamed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.deal.collectionKey = "CONTACTS"
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -97,8 +96,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect root record field removed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.contact.fields =  newSetup.contact.fields.filter(f => f.name !== "name")
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -109,8 +108,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect child record field removed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.deal.fields =  newSetup.deal.fields.filter(f => f.name !== "name")
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -121,8 +120,8 @@ describe("diffHierarchy", () => {
   })
   
   it("should detect record field added", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     const notesField = newSetup.templateApi.getNewField("string")
     notesField.name = "notes"
     newSetup.templateApi.addField(newSetup.contact, notesField)
@@ -136,8 +135,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect 1 record field added and 1 removed (total no. fields unchanged)", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     const notesField = newSetup.templateApi.getNewField("string")
     notesField.name = "notes"
     newSetup.templateApi.addField(newSetup.contact, notesField)
@@ -151,8 +150,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect root record estimated record count changed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.contact.estimatedRecordCount =  987
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -163,8 +162,8 @@ describe("diffHierarchy", () => {
   })
 
   it("should detect root record estimated record count changed", async () => {
-    const oldSetup = (await setup());
+    const oldSetup = await setup()
-    const newSetup = (await setup());
+    const newSetup = await setup()
     newSetup.deal.estimatedRecordCount =  987
     const diff = diffHierarchy(oldSetup.root, newSetup.root)
     expect(diff).toEqual([{
@@ -174,44 +173,97 @@ describe("diffHierarchy", () => {
     }])
   })
 
-  it("should detect root record created", async () => {
+  it("should detect root index created", async () => {
-    const oldHierarchy = (await setup()).root;
+    const oldHierarchy = (await setup()).root
-    const newSetup = (await setup());
+    const newSetup = await setup()
-    const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false)
+    const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
     const diff = diffHierarchy(oldHierarchy, newSetup.root)
     expect(diff).toEqual([{
-      newNode: opportunity,
+      newNode: all_deals,
       oldNode: null,
-      type: HierarchyChangeTypes.recordCreated
+      type: HierarchyChangeTypes.indexCreated
     }])
   })
 
+  it("should detect child index created", async () => {
+    const oldHierarchy = (await setup()).root
+    const newSetup = await setup()
+    const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
+    const diff = diffHierarchy(oldHierarchy, newSetup.root)
+    expect(diff).toEqual([{
+      newNode: all_deals,
+      oldNode: null,
+      type: HierarchyChangeTypes.indexCreated
+    }])
+  })
+
+  it("should detect root index deleted", async () => {
+    const oldSetup = await setup()
+    const newSetup = await setup()
+    newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
+    const diff = diffHierarchy(oldSetup.root, newSetup.root)
+    expect(diff).toEqual([{
+      newNode: null,
+      oldNode: oldSetup.root.indexes.find(i => i.name === "contact_index"),
+      type: HierarchyChangeTypes.indexDeleted
+    }])
+  })
+
+  it("should detect child index deleted", async () => {
+    const oldSetup = await setup()
+    const newSetup = await setup()
+    newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
+    const diff = diffHierarchy(oldSetup.root, newSetup.root)
+    expect(diff).toEqual([{
+      newNode: null,
+      oldNode: oldSetup.contact.indexes.find(i => i.name === "deal_index"),
+      type: HierarchyChangeTypes.indexDeleted
+    }])
+  })
+
+  const testIndexChanged = (parent, makechange) => async () => {
+    const oldSetup = await setup()
+    const newSetup = await setup()
+    makechange(newSetup)
+    const diff = diffHierarchy(oldSetup.root, newSetup.root)
+    expect(diff).toEqual([{
+      newNode: newSetup[parent].indexes[0],
+      oldNode: oldSetup[parent].indexes[0],
+      type: HierarchyChangeTypes.indexChanged
+    }])
+  }
+
+  it("should detect root index map changed", testIndexChanged("root", newSetup => {
+    newSetup.root.indexes[0].map = "new"
+  }))
+
+  it("should detect root index filter changed", testIndexChanged("root", newSetup => {
+    newSetup.root.indexes[0].filter = "new"
+  }))
+
+  it("should detect root index shardName changed", testIndexChanged("root", newSetup => {
+    newSetup.root.indexes[0].getShardName = "new"
+  }))
+
+  it("should detect root index allowedRecordIds changed", testIndexChanged("root", newSetup => {
+    newSetup.root.indexes[0].allowedRecordNodeIds.push(3)
+  }))
+
+  it("should detect child index allowedRecordIds changed", testIndexChanged("contact", newSetup => {
+    newSetup.contact.indexes[0].allowedRecordNodeIds.push(3)
+  }))
+
+  it("should detect child index map changed", testIndexChanged("contact", newSetup => {
+    newSetup.contact.indexes[0].map = "new"
+  }))
+
+  it("should detect child index filter changed", testIndexChanged("contact", newSetup => {
+    newSetup.contact.indexes[0].filter = "new"
+  }))
+
+  it("should detect child index shardName changed", testIndexChanged("contact", newSetup => {
+    newSetup.contact.indexes[0].getShardName = "new"
+  }))
+
 })
 
-const setup = async () => {
-  const { templateApi } = await getMemoryTemplateApi()
-  const root = templateApi.getNewRootLevel()
-  const contact = templateApi.getNewRecordTemplate(root, "contact", true)
-
-  const nameField = templateApi.getNewField("string")
-  nameField.name = "name"
-  const statusField = templateApi.getNewField("string")
-  statusField.name = "status"
-
-  templateApi.addField(contact, nameField)
-  templateApi.addField(contact, statusField)
-
-  const lead = templateApi.getNewRecordTemplate(root, "lead", true)
-  const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
-
-  templateApi.addField(deal, {...nameField})
-  templateApi.addField(deal, {...statusField})
-
-  getFlattenedHierarchy(root)
-  return {
-    root, contact, lead, deal, templateApi,
-    all_contacts: root.indexes[0],
-    all_leads: root.indexes[1],
-    deals_for_contacts: contact.indexes[0]
-  }
-}

packages/core/test/templateApi.upgradeData.spec.js

@@ -0,0 +1,244 @@
+import { 
+  getRecordApiFromTemplateApi,
+  getIndexApiFromTemplateApi,
+} from "./specHelpers"
+import { upgradeData } from "../src/templateApi/upgradeData"
+import { setup } from "./upgradeDataSetup"
+import { $, splitKey } from "../src/common"
+import { keys, filter } from "lodash/fp"
+import { _listItems } from "../src/indexApi/listItems"
+import { _save } from "../src/recordApi/save"
+
+describe("upgradeData", () => {
+
+  it("should delete all records and child records, when root record node deleted", async () => {
+    const { oldSetup, newSetup, recordApi } = await configure()
+    newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const remainingKeys = $(recordApi._storeHandle.data, [
+      keys,
+      filter(k => splitKey(k)[0] === "contacts"),
+    ])
+
+    expect(remainingKeys.length).toBe(0)
+    
+  })
+
+  it("should not delete other root record types, when root record node deleted", async () => {
+    const { oldSetup, newSetup, recordApi } = await configure()
+    newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact")
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const remainingKeys = $(recordApi._storeHandle.data, [
+      keys,
+      filter(k => splitKey(k)[0] === "leads"),
+    ])
+
+    expect(remainingKeys.length > 0).toBe(true)
+    
+  })
+
+  it("should delete all child records, when child record node deleted", async () => {
+    const { oldSetup, newSetup, recordApi } = await configure()
+    newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal")
+
+    const startingKeys = $(recordApi._storeHandle.data, [
+      keys,
+      filter(k => k.includes("/deals/")),
+    ])
+
+    expect(startingKeys.length > 0).toBe(true)
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const remainingKeys = $(recordApi._storeHandle.data, [
+      keys,
+      filter(k => k.includes("/deals/")),
+    ])
+
+    expect(remainingKeys.length).toBe(0)
+  })
+
+  it("should build a new root index", async () => {
+    const { oldSetup, newSetup } = await configure()
+    const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.root)
+    newIndex.name = "more_contacts"
+    newIndex.allowedRecordNodeIds = [newSetup.contact.nodeId]
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const itemsInNewIndex = await _listItems(newSetup.app, "/more_contacts")
+
+    expect(itemsInNewIndex.length).toBe(2)
+  })
+
+  it("should update a root index", async () => {
+    const { oldSetup, newSetup } = await configure()
+    const contact_index = indexByName(newSetup.root, "contact_index")
+    contact_index.filter = "record.name === 'bobby'"
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const itemsInNewIndex = await _listItems(newSetup.app, "/contact_index")
+
+    expect(itemsInNewIndex.length).toBe(1)
+  })
+
+  it("should delete a root index", async () => {
+    const { oldSetup, newSetup } = await configure()
+
+    // no exception
+    await _listItems(newSetup.app, "/contact_index")
+
+    newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index")
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    let er
+    try {
+      await _listItems(newSetup.app, "/contact_index")
+    } catch (e) {
+      er = e
+    }
+
+    expect(er).toBeDefined()
+  })
+
+  it("should build a new child index", async () => {
+    const { oldSetup, newSetup, records } = await configure()
+    const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.contact)
+    newIndex.name = "more_deals"
+    newIndex.allowedRecordNodeIds = [newSetup.deal.nodeId]
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const itemsInNewIndex = await _listItems(newSetup.app, `${records.contact1.key}/more_deals`)
+
+    expect(itemsInNewIndex.length).toBe(2)
+  })
+
+  it("should update a child index", async () => {
+    const { oldSetup, newSetup, records } = await configure()
+    const deal_index = indexByName(newSetup.contact, "deal_index")
+    deal_index.filter = "record.status === 'new'"
+
+    let itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
+    expect(itemsInIndex.length).toBe(2)
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
+    expect(itemsInIndex.length).toBe(1)
+  })
+
+  it("should delete a child index", async () => {
+    const { oldSetup, newSetup, records } = await configure()
+
+    // no exception
+    await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
+
+    newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index")
+
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    let er
+    try {
+      await _listItems(newSetup.app, `${records.contact1.key}/deal_index`)
+    } catch (e) {
+      er = e
+    }
+
+    expect(er).toBeDefined()
+  })
+
+  it("should build a new reference index", async () => {
+    const { oldSetup, newSetup, records, recordApi } = await configure()
+    const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.lead)
+    newIndex.name = "contact_leads"
+    newIndex.allowedRecordNodeIds = [newSetup.lead.nodeId]
+    newIndex.indexType = "reference"
+
+    const leadField = newSetup.templateApi.getNewField("string")
+    leadField.name = "lead"
+    leadField.type = "reference"
+    leadField.typeOptions = {
+      reverseIndexNodeKeys: [ newIndex.nodeKey() ],
+      indexNodeKey: "/lead_index",
+      displayValue: "name"
+    }
+
+    newSetup.templateApi.addField(newSetup.contact, leadField)
+    
+    await upgradeData(oldSetup.app)(newSetup.root)
+
+    const indexKey = `${records.lead1.key}/contact_leads`
+
+    let itemsInNewIndex = await _listItems(newSetup.app, indexKey)
+
+    expect(itemsInNewIndex.length).toBe(0)
+
+    records.contact1.lead = records.lead1
+    records.contact1.isNew = false
+
+    await _save(newSetup.app, records.contact1)
+
+    itemsInNewIndex = await _listItems(newSetup.app, indexKey)
+
+    expect(itemsInNewIndex.length).toBe(1)
+
+  })
+
+})
+
+const configure = async () => {
+  const oldSetup = await setup()
+  
+  const recordApi = await getRecordApiFromTemplateApi(oldSetup.templateApi)
+  const indexApi = await getIndexApiFromTemplateApi(oldSetup.templateApi)
+  
+  const newSetup = await setup(oldSetup.store)
+
+  const records = await createSomeRecords(recordApi)
+
+  return { oldSetup, newSetup, recordApi, records, indexApi }
+}
+
+const createSomeRecords = async recordApi => {
+  const contact1 = recordApi.getNew("/contacts", "contact")
+  contact1.name = "bobby"
+  const contact2 = recordApi.getNew("/contacts", "contact")
+  contact2.name = "poppy"
+
+  await recordApi.save(contact1)
+  await recordApi.save(contact2)
+
+  const deal1 = recordApi.getNew(`${contact1.key}/deals`, "deal")
+  deal1.name = "big mad deal"
+  deal1.status = "new"
+  const deal2 = recordApi.getNew(`${contact1.key}/deals`, "deal")
+  deal2.name = "smaller deal"
+  deal2.status = "old"
+  const deal3 = recordApi.getNew(`${contact2.key}/deals`, "deal")
+  deal3.name = "ok deal"
+  deal3.status = "new"
+
+  await recordApi.save(deal1)
+  await recordApi.save(deal2)
+  await recordApi.save(deal3)
+
+  const lead1 = recordApi.getNew("/leads", "lead")
+  lead1.name = "big new lead"
+
+  await recordApi.save(lead1)
+
+
+
+  return {
+    contact1, contact2, deal1, deal2, deal3, lead1,
+  }
+}
+
+const indexByName = (parent, name) => parent.indexes.find(i => i.name === name)

packages/core/test/upgradeDataSetup.js

@@ -0,0 +1,47 @@
+import { getMemoryTemplateApi, appFromTempalteApi } from "./specHelpers"
+import { getFlattenedHierarchy } from "../src/templateApi/hierarchy"
+import { initialiseData } from "../src/appInitialise/initialiseData"
+
+export const setup = async (store) => {
+  const { templateApi } = await getMemoryTemplateApi(store)
+  const root = templateApi.getNewRootLevel()
+  const contact = templateApi.getNewRecordTemplate(root, "contact", true)
+  contact.collectionName = "contacts"
+
+  const nameField = templateApi.getNewField("string")
+  nameField.name = "name"
+  const statusField = templateApi.getNewField("string")
+  statusField.name = "status"
+
+  templateApi.addField(contact, nameField)
+  templateApi.addField(contact, statusField)
+
+  const lead = templateApi.getNewRecordTemplate(root, "lead", true)
+  lead.collectionName = "leads"
+  const deal = templateApi.getNewRecordTemplate(contact, "deal", true)
+  deal.collectionName = "deals"
+
+  templateApi.addField(deal, {...nameField})
+  templateApi.addField(deal, {...statusField})
+
+  templateApi.addField(lead, {...nameField})
+
+  getFlattenedHierarchy(root)
+
+  if (!store)
+    await initialiseData(templateApi._storeHandle, {
+      hierarchy: root,
+      actions: [],
+      triggers: [],
+    })
+  const app = await appFromTempalteApi(templateApi)
+  app.hierarchy = root
+
+  return {
+    root, contact, lead, app,
+    deal, templateApi, store: templateApi._storeHandle,
+    all_contacts: root.indexes[0],
+    all_leads: root.indexes[1],
+    deals_for_contacts: contact.indexes[0],
+  }
+}

packages/server/middleware/routeHandlers/index.js

@@ -18,6 +18,7 @@ const lookupField = require("./lookupField")
 const getRecord = require("./getRecord")
 const deleteRecord = require("./deleteRecord")
 const saveAppHierarchy = require("./saveAppHierarchy")
+const upgradeData = require("./saveAppHierarchy")
 
 module.exports = {
   authenticate,
@@ -40,4 +41,5 @@ module.exports = {
   getRecord,
   deleteRecord,
   saveAppHierarchy,
+  upgradeData,
 }

packages/server/middleware/routeHandlers/upgradeData.js

@@ -0,0 +1,6 @@
+const StatusCodes = require("../../utilities/statusCodes")
+
+module.exports = async ctx => {
+  await ctx.instance.templateApi.upgradeData(ctx.request.body.newHierarchy)
+  ctx.response.status = StatusCodes.OK
+}

packages/server/middleware/routers.js

@@ -238,6 +238,10 @@ module.exports = (config, app) => {
         ctx.response.status = StatusCodes.UNAUTHORIZED
       }
     })
+    .post(
+      "/_builder/instance/:appname/:instanceid/api/upgradeData",
+      routeHandlers.upgradeData
+    )
     .post("/:appname/api/changeMyPassword", routeHandlers.changeMyPassword)
     .post(
       "/_builder/instance/:appname/:instanceid/api/changeMyPassword",