From c469d765c5e991083e7c861926f4d182237ddde5 Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Mon, 16 Mar 2020 18:18:42 +0000 Subject: [PATCH 1/8] global ui changes Changes include: Label font weights Secondary color Icon background colors Positioning of icons --- packages/builder/src/budibase.css | 7 ++--- packages/builder/src/global.css | 17 ++++++------ packages/builder/src/nav/BackendNav.svelte | 2 +- .../ComponentPropertiesPanel.svelte | 27 +++++++++++++------ .../ComponentsHierarchyChildren.svelte | 2 +- .../ComponentsPaneSwitcher.svelte | 4 +-- .../src/userInterface/LayoutEditor.svelte | 13 ++++----- .../LayoutTemplateControls.svelte | 10 +++---- .../src/userInterface/PagesList.svelte | 4 +-- .../src/userInterface/PropControl.svelte | 6 ++--- .../PropertyCascader/PropertyCascader.svelte | 4 +-- .../userInterface/UserInterfaceRoot.svelte | 24 ++++++++++++----- 12 files changed, 72 insertions(+), 48 deletions(-) diff --git a/packages/builder/src/budibase.css b/packages/builder/src/budibase.css index 30fb9fd02b..582e5bff7f 100644 --- a/packages/builder/src/budibase.css +++ b/packages/builder/src/budibase.css @@ -1,7 +1,8 @@ /* Budibase Component Styles */ .header { font-size: 0.75rem; - color: #999; + color: #000333; + opacity: 0.4; text-transform: uppercase; margin-top: 1rem; font-weight: 500; @@ -68,7 +69,7 @@ .budibase__nav-item.selected { color: var(--button-text); - background: var(--background-button) !important; + background: #fafafa !important; } .budibase__nav-item:hover { @@ -82,7 +83,7 @@ border: 1px solid #DBDBDB; text-align: left; letter-spacing: 0.7px; - color: #163057; + color: #000333; font-size: 16px; padding-left: 5px; } diff --git a/packages/builder/src/global.css b/packages/builder/src/global.css index 745dee014c..15df05b3dc 100644 --- a/packages/builder/src/global.css +++ b/packages/builder/src/global.css @@ -9,12 +9,13 @@ --primary5: #454ca00c; --primarydark: #3F448A; - --secondary100:#828fa5; - --secondary75: #162B4DBF; - --secondary50: #162B4D80; - --secondary25: #162B4D40; - --secondary10: #162B4D1A; - --secondary5:#fff; + --secondary100:#000333; + --secondary80: rgba(0, 3, 51, 0.8); + --secondary60: rgba(0, 3, 51, 0.6); + --secondary40: rgba(0, 3, 51, 0.4); + --secondary20: rgba(0, 3, 51, 0.2); + --secondary10: rgba(0, 3, 51, 0.1); + --secondary5: rgba(0, 3, 51, 0.05); --secondarydark: #3F448A; --tertiary: #F2F5F7; @@ -61,7 +62,7 @@ html, body { font-family: var(--fontnormal); - color: var(--secondary100); + color: var(--secondary80); padding: 0; margin: 0; height:100%; @@ -83,7 +84,7 @@ h2 { h3 { font-family: var(--fontbold); font-size: 24pt; - color: var(--darkslate); + color: var(--secondary60); } h4 { diff --git a/packages/builder/src/nav/BackendNav.svelte b/packages/builder/src/nav/BackendNav.svelte index 1da512d4eb..6956a8e2d8 100644 --- a/packages/builder/src/nav/BackendNav.svelte +++ b/packages/builder/src/nav/BackendNav.svelte @@ -99,7 +99,7 @@ flex-direction: column; max-height: 100%; height: 100%; - background-color: var(--secondary5); + background-color: var(--white); } .nav-group-header { diff --git a/packages/builder/src/userInterface/ComponentPropertiesPanel.svelte b/packages/builder/src/userInterface/ComponentPropertiesPanel.svelte index 4fa7b092bd..4e8f5a40f8 100644 --- a/packages/builder/src/userInterface/ComponentPropertiesPanel.svelte +++ b/packages/builder/src/userInterface/ComponentPropertiesPanel.svelte @@ -162,38 +162,49 @@ .component-props-container { margin-top: 10px; flex: 1 1 auto; - overflow-y: auto; } ul { list-style: none; display: flex; + justify-content: space-between; padding: 0; } li { - margin-right: 20px; background: none; - border-radius: 5px; + border-radius: 3px; width: 48px; height: 48px; } + + li button { - width: 100%; - height: 100%; + width: 48px; + height: 48px; background: none; border: none; - border-radius: 5px; - padding: 12px; + border-radius: 3px; + padding: 7px; outline: none; cursor: pointer; position: relative; } + li:nth-last-child(1) { + margin-right: 0px; + background: none; + border-radius: 3px; + width: 48px; + height: 48px; + } + .selected { color: var(--button-text); - background: var(--background-button) !important; + background: #f9f9f9 !important; + width: 48px; + height: 48px; } .button-indicator { diff --git a/packages/builder/src/userInterface/ComponentsHierarchyChildren.svelte b/packages/builder/src/userInterface/ComponentsHierarchyChildren.svelte index 8a168ec11e..d62886ac55 100644 --- a/packages/builder/src/userInterface/ComponentsHierarchyChildren.svelte +++ b/packages/builder/src/userInterface/ComponentsHierarchyChildren.svelte @@ -94,7 +94,7 @@ border-radius: 3px; height: 35px; align-items: center; - font-weight: normal; + font-weight: 400; } .item button { diff --git a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte index d7abe25b6d..6cde6369ab 100644 --- a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte +++ b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte @@ -46,7 +46,7 @@ {/if} - + diff --git a/packages/builder/src/userInterface/LayoutEditor.svelte b/packages/builder/src/userInterface/LayoutEditor.svelte index 44ae3c0d40..f48c1f3c77 100644 --- a/packages/builder/src/userInterface/LayoutEditor.svelte +++ b/packages/builder/src/userInterface/LayoutEditor.svelte @@ -136,7 +136,8 @@ text-transform: uppercase; font-size: 12px; font-weight: 700; - color: #8997ab; + color: #000333; + opacity: 0.6; margin-bottom: 10px; } @@ -144,16 +145,16 @@ text-transform: uppercase; font-size: 10px; font-weight: 700; - color: #163057; - opacity: 0.3; + color: #000333; + opacity: 0.4; margin-bottom: 15px; } h5 { font-size: 12px; - font-weight: 700; - color: #163057; - opacity: 0.6; + font-weight: 400; + color: #000333; + opacity: 0.8; padding-top: 12px; margin-bottom: 0; } diff --git a/packages/builder/src/userInterface/LayoutTemplateControls.svelte b/packages/builder/src/userInterface/LayoutTemplateControls.svelte index 21c375abcc..dc1395bf0e 100644 --- a/packages/builder/src/userInterface/LayoutTemplateControls.svelte +++ b/packages/builder/src/userInterface/LayoutTemplateControls.svelte @@ -51,26 +51,26 @@ diff --git a/packages/builder/src/userInterface/PropControl.svelte b/packages/builder/src/userInterface/PropControl.svelte index aabd245ac2..63bcc22550 100644 --- a/packages/builder/src/userInterface/PropControl.svelte +++ b/packages/builder/src/userInterface/PropControl.svelte @@ -39,9 +39,9 @@ h5 { word-wrap: break-word; font-size: 12px; - font-weight: 700; - color: #163057; - opacity: 0.6; + font-weight: 400; + color: #000333; + opacity: 0.8; padding-top: 12px; margin-bottom: 0; } diff --git a/packages/builder/src/userInterface/PropertyCascader/PropertyCascader.svelte b/packages/builder/src/userInterface/PropertyCascader/PropertyCascader.svelte index 27f76b9356..4b51cab58b 100644 --- a/packages/builder/src/userInterface/PropertyCascader/PropertyCascader.svelte +++ b/packages/builder/src/userInterface/PropertyCascader/PropertyCascader.svelte @@ -91,9 +91,7 @@ cursor: pointer; outline: none; border: none; - border-radius: 5px; - background: rgba(249, 249, 249, 1); - + border-radius: 3px; font-size: 1.6rem; font-weight: 700; color: rgba(22, 48, 87, 1); diff --git a/packages/builder/src/userInterface/UserInterfaceRoot.svelte b/packages/builder/src/userInterface/UserInterfaceRoot.svelte index 18ad131923..d946712df1 100644 --- a/packages/builder/src/userInterface/UserInterfaceRoot.svelte +++ b/packages/builder/src/userInterface/UserInterfaceRoot.svelte @@ -108,17 +108,28 @@ padding: 0; } + +.root { + display: grid; + grid-template-columns: 275px 1fr 275px; + height: 100%; + width: 100%; + background: #fafafa; +} + +@media only screen and (min-width: 1800px) { .root { display: grid; - grid-template-columns: 290px 1fr 350px; + grid-template-columns: 300px 1fr 300px; height: 100%; width: 100%; background: #fafafa; } +} .ui-nav { grid-column: 1; - background-color: var(--secondary5); + background-color: var(--white); height: calc(100vh - 49px); padding: 0; overflow: hidden; @@ -136,17 +147,18 @@ .components-pane { grid-column: 3; - background-color: var(--secondary5); + background-color: var(--white); min-height: 0px; overflow-y: hidden; } .components-nav-header { - font-size: 0.75rem; - color: #999; + font-size: 12px; + color: #000333; text-transform: uppercase; margin-top: 1rem; - font-weight: 500; + font-weight: 700; + opacity: 0.6; } .nav-group-header { From 74caf8496f2349a81d69412e22dd47f414040d21 Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Tue, 17 Mar 2020 09:57:18 +0000 Subject: [PATCH 2/8] component nav tidy up --- packages/builder/src/common/PlusButton.svelte | 4 +--- packages/builder/src/common/Select.svelte | 9 ++++----- packages/builder/src/global.css | 6 +++--- .../userInterface/ComponentSelectionList.svelte | 12 +++++------- .../userInterface/ComponentsPaneSwitcher.svelte | 15 ++++++++------- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/packages/builder/src/common/PlusButton.svelte b/packages/builder/src/common/PlusButton.svelte index 5baca63715..6e2e4cf58a 100644 --- a/packages/builder/src/common/PlusButton.svelte +++ b/packages/builder/src/common/PlusButton.svelte @@ -8,8 +8,6 @@ outline: none; border: none; border-radius: 5px; - background: rgba(249, 249, 249, 1); - min-width: 1.8rem; min-height: 1.8rem; padding-bottom: 10px; @@ -20,6 +18,6 @@ font-size: 1.2rem; font-weight: 700; - color: rgba(22, 48, 87, 1); + color: var(--secondary100); } diff --git a/packages/builder/src/common/Select.svelte b/packages/builder/src/common/Select.svelte index e360c1c4d6..de0c57fc41 100644 --- a/packages/builder/src/common/Select.svelte +++ b/packages/builder/src/common/Select.svelte @@ -23,11 +23,11 @@ } select { - height: 35px; + height: 40px; display: block; font-family: sans-serif; - font-weight: 500; - color: #163057; + font-weight: 400; + color: #000333; padding: 0 2.6em 0em 1.4em; width: 100%; max-width: 100%; @@ -36,8 +36,7 @@ -moz-appearance: none; -webkit-appearance: none; appearance: none; - background: #fff; - border: 1px solid #ccc; + background: var(--lightslate); } .arrow { diff --git a/packages/builder/src/global.css b/packages/builder/src/global.css index 15df05b3dc..003294ee26 100644 --- a/packages/builder/src/global.css +++ b/packages/builder/src/global.css @@ -36,8 +36,8 @@ --white: #FFFFFF; --darkslate: #1a202c; - --slate: #a0aec0; - --lightslate: #f7fafc; + --slate: #d8d8d8; + --lightslate: #f9f9f9; --borderradius: 2px; --borderradiusall: 2px 2px 2px 2px; @@ -56,7 +56,7 @@ --quotation: var(--fontnormal) "italics" var(--darkslate) 16pt; --smallheavybodytext: var(--fontbold) "regular" var(--secondary100) 14pt; - --background-button: #e6eeff; + --background-button: #f9f9f9; --button-text: #0055ff; } diff --git a/packages/builder/src/userInterface/ComponentSelectionList.svelte b/packages/builder/src/userInterface/ComponentSelectionList.svelte index b123350f10..34e0ca8d33 100644 --- a/packages/builder/src/userInterface/ComponentSelectionList.svelte +++ b/packages/builder/src/userInterface/ComponentSelectionList.svelte @@ -239,12 +239,12 @@ position: relative; padding: 0 15px; cursor: pointer; - border: 1px solid #ebebeb; + border: 1px solid #d8d8d8; border-radius: 2px; margin: 5px 0; height: 40px; box-sizing: border-box; - color: #163057; + color: #000333; display: flex; align-items: center; flex: 1; @@ -256,11 +256,10 @@ } .component > .name { - color: #163057; + color: #000333; display: inline-block; font-size: 12px; - font-weight: bold; - opacity: 0.6; + opacity: 0.8; } ul { @@ -279,12 +278,11 @@ background: #fafafa; padding: 10px; border-radius: 2px; - color: rgba(22, 48, 87, 0.6); + color:var(--secondary80); } .preset-menu > span { font-size: 12px; - font-weight: bold; text-transform: uppercase; margin-top: 5px; } diff --git a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte index 6cde6369ab..504ff18372 100644 --- a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte +++ b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte @@ -54,14 +54,15 @@ height: 100%; display: flex; flex-direction: column; - padding: 2rem 0; + padding: 20px 0; } .switcher { display: flex; justify-content: space-between; - margin-bottom: 25px; - padding: 0 1.5rem; + margin-bottom: 20px; + padding: 0 20px 20px; + border-bottom: 1px solid #d8d8d8; } .switcher > button { @@ -70,15 +71,15 @@ margin: 0; padding: 0; cursor: pointer; - font-weight: 600; font-size: 0.85rem; + font-weight: 400; text-transform: uppercase; - color: #999; - background-color: rgba(0, 0, 0, 0); + color: var(--secondary60); } .switcher > .selected { - color: #333; + color: var(--secondary100); + font-weight: 500; } .panel { From 54317f3d53c1f74a22befbbb9d067cb05161edb6 Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Tue, 17 Mar 2020 11:16:49 +0000 Subject: [PATCH 3/8] color change --- packages/standard-components/src/Button.svelte | 5 ++--- packages/standard-components/src/Login.svelte | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/standard-components/src/Button.svelte b/packages/standard-components/src/Button.svelte index d4bcd7b354..10f66d9311 100644 --- a/packages/standard-components/src/Button.svelte +++ b/packages/standard-components/src/Button.svelte @@ -78,13 +78,12 @@ box-sizing: border-box; border: 1px solid #ccc; border-radius: 2px; - color: #333; - background-color: #f4f4f4; + color: #000333; outline: none; } .default:active { - background-color: #ddd; + background-color: #f9f9f9; } .default:focus { diff --git a/packages/standard-components/src/Login.svelte b/packages/standard-components/src/Login.svelte index b3f6f36779..501023d83e 100644 --- a/packages/standard-components/src/Login.svelte +++ b/packages/standard-components/src/Login.svelte @@ -155,16 +155,15 @@ box-sizing: border-box; border: 1px solid #ccc; border-radius: 2px; - color: #333; - background-color: #f4f4f4; + color: #000333; outline: none; } .default-button:active { - background-color: #ddd; + background-color: #f9f9f9; } .default-button:focus { - border-color: #666; + border-color: #f9f9f9; } From 217906c7df0e5f61c26db3b0a1e8754ffda62eea Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Thu, 19 Mar 2020 09:34:23 +0000 Subject: [PATCH 4/8] navigator panel touch ups the current navigator panel design contains the wrong colors, padding, etc. This resolves that. --- packages/builder/src/budibase.css | 2 +- .../ComponentsPaneSwitcher.svelte | 2 +- .../src/userInterface/PropControl.svelte | 1 + .../userInterface/UserInterfaceRoot.svelte | 38 +++++++++++-------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/budibase.css b/packages/builder/src/budibase.css index 582e5bff7f..7a82999241 100644 --- a/packages/builder/src/budibase.css +++ b/packages/builder/src/budibase.css @@ -59,7 +59,7 @@ cursor: pointer; padding: 0 7px 0 3px; height: 35px; - margin: 5px 0; + margin: 5px 20px 5px 0px; border-radius: 0 5px 5px 0; display: flex; align-items: center; diff --git a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte index 504ff18372..12c0206407 100644 --- a/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte +++ b/packages/builder/src/userInterface/ComponentsPaneSwitcher.svelte @@ -71,7 +71,7 @@ margin: 0; padding: 0; cursor: pointer; - font-size: 0.85rem; + font-size: 14px; font-weight: 400; text-transform: uppercase; color: var(--secondary60); diff --git a/packages/builder/src/userInterface/PropControl.svelte b/packages/builder/src/userInterface/PropControl.svelte index 63bcc22550..7106245428 100644 --- a/packages/builder/src/userInterface/PropControl.svelte +++ b/packages/builder/src/userInterface/PropControl.svelte @@ -34,6 +34,7 @@ grid-template-rows: 1fr; grid-template-columns: 70px 1fr; grid-gap: 10px; + align-items: baseline; } h5 { diff --git a/packages/builder/src/userInterface/UserInterfaceRoot.svelte b/packages/builder/src/userInterface/UserInterfaceRoot.svelte index d946712df1..8fd3039eb5 100644 --- a/packages/builder/src/userInterface/UserInterfaceRoot.svelte +++ b/packages/builder/src/userInterface/UserInterfaceRoot.svelte @@ -43,7 +43,9 @@
Navigator - Pages +
+ + Pages
@@ -152,25 +154,29 @@ overflow-y: hidden; } - .components-nav-header { + .components-nav-page { font-size: 12px; color: #000333; text-transform: uppercase; - margin-top: 1rem; + padding-left: 20px; + margin-top: 20px; font-weight: 700; opacity: 0.6; } - .nav-group-header { - font-size: 0.9rem; - padding-left: 1rem; + .components-nav-header { + font-size: 12px; + color: #000333; + text-transform: uppercase; + margin-top: 20px; + font-weight: 700; + opacity: 0.6; } .nav-header { display: flex; flex-direction: column; - margin-top: 1.5rem; - padding: 0 1.8rem; + margin-top: 20px; } .nav-items-container { @@ -179,7 +185,7 @@ .nav-group-header { display: flex; - padding: 1.5rem 0 0 1.8rem; + padding: 0px 20px 0px 20px; font-size: 0.9rem; font-weight: bold; justify-content: space-between; @@ -212,20 +218,20 @@ } .navigator-title { + font-size: 14px; + color: var(--secondary100); + font-weight: 500; text-transform: uppercase; - font-weight: 400; - color: #999; - font-size: 0.9rem; + padding: 0 20px 20px 20px; + line-height: 1rem !important; } .border-line { - border-bottom: 1px solid #ddd; - margin-top: 1.5rem; - width: calc(100% + 1.5rem); + border-bottom: 1px solid #d8d8d8; } .components-list-container { overflow: auto; - padding: 0 30px 0 0; + padding: 20px 0px 0 0; } From a166f3e510fe87ed7a7e2fcd90b11362b10fbfde Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Thu, 19 Mar 2020 12:31:23 +0000 Subject: [PATCH 5/8] primary button and new screen modal redesigned Changes include: Primary button redesigned Cancel button added Button positiioning changed New screen modal redesigned --- .../builder/src/common/ActionButton.svelte | 16 ++++++--- .../builder/src/common/ButtonGroup.svelte | 7 ++-- .../builder/src/common/ConfirmDialog.svelte | 17 ++++++++-- packages/builder/src/global.css | 17 +++++----- .../src/userInterface/NewScreen.svelte | 33 +++++++++++++++++++ 5 files changed, 72 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/common/ActionButton.svelte b/packages/builder/src/common/ActionButton.svelte index e1230692d1..f4f13c0757 100644 --- a/packages/builder/src/common/ActionButton.svelte +++ b/packages/builder/src/common/ActionButton.svelte @@ -2,6 +2,7 @@ export let disabled = false export let hidden = false export let primary = true + export let cancel = false export let alert = false export let warning = false @@ -12,6 +13,7 @@ class:hidden class:primary class:alert + class:cancel class:warning {disabled}> @@ -19,8 +21,8 @@ diff --git a/packages/builder/src/common/ConfirmDialog.svelte b/packages/builder/src/common/ConfirmDialog.svelte index 5d1002c483..8b69a120ad 100644 --- a/packages/builder/src/common/ConfirmDialog.svelte +++ b/packages/builder/src/common/ConfirmDialog.svelte @@ -41,13 +41,26 @@

{title}

- {body} + {body}
+ {cancelText} {okText} - {cancelText}
+ + diff --git a/packages/builder/src/global.css b/packages/builder/src/global.css index 003294ee26..a05adfc7c9 100644 --- a/packages/builder/src/global.css +++ b/packages/builder/src/global.css @@ -1,13 +1,14 @@ @import "./budibase.css"; :root { - --primary100: #173157FF; - --primary75: #454CA0BF; - --primary50: #454CA080; - --primary25: #454CA040; - --primary10: #454CA01A; - --primary5: #454ca00c; - --primarydark: #3F448A; + --primary100: #0055ff; + --primary80: rgba(0, 85, 255, 0.8); + --primary60: #rgba(0, 85, 255, 0.6); + --primary40: #rgba(0, 85, 255, 0.4); + --primary20: #rgba(0, 85, 255, 0.2); + --primary10: #rgba(0, 85, 255, 0.1); + --primary5: #rgba(0, 85, 255, 0.05); + --primarydark: #0044cc; --secondary100:#000333; --secondary80: rgba(0, 3, 51, 0.8); @@ -16,7 +17,7 @@ --secondary20: rgba(0, 3, 51, 0.2); --secondary10: rgba(0, 3, 51, 0.1); --secondary5: rgba(0, 3, 51, 0.05); - --secondarydark: #3F448A; + --secondarydark: #00021a; --tertiary: #F2F5F7; diff --git a/packages/builder/src/userInterface/NewScreen.svelte b/packages/builder/src/userInterface/NewScreen.svelte index b5e2a32570..e247c19417 100644 --- a/packages/builder/src/userInterface/NewScreen.svelte +++ b/packages/builder/src/userInterface/NewScreen.svelte @@ -126,3 +126,36 @@ + + From 4d9949a4290fffaf8806cbfafc2aacd0c49fa6f4 Mon Sep 17 00:00:00 2001 From: Michael Shanks Date: Fri, 20 Mar 2020 13:39:38 +0000 Subject: [PATCH 6/8] #8 Diff Hierarchy & upgrade data --- packages/core/src/appInitialise/cloneApp.js | 8 + .../core/src/appInitialise/initialiseData.js | 26 +- packages/core/src/index.js | 6 +- packages/core/src/indexApi/buildIndex.js | 83 +----- packages/core/src/indexApi/listItems.js | 2 +- packages/core/src/indexing/allIds.js | 3 +- packages/core/src/indexing/initialiseIndex.js | 14 +- packages/core/src/indexing/read.js | 12 + .../core/src/recordApi/initialiseChildren.js | 81 ++++++ packages/core/src/recordApi/save.js | 42 +-- .../templateApi/deleteAllIndexFilesForNode.js | 34 +++ .../templateApi/deleteAllRecordsForNode.js | 32 +++ .../core/src/templateApi/diffHierarchy.js | 7 +- packages/core/src/templateApi/hierarchy.js | 24 +- packages/core/src/templateApi/index.js | 2 + .../src/templateApi/initialiseNewIndex.js | 27 ++ packages/core/src/templateApi/upgradeData.js | 209 +++++++++++++-- packages/core/src/transactions/cleanup.js | 15 +- packages/core/src/transactions/execute.js | 31 ++- packages/core/src/transactions/retrieve.js | 49 ++-- .../core/src/transactions/setCleanupFunc.js | 14 + .../src/transactions/transactionsCommon.js | 4 +- packages/core/test/specHelpers.js | 14 +- .../test/templateApi.diffHierarchy.spec.js | 168 +++++++----- .../core/test/templateApi.upgradeData.spec.js | 244 ++++++++++++++++++ packages/core/test/upgradeDataSetup.js | 47 ++++ .../middleware/routeHandlers/upgradeData.js | 6 + packages/server/middleware/routers.js | 4 + 28 files changed, 962 insertions(+), 246 deletions(-) create mode 100644 packages/core/src/appInitialise/cloneApp.js create mode 100644 packages/core/src/recordApi/initialiseChildren.js create mode 100644 packages/core/src/templateApi/deleteAllIndexFilesForNode.js create mode 100644 packages/core/src/templateApi/deleteAllRecordsForNode.js create mode 100644 packages/core/src/templateApi/initialiseNewIndex.js create mode 100644 packages/core/src/transactions/setCleanupFunc.js create mode 100644 packages/core/test/templateApi.upgradeData.spec.js create mode 100644 packages/core/test/upgradeDataSetup.js create mode 100644 packages/server/middleware/routeHandlers/upgradeData.js diff --git a/packages/core/src/appInitialise/cloneApp.js b/packages/core/src/appInitialise/cloneApp.js new file mode 100644 index 0000000000..0c947cd0e8 --- /dev/null +++ b/packages/core/src/appInitialise/cloneApp.js @@ -0,0 +1,8 @@ +import { setCleanupFunc } from "../transactions/setCleanupFunc" + +export const cloneApp = (app, mergeWith) => { + const newApp = { ...app } + Object.assign(newApp, mergeWith) + setCleanupFunc(newApp) + return newApp +} \ No newline at end of file diff --git a/packages/core/src/appInitialise/initialiseData.js b/packages/core/src/appInitialise/initialiseData.js index 1daf2a6f94..e74b70447f 100644 --- a/packages/core/src/appInitialise/initialiseData.js +++ b/packages/core/src/appInitialise/initialiseData.js @@ -21,22 +21,29 @@ export const initialiseData = async ( applicationDefinition, accessLevels ) => { - await datastore.createFolder(configFolder) - await datastore.createJson(appDefinitionFile, applicationDefinition) + if (!await datastore.exists(configFolder)) + await datastore.createFolder(configFolder) + + if (!await datastore.exists(appDefinitionFile)) + await datastore.createJson(appDefinitionFile, applicationDefinition) await initialiseRootCollections(datastore, applicationDefinition.hierarchy) await initialiseRootIndexes(datastore, applicationDefinition.hierarchy) - await datastore.createFolder(TRANSACTIONS_FOLDER) + if (!await datastore.exists(TRANSACTIONS_FOLDER)) + await datastore.createFolder(TRANSACTIONS_FOLDER) - await datastore.createFolder(AUTH_FOLDER) + if (!await datastore.exists(AUTH_FOLDER)) + await datastore.createFolder(AUTH_FOLDER) - await datastore.createJson(USERS_LIST_FILE, []) + if (!await datastore.exists(USERS_LIST_FILE)) + await datastore.createJson(USERS_LIST_FILE, []) - await datastore.createJson( - ACCESS_LEVELS_FILE, - accessLevels ? accessLevels : { version: 0, levels: [] } - ) + if (!await datastore.exists(ACCESS_LEVELS_FILE)) + await datastore.createJson( + ACCESS_LEVELS_FILE, + accessLevels ? accessLevels : { version: 0, levels: [] } + ) await initialiseRootSingleRecords(datastore, applicationDefinition.hierarchy) } @@ -64,6 +71,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => { const singleRecords = $(flathierarchy, [filter(isSingleRecord)]) for (let record of singleRecords) { + if (await datastore.exists(record.nodeKey())) continue await datastore.createFolder(record.nodeKey()) const result = _getNew(record, "") await _save(app, result) diff --git a/packages/core/src/index.js b/packages/core/src/index.js index 28a9afd93a..c84a82e58f 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -7,7 +7,7 @@ import getActionsApi from "./actionsApi" import { setupDatastore, createEventAggregator } from "./appInitialise" import { initialiseActions } from "./actionsApi/initialise" import { isSomething, crypto } from "./common" -import { cleanup } from "./transactions/cleanup" +import { setCleanupFunc } from "./transactions/setCleanupFunc" import { generateFullPermissions } from "./authApi/generateFullPermissions" import { getApplicationDefinition } from "./templateApi/getApplicationDefinition" import common from "./common" @@ -40,9 +40,7 @@ export const getAppApis = async ( const templateApi = getTemplateApi(app) - app.cleanupTransactions = isSomething(cleanupTransactions) - ? cleanupTransactions - : async () => await cleanup(app) + setCleanupFunc(app, cleanupTransactions) app.getEpochTime = isSomething(getEpochTime) ? getEpochTime diff --git a/packages/core/src/indexApi/buildIndex.js b/packages/core/src/indexApi/buildIndex.js index 954aa2c2af..5f368a8a4f 100644 --- a/packages/core/src/indexApi/buildIndex.js +++ b/packages/core/src/indexApi/buildIndex.js @@ -6,8 +6,10 @@ import { getNode, isIndex, isRecord, + getActualKeyOfParent, getAllowedRecordNodesForIndex, fieldReversesReferenceToIndex, + isTopLevelIndex, } from "../templateApi/hierarchy" import { joinKey, apiWrapper, events, $ } from "../common" import { @@ -16,6 +18,8 @@ import { } from "../transactions/create" import { permission } from "../authApi/permissions" import { BadRequestError } from "../common/errors" +import { initialiseIndex } from "../indexing/initialiseIndex" +import { getRecordInfo } from "../recordApi/recordInfo" /** rebuilds an index * @param {object} app - the application container @@ -32,7 +36,7 @@ export const buildIndex = app => async indexNodeKey => indexNodeKey ) -const _buildIndex = async (app, indexNodeKey) => { +export const _buildIndex = async (app, indexNodeKey) => { const indexNode = getNode(app.hierarchy, indexNodeKey) await createBuildIndexFolder(app.datastore, indexNodeKey) @@ -89,12 +93,6 @@ const buildReverseReferenceIndex = async (app, indexNode) => { } } -/* -const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [ - map(n => n.parent()), -]); -*/ - const buildHeirarchalIndex = async (app, indexNode) => { let recordCount = 0 @@ -127,7 +125,7 @@ const buildHeirarchalIndex = async (app, indexNode) => { ) let allIds = await allIdsIterator() - while (allIds.done === false) { + while (allIds.done === false) { await createTransactionsForIds( allIds.result.collectionKey, allIds.result.ids @@ -139,77 +137,8 @@ const buildHeirarchalIndex = async (app, indexNode) => { return recordCount } -// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children); - const recordNodeApplies = indexNode => recordNode => includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds) -/* -const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [ - getFlattenedHierarchy, - filter( - allTrue( - isRecord, - isDecendant(ancestorNode), - recordNodeApplies(indexNode), - ), - ), -]); -*/ - -/* -const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey, - indexNode, indexKey, currentIndexedData, - currentIndexedDataKey, recordCount = 0) => { - const collectionNode = getCollectionNodeByKeyOrNodeKey( - app.hierarchy, - collection_Key_or_NodeKey, - ); - - const allIdsIterator = await getAllIdsIterator(app)(collection_Key_or_NodeKey); - - - const createTransactionsForIds = async (collectionKey, allIds) => { - for (const recordId of allIds) { - const recordKey = joinKey(collectionKey, recordId); - - const recordNode = chooseChildRecordNodeByKey( - collectionNode, - recordId, - ); - - if (recordNodeApplies(indexNode)(recordNode)) { - await transactionForBuildIndex( - app, indexNode.nodeKey(), - recordKey, recordCount, - ); - recordCount++; - } - - if (hasApplicableDecendant(app.hierarchy, recordNode, indexNode)) { - for (const childCollectionNode of recordNode.children) { - recordCount = await applyAllDecendantRecords( - app, - joinKey(recordKey, childCollectionNode.collectionName), - indexNode, indexKey, currentIndexedData, - currentIndexedDataKey, recordCount, - ); - } - } - } - }; - - let allIds = await allIdsIterator(); - while (allIds.done === false) { - await createTransactionsForIds( - allIds.result.collectionKey, - allIds.result.ids, - ); - allIds = await allIdsIterator(); - } - - return recordCount; -}; -*/ export default buildIndex diff --git a/packages/core/src/indexApi/listItems.js b/packages/core/src/indexApi/listItems.js index 6dea58b812..5bafa1b840 100644 --- a/packages/core/src/indexApi/listItems.js +++ b/packages/core/src/indexApi/listItems.js @@ -33,7 +33,7 @@ const defaultOptions = { searchPhrase: null, } -const _listItems = async (app, indexKey, options = defaultOptions) => { +export const _listItems = async (app, indexKey, options = defaultOptions) => { const { searchPhrase, rangeStartParams, rangeEndParams } = $({}, [ merge(options), merge(defaultOptions), diff --git a/packages/core/src/indexing/allIds.js b/packages/core/src/indexing/allIds.js index 1951ba3086..85213f5955 100644 --- a/packages/core/src/indexing/allIds.js +++ b/packages/core/src/indexing/allIds.js @@ -2,6 +2,7 @@ import { flatten, orderBy, filter, isUndefined } from "lodash/fp" import { getFlattenedHierarchy, getCollectionNodeByKeyOrNodeKey, + getNodeByKeyOrNodeKey, isCollectionRecord, isAncestor, } from "../templateApi/hierarchy" @@ -60,7 +61,7 @@ export const getAllIdsIterator = app => async collection_Key_or_NodeKey => { const recordNode = getCollectionNodeByKeyOrNodeKey( app.hierarchy, collection_Key_or_NodeKey - ) + ) || getNodeByKeyOrNodeKey(app.hierarchy, collection_Key_or_NodeKey) const getAllIdsIteratorForCollectionKey = async ( recordNode, diff --git a/packages/core/src/indexing/initialiseIndex.js b/packages/core/src/indexing/initialiseIndex.js index afca51376f..da4fde1824 100644 --- a/packages/core/src/indexing/initialiseIndex.js +++ b/packages/core/src/indexing/initialiseIndex.js @@ -9,11 +9,19 @@ import { export const initialiseIndex = async (datastore, dir, index) => { const indexDir = joinKey(dir, index.name) - await datastore.createFolder(indexDir) + let newDir = false + if (!await datastore.exists(indexDir)) { + await datastore.createFolder(indexDir) + newDir = true + } if (isShardedIndex(index)) { - await datastore.createFile(getShardMapKey(indexDir), "[]") + const shardFile = getShardMapKey(indexDir) + if (newDir || !await datastore.exists(shardFile)) + await datastore.createFile(shardFile, "[]") } else { - await createIndexFile(datastore, getUnshardedIndexDataKey(indexDir), index) + const indexFile = getUnshardedIndexDataKey(indexDir) + if (newDir || !await datastore.exists(indexFile)) + await createIndexFile(datastore, indexFile, index) } } diff --git a/packages/core/src/indexing/read.js b/packages/core/src/indexing/read.js index 9519a49e27..2256595bc2 100644 --- a/packages/core/src/indexing/read.js +++ b/packages/core/src/indexing/read.js @@ -3,6 +3,9 @@ import { promiseReadableStream } from "./promiseReadableStream" import { createIndexFile } from "./sharding" import { generateSchema } from "./indexSchemaCreator" import { getIndexReader, CONTINUE_READING_RECORDS } from "./serializer" +import { getAllowedRecordNodesForIndex, getRecordNodeId } from "../templateApi/hierarchy" +import { $ } from "../common" +import { filter, includes, find } from "lodash/fp" export const readIndex = async ( hierarchy, @@ -11,8 +14,10 @@ export const readIndex = async ( indexedDataKey ) => { const records = [] + const getType = typeLoader(index, hierarchy) const doRead = iterateIndex( async item => { + item.type = getType(item.key) records.push(item) return CONTINUE_READING_RECORDS }, @@ -31,8 +36,10 @@ export const searchIndex = async ( ) => { const records = [] const schema = generateSchema(hierarchy, index) + const getType = typeLoader(index, hierarchy) const doRead = iterateIndex( async item => { + item.type = getType(item.key) const idx = lunr(function() { this.ref("key") for (const field of schema) { @@ -76,3 +83,8 @@ export const iterateIndex = (onGetItem, getFinalResult) => async ( return [] } } + +const typeLoader = (index, hierarchy) => { + const allowedNodes = getAllowedRecordNodesForIndex(hierarchy, index) + return key => find(n => getRecordNodeId(key) === n.nodeId)(allowedNodes).name +} diff --git a/packages/core/src/recordApi/initialiseChildren.js b/packages/core/src/recordApi/initialiseChildren.js new file mode 100644 index 0000000000..bc8edc8637 --- /dev/null +++ b/packages/core/src/recordApi/initialiseChildren.js @@ -0,0 +1,81 @@ +import { isString, flatten, map, filter } from "lodash/fp" +import { initialiseChildCollections } from "../collectionApi/initialise" +import { _loadFromInfo } from "./load" +import { $ } from "../common" +import { + getFlattenedHierarchy, + isRecord, + getNode, + isTopLevelRecord, + fieldReversesReferenceToNode, +} from "../templateApi/hierarchy" +import { initialiseIndex } from "../indexing/initialiseIndex" +import { getRecordInfo } from "./recordInfo" + +export const initialiseChildren = async (app, recordInfoOrKey) => { + const recordInfo = isString(recordInfoOrKey) + ? getRecordInfo(app.hierarchy, recordInfoOrKey) + : recordInfoOrKey + await initialiseReverseReferenceIndexes(app, recordInfo) + await initialiseAncestorIndexes(app, recordInfo) + await initialiseChildCollections(app, recordInfo) +} + +export const initialiseChildrenForNode = async (app, recordNode) => { + + if (isTopLevelRecord(recordNode)) { + await initialiseChildren( + app, recordNode.nodeKey()) + return + } + + const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey()) + let iterateResult = await iterate() + while (!iterateResult.done) { + const { result } = iterateResult + for (const id of result.ids) { + const initialisingRecordKey = joinKey( + result.collectionKey, id) + await initialiseChildren(app, initialisingRecordKey) + } + iterateResult = await iterate() + } +} + +const initialiseAncestorIndexes = async (app, recordInfo) => { + for (const index of recordInfo.recordNode.indexes) { + const indexKey = recordInfo.child(index.name) + if (!(await app.datastore.exists(indexKey))) { + await initialiseIndex(app.datastore, recordInfo.dir, index) + } + } +} + +const initialiseReverseReferenceIndexes = async (app, recordInfo) => { + const indexNodes = $( + fieldsThatReferenceThisRecord(app, recordInfo.recordNode), + [ + map(f => + $(f.typeOptions.reverseIndexNodeKeys, [ + map(n => getNode(app.hierarchy, n)), + ]) + ), + flatten, + ] + ) + + for (const indexNode of indexNodes) { + await initialiseIndex(app.datastore, recordInfo.dir, indexNode) + } +} + +const fieldsThatReferenceThisRecord = (app, recordNode) => + $(app.hierarchy, [ + getFlattenedHierarchy, + filter(isRecord), + map(n => n.fields), + flatten, + filter(fieldReversesReferenceToNode(recordNode)), + ]) + + diff --git a/packages/core/src/recordApi/save.js b/packages/core/src/recordApi/save.js index 51c7e55373..d69e0e205d 100644 --- a/packages/core/src/recordApi/save.js +++ b/packages/core/src/recordApi/save.js @@ -1,5 +1,4 @@ import { cloneDeep, take, takeRight, flatten, map, filter } from "lodash/fp" -import { initialiseChildCollections } from "../collectionApi/initialise" import { validate } from "./validate" import { _loadFromInfo } from "./load" import { apiWrapper, events, $, joinKey } from "../common" @@ -17,6 +16,7 @@ import { permission } from "../authApi/permissions" import { initialiseIndex } from "../indexing/initialiseIndex" import { BadRequestError } from "../common/errors" import { getRecordInfo } from "./recordInfo" +import { initialiseChildren } from "./initialiseChildren" export const save = app => async (record, context) => apiWrapper( @@ -59,9 +59,7 @@ export const _save = async (app, record, context, skipValidation = false) => { await createRecordFolderPath(app.datastore, pathInfo) await app.datastore.createFolder(files) await app.datastore.createJson(recordJson, recordClone) - await initialiseReverseReferenceIndexes(app, recordInfo) - await initialiseAncestorIndexes(app, recordInfo) - await initialiseChildCollections(app, recordInfo) + await initialiseChildren(app, recordInfo) await app.publish(events.recordApi.save.onRecordCreated, { record: recordClone, }) @@ -87,42 +85,6 @@ export const _save = async (app, record, context, skipValidation = false) => { return returnedClone } -const initialiseAncestorIndexes = async (app, recordInfo) => { - for (const index of recordInfo.recordNode.indexes) { - const indexKey = recordInfo.child(index.name) - if (!(await app.datastore.exists(indexKey))) { - await initialiseIndex(app.datastore, recordInfo.dir, index) - } - } -} - -const initialiseReverseReferenceIndexes = async (app, recordInfo) => { - const indexNodes = $( - fieldsThatReferenceThisRecord(app, recordInfo.recordNode), - [ - map(f => - $(f.typeOptions.reverseIndexNodeKeys, [ - map(n => getNode(app.hierarchy, n)), - ]) - ), - flatten, - ] - ) - - for (const indexNode of indexNodes) { - await initialiseIndex(app.datastore, recordInfo.dir, indexNode) - } -} - -const fieldsThatReferenceThisRecord = (app, recordNode) => - $(app.hierarchy, [ - getFlattenedHierarchy, - filter(isRecord), - map(n => n.fields), - flatten, - filter(fieldReversesReferenceToNode(recordNode)), - ]) - const createRecordFolderPath = async (datastore, pathInfo) => { const recursiveCreateFolder = async ( subdirs, diff --git a/packages/core/src/templateApi/deleteAllIndexFilesForNode.js b/packages/core/src/templateApi/deleteAllIndexFilesForNode.js new file mode 100644 index 0000000000..1cd63b9567 --- /dev/null +++ b/packages/core/src/templateApi/deleteAllIndexFilesForNode.js @@ -0,0 +1,34 @@ +import { getAllIdsIterator } from "../indexing/allIds" +import { getRecordInfo } from "../recordApi/recordInfo" +import { isTopLevelIndex, getParentKey, getLastPartInKey } from "./hierarchy" +import { safeKey, joinKey } from "../common" + +export const deleteAllIndexFilesForNode = async (app, indexNode) => { + + if (isTopLevelIndex(indexNode)) { + await app.datastore.deleteFolder(indexNode.nodeKey()) + return + } + + const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey()) + let iterateResult = await iterate() + while (!iterateResult.done) { + const { result } = iterateResult + for (const id of result.ids) { + const deletingIndexKey = joinKey( + result.collectionKey, id, indexNode.name) + await deleteIndexFolder(app, deletingIndexKey) + } + iterateResult = await iterate() + } + +} + +const deleteIndexFolder = async (app, indexKey) => { + indexKey = safeKey(indexKey) + const indexName = getLastPartInKey(indexKey) + const parentRecordKey = getParentKey(indexKey) + const recordInfo = getRecordInfo(app.hierarchy, parentRecordKey) + await app.datastore.deleteFolder( + joinKey(recordInfo.dir, indexName)) +} \ No newline at end of file diff --git a/packages/core/src/templateApi/deleteAllRecordsForNode.js b/packages/core/src/templateApi/deleteAllRecordsForNode.js new file mode 100644 index 0000000000..951decea31 --- /dev/null +++ b/packages/core/src/templateApi/deleteAllRecordsForNode.js @@ -0,0 +1,32 @@ +import { getAllIdsIterator } from "../indexing/allIds" +import { getCollectionDir } from "../recordApi/recordInfo" +import { isTopLevelRecord, getCollectionKey } from "./hierarchy" +import { safeKey, joinKey } from "../common" + +export const deleteAllRecordsForNode = async (app, recordNode) => { + + if (isTopLevelRecord(recordNode)) { + await deleteRecordCollection( + app, recordNode.collectionName) + return + } + + const iterate = await getAllIdsIterator(app)(recordNode.parent().nodeKey()) + let iterateResult = await iterate() + while (!iterateResult.done) { + const { result } = iterateResult + for (const id of result.ids) { + const deletingCollectionKey = joinKey( + result.collectionKey, id, recordNode.collectionName) + await deleteRecordCollection(app, deletingCollectionKey) + } + iterateResult = await iterate() + } + +} + +const deleteRecordCollection = async (app, collectionKey) => { + collectionKey = safeKey(collectionKey) + await app.datastore.deleteFolder( + getCollectionDir(app.hierarchy, collectionKey)) +} \ No newline at end of file diff --git a/packages/core/src/templateApi/diffHierarchy.js b/packages/core/src/templateApi/diffHierarchy.js index 5206a3526a..a4b933b0b1 100644 --- a/packages/core/src/templateApi/diffHierarchy.js +++ b/packages/core/src/templateApi/diffHierarchy.js @@ -1,6 +1,6 @@ import { getFlattenedHierarchy, isRecord, isIndex, isAncestor } from "./hierarchy" import { $, none } from "../common" -import { map, filter, some, find } from "lodash/fp" +import { map, filter, some, find, difference } from "lodash/fp" export const HierarchyChangeTypes = { recordCreated: "Record Created", @@ -10,7 +10,7 @@ export const HierarchyChangeTypes = { recordEstimatedRecordTypeChanged: "Record's Estimated Record Count Changed", indexCreated: "Index Created", indexDeleted: "Index Deleted", - indexChanged: "index Changed", + indexChanged: "Index Changed", } export const diffHierarchy = (oldHierarchy, newHierarchy) => { @@ -123,7 +123,7 @@ const findDeletedIndexes = (oldHierarchyFlat, newHierarchyFlat, deletedRecords) const findUpdatedIndexes = (oldHierarchyFlat, newHierarchyFlat) => $(oldHierarchyFlat, [ - filter(isRecord), + filter(isIndex), filter(nodeExistsIn(newHierarchyFlat)), filter(nodeChanged(newHierarchyFlat, indexHasChanged)), map(n => changeItem( @@ -150,6 +150,7 @@ const indexHasChanged = (_new, old) => _new.map !== old.map || _new.filter !== old.filter || _new.getShardName !== old.getShardName + || difference(_new.allowedRecordNodeIds)(old.allowedRecordNodeIds).length > 0 const isFieldSame = f1 => f2 => f1.name === f2.name && f1.type === f2.type diff --git a/packages/core/src/templateApi/hierarchy.js b/packages/core/src/templateApi/hierarchy.js index d42b745169..60a2686b7f 100644 --- a/packages/core/src/templateApi/hierarchy.js +++ b/packages/core/src/templateApi/hierarchy.js @@ -191,6 +191,22 @@ export const getAllowedRecordNodesForIndex = (appHierarchy, indexNode) => { } } +export const getDependantIndexes = (hierarchy, recordNode) => { + const allIndexes = $(hierarchy, [ getFlattenedHierarchy, filter(isIndex)]) + + const allowedAncestors = $(allIndexes, [ + filter(isAncestorIndex), + filter(i => recordNodeIsAllowed(i)(recordNode)), + ]) + + const allowedReference = $(allIndexes, [ + filter(isReferenceIndex), + filter(i => some(fieldReversesReferenceToIndex(i))(recordNode.fields)) + ]) + + return [...allowedAncestors, ...allowedReference] +} + export const getNodeFromNodeKeyHash = hierarchy => hash => $(hierarchy, [ getFlattenedHierarchy, @@ -206,13 +222,19 @@ export const isaggregateGroup = node => export const isShardedIndex = node => isIndex(node) && isNonEmptyString(node.getShardName) export const isRoot = node => isSomething(node) && node.isRoot() +export const findRoot = node => isRoot(node) ? node : findRoot(node.parent()) export const isDecendantOfARecord = hasMatchingAncestor(isRecord) export const isGlobalIndex = node => isIndex(node) && isRoot(node.parent()) export const isReferenceIndex = node => isIndex(node) && node.indexType === indexTypes.reference export const isAncestorIndex = node => isIndex(node) && node.indexType === indexTypes.ancestor - +export const isTopLevelRecord = node => isRoot(node.parent()) && isRecord(node) +export const isTopLevelIndex = node => isRoot(node.parent()) && isIndex(node) +export const getCollectionKey = recordKey => $(recordKey, [ + splitKey, + parts => joinKey(parts.slice(0, parts.length - 1)) +]) export const fieldReversesReferenceToNode = node => field => field.type === "reference" && intersection(field.typeOptions.reverseIndexNodeKeys)( diff --git a/packages/core/src/templateApi/index.js b/packages/core/src/templateApi/index.js index b11d568427..d0a11b41d1 100644 --- a/packages/core/src/templateApi/index.js +++ b/packages/core/src/templateApi/index.js @@ -28,6 +28,7 @@ import { saveApplicationHierarchy } from "./saveApplicationHierarchy" import { saveActionsAndTriggers } from "./saveActionsAndTriggers" import { all } from "../types" import { getBehaviourSources } from "./getBehaviourSources" +import { upgradeData } from "./upgradeData" const api = app => ({ getApplicationDefinition: getApplicationDefinition(app.datastore), @@ -57,6 +58,7 @@ const api = app => ({ validateNode, validateAll, validateTriggers, + upgradeData: upgradeData(app) }) export const getTemplateApi = app => api(app) diff --git a/packages/core/src/templateApi/initialiseNewIndex.js b/packages/core/src/templateApi/initialiseNewIndex.js new file mode 100644 index 0000000000..fce66b48be --- /dev/null +++ b/packages/core/src/templateApi/initialiseNewIndex.js @@ -0,0 +1,27 @@ +import { getAllIdsIterator } from "../indexing/allIds" +import { getRecordInfo } from "../recordApi/recordInfo" +import { isTopLevelIndex } from "./hierarchy" +import { joinKey } from "../common" +import { initialiseIndex } from "../indexing/initialiseIndex" + +export const initialiseNewIndex = async (app, indexNode) => { + + if (isTopLevelIndex(indexNode)) { + await initialiseIndex(app.datastore, "/", indexNode) + return + } + + const iterate = await getAllIdsIterator(app)(indexNode.parent().nodeKey()) + let iterateResult = await iterate() + while (!iterateResult.done) { + const { result } = iterateResult + for (const id of result.ids) { + const recordKey = joinKey(result.collectionKey, id) + await initialiseIndex( + app.datastore, + getRecordInfo(app.hierarchy, recordKey).dir, + indexNode) + } + iterateResult = await iterate() + } +} \ No newline at end of file diff --git a/packages/core/src/templateApi/upgradeData.js b/packages/core/src/templateApi/upgradeData.js index 197252f890..6da0eb1a9f 100644 --- a/packages/core/src/templateApi/upgradeData.js +++ b/packages/core/src/templateApi/upgradeData.js @@ -1,17 +1,194 @@ - /* -const changeActions = { - rebuildIndex: indexNodeKey => ({ - type: "rebuildIndex", - indexNodeKey, - }), - reshardRecords: recordNodeKey => ({ - type: "reshardRecords", - recordNodeKey, - }), - deleteRecords: recordNodeKey => ({ - type: "reshardRecords", - recordNodeKey, - }), - renameRecord +import { diffHierarchy, HierarchyChangeTypes } from "./diffHierarchy" +import { $, switchCase } from "../common" +import { + differenceBy, + isEqual, + some, + map, + filter, + uniqBy, + flatten +} from "lodash/fp" +import { + findRoot, + getDependantIndexes, + isTopLevelRecord, + isAncestorIndex +} from "./hierarchy" +import { generateSchema } from "../indexing/indexSchemaCreator" +import { _buildIndex } from "../indexApi/buildIndex" +import { constructHierarchy } from "./createNodes" +import { deleteAllRecordsForNode } from "./deleteAllRecordsForNode" +import { deleteAllIndexFilesForNode } from "./deleteAllIndexFilesForNode" +import { cloneApp } from "../appInitialise/cloneApp" +import { initialiseData } from "../appInitialise/initialiseData" +import { initialiseChildrenForNode } from "../recordApi/initialiseChildren" +import { initialiseNewIndex } from "./initialiseNewIndex" +import { saveApplicationHierarchy } from "../templateApi/saveApplicationHierarchy" + +export const upgradeData = app => async newHierarchy => { + const diff = diffHierarchy(app.hierarchy, newHierarchy) + const changeActions = gatherChangeActions(diff) + + if (changeActions.length === 0) return + + newHierarchy = constructHierarchy(newHierarchy) + const newApp = newHierarchy && cloneApp(app, { + hierarchy: newHierarchy + }) + await doUpgrade(app, newApp, changeActions) + await saveApplicationHierarchy(newApp)(newHierarchy) } -*/ \ No newline at end of file + +const gatherChangeActions = (diff) => + $(diff, [ + map(actionForChange), + flatten, + uniqBy(a => a.compareKey) + ]) + +const doUpgrade = async (oldApp, newApp, changeActions) => { + for(let action of changeActions) { + await action.run(oldApp, newApp, action.diff) + } +} + +const actionForChange = diff => + switchCase( + + [isChangeType(HierarchyChangeTypes.recordCreated), recordCreatedAction], + + [isChangeType(HierarchyChangeTypes.recordDeleted), deleteRecordsAction], + + [ + isChangeType(HierarchyChangeTypes.recordFieldsChanged), + rebuildAffectedIndexesAction + ], + + [isChangeType(HierarchyChangeTypes.recordRenamed), renameRecordAction], + + [ + isChangeType(HierarchyChangeTypes.recordEstimatedRecordTypeChanged), + reshardRecordsAction + ], + + [isChangeType(HierarchyChangeTypes.indexCreated), newIndexAction], + + [isChangeType(HierarchyChangeTypes.indexDeleted), deleteIndexAction], + + [isChangeType(HierarchyChangeTypes.indexChanged), rebuildIndexAction], + + )(diff) + + +const isChangeType = changeType => change => + change.type === changeType + +const action = (diff, compareKey, run) => ({ + diff, + compareKey, + run, +}) + + +const reshardRecordsAction = diff => + [action(diff, `reshardRecords-${diff.oldNode.nodeKey()}`, runReshardRecords)] + +const rebuildIndexAction = diff => + [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)] + +const newIndexAction = diff => { + if (isAncestorIndex(diff.newNode)) { + return [action(diff, `rebuildIndex-${diff.newNode.nodeKey()}`, runRebuildIndex)] + } else { + return [action(diff, `newIndex-${diff.newNode.nodeKey()}`, runNewIndex)] + } +} + +const deleteIndexAction = diff => + [action(diff, `deleteIndex-${diff.oldNode.nodeKey()}`, runDeleteIndex)] + +const deleteRecordsAction = diff => + [action(diff, `deleteRecords-${diff.oldNode.nodeKey()}`, runDeleteRecords)] + +const renameRecordAction = diff => + [action(diff, `renameRecords-${diff.oldNode.nodeKey()}`, runRenameRecord)] + +const recordCreatedAction = diff => { + if (isTopLevelRecord(diff.newNode)) { + return [action(diff, `initialiseRoot`, runInitialiseRoot)] + } + + return [action(diff, `initialiseChildRecord-${diff.newNode.nodeKey()}`, runInitialiseChildRecord)] +} + +const rebuildAffectedIndexesAction = diff =>{ + const newHierarchy = findRoot(diff.newNode) + const oldHierarchy = findRoot(diff.oldNode) + const indexes = getDependantIndexes(newHierarchy, diff.newNode) + + const changedFields = (() => { + const addedFields = differenceBy(f => f.name) + (diff.oldNode.fields) + (diff.newNode.fields) + + const removedFields = differenceBy(f => f.name) + (diff.newNode.fields) + (diff.oldNode.fields) + + return map(f => f.name)([...addedFields, ...removedFields]) + })() + + const isIndexAffected = i => { + if (!isEqual( + generateSchema(oldHierarchy, i), + generateSchema(newHierarchy, i))) return true + + if (some(f => indexes.filter.indexOf(`record.${f}`) > -1)(changedFields)) + return true + + if (some(f => indexes.getShardName.indexOf(`record.${f}`) > -1)(changedFields)) + return true + + return false + } + + return $(indexes, [ + filter(isIndexAffected), + map(i => action({ newNode:i }, `rebuildIndex-${i.nodeKey()}`, runRebuildIndex)) + ]) +} + +const runReshardRecords = async change => { + throw new Error("Resharding of records is not supported yet") +} + +const runRebuildIndex = async (_, newApp, diff) => { + await _buildIndex(newApp, diff.newNode.nodeKey()) +} + +const runDeleteIndex = async (oldApp, _, diff) => { + await deleteAllIndexFilesForNode(oldApp, diff.oldNode) +} + +const runDeleteRecords = async (oldApp, _, diff) => { + await deleteAllRecordsForNode(oldApp, diff.oldNode) +} + +const runNewIndex = async (_, newApp, diff) => { + await initialiseNewIndex(newApp, diff.newNode) +} + +const runRenameRecord = change => { + /* + Going to disllow this in the builder. once a collection key is set... its done + */ +} + +const runInitialiseRoot = async (_, newApp) => { + await initialiseData(newApp.datastore, newApp) +} + +const runInitialiseChildRecord = async (_, newApp, diff) => { + await initialiseChildrenForNode(newApp.datastore, diff.newNode) +} \ No newline at end of file diff --git a/packages/core/src/transactions/cleanup.js b/packages/core/src/transactions/cleanup.js index eeb7624867..0770af7c69 100644 --- a/packages/core/src/transactions/cleanup.js +++ b/packages/core/src/transactions/cleanup.js @@ -14,8 +14,10 @@ export const cleanup = async app => { const lock = await getTransactionLock(app) if (isNolock(lock)) return - try { + const _cleanupBatch = async () => { + let processed = 0 const transactions = await retrieve(app) + let i = 1 if (transactions.length > 0) { await executeTransactions(app)(transactions) @@ -34,10 +36,21 @@ export const cleanup = async app => { ]) await Promise.all(deleteFiles) + + processed = transactions.length + } + return processed + } + + try { + let count = -1 + while (count !== 0) { + count = await _cleanupBatch() } } finally { await releaseLock(app, lock) } + } const getTransactionLock = async app => diff --git a/packages/core/src/transactions/execute.js b/packages/core/src/transactions/execute.js index d438204246..6180b776ba 100644 --- a/packages/core/src/transactions/execute.js +++ b/packages/core/src/transactions/execute.js @@ -1,6 +1,7 @@ import { filter, map, + reduce, isUndefined, includes, flatten, @@ -10,6 +11,7 @@ import { keys, differenceBy, difference, + some, } from "lodash/fp" import { union } from "lodash" import { @@ -38,14 +40,22 @@ import { fieldReversesReferenceToIndex, isReferenceIndex, getExactNodeForKey, + getParentKey } from "../templateApi/hierarchy" import { getRecordInfo } from "../recordApi/recordInfo" import { getIndexDir } from "../indexApi/getIndexDir" +import { initialiseIndex } from "../indexing/initialiseIndex" export const executeTransactions = app => async transactions => { const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions) for (const shard of keys(recordsByShard)) { + if (recordsByShard[shard].isRebuild) + await initialiseIndex( + app.datastore, + getParentKey(recordsByShard[shard].indexDir), + recordsByShard[shard].indexNode + ) await applyToShard( app.hierarchy, app.datastore, @@ -66,9 +76,9 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => { const indexBuild = getBuildIndexTransactionsByShard(hierarchy, transactions) - const toRemove = [...deletes, ...updates.toRemove] + const toRemove = [...deletes, ...updates.toRemove, ...indexBuild.toRemove] - const toWrite = [...created, ...updates.toWrite, ...indexBuild] + const toWrite = [...created, ...updates.toWrite, ...indexBuild.toWrite] const transByShard = {} @@ -77,6 +87,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => { transByShard[t.indexShardKey] = { writes: [], removes: [], + isRebuild: some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toWrite) + || some(i => i.indexShardKey === t.indexShardKey)(indexBuild.toRemove), indexDir: t.indexDir, indexNodeKey: t.indexNode.nodeKey(), indexNode: t.indexNode, @@ -207,7 +219,7 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => { const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { const buildTransactions = $(transactions, [filter(isBuildIndex)]) - if (!isNonEmptyArray(buildTransactions)) return [] + if (!isNonEmptyArray(buildTransactions)) return { toWrite:[], toRemove:[] } const indexNode = transactions.indexNode const getIndexDirs = t => { @@ -247,8 +259,8 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { return $(buildTransactions, [ map(t => { - const mappedRecord = evaluate(t.record)(indexNode) - if (!mappedRecord.passedFilter) return null + const mappedRecord = evaluate(t.record)(indexNode) + mappedRecord.result = mappedRecord.result || t.record const indexDirs = getIndexDirs(t) return $(indexDirs, [ map(indexDir => ({ @@ -262,9 +274,16 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => { ), })), ]) + }), flatten, - filter(isSomething), + reduce((obj, res) => { + if (res.mappedRecord.passedFilter) + obj.toWrite.push(res) + else + obj.toRemove.push(res) + return obj + }, { toWrite: [], toRemove: [] }) ]) } diff --git a/packages/core/src/transactions/retrieve.js b/packages/core/src/transactions/retrieve.js index d0f24454f0..f32d091321 100644 --- a/packages/core/src/transactions/retrieve.js +++ b/packages/core/src/transactions/retrieve.js @@ -22,32 +22,41 @@ export const retrieve = async app => { TRANSACTIONS_FOLDER ) - let transactions = [] - if (some(isBuildIndexFolder)(transactionFiles)) { - const buildIndexFolder = find(isBuildIndexFolder)(transactionFiles) - - transactions = await retrieveBuildIndexTransactions( - app, - joinKey(TRANSACTIONS_FOLDER, buildIndexFolder) - ) + const buildIndexFolders = filter(isBuildIndexFolder)(transactionFiles) + let currentFolderIndex = 0 + while (currentFolderIndex < buildIndexFolders.length) { + const buildIndexFolder = buildIndexFolders[currentFolderIndex] + const transactions = await retrieveBuildIndexTransactions( + app, + joinKey(TRANSACTIONS_FOLDER, buildIndexFolder) + ) + if(transactions.length === 0) { + await app.datastore.deleteFolder( + joinKey(TRANSACTIONS_FOLDER, buildIndexFolder)) + } else { + return transactions + } + currentFolderIndex += 1 + } + + return [] } - if (transactions.length > 0) return transactions - return await retrieveStandardTransactions(app, transactionFiles) } const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => { const childFolders = await app.datastore.getFolderContents(buildIndexFolder) - if (childFolders.length === 0) { - // cleanup - await app.datastore.deleteFolder(buildIndexFolder) + const childFolderCount = childFolders.length + if (childFolderCount === 0) { return [] } const getTransactionFiles = async (childFolderIndex = 0) => { - if (childFolderIndex >= childFolders.length) return [] + if (childFolderIndex >= childFolders.length) { + return { childFolderKey: "", files: [] } + } const childFolderKey = joinKey( buildIndexFolder, @@ -55,17 +64,19 @@ const retrieveBuildIndexTransactions = async (app, buildIndexFolder) => { ) const files = await app.datastore.getFolderContents(childFolderKey) - if (files.length === 0) { - await app.datastore.deleteFolder(childFolderKey) - return await getTransactionFiles(childFolderIndex + 1) + if (files.length > 0) { + return { childFolderKey, files } } - return { childFolderKey, files } + await app.datastore.deleteFolder(childFolderKey) + return await getTransactionFiles(childFolderIndex + 1) } const transactionFiles = await getTransactionFiles() - if (transactionFiles.files.length === 0) return [] + if (transactionFiles.files.length === 0) { + return [] + } const transactions = $(transactionFiles.files, [map(parseTransactionId)]) diff --git a/packages/core/src/transactions/setCleanupFunc.js b/packages/core/src/transactions/setCleanupFunc.js new file mode 100644 index 0000000000..ede9469c6e --- /dev/null +++ b/packages/core/src/transactions/setCleanupFunc.js @@ -0,0 +1,14 @@ +import { cleanup } from "./cleanup" + +export const setCleanupFunc = (app, cleanupTransactions) => { + if (cleanupTransactions) { + app.cleanupTransactions = cleanupTransactions + return + } + + if (!app.cleanupTransactions || app.cleanupTransactions.isDefault) { + const newCleanup = async () => cleanup(app) + newCleanup.isDefault = true + app.cleanupTransactions = newCleanup + } +} \ No newline at end of file diff --git a/packages/core/src/transactions/transactionsCommon.js b/packages/core/src/transactions/transactionsCommon.js index 542d9cf383..59583906db 100644 --- a/packages/core/src/transactions/transactionsCommon.js +++ b/packages/core/src/transactions/transactionsCommon.js @@ -1,18 +1,20 @@ import { joinKey, keySep, getHashCode } from "../common" import { getLastPartInKey } from "../templateApi/hierarchy" +import { includes } from "lodash/fp" export const TRANSACTIONS_FOLDER = `${keySep}.transactions` export const LOCK_FILENAME = "lock" export const LOCK_FILE_KEY = joinKey(TRANSACTIONS_FOLDER, LOCK_FILENAME) export const idSep = "$" -const isOfType = typ => trans => trans.transactionType === typ +const isOfType = (...typ) => trans => includes(trans.transactionType)(typ) export const CREATE_RECORD_TRANSACTION = "create" export const UPDATE_RECORD_TRANSACTION = "update" export const DELETE_RECORD_TRANSACTION = "delete" export const BUILD_INDEX_TRANSACTION = "build" +export const isUpdate_Or_Rebuild = isOfType(UPDATE_RECORD_TRANSACTION, BUILD_INDEX_TRANSACTION) export const isUpdate = isOfType(UPDATE_RECORD_TRANSACTION) export const isDelete = isOfType(DELETE_RECORD_TRANSACTION) export const isCreate = isOfType(CREATE_RECORD_TRANSACTION) diff --git a/packages/core/test/specHelpers.js b/packages/core/test/specHelpers.js index ea44cdc6c5..fa37c5819c 100644 --- a/packages/core/test/specHelpers.js +++ b/packages/core/test/specHelpers.js @@ -24,7 +24,7 @@ import { filter, find } from "lodash/fp" import { createBehaviourSources } from "../src/actionsApi/buildBehaviourSource" import { createAction, createTrigger } from "../src/templateApi/createActions" import { initialiseActions } from "../src/actionsApi/initialise" -import { cleanup } from "../src/transactions/cleanup" +import { setCleanupFunc } from "../src/transactions/setCleanupFunc" import { permission } from "../src/authApi/permissions" import { generateFullPermissions } from "../src/authApi/generateFullPermissions" import { initialiseData } from "../src/appInitialise/initialiseData" @@ -39,9 +39,9 @@ export const testTemplatesPath = testAreaName => path.join(testFileArea(testAreaName), templateDefinitions) export const getMemoryStore = () => setupDatastore(memory({})) -export const getMemoryTemplateApi = () => { +export const getMemoryTemplateApi = (store) => { const app = { - datastore: getMemoryStore(), + datastore: store || getMemoryStore(), publish: () => {}, getEpochTime: async () => new Date().getTime(), user: { name: "", permissions: [permission.writeTemplates.get()] }, @@ -78,8 +78,9 @@ export const appFromTempalteApi = async ( const fullPermissions = generateFullPermissions(app) app.user.permissions = fullPermissions - if (disableCleanupTransactions) app.cleanupTransactions = async () => {} - else app.cleanupTransactions = async () => await cleanup(app) + if (disableCleanupTransactions) setCleanupFunc(app, async () => {}) + else setCleanupFunc(app) + return app } @@ -100,8 +101,9 @@ export const getRecordApiFromTemplateApi = async ( disableCleanupTransactions = false ) => { const app = await appFromTempalteApi(templateApi, disableCleanupTransactions) - const recordapi = getRecordApi() + const recordapi = getRecordApi(app) recordapi._storeHandle = app.datastore + return recordapi } export const getCollectionApiFromTemplateApi = async ( diff --git a/packages/core/test/templateApi.diffHierarchy.spec.js b/packages/core/test/templateApi.diffHierarchy.spec.js index 6f9cbce4e1..01eb54e69e 100644 --- a/packages/core/test/templateApi.diffHierarchy.spec.js +++ b/packages/core/test/templateApi.diffHierarchy.spec.js @@ -1,6 +1,5 @@ -import { getMemoryTemplateApi } from "./specHelpers" +import { setup } from "./upgradeDataSetup" import { diffHierarchy, HierarchyChangeTypes } from "../src/templateApi/diffHierarchy" -import { getFlattenedHierarchy } from "../src/templateApi/hierarchy" describe("diffHierarchy", () => { @@ -13,7 +12,7 @@ describe("diffHierarchy", () => { it("should detect root record created", async () => { const oldHierarchy = (await setup()).root; - const newSetup = (await setup()); + const newSetup = await setup() const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false) const diff = diffHierarchy(oldHierarchy, newSetup.root) expect(diff).toEqual([{ @@ -25,7 +24,7 @@ describe("diffHierarchy", () => { it("should only detect root record, when newly created root record has children ", async () => { const oldHierarchy = (await setup()).root; - const newSetup = (await setup()); + const newSetup = await setup() const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false) newSetup.templateApi.getNewRecordTemplate(opportunity, "invoice", true) const diff = diffHierarchy(oldHierarchy, newSetup.root) @@ -38,7 +37,7 @@ describe("diffHierarchy", () => { it("should detect child record created", async () => { const oldHierarchy = (await setup()).root; - const newSetup = (await setup()); + const newSetup = await setup() const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.contact, "opportunity", false) const diff = diffHierarchy(oldHierarchy, newSetup.root) expect(diff).toEqual([{ @@ -49,8 +48,8 @@ describe("diffHierarchy", () => { }) it("should detect root record deleted", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact") const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -61,8 +60,8 @@ describe("diffHierarchy", () => { }) it("should detect child record deleted", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal") const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -73,8 +72,8 @@ describe("diffHierarchy", () => { }) it("should detect root record renamed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.contact.collectionKey = "CONTACTS" const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -85,8 +84,8 @@ describe("diffHierarchy", () => { }) it("should detect child record renamed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.deal.collectionKey = "CONTACTS" const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -97,8 +96,8 @@ describe("diffHierarchy", () => { }) it("should detect root record field removed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.contact.fields = newSetup.contact.fields.filter(f => f.name !== "name") const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -109,8 +108,8 @@ describe("diffHierarchy", () => { }) it("should detect child record field removed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.deal.fields = newSetup.deal.fields.filter(f => f.name !== "name") const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -121,8 +120,8 @@ describe("diffHierarchy", () => { }) it("should detect record field added", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() const notesField = newSetup.templateApi.getNewField("string") notesField.name = "notes" newSetup.templateApi.addField(newSetup.contact, notesField) @@ -136,8 +135,8 @@ describe("diffHierarchy", () => { }) it("should detect 1 record field added and 1 removed (total no. fields unchanged)", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() const notesField = newSetup.templateApi.getNewField("string") notesField.name = "notes" newSetup.templateApi.addField(newSetup.contact, notesField) @@ -151,8 +150,8 @@ describe("diffHierarchy", () => { }) it("should detect root record estimated record count changed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.contact.estimatedRecordCount = 987 const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -163,8 +162,8 @@ describe("diffHierarchy", () => { }) it("should detect root record estimated record count changed", async () => { - const oldSetup = (await setup()); - const newSetup = (await setup()); + const oldSetup = await setup() + const newSetup = await setup() newSetup.deal.estimatedRecordCount = 987 const diff = diffHierarchy(oldSetup.root, newSetup.root) expect(diff).toEqual([{ @@ -174,44 +173,97 @@ describe("diffHierarchy", () => { }]) }) - it("should detect root record created", async () => { - const oldHierarchy = (await setup()).root; - const newSetup = (await setup()); - const opportunity = newSetup.templateApi.getNewRecordTemplate(newSetup.root, "opportunity", false) + it("should detect root index created", async () => { + const oldHierarchy = (await setup()).root + const newSetup = await setup() + const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.root) const diff = diffHierarchy(oldHierarchy, newSetup.root) expect(diff).toEqual([{ - newNode: opportunity, + newNode: all_deals, oldNode: null, - type: HierarchyChangeTypes.recordCreated + type: HierarchyChangeTypes.indexCreated }]) }) + it("should detect child index created", async () => { + const oldHierarchy = (await setup()).root + const newSetup = await setup() + const all_deals = newSetup.templateApi.getNewIndexTemplate(newSetup.contact) + const diff = diffHierarchy(oldHierarchy, newSetup.root) + expect(diff).toEqual([{ + newNode: all_deals, + oldNode: null, + type: HierarchyChangeTypes.indexCreated + }]) + }) + + it("should detect root index deleted", async () => { + const oldSetup = await setup() + const newSetup = await setup() + newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index") + const diff = diffHierarchy(oldSetup.root, newSetup.root) + expect(diff).toEqual([{ + newNode: null, + oldNode: oldSetup.root.indexes.find(i => i.name === "contact_index"), + type: HierarchyChangeTypes.indexDeleted + }]) + }) + + it("should detect child index deleted", async () => { + const oldSetup = await setup() + const newSetup = await setup() + newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index") + const diff = diffHierarchy(oldSetup.root, newSetup.root) + expect(diff).toEqual([{ + newNode: null, + oldNode: oldSetup.contact.indexes.find(i => i.name === "deal_index"), + type: HierarchyChangeTypes.indexDeleted + }]) + }) + + const testIndexChanged = (parent, makechange) => async () => { + const oldSetup = await setup() + const newSetup = await setup() + makechange(newSetup) + const diff = diffHierarchy(oldSetup.root, newSetup.root) + expect(diff).toEqual([{ + newNode: newSetup[parent].indexes[0], + oldNode: oldSetup[parent].indexes[0], + type: HierarchyChangeTypes.indexChanged + }]) + } + + it("should detect root index map changed", testIndexChanged("root", newSetup => { + newSetup.root.indexes[0].map = "new" + })) + + it("should detect root index filter changed", testIndexChanged("root", newSetup => { + newSetup.root.indexes[0].filter = "new" + })) + + it("should detect root index shardName changed", testIndexChanged("root", newSetup => { + newSetup.root.indexes[0].getShardName = "new" + })) + + it("should detect root index allowedRecordIds changed", testIndexChanged("root", newSetup => { + newSetup.root.indexes[0].allowedRecordNodeIds.push(3) + })) + + it("should detect child index allowedRecordIds changed", testIndexChanged("contact", newSetup => { + newSetup.contact.indexes[0].allowedRecordNodeIds.push(3) + })) + + it("should detect child index map changed", testIndexChanged("contact", newSetup => { + newSetup.contact.indexes[0].map = "new" + })) + + it("should detect child index filter changed", testIndexChanged("contact", newSetup => { + newSetup.contact.indexes[0].filter = "new" + })) + + it("should detect child index shardName changed", testIndexChanged("contact", newSetup => { + newSetup.contact.indexes[0].getShardName = "new" + })) + }) -const setup = async () => { - const { templateApi } = await getMemoryTemplateApi() - const root = templateApi.getNewRootLevel() - const contact = templateApi.getNewRecordTemplate(root, "contact", true) - - const nameField = templateApi.getNewField("string") - nameField.name = "name" - const statusField = templateApi.getNewField("string") - statusField.name = "status" - - templateApi.addField(contact, nameField) - templateApi.addField(contact, statusField) - - const lead = templateApi.getNewRecordTemplate(root, "lead", true) - const deal = templateApi.getNewRecordTemplate(contact, "deal", true) - - templateApi.addField(deal, {...nameField}) - templateApi.addField(deal, {...statusField}) - - getFlattenedHierarchy(root) - return { - root, contact, lead, deal, templateApi, - all_contacts: root.indexes[0], - all_leads: root.indexes[1], - deals_for_contacts: contact.indexes[0] - } -} \ No newline at end of file diff --git a/packages/core/test/templateApi.upgradeData.spec.js b/packages/core/test/templateApi.upgradeData.spec.js new file mode 100644 index 0000000000..2cbc7ffcb5 --- /dev/null +++ b/packages/core/test/templateApi.upgradeData.spec.js @@ -0,0 +1,244 @@ +import { + getRecordApiFromTemplateApi, + getIndexApiFromTemplateApi, +} from "./specHelpers" +import { upgradeData } from "../src/templateApi/upgradeData" +import { setup } from "./upgradeDataSetup" +import { $, splitKey } from "../src/common" +import { keys, filter } from "lodash/fp" +import { _listItems } from "../src/indexApi/listItems" +import { _save } from "../src/recordApi/save" + +describe("upgradeData", () => { + + it("should delete all records and child records, when root record node deleted", async () => { + const { oldSetup, newSetup, recordApi } = await configure() + newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact") + + await upgradeData(oldSetup.app)(newSetup.root) + + const remainingKeys = $(recordApi._storeHandle.data, [ + keys, + filter(k => splitKey(k)[0] === "contacts"), + ]) + + expect(remainingKeys.length).toBe(0) + + }) + + it("should not delete other root record types, when root record node deleted", async () => { + const { oldSetup, newSetup, recordApi } = await configure() + newSetup.root.children = newSetup.root.children.filter(n => n.name !== "contact") + + await upgradeData(oldSetup.app)(newSetup.root) + + const remainingKeys = $(recordApi._storeHandle.data, [ + keys, + filter(k => splitKey(k)[0] === "leads"), + ]) + + expect(remainingKeys.length > 0).toBe(true) + + }) + + it("should delete all child records, when child record node deleted", async () => { + const { oldSetup, newSetup, recordApi } = await configure() + newSetup.contact.children = newSetup.contact.children.filter(n => n.name !== "deal") + + const startingKeys = $(recordApi._storeHandle.data, [ + keys, + filter(k => k.includes("/deals/")), + ]) + + expect(startingKeys.length > 0).toBe(true) + + await upgradeData(oldSetup.app)(newSetup.root) + + const remainingKeys = $(recordApi._storeHandle.data, [ + keys, + filter(k => k.includes("/deals/")), + ]) + + expect(remainingKeys.length).toBe(0) + }) + + it("should build a new root index", async () => { + const { oldSetup, newSetup } = await configure() + const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.root) + newIndex.name = "more_contacts" + newIndex.allowedRecordNodeIds = [newSetup.contact.nodeId] + + await upgradeData(oldSetup.app)(newSetup.root) + + const itemsInNewIndex = await _listItems(newSetup.app, "/more_contacts") + + expect(itemsInNewIndex.length).toBe(2) + }) + + it("should update a root index", async () => { + const { oldSetup, newSetup } = await configure() + const contact_index = indexByName(newSetup.root, "contact_index") + contact_index.filter = "record.name === 'bobby'" + + await upgradeData(oldSetup.app)(newSetup.root) + + const itemsInNewIndex = await _listItems(newSetup.app, "/contact_index") + + expect(itemsInNewIndex.length).toBe(1) + }) + + it("should delete a root index", async () => { + const { oldSetup, newSetup } = await configure() + + // no exception + await _listItems(newSetup.app, "/contact_index") + + newSetup.root.indexes = newSetup.root.indexes.filter(i => i.name !== "contact_index") + + await upgradeData(oldSetup.app)(newSetup.root) + + let er + try { + await _listItems(newSetup.app, "/contact_index") + } catch (e) { + er = e + } + + expect(er).toBeDefined() + }) + + it("should build a new child index", async () => { + const { oldSetup, newSetup, records } = await configure() + const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.contact) + newIndex.name = "more_deals" + newIndex.allowedRecordNodeIds = [newSetup.deal.nodeId] + + await upgradeData(oldSetup.app)(newSetup.root) + + const itemsInNewIndex = await _listItems(newSetup.app, `${records.contact1.key}/more_deals`) + + expect(itemsInNewIndex.length).toBe(2) + }) + + it("should update a child index", async () => { + const { oldSetup, newSetup, records } = await configure() + const deal_index = indexByName(newSetup.contact, "deal_index") + deal_index.filter = "record.status === 'new'" + + let itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`) + expect(itemsInIndex.length).toBe(2) + + await upgradeData(oldSetup.app)(newSetup.root) + + itemsInIndex = await _listItems(newSetup.app, `${records.contact1.key}/deal_index`) + expect(itemsInIndex.length).toBe(1) + }) + + it("should delete a child index", async () => { + const { oldSetup, newSetup, records } = await configure() + + // no exception + await _listItems(newSetup.app, `${records.contact1.key}/deal_index`) + + newSetup.contact.indexes = newSetup.contact.indexes.filter(i => i.name !== "deal_index") + + await upgradeData(oldSetup.app)(newSetup.root) + + let er + try { + await _listItems(newSetup.app, `${records.contact1.key}/deal_index`) + } catch (e) { + er = e + } + + expect(er).toBeDefined() + }) + + it("should build a new reference index", async () => { + const { oldSetup, newSetup, records, recordApi } = await configure() + const newIndex = newSetup.templateApi.getNewIndexTemplate(newSetup.lead) + newIndex.name = "contact_leads" + newIndex.allowedRecordNodeIds = [newSetup.lead.nodeId] + newIndex.indexType = "reference" + + const leadField = newSetup.templateApi.getNewField("string") + leadField.name = "lead" + leadField.type = "reference" + leadField.typeOptions = { + reverseIndexNodeKeys: [ newIndex.nodeKey() ], + indexNodeKey: "/lead_index", + displayValue: "name" + } + + newSetup.templateApi.addField(newSetup.contact, leadField) + + await upgradeData(oldSetup.app)(newSetup.root) + + const indexKey = `${records.lead1.key}/contact_leads` + + let itemsInNewIndex = await _listItems(newSetup.app, indexKey) + + expect(itemsInNewIndex.length).toBe(0) + + records.contact1.lead = records.lead1 + records.contact1.isNew = false + + await _save(newSetup.app, records.contact1) + + itemsInNewIndex = await _listItems(newSetup.app, indexKey) + + expect(itemsInNewIndex.length).toBe(1) + + }) + +}) + +const configure = async () => { + const oldSetup = await setup() + + const recordApi = await getRecordApiFromTemplateApi(oldSetup.templateApi) + const indexApi = await getIndexApiFromTemplateApi(oldSetup.templateApi) + + const newSetup = await setup(oldSetup.store) + + const records = await createSomeRecords(recordApi) + + return { oldSetup, newSetup, recordApi, records, indexApi } +} + +const createSomeRecords = async recordApi => { + const contact1 = recordApi.getNew("/contacts", "contact") + contact1.name = "bobby" + const contact2 = recordApi.getNew("/contacts", "contact") + contact2.name = "poppy" + + await recordApi.save(contact1) + await recordApi.save(contact2) + + const deal1 = recordApi.getNew(`${contact1.key}/deals`, "deal") + deal1.name = "big mad deal" + deal1.status = "new" + const deal2 = recordApi.getNew(`${contact1.key}/deals`, "deal") + deal2.name = "smaller deal" + deal2.status = "old" + const deal3 = recordApi.getNew(`${contact2.key}/deals`, "deal") + deal3.name = "ok deal" + deal3.status = "new" + + await recordApi.save(deal1) + await recordApi.save(deal2) + await recordApi.save(deal3) + + const lead1 = recordApi.getNew("/leads", "lead") + lead1.name = "big new lead" + + await recordApi.save(lead1) + + + + return { + contact1, contact2, deal1, deal2, deal3, lead1, + } +} + +const indexByName = (parent, name) => parent.indexes.find(i => i.name === name) \ No newline at end of file diff --git a/packages/core/test/upgradeDataSetup.js b/packages/core/test/upgradeDataSetup.js new file mode 100644 index 0000000000..9ed97b0b17 --- /dev/null +++ b/packages/core/test/upgradeDataSetup.js @@ -0,0 +1,47 @@ +import { getMemoryTemplateApi, appFromTempalteApi } from "./specHelpers" +import { getFlattenedHierarchy } from "../src/templateApi/hierarchy" +import { initialiseData } from "../src/appInitialise/initialiseData" + +export const setup = async (store) => { + const { templateApi } = await getMemoryTemplateApi(store) + const root = templateApi.getNewRootLevel() + const contact = templateApi.getNewRecordTemplate(root, "contact", true) + contact.collectionName = "contacts" + + const nameField = templateApi.getNewField("string") + nameField.name = "name" + const statusField = templateApi.getNewField("string") + statusField.name = "status" + + templateApi.addField(contact, nameField) + templateApi.addField(contact, statusField) + + const lead = templateApi.getNewRecordTemplate(root, "lead", true) + lead.collectionName = "leads" + const deal = templateApi.getNewRecordTemplate(contact, "deal", true) + deal.collectionName = "deals" + + templateApi.addField(deal, {...nameField}) + templateApi.addField(deal, {...statusField}) + + templateApi.addField(lead, {...nameField}) + + getFlattenedHierarchy(root) + + if (!store) + await initialiseData(templateApi._storeHandle, { + hierarchy: root, + actions: [], + triggers: [], + }) + const app = await appFromTempalteApi(templateApi) + app.hierarchy = root + + return { + root, contact, lead, app, + deal, templateApi, store: templateApi._storeHandle, + all_contacts: root.indexes[0], + all_leads: root.indexes[1], + deals_for_contacts: contact.indexes[0], + } +} \ No newline at end of file diff --git a/packages/server/middleware/routeHandlers/upgradeData.js b/packages/server/middleware/routeHandlers/upgradeData.js new file mode 100644 index 0000000000..f25019c4c5 --- /dev/null +++ b/packages/server/middleware/routeHandlers/upgradeData.js @@ -0,0 +1,6 @@ +const StatusCodes = require("../../utilities/statusCodes") + +module.exports = async ctx => { + await ctx.instance.templateApi.upgradeData(ctx.request.body.newHierarchy) + ctx.response.status = StatusCodes.OK +} diff --git a/packages/server/middleware/routers.js b/packages/server/middleware/routers.js index dd77706f5f..082ebed01c 100644 --- a/packages/server/middleware/routers.js +++ b/packages/server/middleware/routers.js @@ -238,6 +238,10 @@ module.exports = (config, app) => { ctx.response.status = StatusCodes.UNAUTHORIZED } }) + .post( + "/_builder/instance/:appname/:instanceid/api/upgradeData", + routeHandlers.upgradeData + ) .post("/:appname/api/changeMyPassword", routeHandlers.changeMyPassword) .post( "/_builder/instance/:appname/:instanceid/api/changeMyPassword", From 5ca543f3738353a7d34317dfd34ad0eb46cf45c9 Mon Sep 17 00:00:00 2001 From: Michael Shanks Date: Fri, 20 Mar 2020 13:58:05 +0000 Subject: [PATCH 7/8] diff hierarchy bugfix --- packages/server/middleware/routeHandlers/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/middleware/routeHandlers/index.js b/packages/server/middleware/routeHandlers/index.js index af4ff60605..ea91c4ba12 100644 --- a/packages/server/middleware/routeHandlers/index.js +++ b/packages/server/middleware/routeHandlers/index.js @@ -18,6 +18,7 @@ const lookupField = require("./lookupField") const getRecord = require("./getRecord") const deleteRecord = require("./deleteRecord") const saveAppHierarchy = require("./saveAppHierarchy") +const upgradeData = require("./saveAppHierarchy") module.exports = { authenticate, @@ -40,4 +41,5 @@ module.exports = { getRecord, deleteRecord, saveAppHierarchy, + upgradeData, } From 106fa19a31532636c89e0384d0bb60374f36a8ba Mon Sep 17 00:00:00 2001 From: Michael Shanks Date: Fri, 20 Mar 2020 14:01:10 +0000 Subject: [PATCH 8/8] bugfix: Delete button cut off on windows --- packages/builder/src/database/ActionsHeader.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/database/ActionsHeader.svelte b/packages/builder/src/database/ActionsHeader.svelte index 71e3e8abdc..8bec64e7f1 100644 --- a/packages/builder/src/database/ActionsHeader.svelte +++ b/packages/builder/src/database/ActionsHeader.svelte @@ -58,6 +58,7 @@ padding: 1.5rem; width: 100%; align-items: right; + box-sizing: border-box; } .actions-modal-body {