Merge branch 'develop' of github.com:Budibase/budibase into feature/json-backend

.github/workflows/release-selfhost.yml

@@ -27,15 +27,19 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-1
 
+      - name: 'Get Previous tag'
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+
       - name: Build/release Docker images (Self Host)
         run: | 
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
           yarn build
-          yarn build:docker:production
+          yarn build:docker:selfhost
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: latest
+          BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
       
       - uses: azure/setup-helm@v1
         id: install

LICENSE

@@ -1,11 +1,7 @@
-Copyright 2019-2021, Budibase Inc.
+Copyright 2019-2021, Budibase Ltd.
 
-Each Budibase package has its own license:
+Each Budibase package has its own license, please check the license file in each package.
 
-builder: GPLv3
+You can consider Budibase to be GPLv3 licensed overall.
-server: GPLv3
-client: MPLv2.0
 
-You can consider Budibase to be GPLv3 licensed.
+The apps that you build with Budibase do not package any GPLv3 licensed code, thus do not fall under those restrictions.
-
-The apps that you build with Budibase do not fall under GPLv3 - hence why our components and client library are licensed differently.

hosting/scripts/linux/release-to-docker-hub.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 tag=$1
-production=$2
 
 if [[ ! "$tag" ]]; then
 	echo "No tag present. You must pass a tag to this script"
@@ -13,11 +12,5 @@ echo "Tagging images with tag: $tag"
 docker tag app-service budibase/apps:$tag
 docker tag worker-service budibase/worker:$tag
 
-if [[ "$production" ]]; then
-	echo "Production Deployment. Tagging latest.."
-	docker tag app-service budibase/apps:latest
-	docker tag worker-service budibase/worker:latest
-fi
-
 docker push --all-tags budibase/apps 
 docker push --all-tags budibase/worker

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -44,9 +44,10 @@
     "test:e2e": "lerna run cy:test",
     "test:e2e:ci": "lerna run cy:ci",
     "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:production": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION release && cd -",
+    "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
     "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
+    "build:docs": "lerna run build:docs",
     "release:helm": "./scripts/release_helm_chart.sh",
     "env:multi:enable": "lerna run env:multi:enable",
     "env:multi:disable": "lerna run env:multi:disable",

packages/auth/LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright 2019-2021, Budibase Ltd
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

packages/auth/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",
-  "license": "AGPL-3.0",
+  "license": "GPL-3.0",
   "scripts": {
     "test": "jest",
     "test:watch": "jest --watchAll"

packages/auth/src/security/permissions.js

@@ -131,38 +131,14 @@ exports.getBuiltinPermissionByID = id => {
   return perms.find(perm => perm._id === id)
 }
 
-exports.doesHaveResourcePermission = (
+exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => {
-  permissions,
+  const basePermissions = [
-  permLevel,
+    ...new Set(rolesHierarchy.map(role => role.permissionId)),
-  { resourceId, subResourceId }
+  ]
-) => {
-  // set foundSub to not subResourceId, incase there is no subResource
-  let foundMain = false,
-    foundSub = false
-  for (let [resource, levels] of Object.entries(permissions)) {
-    if (resource === resourceId && levels.indexOf(permLevel) !== -1) {
-      foundMain = true
-    }
-    if (
-      subResourceId &&
-      resource === subResourceId &&
-      levels.indexOf(permLevel) !== -1
-    ) {
-      foundSub = true
-    }
-    // this will escape if foundMain only when no sub resource
-    if (foundMain && foundSub) {
-      break
-    }
-  }
-  return foundMain || foundSub
-}
-
-exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => {
   const builtins = Object.values(BUILTIN_PERMISSIONS)
   let permissions = flatten(
     builtins
-      .filter(builtin => permissionIds.indexOf(builtin._id) !== -1)
+      .filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
       .map(builtin => builtin.permissions)
   )
   for (let permission of permissions) {

packages/auth/src/security/roles.js

@@ -170,47 +170,18 @@ async function getAllUserRoles(appId, userRoleId) {
  * to determine if a user can access something that requires a specific role.
  * @param {string} appId The ID of the application from which roles should be obtained.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
+ * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
  * @returns {Promise<string[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-exports.getUserRoleHierarchy = async (appId, userRoleId) => {
+exports.getUserRoleHierarchy = async (
+  appId,
+  userRoleId,
+  opts = { idOnly: true }
+) => {
   // special case, if they don't have a role then they are a public user
-  return (await getAllUserRoles(appId, userRoleId)).map(role => role._id)
+  const roles = await getAllUserRoles(appId, userRoleId)
-}
+  return opts.idOnly ? roles.map(role => role._id) : roles
-
-/**
- * Get all of the user permissions which could be found across the role hierarchy
- * @param appId The ID of the application from which roles should be obtained.
- * @param userRoleId The user's role ID, this can be found in their access token.
- * @returns {Promise<{basePermissions: string[], permissions: Object}>} the base
- * permission IDs as well as any custom resource permissions.
- */
-exports.getUserPermissions = async (appId, userRoleId) => {
-  const rolesHierarchy = await getAllUserRoles(appId, userRoleId)
-  const basePermissions = [
-    ...new Set(rolesHierarchy.map(role => role.permissionId)),
-  ]
-  const permissions = {}
-  for (let role of rolesHierarchy) {
-    if (role.permissions) {
-      for (let [resource, levels] of Object.entries(role.permissions)) {
-        if (!permissions[resource]) {
-          permissions[resource] = []
-        }
-        const permsSet = new Set(permissions[resource])
-        if (Array.isArray(levels)) {
-          levels.forEach(level => permsSet.add(level))
-        } else {
-          permsSet.add(levels)
-        }
-        permissions[resource] = [...permsSet]
-      }
-    }
-  }
-  return {
-    basePermissions,
-    permissions,
-  }
 }
 
 /**
@@ -246,6 +217,39 @@ exports.getAllRoles = async appId => {
   return roles
 }
 
+/**
+ * This retrieves the required role/
+ * @param appId
+ * @param permLevel
+ * @param resourceId
+ * @param subResourceId
+ * @return {Promise<{permissions}|Object>}
+ */
+exports.getRequiredResourceRole = async (
+  appId,
+  permLevel,
+  { resourceId, subResourceId }
+) => {
+  const roles = await exports.getAllRoles(appId)
+  let main = [],
+    sub = []
+  for (let role of roles) {
+    // no permissions, ignore it
+    if (!role.permissions) {
+      continue
+    }
+    const mainRes = role.permissions[resourceId]
+    const subRes = role.permissions[subResourceId]
+    if (mainRes && mainRes.indexOf(permLevel) !== -1) {
+      main.push(role._id)
+    } else if (subRes && subRes.indexOf(permLevel) !== -1) {
+      sub.push(role._id)
+    }
+  }
+  // for now just return the IDs
+  return main.concat(sub)
+}
+
 class AccessController {
   constructor(appId) {
     this.appId = appId

packages/bbui/LICENSE

@@ -0,0 +1,375 @@
+Mozilla Public License Version 2.0
+==================================
+
+Copyright 2019-2021, Budibase Ltd
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

packages/bbui/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
-  "license": "AGPL-3.0",
+  "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
   "exports": {

packages/bbui/src/Form/Combobox.svelte

@@ -6,6 +6,7 @@
   export let value = null
   export let label = undefined
   export let disabled = false
+  export let readonly = false
   export let labelPosition = "above"
   export let error = null
   export let placeholder = "Choose an option or type"
@@ -33,6 +34,7 @@
     {value}
     {options}
     {placeholder}
+    {readonly}
     {getOptionLabel}
     {getOptionValue}
     on:change={onChange}

packages/bbui/src/Form/Core/Combobox.svelte

@@ -9,6 +9,7 @@
   export let id = null
   export let placeholder = "Choose an option or type"
   export let disabled = false
+  export let readonly = false
   export let error = null
   export let options = []
   export let getOptionLabel = option => option
@@ -73,6 +74,7 @@
       value={value || ""}
       placeholder={placeholder || ""}
       {disabled}
+      {readonly}
       class="spectrum-Textfield-input spectrum-InputGroup-input"
     />
   </div>

packages/bbui/src/Form/Core/TextField.svelte

@@ -16,15 +16,15 @@
   const dispatch = createEventDispatcher()
   let focus = false
 
-  const updateValue = value => {
+  const updateValue = newValue => {
     if (readonly) {
       return
     }
     if (type === "number") {
-      const float = parseFloat(value)
+      const float = parseFloat(newValue)
-      value = isNaN(float) ? null : float
+      newValue = isNaN(float) ? null : float
     }
-    dispatch("change", value)
+    dispatch("change", newValue)
   }
 
   const onFocus = () => {

packages/builder/LICENSE.md

@@ -1,7 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright 2019-2021, Budibase Inc
+ Copyright 2019-2021, Budibase Ltd
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

packages/builder/cypress/integration/addMultiOptionDatatype.spec.js

@@ -14,7 +14,8 @@ context("Add Multi-Option Datatype", () => {
     cy.navigateToFrontend()
     cy.wait(500)
     // Add data provider
-        cy.get(`[data-cy="category-Data Provider"]`).click()
+    cy.get(`[data-cy="category-Data"]`).click()
+    cy.get(`[data-cy="component-Data Provider"]`).click()
     cy.get('[data-cy="dataSource-prop-control"]').click()
     cy.get(".dropdown").contains("Multi Data").click()
     cy.wait(500)
@@ -24,18 +25,20 @@ context("Add Multi-Option Datatype", () => {
     cy.get(".dropdown").contains("Multi Data").click()
     cy.wait(500)
     // Add multi-select picker to form
-        cy.addComponent("Form", "Multi-select Picker").then((componentId) => {
+    cy.addComponent("Form", "Multi-select Picker").then(componentId => {
-            cy.get('[data-cy="field-prop-control"]').type("Test Data").type('{enter}')
+      cy.get('[data-cy="field-prop-control"]').type("Test Data").type("{enter}")
       cy.wait(1000)
       cy.getComponent(componentId).contains("Choose some options").click()
       // Check picker has 5 items
-            cy.getComponent(componentId).find('li').should('have.length', 5)
+      cy.getComponent(componentId).find("li").should("have.length", 5)
       // Select all items
       for (let i = 1; i < 6; i++) {
-                cy.getComponent(componentId).find('li').contains(i).click()
+        cy.getComponent(componentId).find("li").contains(i).click()
       }
       // Check items have been selected
-            cy.getComponent(componentId).find('.spectrum-Picker-label').contains("(5)")
+      cy.getComponent(componentId)
+        .find(".spectrum-Picker-label")
+        .contains("(5)")
     })
   })
 })

packages/builder/cypress/integration/createAutomation.spec.js

@@ -12,7 +12,7 @@ context("Create a automation", () => {
     cy.get("[data-cy='new-screen'] > .spectrum-Icon").click()
     cy.get(".modal-inner-wrapper").within(() => {
       cy.get("input").type("Add Row")
-      cy.contains("Row Created").click()
+      cy.contains("Row Created").click({ force: true })
       cy.wait(500)
       cy.get(".spectrum-Button--cta").click()
     })
@@ -22,9 +22,9 @@ context("Create a automation", () => {
     cy.get(".spectrum-Picker-label").click()
     cy.wait(500)
     cy.contains("dog").click()
-
+    cy.wait(2000)
     // Create action
-    cy.contains("Add Action").click()
+    cy.get(".block > .spectrum-Icon").click()
     cy.get(".modal-inner-wrapper").within(() => {
       cy.wait(1000)
       cy.contains("Create Row").trigger('mouseover').click().click()

packages/builder/cypress/support/commands.js

@@ -182,7 +182,9 @@ Cypress.Commands.add("navigateToFrontend", () => {
   cy.wait(1000)
   cy.contains("Design").click()
   cy.get(".spectrum-Search").type("/")
-  cy.get(".nav-item").contains("Home").click()
+  cy.createScreen("home", "home")
+  cy.addComponent("Elements", "Headline")
+  cy.get(".nav-item").contains("home").click()
 })
 
 Cypress.Commands.add("createScreen", (screenName, route) => {

packages/builder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
-  "license": "AGPL-3.0",
+  "license": "GPL-3.0",
   "private": true,
   "scripts": {
     "build": "routify -b && vite build --emptyOutDir",
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.185-alpha.14",
+    "@budibase/bbui": "^0.9.190-alpha.11",
-    "@budibase/client": "^0.9.185-alpha.14",
+    "@budibase/client": "^0.9.190-alpha.11",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.185-alpha.14",
+    "@budibase/string-templates": "^0.9.190-alpha.11",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/store/automation/index.js

@@ -93,7 +93,9 @@ const automationActions = store => ({
   },
   select: automation => {
     store.update(state => {
+      let testResults = state.selectedAutomation?.testResults
       state.selectedAutomation = new Automation(cloneDeep(automation))
+      state.selectedAutomation.testResults = testResults
       state.selectedBlock = null
       return state
     })

packages/builder/src/builderStore/store/screenTemplates/createFromScratchScreen.js

@@ -4,6 +4,7 @@ export default {
   name: `Create from scratch`,
   id: `createFromScratch`,
   create: () => createScreen(),
+  table: `Create from scratch`,
 }
 
 const createScreen = () => {

packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js

@@ -15,6 +15,7 @@ export default function (tables) {
       name: `${table.name} - New`,
       create: () => createScreen(table),
       id: NEW_ROW_TEMPLATE,
+      table: table.name,
     }
   })
 }

packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js

@@ -17,6 +17,7 @@ export default function (tables) {
       name: `${table.name} - Detail`,
       create: () => createScreen(table),
       id: ROW_DETAIL_TEMPLATE,
+      table: table.name,
     }
   })
 }

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -10,6 +10,7 @@ export default function (tables) {
       name: `${table.name} - List`,
       create: () => createScreen(table),
       id: ROW_LIST_TEMPLATE,
+      table: table.name,
     }
   })
 }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -84,7 +84,6 @@
         class="block"
         animate:flip={{ duration: 500 }}
         in:fly|local={{ x: 500, duration: 1500 }}
-        out:fly|local={{ x: 500, duration: 800 }}
       >
         <FlowItem {testDataModal} {testAutomation} {onSelect} {block} />
       </div>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -25,10 +25,10 @@
   let resultsModal
   let setupToggled
   let blockComplete
+
   $: testResult = $automationStore.selectedAutomation.testResults?.steps.filter(
     step => step.stepId === block.stepId
   )
-
   $: isTrigger = block.type === "TRIGGER"
 
   $: selected = $automationStore.selectedBlock?.id === block.id
@@ -150,15 +150,6 @@
               >Finish and test automation</Button
             >
           {/if}
-          <Button
-            disabled={!hasCompletedInputs}
-            on:click={() => {
-              setupToggled = false
-              actionModal.show()
-            }}
-            primary={!isTrigger}
-            cta={isTrigger}>Add Action</Button
-          >
         {/if}
       </Layout>
     </div>

packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte

@@ -9,6 +9,7 @@
   onMount(() => {
     automationStore.actions.fetch()
   })
+
   function selectAutomation(automation) {
     automationStore.actions.select(automation)
     $goto(`./${automation._id}`)

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -3,7 +3,14 @@
   import { database } from "stores/backend"
   import { automationStore } from "builderStore"
   import { notifications } from "@budibase/bbui"
-  import { Input, ModalContent, Layout, Body, Icon } from "@budibase/bbui"
+  import {
+    Input,
+    InlineAlert,
+    ModalContent,
+    Layout,
+    Body,
+    Icon,
+  } from "@budibase/bbui"
   import analytics, { Events } from "analytics"
 
   let name
@@ -56,6 +63,10 @@
   onConfirm={createAutomation}
   disabled={!selectedTrigger || !name}
 >
+  <InlineAlert
+    header="You must publish your app to activate your automations."
+    message="To test your automation before publishing, you can use the 'Run Test' functionality on the next screen."
+  />
   <Body size="XS"
     >Please name your automation, then select a trigger. Every automation must
     start with a trigger.

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -51,8 +51,7 @@
     : { schema: {} }
   $: schemaFields = table ? Object.values(table.schema) : []
 
-  const onChange = debounce(
+  const onChange = debounce(async function (e, key) {
-    async function (e, key) {
     if (isTestModal) {
       // Special case for webhook, as it requires a body, but the schema already brings back the body's contents
       if (stepId === "WEBHOOK") {
@@ -67,15 +66,16 @@
         [key]: e.detail,
       })
       testData[key] = e.detail
+      await automationStore.actions.save(
+        $automationStore.selectedAutomation?.automation
+      )
     } else {
       block.inputs[key] = e.detail
       await automationStore.actions.save(
         $automationStore.selectedAutomation?.automation
       )
     }
-    },
+  }, 800)
-    isTestModal ? 0 : 800
-  )
 
   function getAvailableBindings(block, automation) {
     if (!block || !automation) {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -10,6 +10,7 @@
     ModalContent,
     Context,
     Modal,
+    notifications,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -26,7 +27,6 @@
     SWITCHABLE_TYPES,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
-  import { notifications } from "@budibase/bbui"
   import ValuesList from "components/common/ValuesList.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { truncate } from "lodash"
@@ -41,6 +41,7 @@
   const STRING_TYPE = FIELDS.STRING.type
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
+  const DATE_TYPE = FIELDS.DATETIME.type
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -76,13 +77,8 @@
   $: invalid =
     !field.name ||
     (field.type === LINK_TYPE && !field.tableId) ||
-    Object.keys($tables.draft?.schema ?? {}).some(
+    Object.keys(errors).length !== 0
-      key => key !== originalName && key === field.name
+  $: errors = checkErrors(field)
-    ) ||
-    columnNameInvalid
-  $: columnNameInvalid = PROHIBITED_COLUMN_NAMES.some(
-    name => field.name === name
-  )
 
   // used to select what different options can be displayed for column type
   $: canBeSearched =
@@ -111,6 +107,10 @@
     if (field.type === AUTO_TYPE) {
       field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
     }
+    if (field.type !== LINK_TYPE) {
+      delete field.fieldName
+    }
+    try {
       await tables.saveField({
         originalName,
         field,
@@ -118,6 +118,9 @@
         indexes,
       })
       dispatch("updatecolumns")
+    } catch (err) {
+      notifications.error(err)
+    }
   }
 
   function deleteColumn() {
@@ -266,6 +269,34 @@
     ) {
       fieldToCheck.constraints.numericality = {}
     }
+    if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
+      fieldToCheck.constraints.datetime = {}
+    }
+  }
+
+  function checkErrors(fieldInfo) {
+    function inUse(tbl, column, ogName = null) {
+      return Object.keys(tbl?.schema || {}).some(
+        key => key !== ogName && key === column
+      )
+    }
+    const newError = {}
+    if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
+      newError.name = `${PROHIBITED_COLUMN_NAMES.join(
+        ", "
+      )} are not allowed as column names`
+    } else if (inUse($tables.draft, fieldInfo.name, originalName)) {
+      newError.name = `Column name already in use.`
+    }
+    if (fieldInfo.fieldName && fieldInfo.tableId) {
+      const relatedTable = $tables.list.find(
+        tbl => tbl._id === fieldInfo.tableId
+      )
+      if (inUse(relatedTable, fieldInfo.fieldName)) {
+        newError.relatedName = `Column name already in use in table ${relatedTable.name}`
+      }
+    }
+    return newError
   }
 </script>
 
@@ -279,9 +310,7 @@
     label="Name"
     bind:value={field.name}
     disabled={uneditable || (linkEditDisabled && field.type === LINK_TYPE)}
-    error={columnNameInvalid
+    error={errors?.name}
-      ? `${PROHIBITED_COLUMN_NAMES.join(", ")} are not allowed as column names`
-      : ""}
   />
 
   <Select
@@ -390,6 +419,7 @@
       disabled={linkEditDisabled}
       label={`Column name in other table`}
       bind:value={field.fieldName}
+      error={errors.relatedName}
     />
   {:else if field.type === FORMULA_TYPE}
     <ModalBindableInput
@@ -398,7 +428,7 @@
       value={field.formula}
       on:change={e => (field.formula = e.detail)}
       bindings={getBindings({ table })}
-      serverSide="true"
+      allowJS
     />
   {:else if field.type === AUTO_TYPE}
     <Select

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -1,6 +1,7 @@
 <script>
-  import { Select, ModalContent } from "@budibase/bbui"
+  import { Select, ModalContent, notifications } from "@budibase/bbui"
   import download from "downloadjs"
+  import { get } from "builderStore/api"
 
   const FORMATS = [
     {
@@ -18,11 +19,18 @@
   let exportFormat = FORMATS[0].key
 
   async function exportView() {
-    download(
+    const uri = encodeURIComponent(view)
-      `/api/views/export?view=${encodeURIComponent(
+    const response = await get(
-        view
+      `/api/views/export?view=${uri}&format=${exportFormat}`
-      )}&format=${exportFormat}`
     )
+    if (response.status === 200) {
+      const data = await response.text()
+      download(data, `export.${exportFormat}`)
+    } else {
+      notifications.error(
+        `Unable to export ${exportFormat.toUpperCase()} data.`
+      )
+    }
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { datasources } from "stores/backend"
+  import { datasources, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -13,10 +13,16 @@
 
   async function deleteDatasource() {
     const wasSelectedSource = $datasources.selected
+    const wasSelectedTable = $tables.selected
     await datasources.delete(datasource)
     notifications.success("Datasource deleted")
     // navigate to first index page if the source you are deleting is selected
-    if (wasSelectedSource === datasource._id) {
+    const entities = Object.values(datasource.entities)
+    if (
+      wasSelectedSource === datasource._id ||
+      (entities &&
+        entities.find(entity => entity._id === wasSelectedTable?._id))
+    ) {
       $goto("./datasource")
     }
   }

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -71,6 +71,9 @@
     if ($touched.toCol && !toRelate.name) {
       errors.toCol = colNotSet
     }
+    if ($touched.primary && !fromPrimary) {
+      errors.primary = "Please pick the primary key"
+    }
     // currently don't support relationships back onto the table itself, needs to relate out
     const tableError = "From/to/through tables must be different"
     if (fromTable && (fromTable === toTable || fromTable === throughTable)) {
@@ -95,6 +98,16 @@
     return errors
   }
 
+  let fromPrimary
+  $: {
+    if (!fromPrimary && fromTable) {
+      fromPrimary = fromTable.primary[0]
+    }
+  }
+  $: isManyToMany =
+    fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY
+  $: isManyToOne =
+    fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_ONE
   $: tableOptions = plusTables.map(table => ({
     label: table.name,
     value: table._id,
@@ -179,13 +192,13 @@
       // foreignKey is what is linking out of the current table.
       relateFrom = {
         ...relateFrom,
-        foreignKey: fromTable.primary[0],
+        foreignKey: fromPrimary,
       }
       relateTo = {
         ...relateTo,
         relationshipType: RelationshipTypes.ONE_TO_MANY,
         foreignKey: relateFrom.fieldName,
-        fieldName: fromTable.primary[0],
+        fieldName: fromPrimary,
       }
     }
 
@@ -264,6 +277,15 @@
     bind:error={errors.from}
     bind:value={toRelationship.tableId}
   />
+  {#if isManyToOne && fromTable}
+    <Select
+      label={`Primary Key (${fromTable?.name})`}
+      options={Object.keys(fromTable?.schema)}
+      on:change={() => ($touched.primary = true)}
+      bind:error={errors.primary}
+      bind:value={fromPrimary}
+    />
+  {/if}
   <Select
     label={"Select to table"}
     options={tableOptions}
@@ -271,7 +293,7 @@
     bind:error={errors.to}
     bind:value={fromRelationship.tableId}
   />
-  {#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY}
+  {#if isManyToMany}
     <Select
       label={"Through"}
       options={tableOptions}
@@ -295,7 +317,7 @@
         bind:value={fromRelationship.throughFrom}
       />
     {/if}
-  {:else if fromRelationship?.relationshipType && toTable}
+  {:else if isManyToOne && toTable}
     <Select
       label={`Foreign Key (${toTable?.name})`}
       options={Object.keys(toTable?.schema).filter(

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -19,7 +19,10 @@
   let fields = []
   let hasValidated = false
 
-  $: valid = !schema || fields.every(column => schema[column].success)
+  $: valid =
+    !schema ||
+    (fields.every(column => schema[column].success) &&
+      (!hasValidated || Object.keys(schema).length > 0))
   $: dataImport = {
     valid,
     schema: buildTableSchema(schema),

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -72,15 +72,17 @@
   }
 </script>
 
+{#if allowDeletion}
   <ActionMenu>
     <div slot="control" class="icon">
       <Icon s hoverable name="MoreSmallList" />
     </div>
+    {#if !external}
       <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
-  {#if allowDeletion}
-    <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
     {/if}
+    <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
   </ActionMenu>
+{/if}
 
 <Modal bind:this={editorModal}>
   <ModalContent

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -1,6 +1,4 @@
 <script context="module">
-  import { Label } from "@budibase/bbui"
-
   export const EditorModes = {
     JS: {
       name: "javascript",
@@ -21,6 +19,7 @@
 </script>
 
 <script>
+  import { Label } from "@budibase/bbui"
   import CodeMirror from "components/integration/codemirror"
   import { themeStore } from "builderStore"
   import { createEventDispatcher, onMount } from "svelte"
@@ -156,4 +155,9 @@
   div :global(.CodeMirror-focused) {
     border-color: var(--spectrum-alias-border-color-mouse-focus);
   }
+
+  /* Ensure hints are always on top */
+  :global(.CodeMirror-hints) {
+    z-index: 999999;
+  }
 </style>

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -22,7 +22,7 @@
 
   const dispatch = createEventDispatcher()
 
-  export let bindableProperties
+  export let bindings
   export let value = ""
   export let valid
   export let allowJS = false
@@ -36,17 +36,23 @@
   let hbsValue = initialValueJS ? null : value
 
   $: usingJS = mode === "JavaScript"
-  $: ({ context } = groupBy("type", bindableProperties))
   $: searchRgx = new RegExp(search, "ig")
-  $: filteredBindings = context?.filter(context => {
+  $: categories = Object.entries(groupBy("category", bindings))
-    return context.readableBinding.match(searchRgx)
+  $: filteredCategories = categories
-  })
+    .map(([name, categoryBindings]) => ({
+      name,
+      bindings: categoryBindings?.filter(binding => {
+        return binding.readableBinding.match(searchRgx)
+      }),
+    }))
+    .filter(category => category.bindings?.length > 0)
   $: filteredHelpers = helpers?.filter(helper => {
     return helper.label.match(searchRgx) || helper.description.match(searchRgx)
   })
+  $: codeMirrorHints = bindings?.map(x => `$("${x.readableBinding}")`)
 
   const updateValue = value => {
-    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+    valid = isValid(readableToRuntimeBinding(bindings, value))
     if (valid) {
       dispatch("change", value)
     }
@@ -91,7 +97,7 @@
   }
 
   onMount(() => {
-    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+    valid = isValid(readableToRuntimeBinding(bindings, value))
   })
 </script>
 
@@ -102,18 +108,29 @@
         <div class="heading">Search</div>
         <Search placeholder="Search" bind:value={search} />
       </section>
-      {#if filteredBindings?.length}
+      {#each filteredCategories as category}
+        {#if category.bindings?.length}
           <section>
-          <div class="heading">Bindable Values</div>
+            <div class="heading">{category.name}</div>
             <ul>
-            {#each filteredBindings as binding}
+              {#each category.bindings as binding}
                 <li on:click={() => addBinding(binding)}>
-                {binding.readableBinding}
+                  <span class="binding__label">{binding.readableBinding}</span>
+                  {#if binding.type}
+                    <span class="binding__type">{binding.type}</span>
+                  {/if}
+                  {#if binding.description}
+                    <br />
+                    <div class="binding__description">
+                      {binding.description || ""}
+                    </div>
+                  {/if}
                 </li>
               {/each}
             </ul>
           </section>
         {/if}
+      {/each}
       {#if filteredHelpers?.length && !usingJS}
         <section>
           <div class="heading">Helpers</div>
@@ -162,7 +179,7 @@
                 height={200}
                 value={decodeJSBinding(jsValue)}
                 on:change={onChangeJSValue}
-                hints={context?.map(x => `$("${x.readableBinding}")`)}
+                hints={codeMirrorHints}
               />
               <Body size="S">
                 JavaScript expressions are executed as functions, so ensure that
@@ -234,6 +251,24 @@
     color: var(--spectrum-global-color-gray-900) !important;
   }
 
+  .binding__label {
+    font-weight: 600;
+    text-transform: capitalize;
+  }
+  .binding__description {
+    color: var(--spectrum-global-color-gray-700);
+    margin: 0.5rem 0 0 0;
+    white-space: normal;
+  }
+  .binding__type {
+    font-family: monospace;
+    background-color: var(--spectrum-global-color-gray-200);
+    border-radius: var(--border-radius-s);
+    padding: 2px 4px;
+    margin-left: 2px;
+    font-weight: 600;
+  }
+
   .helper {
     display: flex;
     flex-direction: column;

packages/builder/src/components/common/bindings/ClientBindingPanel.svelte

@@ -0,0 +1,30 @@
+<script>
+  import BindingPanel from "./BindingPanel.svelte"
+
+  export let bindings = []
+  export let valid
+  export let value = ""
+  export let allowJS = false
+
+  $: enrichedBindings = enrichBindings(bindings)
+
+  // Ensure bindings have the correct categories
+  const enrichBindings = bindings => {
+    if (!bindings?.length) {
+      return bindings
+    }
+    return bindings?.map(binding => ({
+      ...binding,
+      category: "Bindable Values",
+      type: null,
+    }))
+  }
+</script>
+
+<BindingPanel
+  bind:valid
+  bindings={enrichedBindings}
+  {value}
+  {allowJS}
+  on:change
+/>

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -4,11 +4,11 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"
 
-  export let panel = BindingPanel
+  export let panel = ClientBindingPanel
   export let value = ""
   export let bindings = []
   export let title = "Bindings"
@@ -55,6 +55,7 @@
   <Combobox
     {label}
     {disabled}
+    readonly={isJS}
     value={isJS ? "(JavaScript function)" : readableValue}
     on:type={e => onChange(e.detail, false)}
     on:pick={e => onChange(e.detail, true)}
@@ -82,7 +83,7 @@
     value={readableValue}
     close={handleClose}
     on:change={event => (tempValue = event.detail)}
-    bindableProperties={bindings}
+    {bindings}
     {allowJS}
   />
 </Drawer>

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -4,11 +4,11 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"
 
-  export let panel = BindingPanel
+  export let panel = ClientBindingPanel
   export let value = ""
   export let bindings = []
   export let title = "Bindings"
@@ -40,6 +40,7 @@
   <Input
     {label}
     {disabled}
+    readonly={isJS}
     value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
@@ -63,7 +64,7 @@
     bind:valid
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
-    bindableProperties={bindings}
+    {bindings}
     {allowJS}
   />
 </Drawer>

packages/builder/src/components/common/bindings/ModalBindableInput.svelte

@@ -6,20 +6,23 @@
   } from "builderStore/dataBinding"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = ServerBindingPanel
   export let value = ""
   export let bindings = []
-  export let thin = true
   export let title = "Bindings"
   export let placeholder
   export let label
+  export let allowJS = false
 
   const dispatch = createEventDispatcher()
   let bindingModal
   let valid = true
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
     onChange(tempValue)
@@ -34,8 +37,8 @@
 <div class="control">
   <Input
     {label}
-    {thin}
+    readonly={isJS}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
   />
@@ -55,7 +58,8 @@
         value={readableValue}
         bind:valid
         on:change={e => (tempValue = e.detail)}
-        bindableProperties={bindings}
+        {bindings}
+        {allowJS}
       />
     </div>
   </ModalContent>

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -1,209 +1,27 @@
 <script>
-  import groupBy from "lodash/fp/groupBy"
+  import BindingPanel from "./BindingPanel.svelte"
-  import { Search, TextArea, DrawerContent } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
-  import { isValid } from "@budibase/string-templates"
-  import { handlebarsCompletions } from "constants/completions"
-  import { readableToRuntimeBinding } from "builderStore/dataBinding"
-  import { addHBSBinding } from "./utils"
 
-  const dispatch = createEventDispatcher()
+  export let bindings = []
-
+  export let valid
-  export let bindableProperties = []
-  export let valid = true
   export let value = ""
+  export let allowJS = false
 
-  let helpers = handlebarsCompletions()
+  $: enrichedBindings = enrichBindings(bindings)
-  let getCaretPosition
-  let search = ""
 
-  $: categories = Object.entries(groupBy("category", bindableProperties))
+  // Ensure bindings have the correct properties
-  $: valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  const enrichBindings = bindings => {
-  $: dispatch("change", value)
+    return bindings?.map(binding => ({
-  $: searchRgx = new RegExp(search, "ig")
+      ...binding,
-  $: filteredCategories = categories.map(([categoryName, bindings]) => {
+      readableBinding: binding.label || binding.readableBinding,
-    const filteredBindings = bindings.filter(binding => {
+      runtimeBinding: binding.path || binding.runtimeBinding,
-      return binding.label.match(searchRgx)
+    }))
-    })
+  }
-    return [categoryName, filteredBindings]
-  })
-  $: filteredHelpers = helpers?.filter(helper => {
-    return helper.label.match(searchRgx) || helper.description.match(searchRgx)
-  })
 </script>
 
-<DrawerContent>
+<BindingPanel
-  <svelte:fragment slot="sidebar">
+  bind:valid
-    <div class="container">
+  bindings={enrichedBindings}
-      <section>
+  {value}
-        <div class="heading">Search</div>
+  {allowJS}
-        <Search placeholder="Search" bind:value={search} />
+  on:change
-      </section>
-      {#each filteredCategories as [categoryName, bindings]}
-        {#if bindings.length}
-          <section>
-            <div class="heading">{categoryName}</div>
-            <ul>
-              {#each bindings as binding}
-                <li
-                  on:click={() => {
-                    value = addHBSBinding(value, getCaretPosition(), binding)
-                  }}
-                >
-                  <span class="binding__label">{binding.label}</span>
-                  <span class="binding__type">{binding.type}</span>
-                  {#if binding.description}
-                    <br />
-                    <div class="binding__description">
-                      {binding.description || ""}
-                    </div>
-                  {/if}
-                </li>
-              {/each}
-            </ul>
-          </section>
-        {/if}
-      {/each}
-      {#if filteredHelpers?.length}
-        <section>
-          <div class="heading">Helpers</div>
-          <ul>
-            {#each filteredHelpers as helper}
-              <li
-                on:click={() => {
-                  value = addHBSBinding(value, getCaretPosition(), helper.text)
-                }}
-              >
-                <div class="helper">
-                  <div class="helper__name">{helper.displayText}</div>
-                  <div class="helper__description">
-                    {@html helper.description}
-                  </div>
-                  <pre class="helper__example">{helper.example || ''}</pre>
-                </div>
-              </li>
-            {/each}
-          </ul>
-        </section>
-      {/if}
-    </div>
-  </svelte:fragment>
-  <div class="main">
-    <TextArea
-      bind:getCaretPosition
-      bind:value
-      placeholder="Add text, or click the objects on the left to add them to the textbox."
 />
-    {#if !valid}
-      <p class="syntax-error">
-        Current Handlebars syntax is invalid, please check the guide
-        <a href="https://handlebarsjs.com/guide/">here</a>
-        for more details.
-      </p>
-    {/if}
-  </div>
-</DrawerContent>
-
-<style>
-  .main :global(textarea) {
-    min-height: 150px !important;
-  }
-
-  .container {
-    margin: calc(-1 * var(--spacing-xl));
-  }
-  .heading {
-    font-size: var(--font-size-s);
-    font-weight: 600;
-    text-transform: uppercase;
-    color: var(--spectrum-global-color-gray-600);
-    padding: var(--spacing-xl) 0 var(--spacing-m) 0;
-  }
-
-  section {
-    padding: 0 var(--spacing-xl) var(--spacing-xl) var(--spacing-xl);
-  }
-  section:not(:first-child) {
-    border-top: var(--border-light);
-  }
-  ul {
-    list-style: none;
-    padding: 0;
-    margin: 0;
-  }
-
-  li {
-    font-size: var(--font-size-s);
-    padding: var(--spacing-m);
-    border-radius: 4px;
-    border: var(--border-light);
-    transition: background-color 130ms ease-in-out, color 130ms ease-in-out,
-      border-color 130ms ease-in-out;
-  }
-  li:not(:last-of-type) {
-    margin-bottom: var(--spacing-s);
-  }
-  li :global(*) {
-    transition: color 130ms ease-in-out;
-  }
-  li:hover {
-    color: var(--spectrum-global-color-gray-900);
-    background-color: var(--spectrum-global-color-gray-50);
-    border-color: var(--spectrum-global-color-gray-500);
-    cursor: pointer;
-  }
-  li:hover :global(*) {
-    color: var(--spectrum-global-color-gray-900) !important;
-  }
-
-  .helper {
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: flex-start;
-    gap: var(--spacing-xs);
-  }
-  .helper__name {
-    font-weight: bold;
-  }
-  .helper__description,
-  .helper__description :global(*) {
-    color: var(--spectrum-global-color-gray-700);
-  }
-  .helper__example {
-    white-space: normal;
-    margin: 0.5rem 0 0 0;
-    font-weight: 700;
-  }
-  .helper__description :global(p) {
-    margin: 0;
-  }
-
-  .syntax-error {
-    padding-top: var(--spacing-m);
-    color: var(--red);
-    font-size: 12px;
-  }
-  .syntax-error a {
-    color: var(--red);
-    text-decoration: underline;
-  }
-
-  .binding__label {
-    font-weight: 600;
-    text-transform: capitalize;
-  }
-  .binding__description {
-    color: var(--spectrum-global-color-gray-700);
-    margin: 0.5rem 0 0 0;
-    white-space: normal;
-  }
-  .binding__type {
-    font-family: monospace;
-    background-color: var(--spectrum-global-color-gray-200);
-    border-radius: var(--border-radius-s);
-    padding: 2px 4px;
-    margin-left: 2px;
-    font-weight: 600;
-  }
-</style>

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -8,12 +8,24 @@
       "repeaterblock"
     ]
   },
-  "section",
+  {
+    "name": "Layout",
+    "icon": "ClassicGridView",
+    "children": [
       "container",
+      "section"
+    ]
+  },
+  {
+    "name": "Data",
+    "icon": "Data",
+    "children": [
       "dataprovider",
-  "table",
       "repeater",
-  "button",
+      "table",
+      "dynamicfilter"
+    ]
+  },
   {
     "name": "Form",
     "icon": "Form",
@@ -60,6 +72,7 @@
     "children": [
       "heading",
       "text",
+      "button",
       "divider",
       "image",
       "backgroundimage",

packages/builder/src/components/design/NavigationPanel/FrontendNavigatePane.svelte

@@ -25,8 +25,7 @@
       key: "layout",
     },
   ]
-
+  let newLayoutModal
-  let modal
   $: selected = tabs.find(t => t.key === $params.assetType)?.title || "Screens"
 
   const navigate = ({ detail }) => {
@@ -93,14 +92,18 @@
         {#each $store.layouts as layout, idx (layout._id)}
           <Layout {layout} border={idx > 0} />
         {/each}
-        <Modal bind:this={modal}>
+        <Modal bind:this={newLayoutModal}>
           <NewLayoutModal />
         </Modal>
       </div>
     </Tab>
   </Tabs>
   <div class="add-button">
-    <Icon hoverable name="AddCircle" on:click={showModal()} />
+    <Icon
+      hoverable
+      name="AddCircle"
+      on:click={selected === "Layouts" ? newLayoutModal.show() : showModal()}
+    />
   </div>
 </div>
 

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -1,37 +1,72 @@
 <script>
   import { store } from "builderStore"
   import { tables } from "stores/backend"
-  import { ModalContent, Body, Detail, Layout, Icon } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Body,
+    Detail,
+    Layout,
+    Icon,
+    ProgressCircle,
+  } from "@budibase/bbui"
   import getTemplates from "builderStore/store/screenTemplates"
+  import { onDestroy } from "svelte"
+
+  import { createEventDispatcher } from "svelte"
 
-  export let selectedScreens = []
   export let chooseModal
   export let save
+  export let showProgressCircle = false
+
+  let selectedScreens = []
 
   const blankScreen = "createFromScratch"
+  const dispatch = createEventDispatcher()
+
+  function setScreens() {
+    dispatch("save", {
+      screens: selectedScreens,
+    })
+  }
 
   $: blankSelected = selectedScreens?.length === 1
   $: autoSelected = selectedScreens?.length > 0 && !blankSelected
+
   let templates = getTemplates($store, $tables.list)
+
+  const confirm = async () => {
+    if (autoSelected) {
+      setScreens()
+      await save()
+    } else {
+      setScreens()
+      chooseModal(1)
+    }
+  }
   const toggleScreenSelection = table => {
-    if (selectedScreens.find(s => s.name.includes(table.name))) {
+    if (selectedScreens.find(s => s.table === table.name)) {
       selectedScreens = selectedScreens.filter(
-        screen => !screen.name.includes(table.name)
+        screen => screen.table !== table.name
       )
     } else {
       let partialTemplates = getTemplates($store, $tables.list).filter(
-        template => template.name.includes(table.name)
+        template => template.table === table.name
       )
       selectedScreens = [...partialTemplates, ...selectedScreens]
     }
   }
+
+  onDestroy(() => {
+    selectedScreens = []
+  })
 </script>
 
+<div>
   <ModalContent
     title="Add screens"
     confirmText="Add Screens"
     cancelText="Cancel"
-  onConfirm={() => (autoSelected ? save() : chooseModal(1))}
+    onConfirm={() => confirm()}
     disabled={!selectedScreens.length}
     size="L"
   >
@@ -50,56 +85,84 @@
         class:disabled={autoSelected}
       >
         <div data-cy="blank-screen" class="content">
-        <Body size="S">Blank</Body>
+          <div class="text">Blank</div>
-      </div>
-      <div style="color: var(--spectrum-global-color-green-600); float: right">
-        {#if selectedScreens.find(x => x.id === blankScreen)}
-          <Icon size="S" name="CheckmarkCircleOutline" />
-        {/if}
-      </div>
-    </div>
-    <Detail size="S">Autogenerated Screens</Detail>
-
-    {#each $tables.list.filter(table => table.type !== "external" && table._id !== "ta_users") as table}
-      <div
-        class:disabled={blankSelected}
-        class:selected={selectedScreens.find(x => x.name.includes(table.name))}
-        on:click={() => toggleScreenSelection(table)}
-        class="item"
-      >
-        <div class="content">
-          {table.name}
         </div>
         <div
           style="color: var(--spectrum-global-color-green-600); float: right"
         >
-          {#if selectedScreens.find(x => x.name.includes(table.name))}
+          {#if selectedScreens.find(x => x.id === blankScreen)}
+            <div class="checkmark-spacing">
               <Icon size="S" name="CheckmarkCircleOutline" />
+            </div>
+          {/if}
+        </div>
+      </div>
+      {#if $tables.list.filter(table => table._id !== "ta_users").length > 0}
+        <Detail size="S">Autogenerated Screens</Detail>
+
+        {#each $tables.list.filter(table => table._id !== "ta_users") as table}
+          <div
+            class:disabled={blankSelected}
+            class:selected={selectedScreens.find(x => x.table === table.name)}
+            on:click={() => toggleScreenSelection(table)}
+            class="item"
+          >
+            <div class="content">
+              <div class="text">{table.name}</div>
+            </div>
+            <div
+              style="color: var(--spectrum-global-color-green-600); float: right"
+            >
+              {#if selectedScreens.find(x => x.table === table.name)}
+                <div class="checkmark-spacing">
+                  <Icon size="S" name="CheckmarkCircleOutline" />
+                </div>
               {/if}
             </div>
           </div>
         {/each}
+      {/if}
     </Layout>
+    <div slot="footer">
+      {#if showProgressCircle}
+        <div class="footer-progress"><ProgressCircle size="S" /></div>
+      {/if}
+    </div>
   </ModalContent>
+</div>
 
 <style>
   .disabled {
     opacity: 0.3;
     pointer-events: none;
   }
+  .checkmark-spacing {
+    margin-right: var(--spacing-m);
+  }
 
   .content {
     letter-spacing: 0px;
-    color: #2c2c2c;
   }
+
+  .footer-progress {
+    margin-top: var(--spacing-s);
+  }
+
+  .text {
+    font-weight: 600;
+    margin-left: var(--spacing-m);
+    font-size: 14px;
+    text-transform: capitalize;
+  }
+
   .item {
     cursor: pointer;
     grid-gap: var(--spectrum-alias-grid-margin-xsmall);
     padding: var(--spectrum-alias-item-padding-s);
-    background: var(--background);
+    background: var(--spectrum-alias-background-color-primary);
     transition: 0.3s all;
-    border: solid var(--spectrum-alias-border-color);
+    border: 1px solid var(--spectrum-global-color-gray-300);
-    border-radius: 2px;
+    border-radius: 4px;
     box-sizing: border-box;
     border-width: 1px;
     display: flex;

packages/builder/src/components/design/NavigationPanel/ScreenDetailsModal.svelte

@@ -1,12 +1,15 @@
 <script>
-  import { ModalContent, Input } from "@budibase/bbui"
+  import { ModalContent, Input, ProgressCircle } from "@budibase/bbui"
   import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
   import { selectedAccessRole, allScreens } from "builderStore"
+  import { onDestroy } from "svelte"
 
   export let screenName
   export let url
   export let chooseModal
   export let save
+  export let showProgressCircle = false
+
   let routeError
   let roleId = $selectedAccessRole || "BASIC"
 
@@ -30,6 +33,11 @@
         screen.routing.roleId === roleId
     )
   }
+
+  onDestroy(() => {
+    screenName = ""
+    url = ""
+  })
 </script>
 
 <ModalContent
@@ -48,4 +56,15 @@
     bind:value={url}
     on:change={routeChanged}
   />
+  <div slot="footer">
+    {#if showProgressCircle}
+      <div class="footer-progress"><ProgressCircle size="S" /></div>
+    {/if}
+  </div>
 </ModalContent>
+
+<style>
+  .footer-progress {
+    margin-top: var(--spacing-s);
+  }
+</style>

packages/builder/src/components/design/NavigationPanel/ScreenWizard.svelte

@@ -4,7 +4,7 @@
   import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
   import { Modal } from "@budibase/bbui"
   import { store, selectedAccessRole, allScreens } from "builderStore"
-  import { onDestroy } from "svelte"
+  import analytics, { Events } from "analytics"
 
   let newScreenModal
   let navigationSelectionModal
@@ -12,32 +12,39 @@
   let screenName = ""
   let url = ""
   let selectedScreens = []
-
   let roleId = $selectedAccessRole || "BASIC"
-
+  let showProgressCircle = false
   let routeError
   let createdScreens = []
-  $: {
+
-    selectedScreens?.forEach(screen => {
+  const createScreens = async () => {
-      createdScreens = [...createdScreens, screen.create()]
+    for (let screen of selectedScreens) {
+      let test = screen.create()
+      createdScreens.push(test)
+      analytics.captureEvent(Events.SCREEN.CREATED, {
+        template: screen.id || screen.name,
       })
     }
+  }
 
   const save = async () => {
+    showProgressCircle = true
+    await createScreens()
     for (let screen of createdScreens) {
       await saveScreens(screen)
     }
-
     await store.actions.routing.fetch()
     selectedScreens = []
+    createdScreens = []
     screenName = ""
     url = ""
+    showProgressCircle = false
   }
+
   const saveScreens = async draftScreen => {
     let existingScreenCount = $store.screens.filter(
       s => s.props._instanceName == draftScreen.props._instanceName
     ).length
-
     if (existingScreenCount > 0) {
       let oldUrlArr = draftScreen.routing.route.split("/")
       oldUrlArr[1] = `${oldUrlArr[1]}-${existingScreenCount + 1}`
@@ -82,16 +89,14 @@
     )
   }
 
-  onDestroy(() => {
-    selectedScreens = []
-    screenName = ""
-    url = ""
-  })
-
   export const showModal = () => {
     newScreenModal.show()
   }
 
+  const setScreens = evt => {
+    selectedScreens = evt.detail.screens
+  }
+
   const chooseModal = index => {
     /*
     0 = newScreenModal
@@ -109,9 +114,20 @@
 </script>
 
 <Modal bind:this={newScreenModal}>
-  <NewScreenModal bind:selectedScreens {save} {chooseModal} />
+  <NewScreenModal
+    on:save={setScreens}
+    {showProgressCircle}
+    {save}
+    {chooseModal}
+  />
 </Modal>
 
 <Modal bind:this={screenDetailsModal}>
-  <ScreenDetailsModal bind:screenName bind:url {save} {chooseModal} />
+  <ScreenDetailsModal
+    bind:screenName
+    bind:url
+    {showProgressCircle}
+    {save}
+    {chooseModal}
+  />
 </Modal>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -11,14 +11,14 @@
     Select,
   } from "@budibase/bbui"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
-  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { generate } from "shortid"
   import { getValidOperatorsForType, OperatorOptions } from "constants/lucene"
 
   export let schemaFields
   export let filters = []
   export let bindings = []
-  export let panel = BindingPanel
+  export let panel = ClientBindingPanel
   export let allowBindings = true
 
   const BannedTypes = ["link", "attachment", "formula"]

packages/builder/src/components/integration/QueryViewer.svelte

@@ -32,31 +32,22 @@
   import { onMount } from "svelte"
 
   export let query
-  export let fields = []
 
+  let fields = query.schema ? schemaToFields(query.schema) : []
   let parameters
   let data = []
   let roleId
   const transformerDocs =
     "https://docs.budibase.com/building-apps/data/transformers"
   const typeOptions = [
-    { label: "Text", value: "STRING" },
+    { label: "Text", value: "string" },
-    { label: "Number", value: "NUMBER" },
+    { label: "Number", value: "number" },
-    { label: "Boolean", value: "BOOLEAN" },
+    { label: "Boolean", value: "boolean" },
-    { label: "Datetime", value: "DATETIME" },
+    { label: "Datetime", value: "datetime" },
   ]
 
   $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
-  $: query.schema = fields.reduce(
+  $: query.schema = fieldsToSchema(fields)
-    (acc, next) => ({
-      ...acc,
-      [next.name]: {
-        name: next.name,
-        type: "string",
-      },
-    }),
-    {}
-  )
   $: datasourceType = datasource?.source
   $: integrationInfo = $integrations[datasourceType]
   $: queryConfig = integrationInfo?.query
@@ -135,7 +126,7 @@
       // unique fields returned by the server
       fields = json.schemaFields.map(field => ({
         name: field,
-        type: "STRING",
+        type: "string",
       }))
     } catch (err) {
       notifications.error(`Query Error: ${err.message}`)
@@ -155,6 +146,26 @@
     }
   }
 
+  function schemaToFields(schema) {
+    return Object.keys(schema).map(key => ({
+      name: key,
+      type: query.schema[key].type,
+    }))
+  }
+
+  function fieldsToSchema(fieldsToConvert) {
+    return fieldsToConvert.reduce(
+      (acc, next) => ({
+        ...acc,
+        [next.name]: {
+          name: next.name,
+          type: next.type,
+        },
+      }),
+      {}
+    )
+  }
+
   onMount(async () => {
     if (!query || !query._id) {
       roleId = Roles.BASIC

packages/builder/src/components/start/CreateAppModal.svelte

@@ -83,12 +83,11 @@
   }
 
   async function createNewApp() {
-    const letTemplateToUse =
+    const templateToUse = Object.keys(template).length === 0 ? null : template
-      Object.keys(template).length === 0 ? null : template
     submitting = true
 
     // Check a template exists if we are important
-    if (letTemplateToUse?.fromFile && !$values.file) {
+    if (templateToUse?.fromFile && !$values.file) {
       $errors.file = "Please choose a file to import"
       valid = false
       submitting = false
@@ -99,10 +98,10 @@
       // Create form data to create app
       let data = new FormData()
       data.append("name", $values.name.trim())
-      data.append("useTemplate", letTemplateToUse != null)
+      data.append("useTemplate", templateToUse != null)
-      if (letTemplateToUse) {
+      if (templateToUse) {
-        data.append("templateName", letTemplateToUse.name)
+        data.append("templateName", templateToUse.name)
-        data.append("templateKey", letTemplateToUse.key)
+        data.append("templateKey", templateToUse.key)
         data.append("templateFile", $values.file)
       }
 
@@ -116,7 +115,7 @@
       analytics.captureEvent(Events.APP.CREATED, {
         name: $values.name,
         appId: appJson.instance._id,
-        letTemplateToUse,
+        templateToUse,
       })
 
       // Select Correct Application/DB in prep for creating user

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -13,9 +13,11 @@
     notifications,
     Search,
   } from "@budibase/bbui"
+  import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
   import UpdateAppModal from "components/start/UpdateAppModal.svelte"
-  import { del } from "builderStore/api"
+  import { store, automationStore } from "builderStore"
+  import api, { del, post, get } from "builderStore/api"
   import { onMount } from "svelte"
   import { apps, auth, admin } from "stores/portal"
   import download from "downloadjs"
@@ -24,6 +26,7 @@
   import AppCard from "components/start/AppCard.svelte"
   import AppRow from "components/start/AppRow.svelte"
   import { AppStatus } from "constants"
+  import analytics, { Events } from "analytics"
 
   let layout = "grid"
   let sortBy = "name"
@@ -38,6 +41,7 @@
   let searchTerm = ""
   let cloud = $admin.cloud
   let appName = ""
+  let creatingFromTemplate = false
 
   $: enrichedApps = enrichApps($apps, $auth.user, sortBy)
   $: filteredApps = enrichedApps.filter(app =>
@@ -92,6 +96,62 @@
     creatingApp = true
   }
 
+  const autoCreateApp = async () => {
+    try {
+      // Auto name app if has same name
+      let appName = template.key
+      const appsWithSameName = $apps.filter(app =>
+        app.name?.startsWith(appName)
+      )
+      appName = `${appName}-${appsWithSameName.length + 1}`
+
+      // Create form data to create app
+      let data = new FormData()
+      data.append("name", appName)
+      data.append("useTemplate", true)
+      data.append("templateKey", template.key)
+
+      // Create App
+      const appResp = await post("/api/applications", data, {})
+      const appJson = await appResp.json()
+      if (!appResp.ok) {
+        throw new Error(appJson.message)
+      }
+
+      analytics.captureEvent(Events.APP.CREATED, {
+        name: appName,
+        appId: appJson.instance._id,
+        template,
+        fromTemplateMarketplace: true,
+      })
+
+      // Select Correct Application/DB in prep for creating user
+      const applicationPkg = await get(
+        `/api/applications/${appJson.instance._id}/appPackage`
+      )
+      const pkg = await applicationPkg.json()
+      if (applicationPkg.ok) {
+        await store.actions.initialise(pkg)
+        await automationStore.actions.fetch()
+        // update checklist - incase first app
+        await admin.init()
+      } else {
+        throw new Error(pkg)
+      }
+
+      // Create user
+      const userResp = await api.post(`/api/users/metadata/self`, {
+        roleId: "BASIC",
+      })
+      await userResp.json()
+      await auth.setInitInfo({})
+      $goto(`/builder/app/${appJson.instance._id}`)
+    } catch (error) {
+      console.error(error)
+      notifications.error(error)
+    }
+  }
+
   const stopAppCreation = () => {
     template = null
     creatingApp = false
@@ -194,7 +254,7 @@
       template = {
         key: templateKey,
       }
-      initiateAppCreation()
+      autoCreateApp()
     } else {
       notifications.error("Your Template URL is invalid. Please try another.")
     }
@@ -202,12 +262,14 @@
 
   onMount(async () => {
     await apps.load()
-    loaded = true
     // if the portal is loaded from an external URL with a template param
     const initInfo = await auth.getInitInfo()
-    if (initInfo.init_template) {
+    if (initInfo?.init_template) {
+      creatingFromTemplate = true
       createAppFromTemplateUrl(initInfo.init_template)
+      return
     }
+    loaded = true
   })
 </script>
 
@@ -285,6 +347,12 @@
       </Modal>
     </div>
   {/if}
+  {#if creatingFromTemplate}
+    <div class="empty-wrapper">
+      <p>Creating your Budibase app from your selected template...</p>
+      <Spinner size="10" />
+    </div>
+  {/if}
 </Page>
 <Modal
   bind:this={creationModal}

packages/builder/src/stores/backend/tables.js

@@ -66,6 +66,9 @@ export function createTablesStore() {
     }
 
     const response = await api.post(`/api/tables`, updatedTable)
+    if (response.status !== 200) {
+      throw (await response.json()).message
+    }
     const savedTable = await response.json()
     await fetch()
     if (table.type === "external") {
@@ -119,7 +122,7 @@ export function createTablesStore() {
       update(state => {
         // delete the original if renaming
         // need to handle if the column had no name, empty string
-        if (originalName || originalName === "") {
+        if (originalName != null && originalName !== field.name) {
           delete state.draft.schema[originalName]
           state.draft._rename = {
             old: originalName,

packages/builder/src/stores/backend/tests/tables.spec.js

@@ -42,7 +42,7 @@ describe("Tables Store", () => {
   })
 
   it("saving a table also selects it", async () => {
-    api.post.mockReturnValue({ json: () => SAVE_TABLES_RESPONSE})    
+    api.post.mockReturnValue({ status: 200, json: () => SAVE_TABLES_RESPONSE})
 
     await store.save(A_TABLE)
 
@@ -50,7 +50,7 @@ describe("Tables Store", () => {
   })
 
   it("saving the table returns a response", async () => {
-    api.post.mockReturnValue({ json: () => SAVE_TABLES_RESPONSE})    
+    api.post.mockReturnValue({ status: 200, json: () => SAVE_TABLES_RESPONSE})
 
     const response = await store.save(A_TABLE)
 

packages/cli/LICENSE.md

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright 2019-2021, Budibase Ltd
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

packages/cli/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
     "budi": "src/index.js"
   },
   "author": "Budibase",
-  "license": "AGPL-3.0-or-later",
+  "license": "GPL-3.0",
   "scripts": {
     "build": "pkg . --out-path build"
   },

packages/client/LICENSE

@@ -1,7 +1,7 @@
 Mozilla Public License Version 2.0
 ==================================
 
-Copyright 2019-2021, Budibase Inc
+Copyright 2019-2021, Budibase Ltd
 
 1. Definitions
 --------------

packages/client/manifest.json

@@ -247,7 +247,6 @@
     "description": "A basic html button that is ready for styling",
     "icon": "Button",
     "editable": true,
-    "illegalChildren": ["section"],
     "showSettingsBar": true,
     "settings": [
       {
@@ -2658,6 +2657,49 @@
       }
     ]
   },
+  "dynamicfilter": {
+    "name": "Dynamic Filter",
+    "icon": "FilterEdit",
+    "showSettingsBar": true,
+    "settings": [
+      {
+        "type": "dataProvider",
+        "label": "Provider",
+        "key": "dataProvider"
+      },
+      {
+        "type": "multifield",
+        "label": "Allowed filter fields",
+        "key": "allowedFields",
+        "placeholder": "All fields"
+      },
+      {
+        "type": "select",
+        "label": "Button size",
+        "showInBar": true,
+        "key": "size",
+        "options": [
+          {
+            "label": "Small",
+            "value": "S"
+          },
+          {
+            "label": "Medium",
+            "value": "M"
+          },
+          {
+            "label": "Large",
+            "value": "L"
+          },
+          {
+            "label": "Extra large",
+            "value": "XL"
+          }
+        ],
+        "defaultValue": "M"
+      }
+    ]
+  },
   "tableblock": {
     "block": true,
     "name": "Table block",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.185-alpha.14",
+    "@budibase/bbui": "^0.9.190-alpha.11",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.185-alpha.14",
+    "@budibase/string-templates": "^0.9.190-alpha.11",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/components/ClientApp.svelte

@@ -121,6 +121,9 @@
                   -->
                   <div id="flatpickr-root" />
 
+                  <!-- Modal container to ensure they sit on top -->
+                  <div class="modal-container" />
+
                   <!-- Layers on top of app -->
                   <NotificationDisplay />
                   <ConfirmationDisplay />

packages/client/src/components/Component.svelte

@@ -72,13 +72,18 @@
   $: inSelectedPath = $builderStore.selectedComponentPath?.includes(id)
   $: inDragPath = inSelectedPath && $builderStore.editMode
 
+  // Derive definition properties which can all be optional, so need to be
+  // coerced to booleans
+  $: editable = !!definition?.editable
+  $: hasChildren = !!definition?.hasChildren
+  $: showEmptyState = definition?.showEmptyState !== false
+
   // Interactive components can be selected, dragged and highlighted inside
   // the builder preview
   $: interactive =
     $builderStore.inBuilder &&
     ($builderStore.previewType === "layout" || insideScreenslot) &&
     !isBlock
-  $: editable = definition?.editable
   $: editing = editable && selected && $builderStore.editMode
   $: draggable = !inDragPath && interactive && !isLayout && !isScreen
   $: droppable = interactive && !isLayout && !isScreen
@@ -86,8 +91,8 @@
   // Empty components are those which accept children but do not have any.
   // Empty states can be shown for these components, but can be disabled
   // in the component manifest.
-  $: empty = interactive && !children.length && definition?.hasChildren
+  $: empty = interactive && !children.length && hasChildren
-  $: emptyState = empty && definition?.showEmptyState !== false
+  $: emptyState = empty && showEmptyState
 
   // Raw settings are all settings excluding internal props and children
   $: rawSettings = getRawSettings(instance)
@@ -103,6 +108,9 @@
   // Build up the final settings object to be passed to the component
   $: cacheSettings(enrichedSettings, nestedSettings, conditionalSettings)
 
+  // Render key is used to determine when components need to fully remount
+  $: renderKey = getRenderKey(id, editing)
+
   // Update component context
   $: componentStore.set({
     id,
@@ -268,8 +276,15 @@
       })
     }
   }
+
+  // Generates a key used to determine when components need to fully remount.
+  // Currently only toggling editing requires remounting.
+  const getRenderKey = (id, editing) => {
+    return hashString(`${id}-${editing}`)
+  }
 </script>
 
+{#key renderKey}
   {#if constructor && cachedSettings && (visible || inSelectedPath)}
     <!-- The ID is used as a class because getElementsByClassName is O(1) -->
     <!-- and the performance matters for the selection indicators -->
@@ -297,6 +312,7 @@
       </svelte:component>
     </div>
   {/if}
+{/key}
 
 <style>
   .component {

packages/client/src/components/app/Button.svelte

@@ -12,6 +12,10 @@
   export let type = "primary"
   export let quiet = false
 
+  // For internal use only for now - not defined in the manifest
+  export let icon = null
+  export let active = false
+
   let node
 
   $: $component.editing && node?.focus()
@@ -25,7 +29,7 @@
   }
 
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    builderStore.actions.updateProp("text", e.target.textContent.trim())
   }
 </script>
 
@@ -35,10 +39,22 @@
   {disabled}
   use:styleable={$component.styles}
   on:click={onClick}
-  contenteditable={$component.editing}
+  contenteditable={$component.editing && !icon}
   on:blur={$component.editing ? updateText : null}
   bind:this={node}
+  class:active
 >
+  {#if icon}
+    <svg
+      class:hasText={componentText?.length > 0}
+      class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
+      focusable="false"
+      aria-hidden="true"
+      aria-label={icon}
+    >
+      <use xlink:href="#spectrum-icon-18-{icon}" />
+    </svg>
+  {/if}
   {componentText}
 </button>
 
@@ -56,4 +72,10 @@
   .spectrum-Button::after {
     display: none;
   }
+  .spectrum-Icon.hasText {
+    margin-right: var(--spectrum-button-primary-icon-gap);
+  }
+  .active {
+    color: var(--spectrum-global-color-blue-600);
+  }
 </style>

packages/client/src/components/app/DataProvider.svelte

@@ -275,11 +275,10 @@
     allRows = res.rows
   }
 
-  const addQueryExtension = (key, operator, field, value) => {
+  const addQueryExtension = (key, extension) => {
-    if (!key || !operator || !field) {
+    if (!key || !extension) {
       return
     }
-    const extension = { operator, field, value }
     queryExtensions = { ...queryExtensions, [key]: extension }
   }
 
@@ -295,12 +294,14 @@
   const extendQuery = (defaultQuery, extensions) => {
     const extensionValues = Object.values(extensions || {})
     let extendedQuery = { ...defaultQuery }
-    extensionValues.forEach(({ operator, field, value }) => {
+    extensionValues.forEach(extension => {
+      Object.entries(extension || {}).forEach(([operator, fields]) => {
         extendedQuery[operator] = {
           ...extendedQuery[operator],
-        [field]: value,
+          ...fields,
         }
       })
+    })
 
     if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) {
       query = extendedQuery

packages/client/src/components/app/DateRangePicker.svelte

@@ -13,15 +13,6 @@
 
   const component = getContext("component")
   const { styleable, ActionTypes, getAction } = getContext("sdk")
-
-  $: addExtension = getAction(
-    dataProvider?.id,
-    ActionTypes.AddDataProviderQueryExtension
-  )
-  $: removeExtension = getAction(
-    dataProvider?.id,
-    ActionTypes.RemoveDataProviderQueryExtension
-  )
   const options = [
     "Last 1 day",
     "Last 7 days",
@@ -32,10 +23,23 @@
   ]
   let value = options.includes(defaultValue) ? defaultValue : "Last 30 days"
 
-  $: queryExtension = getQueryExtension(value)
+  $: dataProviderId = dataProvider?.id
-  $: addExtension?.($component.id, "range", field, queryExtension)
+  $: addExtension = getAction(
+    dataProviderId,
+    ActionTypes.AddDataProviderQueryExtension
+  )
+  $: removeExtension = getAction(
+    dataProviderId,
+    ActionTypes.RemoveDataProviderQueryExtension
+  )
+  $: queryExtension = getQueryExtension(field, value)
+  $: addExtension?.($component.id, queryExtension)
+
+  const getQueryExtension = (field, value) => {
+    if (!field || !value) {
+      return null
+    }
 
-  const getQueryExtension = value => {
     let low = dayjs.utc().subtract(1, "year")
     let high = dayjs.utc().add(1, "day")
 
@@ -51,7 +55,14 @@
       low = dayjs.utc().subtract(6, "months")
     }
 
-    return { low: low.format(), high: high.format() }
+    return {
+      range: {
+        [field]: {
+          low: low.format(),
+          high: high.format(),
+        },
+      },
+    }
   }
 
   onDestroy(() => {

packages/client/src/components/app/Heading.svelte

@@ -47,7 +47,7 @@
 
   // Convert contenteditable HTML to text and save
   const updateText = e => {
-    const sanitized = e.target.innerHTML.replace(/<br>/gi, "\n")
+    const sanitized = e.target.innerHTML.replace(/<br>/gi, "\n").trim()
     builderStore.actions.updateProp("text", sanitized)
   }
 </script>

packages/client/src/components/app/Link.svelte

@@ -47,7 +47,7 @@
   }
 
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    builderStore.actions.updateProp("text", e.target.textContent.trim())
   }
 </script>
 

packages/client/src/components/app/Text.svelte

@@ -46,7 +46,7 @@
 
   // Convert contenteditable HTML to text and save
   const updateText = e => {
-    const sanitized = e.target.innerHTML.replace(/<br>/gi, "\n")
+    const sanitized = e.target.innerHTML.replace(/<br>/gi, "\n").trim()
     builderStore.actions.updateProp("text", sanitized)
   }
 </script>

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -182,7 +182,7 @@
           }}
           styles={{
             display: "grid",
-            "grid-template-columns": `repeat(auto-fill, minmax(${cardWidth}px, 1fr))`,
+            "grid-template-columns": `repeat(auto-fill, minmax(min(${cardWidth}px, 100%), 1fr))`,
           }}
         >
           <BlockComponent

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -0,0 +1,85 @@
+<script>
+  import { get } from "svelte/store"
+  import { getContext, onDestroy } from "svelte"
+  import { ModalContent, Modal } from "@budibase/bbui"
+  import FilterModal from "./FilterModal.svelte"
+  import { buildLuceneQuery } from "builder/src/helpers/lucene"
+  import Button from "../Button.svelte"
+
+  export let dataProvider
+  export let allowedFields
+  export let size = "M"
+
+  const component = getContext("component")
+  const { builderStore, ActionTypes, getAction } = getContext("sdk")
+
+  let modal
+  let tmpFilters = []
+  let filters = []
+
+  $: dataProviderId = dataProvider?.id
+  $: addExtension = getAction(
+    dataProviderId,
+    ActionTypes.AddDataProviderQueryExtension
+  )
+  $: removeExtension = getAction(
+    dataProviderId,
+    ActionTypes.RemoveDataProviderQueryExtension
+  )
+  $: schema = dataProvider?.schema
+  $: schemaFields = getSchemaFields(schema, allowedFields)
+
+  // Add query extension to data provider
+  $: {
+    if (filters?.length) {
+      const queryExtension = buildLuceneQuery(filters)
+      addExtension?.($component.id, queryExtension)
+    } else {
+      removeExtension?.($component.id)
+    }
+  }
+
+  const getSchemaFields = (schema, allowedFields) => {
+    let clonedSchema = {}
+    if (!allowedFields?.length) {
+      clonedSchema = schema
+    } else {
+      allowedFields?.forEach(field => {
+        clonedSchema[field] = schema[field]
+      })
+    }
+    return Object.values(clonedSchema || {})
+  }
+
+  const openEditor = () => {
+    if (get(builderStore).inBuilder) {
+      return
+    }
+    tmpFilters = [...filters]
+    modal.show()
+  }
+
+  const updateQuery = () => {
+    filters = [...tmpFilters]
+  }
+
+  onDestroy(() => {
+    removeExtension?.($component.id)
+  })
+</script>
+
+<Button
+  onClick={openEditor}
+  icon="Properties"
+  text="Filter"
+  {size}
+  type="secondary"
+  quiet
+  active={filters?.length > 0}
+/>
+
+<Modal bind:this={modal}>
+  <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
+    <FilterModal bind:filters={tmpFilters} {schemaFields} />
+  </ModalContent>
+</Modal>

packages/client/src/components/app/dynamic-filter/FilterModal.svelte

@@ -0,0 +1,176 @@
+<script>
+  import {
+    Body,
+    Button,
+    Combobox,
+    DatePicker,
+    DrawerContent,
+    Icon,
+    Input,
+    Layout,
+    Select,
+  } from "@budibase/bbui"
+  import { generate } from "shortid"
+  import {
+    getValidOperatorsForType,
+    OperatorOptions,
+  } from "builder/src/constants/lucene"
+
+  export let schemaFields
+  export let filters = []
+
+  const BannedTypes = ["link", "attachment", "formula"]
+
+  $: fieldOptions = (schemaFields ?? [])
+    .filter(field => !BannedTypes.includes(field.type))
+    .map(field => field.name)
+
+  const addFilter = () => {
+    filters = [
+      ...filters,
+      {
+        id: generate(),
+        field: null,
+        operator: OperatorOptions.Equals.value,
+        value: null,
+        valueType: "Value",
+      },
+    ]
+  }
+
+  const removeFilter = id => {
+    filters = filters.filter(field => field.id !== id)
+  }
+
+  const duplicateFilter = id => {
+    const existingFilter = filters.find(filter => filter.id === id)
+    const duplicate = { ...existingFilter, id: generate() }
+    filters = [...filters, duplicate]
+  }
+
+  const onFieldChange = (expression, field) => {
+    // Update the field type
+    expression.type = schemaFields.find(x => x.name === field)?.type
+
+    // Ensure a valid operator is set
+    const validOperators = getValidOperatorsForType(expression.type).map(
+      x => x.value
+    )
+    if (!validOperators.includes(expression.operator)) {
+      expression.operator = validOperators[0] ?? OperatorOptions.Equals.value
+      onOperatorChange(expression, expression.operator)
+    }
+
+    // if changed to an array, change default value to empty array
+    const idx = filters.findIndex(x => x.field === field)
+    if (expression.type === "array") {
+      filters[idx].value = []
+    } else {
+      filters[idx].value = null
+    }
+  }
+
+  const onOperatorChange = (expression, operator) => {
+    const noValueOptions = [
+      OperatorOptions.Empty.value,
+      OperatorOptions.NotEmpty.value,
+    ]
+    expression.noValue = noValueOptions.includes(operator)
+    if (expression.noValue) {
+      expression.value = null
+    }
+  }
+
+  const getFieldOptions = field => {
+    const schema = schemaFields.find(x => x.name === field)
+    return schema?.constraints?.inclusion || []
+  }
+</script>
+
+<DrawerContent>
+  <div class="container">
+    <Layout noPadding>
+      <Body size="S">
+        {#if !filters?.length}
+          Add your first filter expression.
+        {:else}
+          Results are filtered to only those which match all of the following
+          constraints.
+        {/if}
+      </Body>
+      {#if filters?.length}
+        <div class="fields">
+          {#each filters as filter, idx}
+            <Select
+              bind:value={filter.field}
+              options={fieldOptions}
+              on:change={e => onFieldChange(filter, e.detail)}
+              placeholder="Column"
+            />
+            <Select
+              disabled={!filter.field}
+              options={getValidOperatorsForType(filter.type)}
+              bind:value={filter.operator}
+              on:change={e => onOperatorChange(filter, e.detail)}
+              placeholder={null}
+            />
+            {#if ["string", "longform", "number"].includes(filter.type)}
+              <Input disabled={filter.noValue} bind:value={filter.value} />
+            {:else if ["options", "array"].includes(filter.type)}
+              <Combobox
+                disabled={filter.noValue}
+                options={getFieldOptions(filter.field)}
+                bind:value={filter.value}
+              />
+            {:else if filter.type === "boolean"}
+              <Combobox
+                disabled={filter.noValue}
+                options={[
+                  { label: "True", value: "true" },
+                  { label: "False", value: "false" },
+                ]}
+                bind:value={filter.value}
+              />
+            {:else if filter.type === "datetime"}
+              <DatePicker disabled={filter.noValue} bind:value={filter.value} />
+            {:else}
+              <Input disabled />
+            {/if}
+            <Icon
+              name="Duplicate"
+              hoverable
+              size="S"
+              on:click={() => duplicateFilter(filter.id)}
+            />
+            <Icon
+              name="Close"
+              hoverable
+              size="S"
+              on:click={() => removeFilter(filter.id)}
+            />
+          {/each}
+        </div>
+      {/if}
+      <div>
+        <Button icon="AddCircle" size="M" secondary on:click={addFilter}>
+          Add filter
+        </Button>
+      </div>
+    </Layout>
+  </div>
+</DrawerContent>
+
+<style>
+  .container {
+    width: 100%;
+    max-width: 1000px;
+    margin: 0 auto;
+  }
+  .fields {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    align-items: center;
+    grid-template-columns: 1fr 120px 1fr auto auto;
+  }
+</style>

packages/client/src/components/app/dynamic-filter/index.js

@@ -0,0 +1 @@
+export { default as dynamicfilter } from "./DynamicFilter.svelte"

packages/client/src/components/app/forms/Field.svelte

@@ -49,7 +49,7 @@
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
   const updateLabel = e => {
-    builderStore.actions.updateProp("label", e.target.textContent)
+    builderStore.actions.updateProp("label", e.target.textContent.trim())
   }
 </script>
 

packages/client/src/components/app/index.js

@@ -33,6 +33,7 @@ export * from "./charts"
 export * from "./forms"
 export * from "./table"
 export * from "./blocks"
+export * from "./dynamic-filter"
 
 // Deprecated component left for compatibility in old apps
 export { default as navigation } from "./deprecated/Navigation.svelte"

packages/client/src/stores/builder.js

@@ -10,7 +10,6 @@ const dispatchEvent = (type, data = {}) => {
 const createBuilderStore = () => {
   const initialState = {
     inBuilder: false,
-    appId: null,
     layout: null,
     screen: null,
     selectedComponentId: null,
@@ -94,6 +93,7 @@ const createBuilderStore = () => {
   }
   return {
     ...writableStore,
+    set: state => writableStore.set({ ...initialState, ...state }),
     subscribe: derivedStore.subscribe,
     actions,
   }

packages/server/LICENSE

@@ -1,7 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright 2019-2021, Budibase Inc
+ Copyright 2019-2021, Budibase Ltd
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

packages/server/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.190-alpha.11",
   "description": "Budibase Web Server",
-  "main": "src/index.js",
+  "main": "src/index.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/Budibase/budibase.git"
@@ -14,7 +14,8 @@
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service",
+    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -66,11 +67,11 @@
     "budibase"
   ],
   "author": "Budibase",
-  "license": "AGPL-3.0-or-later",
+  "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^0.9.185-alpha.14",
+    "@budibase/auth": "^0.9.190-alpha.11",
-    "@budibase/client": "^0.9.185-alpha.14",
+    "@budibase/client": "^0.9.190-alpha.11",
-    "@budibase/string-templates": "^0.9.185-alpha.14",
+    "@budibase/string-templates": "^0.9.190-alpha.11",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -106,7 +107,7 @@
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
     "node-fetch": "2.6.0",
-    "open": "7.3.0",
+    "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
@@ -130,6 +131,7 @@
     "@babel/preset-env": "^7.14.4",
     "@budibase/standard-components": "^0.9.139",
     "@jest/test-sequencer": "^24.8.0",
+    "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
@@ -137,12 +139,14 @@
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
+    "apidoc": "^0.50.2",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
     "supertest": "^4.0.2",

packages/server/scripts/docs/README.md

@@ -0,0 +1,31 @@
+### Documentation
+
+This directory contains the scripts required to generate the APIDoc based documentation.
+You can find the docs about comment structure at the [APIDocs page](https://apidocjs.com/).
+
+In general most API endpoints will look like:
+```js
+  /**
+   * @api {post} /api/:param/url Give it a name 
+   * @apiName Give it a name
+   * @apiGroup group
+   * @apiPermission permission
+   * @apiDescription Describe what the endpoint does, any special cases the user
+   * should be aware of.
+   *
+   * @apiParam {string} param describe a URL parameter.
+   *
+   * @apiParam (Body) input describe a field on the body.
+   *
+   * @apiSuccess {object} output describe the output.
+   */
+```
+
+There are a few key points to note when writing API docs:
+1. Don't use `@apiBody` - this isn't currently supported by our swagger converter.
+2. Make sure to always have an `@api` definition at the start, which must always have the
+HTTP verb, the endpoint URL and the name.
+3. There are three ways you can specify parameters used as inputs for your endpoint,
+`@apiParam` for a URL param, `@apiParam (Body)` for a field on the request body and `@apiParam (Query)`
+for query string parameters.
+4. The `@apiGroup` should be the same for all API Doc comments in a route file.

packages/server/scripts/docs/generate.js

@@ -0,0 +1,74 @@
+const fs = require("fs")
+const { join } = require("path")
+const { createDoc } = require("apidoc")
+const packageJson = require("../../package.json")
+const toSwagger = require("./toSwagger")
+const open = require("open")
+
+const config = {
+  name: "Budibase API",
+  version: packageJson.version,
+  description: "Documenting the Budibase backend API",
+  title: "Budibase app service API",
+}
+
+const shouldOpen = process.argv[2]
+const disallowed = []
+
+function filter(parsedRouteFiles) {
+  const tagToSearch = "url"
+  for (let routeFile of parsedRouteFiles) {
+    for (let route of routeFile) {
+      let routeInfo = route["local"]
+      if (disallowed.includes(routeInfo[tagToSearch])) {
+        const idx = routeFile.indexOf(route)
+        routeFile.splice(idx, 1)
+      }
+    }
+  }
+}
+
+async function generate() {
+  // start by writing a config file
+  const configPath = join(__dirname, "config.json")
+  fs.writeFileSync(configPath, JSON.stringify(config))
+  const mainPath = join(__dirname, "..", "..")
+  const srcPath = join(mainPath, "src", "api", "routes")
+  const assetsPath = join(mainPath, "builder", "assets", "docs")
+  if (!fs.existsSync(assetsPath)) {
+    fs.mkdirSync(assetsPath, { recursive: true })
+  }
+  const options = {
+    src: [srcPath],
+    dest: assetsPath,
+    filters: {
+      main: {
+        postFilter: filter,
+      },
+    },
+    config: configPath,
+  }
+  const doc = createDoc(options)
+  if (typeof doc !== "boolean") {
+    const swagger = toSwagger(JSON.parse(doc.data), JSON.parse(doc.project))
+    fs.writeFileSync(join(assetsPath, "swagger.json"), JSON.stringify(swagger))
+    fs.writeFileSync(join(assetsPath, "apidoc.json"), doc.data)
+    fs.writeFileSync(join(assetsPath, "project.json"), doc.project)
+    console.log(
+      `Docs generated successfully, find in ${assetsPath}, swagger.json, apidoc.json and project.json`
+    )
+  } else {
+    throw "Unable to generate docs."
+  }
+  // delete the temporary config file
+  fs.unlinkSync(configPath)
+  setTimeout(async () => {
+    if (shouldOpen === "open") {
+      await open(join(assetsPath, "index.html"), { wait: false })
+    }
+  }, 2000)
+}
+
+generate().catch(err => {
+  console.error(err)
+})

packages/server/scripts/docs/toSwagger.js

@@ -0,0 +1,320 @@
+let _ = require("lodash")
+let { pathToRegexp } = require("path-to-regexp")
+
+/********************************************************
+ * Based on: https://github.com/fsbahman/apidoc-swagger *
+ ********************************************************/
+
+let swagger = {
+  swagger: "2.0",
+  info: {},
+  paths: {},
+  definitions: {},
+}
+
+function toSwagger(apidocJson, projectJson) {
+  swagger.info = addInfo(projectJson)
+  swagger.paths = extractPaths(apidocJson)
+  return swagger
+}
+
+let tagsRegex = /(<([^>]+)>)/gi
+// Removes <p> </p> tags from text
+function removeTags(text) {
+  return text ? text.replace(tagsRegex, "") : text
+}
+
+function addInfo(projectJson) {
+  let info = {}
+  info["title"] = projectJson.title || projectJson.name
+  info["version"] = projectJson.version
+  info["description"] = projectJson.description
+  return info
+}
+
+/**
+ * Extracts paths provided in json format
+ * post, patch, put request parameters are extracted in body
+ * get and delete are extracted to path parameters
+ * @param apidocJson
+ * @returns {{}}
+ */
+function extractPaths(apidocJson) {
+  let apiPaths = groupByUrl(apidocJson)
+  let paths = {}
+  for (let i = 0; i < apiPaths.length; i++) {
+    let verbs = apiPaths[i].verbs
+    let url = verbs[0].url
+    let pattern = pathToRegexp(url, null)
+    let matches = pattern.exec(url)
+
+    // Surrounds URL parameters with curly brackets -> :email with {email}
+    let pathKeys = []
+    for (let j = 1; j < matches.length; j++) {
+      let key = matches[j].substr(1)
+      url = url.replace(matches[j], "{" + key + "}")
+      pathKeys.push(key)
+    }
+
+    for (let j = 0; j < verbs.length; j++) {
+      let verb = verbs[j]
+      let type = verb.type
+
+      let obj = (paths[url] = paths[url] || {})
+
+      if (type === "post" || type === "patch" || type === "put") {
+        _.extend(
+          obj,
+          createPostPushPutOutput(verb, swagger.definitions, pathKeys)
+        )
+      } else {
+        _.extend(obj, createGetDeleteOutput(verb, swagger.definitions))
+      }
+    }
+  }
+  return paths
+}
+
+function createPostPushPutOutput(verbs, definitions, pathKeys) {
+  let pathItemObject = {}
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+
+  let params = []
+  let pathParams = createPathParameters(verbs, pathKeys)
+  pathParams = _.filter(pathParams, function (param) {
+    let hasKey = pathKeys.indexOf(param.name) !== -1
+    return !(param.in === "path" && !hasKey)
+  })
+
+  params = params.concat(pathParams)
+  let required =
+    verbs.parameter &&
+    verbs.parameter.fields &&
+    verbs.parameter.fields.Parameter &&
+    verbs.parameter.fields.Parameter.length > 0
+
+  params.push({
+    in: "body",
+    name: "body",
+    description: removeTags(verbs.description),
+    required: required,
+    schema: {
+      $ref: "#/definitions/" + verbDefinitionResult.topLevelParametersRef,
+    },
+  })
+
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: params,
+  }
+
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+
+  return pathItemObject
+}
+
+function createVerbDefinitions(verbs, definitions) {
+  let result = {
+    topLevelParametersRef: null,
+    topLevelSuccessRef: null,
+    topLevelSuccessRefType: null,
+  }
+  let defaultObjectName = verbs.name
+
+  let fieldArrayResult = {}
+  if (verbs && verbs.parameter && verbs.parameter.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.parameter.fields.Parameter,
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelParametersRef = fieldArrayResult.topLevelRef
+  }
+
+  if (verbs && verbs.success && verbs.success.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.success.fields["Success 200"],
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelSuccessRef = fieldArrayResult.topLevelRef
+    result.topLevelSuccessRefType = fieldArrayResult.topLevelRefType
+  }
+
+  return result
+}
+
+function createFieldArrayDefinitions(
+  fieldArray,
+  definitions,
+  topLevelRef,
+  defaultObjectName
+) {
+  let result = {
+    topLevelRef: topLevelRef,
+    topLevelRefType: null,
+  }
+
+  if (!fieldArray) {
+    return result
+  }
+
+  for (let i = 0; i < fieldArray.length; i++) {
+    let parameter = fieldArray[i]
+
+    let nestedName = createNestedName(parameter.field)
+    let objectName = nestedName.objectName
+    if (!objectName) {
+      objectName = defaultObjectName
+    }
+    let type = parameter.type
+    if (i === 0) {
+      result.topLevelRefType = type
+      if (parameter.type === "Object") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+      } else if (parameter.type === "Array") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+        result.topLevelRefType = "array"
+      }
+      result.topLevelRef = objectName
+    }
+
+    definitions[objectName] = definitions[objectName] || {
+      properties: {},
+      required: [],
+    }
+
+    if (nestedName.propertyName) {
+      let prop = {
+        type: (parameter.type || "").toLowerCase(),
+        description: removeTags(parameter.description),
+      }
+      if (parameter.type === "Object") {
+        prop.$ref = "#/definitions/" + parameter.field
+      }
+
+      let typeIndex = type.indexOf("[]")
+      if (typeIndex !== -1 && typeIndex === type.length - 2) {
+        prop.type = "array"
+        prop.items = {
+          type: type.slice(0, type.length - 2),
+        }
+      }
+
+      definitions[objectName]["properties"][nestedName.propertyName] = prop
+      if (!parameter.optional) {
+        let arr = definitions[objectName]["required"]
+        if (arr.indexOf(nestedName.propertyName) === -1) {
+          arr.push(nestedName.propertyName)
+        }
+      }
+    }
+  }
+
+  return result
+}
+
+function createNestedName(field) {
+  let propertyName = field
+  let objectName
+  let propertyNames = field.split(".")
+  if (propertyNames && propertyNames.length > 1) {
+    propertyName = propertyNames[propertyNames.length - 1]
+    propertyNames.pop()
+    objectName = propertyNames.join(".")
+  }
+
+  return {
+    propertyName: propertyName,
+    objectName: objectName,
+  }
+}
+
+/**
+ * Generate get, delete method output
+ * @param verbs
+ * @param definitions
+ * @returns {{}}
+ */
+function createGetDeleteOutput(verbs, definitions) {
+  let pathItemObject = {}
+  verbs.type = verbs.type === "del" ? "delete" : verbs.type
+
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: createPathParameters(verbs),
+  }
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+  return pathItemObject
+}
+
+/**
+ * Iterate through all method parameters and create array of parameter objects which are stored as path parameters
+ * @param verbs
+ * @returns {Array}
+ */
+function createPathParameters(verbs) {
+  let pathItemObject = []
+  if (verbs.parameter && verbs.parameter.fields.Parameter) {
+    for (let i = 0; i < verbs.parameter.fields.Parameter.length; i++) {
+      let param = verbs.parameter.fields.Parameter[i]
+      let field = param.field
+      let type = param.type
+      pathItemObject.push({
+        name: field,
+        in: type === "file" ? "formData" : "path",
+        required: !param.optional,
+        type: param.type.toLowerCase(),
+        description: removeTags(param.description),
+      })
+    }
+  }
+  return pathItemObject
+}
+
+function groupByUrl(apidocJson) {
+  return _.chain(apidocJson)
+    .groupBy("url")
+    .toPairs()
+    .map(function (element) {
+      return _.zipObject(["url", "verbs"], element)
+    })
+    .value()
+}
+
+module.exports = toSwagger

packages/server/scripts/integrations/oracle/docker-compose.yml

@@ -4,7 +4,7 @@
 version: "3.8"
 services:
   db:
-    container_name: oracle-xe
+    restart: always
     platform: linux/x86_64
     image: container-registry.oracle.com/database/express:18.4.0-xe
     environment:

packages/server/src/api/controllers/application.js

@@ -15,14 +15,12 @@ const {
   generateAppID,
   getLayoutParams,
   getScreenParams,
-  generateScreenID,
   generateDevAppID,
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
 const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
-const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const {
@@ -409,9 +407,5 @@ const createEmptyAppPackage = async (ctx, app) => {
     screensAndLayouts.push(await processObject(cloned, app))
   }
 
-  const homeScreen = createHomeScreen(app)
-  homeScreen._id = generateScreenID()
-  screensAndLayouts.push(homeScreen)
-
   await db.bulkDocs(screensAndLayouts)
 }

packages/server/src/api/controllers/query.js

@@ -1,8 +1,11 @@
 const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
-const { generateQueryID, getQueryParams } = require("../../db/utils")
+const {
+  generateQueryID,
+  getQueryParams,
+  isProdAppID,
+} = require("../../db/utils")
 const { BaseQueryVerbs } = require("../../constants")
-const env = require("../../environment")
 const { Thread, ThreadType } = require("../../threads")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
@@ -90,10 +93,9 @@ exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
-  if (env.isProd()) {
+  if (isProdAppID(ctx.appId)) {
     delete query.fields
     delete query.parameters
-    delete query.schema
   }
   ctx.body = query
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,4 +1,5 @@
 import {
+  FilterTypes,
   IncludeRelationships,
   Operation,
   PaginationJson,
@@ -118,8 +119,13 @@ module External {
     }
     // check the row and filters to make sure they aren't a key of some sort
     if (config.filters) {
-      for (let filter of Object.values(config.filters)) {
+      for (let [key, filter] of Object.entries(config.filters)) {
-        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+        // oneOf is an array, don't iterate it
+        if (
+          typeof filter !== "object" ||
+          Object.keys(filter).length === 0 ||
+          key === FilterTypes.ONE_OF
+        ) {
           continue
         }
         iterateObject(filter)
@@ -197,9 +203,9 @@ module External {
     return row
   }
 
-  function isMany(field: FieldSchema) {
+  function isOneSide(field: FieldSchema) {
     return (
-      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+      field.relationshipType && field.relationshipType.split("-")[0] === "one"
     )
   }
 
@@ -274,25 +280,37 @@ module External {
         const linkTable = this.tables[linkTableName]
         // @ts-ignore
         const linkTablePrimary = linkTable.primary[0]
-        if (!isMany(field)) {
+        // one to many
+        if (isOneSide(field)) {
           newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
             row[key][0]
           )[0]
-        } else {
+        }
+        // many to many
+        else if (field.through) {
           // we're not inserting a doc, will be a bunch of update calls
-          const isUpdate = !field.through
+          const otherKey: string = field.throughFrom || linkTablePrimary
-          const thisKey: string = isUpdate
+          const thisKey: string = field.throughTo || tablePrimary
-            ? "id"
-            : field.throughTo || linkTablePrimary
-          // @ts-ignore
-          const otherKey: string = isUpdate
-            ? field.fieldName
-            : field.throughFrom || tablePrimary
           row[key].map((relationship: any) => {
-            // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
               tableId: field.through || field.tableId,
-              isUpdate,
+              isUpdate: false,
+              key: otherKey,
+              [otherKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [thisKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+        // many to one
+        else {
+          const thisKey: string = "id"
+          // @ts-ignore
+          const otherKey: string = field.fieldName
+          row[key].map((relationship: any) => {
+            manyRelationships.push({
+              tableId: field.tableId,
+              isUpdate: true,
               key: otherKey,
               [thisKey]: breakRowIdField(relationship)[0],
               // leave the ID for enrichment later
@@ -425,8 +443,8 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = field.throughFrom || table.primary[0]
+          definition.from = field.throughTo || table.primary[0]
-          definition.to = field.throughTo || linkTable.primary[0]
+          definition.to = field.throughFrom || linkTable.primary[0]
           definition.fromPrimary = table.primary[0]
           definition.toPrimary = linkTable.primary[0]
         }
@@ -448,24 +466,36 @@ module External {
       // make a new request to get the row with all its relationships
       // we need this to work out if any relationships need removed
       for (let field of Object.values(table.schema)) {
-        if (field.type !== FieldTypes.LINK || !field.fieldName) {
+        if (
+          field.type !== FieldTypes.LINK ||
+          !field.fieldName ||
+          isOneSide(field)
+        ) {
           continue
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const manyKey = field.throughFrom || primaryKey
+        const { tableName: relatedTableName } = breakExternalTableId(tableId)
+        // @ts-ignore
+        const linkPrimaryKey = this.tables[relatedTableName].primary[0]
+        const manyKey = field.throughTo || primaryKey
+        const lookupField = isMany ? primaryKey : field.foreignKey
         const fieldName = isMany ? manyKey : field.fieldName
+        if (!lookupField || !row[lookupField]) {
+          continue
+        }
         const response = await getDatasourceAndQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
-              [fieldName]: row[primaryKey],
+              [fieldName]: row[lookupField],
             },
           },
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        related[fieldName] = { rows, isMany, tableId }
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        related[storeTo] = { rows, isMany, tableId }
       }
       return related
     }

packages/server/src/api/controllers/row/external.js

@@ -57,7 +57,8 @@ exports.save = async ctx => {
 exports.fetchView = async ctx => {
   // there are no views in external data sources, shouldn't ever be called
   // for now just fetch
-  ctx.params.tableId = ctx.params.viewName.split("all_")[1]
+  const split = ctx.params.viewName.split("all_")
+  ctx.params.tableId = split[1] ? split[1] : split[0]
   return exports.fetch(ctx)
 }
 

packages/server/src/api/controllers/table/internal.js

@@ -44,14 +44,10 @@ exports.save = async function (ctx) {
   // the column if you want to change the type
   if (oldTable && oldTable.schema) {
     for (let propKey of Object.keys(tableToSave.schema)) {
-      let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
       if (oldColumn && oldColumn.type === "internal") {
         oldColumn.type = "auto"
       }
-      if (oldColumn && oldColumn.type !== column.type) {
-        ctx.throw(400, "Cannot change the type of a column")
-      }
     }
   }
 

packages/server/src/api/controllers/view/exporters.js

@@ -16,3 +16,8 @@ exports.csv = function (headers, rows) {
 exports.json = function (headers, rows) {
   return JSON.stringify(rows, undefined, 2)
 }
+
+exports.ExportFormats = {
+  CSV: "csv",
+  JSON: "json",
+}

packages/server/src/api/controllers/view/index.js

@@ -4,6 +4,7 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
+const { getTable } = require("../table/utils")
 
 exports.fetch = async ctx => {
   const db = new CouchDB(ctx.appId)
@@ -56,7 +57,7 @@ exports.exportView = async ctx => {
   const view = await getView(db, viewName)
 
   const format = ctx.query.format
-  if (!format) {
+  if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
     ctx.throw(400, "Format must be specified, either csv or json")
   }
 
@@ -80,10 +81,22 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await db.get(tableId)
+    const table = await getTable(ctx.appId, tableId)
     schema = table.schema
   }
 
+  // make sure no "undefined" entries appear in the CSV
+  if (format === exporters.ExportFormats.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      for (let row of ctx.body) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
   // Export part
   let headers = Object.keys(schema)
   const exporter = exporters[format]

packages/server/src/api/routes/row.js

@@ -14,24 +14,130 @@ const {
 const router = Router()
 
 router
+  /**
+   * @api {get} /api/:tableId/:rowId/enrich Get an enriched row
+   * @apiName Get an enriched row
+   * @apiGroup rows
+   * @apiPermission table read access
+   * @apiDescription This API is only useful when dealing with rows that have relationships.
+   * Normally when a row is a returned from the API relationships will only have the structure
+   * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows
+   * for each relationship instead.
+   *
+   * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched.
+   *
+   * @apiSuccess {object} row The response body will be the enriched row.
+   */
   .get(
     "/api/:tableId/:rowId/enrich",
     paramSubResource("tableId", "rowId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.fetchEnrichedRow
   )
+  /**
+   * @api {get} /api/:tableId/rows Get all rows in a table
+   * @apiName Get all rows in a table
+   * @apiGroup rows
+   * @apiPermission table read access
+   * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint.
+   * This endpoint gets all of the rows within the specified table - it is not heavily used
+   * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
+   * will simply stop.
+   *
+   * @apiParam {string} tableId The ID of the table to retrieve all rows within.
+   *
+   * @apiSuccess {object[]} rows The response body will be an array of all rows found.
+   */
   .get(
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.fetch
   )
+  /**
+   * @api {get} /api/:tableId/rows/:rowId Retrieve a single row
+   * @apiName Retrieve a single row
+   * @apiGroup rows
+   * @apiPermission table read access
+   * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
+   * a row by anything other than its _id field, use the search endpoint.
+   *
+   * @apiParam {string} tableId The ID of the table to retrieve a row from.
+   * @apiParam {string} rowId The ID of the row to retrieve.
+   *
+   * @apiSuccess {object} body The response body will be the row that was found.
+   */
   .get(
     "/api/:tableId/rows/:rowId",
     paramSubResource("tableId", "rowId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.find
   )
+  /**
+   * @api {post} /api/:tableId/search Search for rows in a table
+   * @apiName Search for rows in a table
+   * @apiGroup rows
+   * @apiPermission table read access
+   * @apiDescription This is the primary method of accessing rows in Budibase, the data provider
+   * and data UI in the builder are built atop this. All filtering, sorting and pagination is
+   * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
+   *
+   * @apiParam {string} tableId The ID of the table to retrieve rows from.
+   *
+   * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
+   * defaults to false.
+   * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none
+   * specified then the request will be unfiltered. An example with all of the possible query
+   * options has been supplied below.
+   * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned,
+   * this will be implemented at the database level if supported for performance reasons. This
+   * is useful when paginating to set exactly how many rows per page.
+   * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned
+   * with each successful search request, this should be supplied back to get the next page.
+   * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to
+   * sort on.
+   * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or
+   * "ascending" as required.
+   * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search
+   * being used, either "string" or "number". This is only used for internal tables.
+   *
+   * @apiParamExample {json} Example:
+   * {
+   *  "tableId": "ta_70260ff0b85c467ca74364aefc46f26d",
+   *  "query": {
+   *    "string": {},
+   *    "fuzzy": {},
+   *    "range": {
+   *      "columnName": {
+   *        "high": 20,
+   *        "low": 10,
+   *      }
+   *    },
+   *    "equal": {
+   *      "columnName": "someValue"
+   *    },
+   *    "notEqual": {},
+   *    "empty": {},
+   *    "notEmpty": {},
+   *    "contains": {},
+   *    "notContains": {}
+   *    "oneOf": {
+   *      "columnName": ["value"]
+   *    }
+   *  },
+   *  "limit": 10,
+   *  "sort": "name",
+   *  "sortOrder": "descending",
+   *  "sortType": "string",
+   *  "paginate": true
+   * }
+   *
+   * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters.
+   * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or
+   * not there is another page after this request.
+   * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next
+   * page.
+   */
   .post(
     "/api/:tableId/search",
     paramResource("tableId"),
@@ -46,6 +152,30 @@ router
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.search
   )
+  /**
+   * @api {post} /api/:tableId/rows Creates a new row
+   * @apiName Creates a new row
+   * @apiGroup rows
+   * @apiPermission table write access
+   * @apiDescription This API will create a new row based on the supplied body. If the
+   * body includes an "_id" field then it will update an existing row if the field
+   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
+   * already used by Budibase tables and cannot be used for columns.
+   *
+   * @apiParam {string} tableId The ID of the table to save a row to.
+   *
+   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
+   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
+   * must also be provided.
+   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
+   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
+   * a column in the specified table. All other fields will be dropped and not stored.
+   *
+   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
+   * is the rows new ID.
+   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
+   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
+   */
   .post(
     "/api/:tableId/rows",
     paramResource("tableId"),
@@ -53,18 +183,66 @@ router
     usage,
     rowController.save
   )
+  /**
+   * @api {patch} /api/:tableId/rows Updates a row
+   * @apiName Update a row
+   * @apiGroup rows
+   * @apiPermission table write access
+   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
+   * error if an _id isn't provided, it will only function for existing rows.
+   */
   .patch(
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     rowController.patch
   )
+  /**
+   * @api {post} /api/:tableId/rows/validate Validate inputs for a row
+   * @apiName Validate inputs for a row
+   * @apiGroup rows
+   * @apiPermission table write access
+   * @apiDescription When attempting to save a row you may want to check if the row is valid
+   * given the table schema, this will iterate through all the constraints on the table and
+   * check if the request body is valid.
+   *
+   * @apiParam {string} tableId The ID of the table the row is to be validated for.
+   *
+   * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
+   * against the table schema and constraints.
+   *
+   * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this
+   * will be true, if it is not then then errors property will be populated.
+   * @apiSuccess {object} [errors] A key value map of information about fields on the input
+   * which do not match the table schema. The key name will be the column names that have breached
+   * the schema.
+   */
   .post(
     "/api/:tableId/rows/validate",
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     rowController.validate
   )
+  /**
+   * @api {delete} /api/:tableId/rows Delete rows
+   * @apiName Delete rows
+   * @apiGroup rows
+   * @apiPermission table write access
+   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
+   * fashion.
+   *
+   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
+   *
+   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
+   * key of the request body that are to be deleted.
+   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
+   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
+   * revision here.
+   *
+   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
+   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
+   * is the deleted row.
+   */
   .delete(
     "/api/:tableId/rows",
     paramResource("tableId"),

packages/server/src/api/routes/table.js

@@ -27,13 +27,110 @@ function generateSaveValidator() {
 }
 
 router
+  /**
+   * @api {get} /api/tables Fetch all tables
+   * @apiName Fetch all tables
+   * @apiGroup tables
+   * @apiPermission table read access
+   * @apiDescription This endpoint retrieves all of the tables which have been created in
+   * an app. This includes all of the external and internal tables; to tell the difference
+   * between these look for the "type" property on each table, either being "internal" or "external".
+   *
+   * @apiSuccess {object[]} body The response body will be the list of tables that was found - as
+   * this does not take any parameters the only error scenario is no access.
+   */
   .get("/api/tables", authorized(BUILDER), tableController.fetch)
+  /**
+   * @api {get} /api/tables/:id Fetch a single table
+   * @apiName Fetch a single table
+   * @apiGroup tables
+   * @apiPermission table read access
+   * @apiDescription Retrieves a single table this could be be internal or external based on
+   * the provided table ID.
+   *
+   * @apiParam {string} id The ID of the table which is to be retrieved.
+   *
+   * @apiSuccess {object[]} body The response body will be the table that was found.
+   */
   .get(
     "/api/tables/:id",
     paramResource("id"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     tableController.find
   )
+  /**
+   * @api {post} /api/tables Save a table
+   * @apiName Save a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription Create or update a table with this endpoint, this will function for both internal
+   * external tables.
+   *
+   * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified.
+   * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
+   * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
+   * this will default to internal.
+   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the data source ID. If
+   * building an internal table this does not need to be set, although it will be returned as "bb_internal".
+   * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
+   * supply the table structure to this endpoint with the name changed.
+   * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this
+   * object. For each column a "type" and "constraints" must be specified, with some types requiring further information.
+   * More information about the schema structure can be found in the Typescript definitions.
+   * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows
+   * from this table as relationships.
+   * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction
+   * of lucene indexes. This functionality is only available for internal tables.
+   * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this
+   * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
+   * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
+   * the rows in the table. This functionality is only available for internal tables.
+   * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
+   * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
+   * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
+   * or for external tables.
+   *
+   * @apiParamExample {json} Example:
+   * {
+   *   "_id": "ta_05541307fa0f4044abee071ca2a82119",
+   *   "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807",
+   *   "type": "internal",
+   *   "views": {},
+   *   "name": "tableName",
+   *   "schema": {
+   *     "column": {
+   *       "type": "string",
+   *       "constraints": {
+   *         "type": "string",
+   *         "length": {
+   *           "maximum": null
+   *         },
+   *         "presence": false
+   *       },
+   *       "name": "column"
+   *     },
+   *   },
+   *   "primaryDisplay": "column",
+   *   "indexes": [],
+   *   "sourceId": "bb_internal",
+   *   "_rename": {
+   *     "old": "columnName",
+   *     "updated": "newColumnName",
+   *   },
+   *   "dataImport": {
+   *     "csvString": "column\nvalue",
+   *     "primaryDisplay": "column",
+   *     "schema": {
+   *       "column": {
+   *         "type": "string"
+   *       }
+   *     }
+   *   }
+   * }
+   *
+   * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
+   * saved to the database.
+   */
   .post(
     "/api/tables",
     // allows control over updating a table
@@ -42,22 +139,68 @@ router
     generateSaveValidator(),
     tableController.save
   )
+  /**
+   * @api {post} /api/tables/csv/validate Validate a CSV for a table
+   * @apiName Validate a CSV for a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
+   * converted into a Budibase schema; this endpoint does this.
+   *
+   * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
+   * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
+   * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
+   * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
+   * using a number instead of strings.
+   * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
+   * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
+   * importing a CSV to an existing table only fields that are present on the table can be imported.
+   *
+   * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
+   * the CSV - this will be in the same format used for table schema.s
+   */
   .post(
     "/api/tables/csv/validate",
     authorized(BUILDER),
     tableController.validateCSVSchema
   )
+  /**
+   * @api {post} /api/tables/:tableId/:revId Delete a table
+   * @apiName Delete a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is
+   * quite dangerous - it will work for internal and external tables.
+   *
+   * @apiParam {string} tableId The ID of the table which is to be deleted.
+   * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for
+   * external tables this can simply be set to anything, e.g. "external".
+   *
+   * @apiSuccess {string} message A message stating that the table was deleted successfully.
+   */
   .delete(
     "/api/tables/:tableId/:revId",
     paramResource("tableId"),
     authorized(BUILDER),
     tableController.destroy
   )
-  // this is currently builder only, but in the future
+  /**
-  // it could be carried out by an end user in app,
+   * @api {post} /api/tables/:tableId/:revId Import CSV to existing table
-  // however some thought will need to be had about
+   * @apiName Import CSV to existing table
-  // implications for automations (triggers)
+   * @apiGroup tables
-  // new trigger type, bulk rows created
+   * @apiPermission builder
+   * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination
+   * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to
+   * import the data; please note this will only import fields that already exist on the table/match the type.
+   *
+   * @apiParam {string} tableId The ID of the table which the data should be imported to.
+   *
+   * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
+   * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
+   * turned into rows.
+   *
+   * @apiSuccess {string} message A message stating that the data was imported successfully.
+   */
   .post(
     "/api/tables/:tableId/import",
     paramResource("tableId"),

packages/server/src/api/routes/tests/application.spec.js

@@ -75,7 +75,6 @@ describe("/applications", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       // should have empty packages
-      expect(res.body.screens.length).toEqual(1)
       expect(res.body.layouts.length).toEqual(2)
     })
   })
@@ -88,7 +87,6 @@ describe("/applications", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.application).toBeDefined()
-      expect(res.body.screens.length).toEqual(1)
       expect(res.body.layouts.length).toEqual(2)
     })
   })

packages/server/src/api/routes/tests/query.spec.js

@@ -1,6 +1,13 @@
-// mock out postgres for this
+// Mock out postgres for this
 jest.mock("pg")
 
+// Mock isProdAppID to we can later mock the implementation and pretend we are
+// using prod app IDs
+const authDb = require("@budibase/auth/db")
+const { isProdAppID } = authDb
+const mockIsProdAppID = jest.fn(isProdAppID)
+authDb.isProdAppID = mockIsProdAppID
+
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures
@@ -19,10 +26,12 @@ describe("/queries", () => {
   })
 
   async function createInvalidIntegration() {
-    const datasource = await config.createDatasource({datasource: {
+    const datasource = await config.createDatasource({
+      datasource: {
         ...basicDatasource().datasource,
         source: "INVALID_INTEGRATION",
-    }})
+      },
+    })
     const query = await config.createQuery()
     return { datasource, query }
   }
@@ -96,10 +105,31 @@ describe("/queries", () => {
           .set(await config.defaultHeaders())
           .expect(200)
           .expect("Content-Type", /json/)
+        expect(res.body.fields).toBeDefined()
+        expect(res.body.parameters).toBeDefined()
+        expect(res.body.schema).toBeDefined()
+      })
+    })
+
+    it("should remove sensitive info for prod apps", async () => {
+      // Mock isProdAppID to pretend we are using a prod app
+      mockIsProdAppID.mockClear()
+      mockIsProdAppID.mockImplementation(() => true)
+
+      const query = await config.createQuery()
+      const res = await request
+        .get(`/api/queries/${query._id}`)
+        .set(await config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body._id).toEqual(query._id)
       expect(res.body.fields).toBeUndefined()
       expect(res.body.parameters).toBeUndefined()
-        expect(res.body.schema).toBeUndefined()
+      expect(res.body.schema).toBeDefined()
-      })
+
+      // Reset isProdAppID mock
+      expect(mockIsProdAppID).toHaveBeenCalledTimes(1)
+      mockIsProdAppID.mockImplementation(isProdAppID)
     })
   })
 

packages/server/src/api/routes/tests/screen.spec.js

@@ -21,7 +21,7 @@ describe("/screens", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.body.length).toEqual(2)
+      expect(res.body.length).toEqual(1)
       expect(res.body.some(s => s._id === screen._id)).toEqual(true)
     })
 

packages/server/src/automations/automationUtils.js

@@ -64,5 +64,14 @@ exports.cleanUpRow = async (appId, tableId, row) => {
 }
 
 exports.getError = err => {
+  if (err == null) {
+    return "No error provided."
+  }
+  if (
+    typeof err === "object" &&
+    (err.toString == null || err.toString() === "[object Object]")
+  ) {
+    return JSON.stringify(err)
+  }
   return typeof err !== "string" ? err.toString() : err
 }

packages/server/src/definitions/datasource.ts

@@ -54,6 +54,17 @@ export enum IncludeRelationships {
   EXCLUDE = 0,
 }
 
+export enum FilterTypes {
+  STRING = "string",
+  FUZZY = "fuzzy",
+  RANGE = "range",
+  EQUAL = "equal",
+  NOT_EQUAL = "notEqual",
+  EMPTY = "empty",
+  NOT_EMPTY = "notEmpty",
+  ONE_OF = "oneOf",
+}
+
 export interface QueryDefinition {
   type: QueryTypes
   displayName?: string

packages/server/src/integrations/base/sql.ts

@@ -21,6 +21,9 @@ function parse(input: any) {
   if (Array.isArray(input)) {
     return JSON.stringify(input)
   }
+  if (input == undefined) {
+    return null
+  }
   if (typeof input !== "string") {
     return input
   }
@@ -43,7 +46,10 @@ function parseBody(body: any) {
   return body
 }
 
-function parseFilters(filters: SearchFilters): SearchFilters {
+function parseFilters(filters: SearchFilters | undefined): SearchFilters {
+  if (!filters) {
+    return {}
+  }
   for (let [key, value] of Object.entries(filters)) {
     let parsed
     if (typeof value === "object") {
@@ -87,7 +93,7 @@ class InternalBuilder {
     if (filters.oneOf) {
       iterate(filters.oneOf, (key, array) => {
         const fnc = allOr ? "orWhereIn" : "whereIn"
-        query = query[fnc](key, array)
+        query = query[fnc](key, Array.isArray(array) ? array : [array])
       })
     }
     if (filters.string) {
@@ -152,6 +158,23 @@ class InternalBuilder {
     return query
   }
 
+  addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
+    let { sort, paginate } = json
+    if (!sort) {
+      return query
+    }
+    const table = json.meta?.table
+    for (let [key, value] of Object.entries(sort)) {
+      const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
+      query = query.orderBy(`${table?.name}.${key}`, direction)
+    }
+    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
+      // @ts-ignore
+      query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
+    }
+    return query
+  }
+
   addRelationships(
     knex: Knex,
     query: KnexQuery,
@@ -249,22 +272,18 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    if (sort) {
-      for (let [key, value] of Object.entries(sort)) {
-        const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
-        query = query.orderBy(key, direction)
-      }
-    }
-    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
-      // @ts-ignore
-      query = query.orderBy(json.meta?.table?.primary[0])
-    }
     query = this.addFilters(tableName, query, filters)
+    // add sorting to pre-query
+    query = this.addSorting(query, json)
     // @ts-ignore
     let preQuery: KnexQuery = knex({
       // @ts-ignore
       [tableName]: query,
     }).select(selectStatement)
+    // have to add after as well (this breaks MS-SQL)
+    if (this.client !== SqlClients.MS_SQL) {
+      preQuery = this.addSorting(preQuery, json)
+    }
     // handle joins
     return this.addRelationships(
       knex,
@@ -416,8 +435,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         id = results?.[0].id
       } else if (sqlClient === SqlClients.MY_SQL) {
         id = results?.insertId
-      } else if (sqlClient === SqlClients.ORACLE) {
-        id = response.outBinds[0][0]
       }
       row = processFn(
         await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))

packages/server/src/integrations/index.ts

@@ -40,7 +40,7 @@ const INTEGRATIONS = {
 }
 
 // optionally add oracle integration if the oracle binary can be installed
-if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+if (!(process.arch === "arm64" && process.platform === "darwin")) {
   const oracle = require("./oracle")
   DEFINITIONS[SourceNames.ORACLE] = oracle.schema
   INTEGRATIONS[SourceNames.ORACLE] = oracle.integration

packages/server/src/integrations/mysql.ts

@@ -134,7 +134,9 @@ module MySQLModule {
         false
       )
       const tableNames = tablesResp.map(
-        (obj: any) => obj[`Tables_in_${database.toLowerCase()}`]
+        (obj: any) =>
+          obj[`Tables_in_${database}`] ||
+          obj[`Tables_in_${database.toLowerCase()}`]
       )
       for (let tableName of tableNames) {
         const primaryKeys = []

packages/server/src/integrations/oracle.ts

@@ -348,18 +348,7 @@ module OracleModule {
       this.schemaErrors = final.errors
     }
 
-    /**
+    private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
-     * Knex default returning behaviour does not work with oracle
-     * Manually add the behaviour for the return column
-     */
-    private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
-      if (bindings instanceof Array) {
-        bindings.push({ dir: oracledb.BIND_OUT })
-        query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
-      }
-    }
-
-    private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
       let connection
       try {
         connection = await this.getConnection()
@@ -367,10 +356,6 @@ module OracleModule {
         const options: ExecuteOptions = { autoCommit: true }
         const bindings: BindParameters = query.bindings || []
 
-        if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
-          this.addReturning(query, bindings, returnColum)
-        }
-
         const result: Result<T> = await connection.execute<T>(
           query.sql,
           bindings,
@@ -429,12 +414,46 @@ module OracleModule {
     }
 
     async query(json: QueryJson) {
-      const primaryKeys = json.meta!.table!.primary
+      const operation = this._operation(json)
-      const primaryKey = primaryKeys ? primaryKeys[0] : undefined
+      const input = this._query(json, { disableReturning: true })
-      const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
+      if (Array.isArray(input)) {
-      const processFn = (response: any) => response.rows ? response.rows : []
+        const responses = []
-      const output = await this.queryWithReturning(json, queryFn, processFn)
+        for (let query of input) {
-      return output
+          responses.push(await this.internalQuery(query))
+        }
+        return responses
+      } else {
+        // read the row to be deleted up front for the return
+        let deletedRows
+        if (operation === Operation.DELETE) {
+          const queryFn = (query: any) => this.internalQuery(query)
+          deletedRows = await this.getReturningRow(queryFn, json)
+        }
+
+        // run the query
+        const response = await this.internalQuery(input)
+
+        // get the results or return the created / updated / deleted row
+        if (deletedRows?.rows?.length) {
+          return deletedRows.rows
+        } else if (response.rows?.length) {
+          return response.rows
+        } else {
+          // get the last row that was updated
+          if (
+            response.lastRowid &&
+            json.endpoint?.entityId &&
+            operation !== Operation.DELETE
+          ) {
+            const lastRow = await this.internalQuery({
+              sql: `SELECT * FROM \"${json.endpoint.entityId}\" WHERE ROWID = '${response.lastRowid}'`,
+            })
+            return lastRow.rows
+          } else {
+            return [{ [ operation.toLowerCase() ]: true }]
+          }
+        }
+      }
     }
   }
 

packages/server/src/integrations/utils.ts

@@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants")
+const {
+  FieldTypes,
+  BuildSchemaErrors,
+  InvalidColumns,
+} = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -42,7 +46,7 @@ export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
   MY_SQL = "mysql",
-  ORACLE = "oracledb"
+  ORACLE = "oracledb",
 }
 
 export function isExternalTable(tableId: string) {

packages/server/src/middleware/authorized.js

@@ -1,7 +1,10 @@
-const { getUserPermissions } = require("@budibase/auth/roles")
+const {
+  getUserRoleHierarchy,
+  getRequiredResourceRole,
+  BUILTIN_ROLE_IDS,
+} = require("@budibase/auth/roles")
 const {
   PermissionTypes,
-  doesHaveResourcePermission,
   doesHaveBasePermission,
 } = require("@budibase/auth/permissions")
 const builderMiddleware = require("./builder")
@@ -29,13 +32,7 @@ module.exports =
     await builderMiddleware(ctx, permType)
 
     const isAuthed = ctx.isAuthenticated
-    const { basePermissions, permissions } = await getUserPermissions(
-      ctx.appId,
-      ctx.roleId
-    )
-
     // builders for now have permission to do anything
-    // TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check
     let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
     const isBuilderApi = permType === PermissionTypes.BUILDER
     if (isBuilder) {
@@ -44,20 +41,30 @@ module.exports =
       return ctx.throw(403, "Not Authorized")
     }
 
-    if (
+    // need to check this first, in-case public access, don't check authed until last
-      hasResource(ctx) &&
+    const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-      doesHaveResourcePermission(permissions, permLevel, ctx)
+    const hierarchy = await getUserRoleHierarchy(ctx.appId, roleId, {
-    ) {
+      idOnly: false,
-      return next()
+    })
+    const permError = "User does not have permission"
+    let possibleRoleIds = []
+    if (hasResource(ctx)) {
+      possibleRoleIds = await getRequiredResourceRole(ctx.appId, permLevel, ctx)
+    }
+    // check if we found a role, if not fallback to base permissions
+    if (possibleRoleIds.length > 0) {
+      const found = hierarchy.find(
+        role => possibleRoleIds.indexOf(role._id) !== -1
+      )
+      return found ? next() : ctx.throw(403, permError)
+    } else if (!doesHaveBasePermission(permType, permLevel, hierarchy)) {
+      ctx.throw(403, permError)
     }
 
+    // if they are not authed, then anything using the authorized middleware will fail
     if (!isAuthed) {
       ctx.throw(403, "Session not authenticated")
     }
 
-    if (!doesHaveBasePermission(permType, permLevel, basePermissions)) {
-      ctx.throw(403, "User does not have permission")
-    }
-
     return next()
   }