budibase/.github/workflows/budibase_ci.yml

name: Budibase CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  # Trigger the workflow on push or pull request,
  # but only for the master branch
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
  workflow_call:
    inputs:
      run_as_oss:
        type: boolean
        required: false
        description: Force running checks as if it was an OSS contributor
        default: false

env:
  BRANCH: ${{ github.event.pull_request.head.ref }}
  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
  NX_BASE_BRANCH: origin/${{ github.base_ref }}
  ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }}
  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}
  DEBUG: testcontainers*

jobs:
  lint:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - run: yarn lint

  build:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile

      # Run build all the projects
      - name: Build
        run: yarn build
      # Check the types of the projects built via esbuild
      - name: Check types
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn check:types
          fi

  helm-lint:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: azure/setup-helm@v3
      - run: cd charts/budibase && helm lint .

  test-libraries:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - name: Pull testcontainers images
        run: |
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
          docker pull redis &

          wait $(jobs -p)

      - run: yarn --frozen-lockfile
      - name: Test
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
            yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
            yarn test -- --scope=@budibase/builder --no-prefix
          fi

  test-worker:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - name: Test worker
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
            if [ -z "$AFFECTED" ]; then
              echo "No affected tests to run"
              exit 0
            fi
          fi

          cd packages/worker
          yarn test --verbose --reporters=default --reporters=github-actions

  test-server:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        datasource:
          [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn

      - name: Load dotenv
        id: dotenv
        uses: falti/dotenv-action@v1.1.3
        with:
          path: ./packages/server/datasource-sha.env

      - name: Pull testcontainers images
        run: |
          if [ "${{ matrix.datasource }}" == "mssql" ]; then
            docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
          elif [ "${{ matrix.datasource }}" == "mysql" ]; then
            docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
          elif [ "${{ matrix.datasource }}" == "postgres" ]; then
            docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
          elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
            docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
          elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
            docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
          elif [ "${{ matrix.datasource }}" == "oracle" ]; then
            docker pull budibase/oracle-database:23.2-slim-faststart
          elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
            docker pull postgres:9.5.25
          fi
          docker pull minio/minio &
          docker pull redis &
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &

          wait $(jobs -p)

      - run: yarn --frozen-lockfile

      - name: Set up PostgreSQL 16
        if: matrix.datasource == 'postgres'
        run: |
          sudo systemctl stop postgresql
          sudo apt-get remove --purge -y postgresql* libpq-dev
          sudo rm -rf /etc/postgresql /var/lib/postgresql
          sudo apt-get autoremove -y
          sudo apt-get autoclean

          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
          sudo apt-get update
          sudo apt-get install -y postgresql-16

      - name: Test server
        env:
          DATASOURCE: ${{ matrix.datasource }}
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
            if [ -z "$AFFECTED" ]; then
              echo "No affected tests to run"
              exit 0
            fi
          fi

          FILTER="./src/tests/filters/datasource-tests.js"
          if [ "${{ matrix.datasource }}" == "none" ]; then
            FILTER="./src/tests/filters/non-datasource-tests.js"
          fi

          cd packages/server
          yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions

  check-pro-submodule:
    runs-on: ubuntu-22.04
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo and submodules
        uses: actions/checkout@v4
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Check pro commit
        id: get_pro_commits
        run: |
          cd packages/pro
          pro_commit=$(git rev-parse HEAD)

          branch="${{ github.base_ref || github.ref_name  }}"
          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"

          base_commit=$(git rev-parse origin/master)

          if [[ ! -z $base_commit ]]; then
            echo "target_branch=$branch"
            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
            echo "pro_commit=$pro_commit"
            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
            echo "base_commit=$base_commit"
            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"

            base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit)
            echo "base_commit_excluding_merges=$base_commit_excluding_merges"
            echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing to do - branch to branch merge."
          fi

      - name: Check submodule merged and latest on base branch
        if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }}
        run: |
          cd packages/pro
          base_commit='${{ steps.get_pro_commits.outputs.base_commit }}'
          base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}'
          pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}'

          any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit)

          if [ -n "$any_commit" ] && [ "$base_commit" != "$pro_commit" ]; then
            echo $any_commit

            echo "An error occurred: <error_message>"
            echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'
            echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md'

            exit 1
          else
            echo 'All good, the submodule had been merged and setup correctly!'
          fi

  check-lockfile:
    runs-on: ubuntu-22.04
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn install
      - name: Check for yarn.lock changes
        run: |
          if [[ $(git status --porcelain) == *"yarn.lock"* ]]; then
            echo "yarn.lock file needs to be modified. Please update it locally and commit the changes."
            exit 1
          else
            echo "yarn.lock file is unchanged."
          fi