budibase/.github/workflows/budibase_ci.yml

name: Budibase CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  # Trigger the workflow on push or pull request,
  # but only for the master branch
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
  workflow_call:
    inputs:
      run_as_oss:
        type: boolean
        required: false
        description: Force running checks as if it was an OSS contributor
        default: false

env:
  BRANCH: ${{ github.event.pull_request.head.ref }}
  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
  NX_BASE_BRANCH: origin/${{ github.base_ref }}
  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - run: yarn lint

  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile

      # Run build all the projects
      - name: Build OSS
        run: yarn build:oss
      - name: Build account portal
        run: yarn build:account-portal
        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
      # Check the types of the projects built via esbuild
      - name: Check types
        run: |
          if ${{ env.USE_NX_AFFECTED }}; then
            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn check:types
          fi

  helm-lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: azure/setup-helm@v3
      - run: cd charts/budibase && helm lint .

  test-libraries:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - name: Pull testcontainers images
        run: |
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.2.1-sql &
          docker pull redis &

          wait $(jobs -p)

      - run: yarn --frozen-lockfile
      - name: Test
        run: |
          if ${{ env.USE_NX_AFFECTED }}; then
            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test --ignore=@budibase/worker --ignore=@budibase/server
          fi

  test-worker:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - name: Test worker
        run: |
          if ${{ env.USE_NX_AFFECTED }}; then
            yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test --scope=@budibase/worker
          fi

  test-server:
    runs-on: budi-tubby-tornado-quad-core-150gb
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn

      - name: Pull testcontainers images
        run: |
          docker pull mcr.microsoft.com/mssql/server:2022-latest &
          docker pull mysql:8.3 &
          docker pull postgres:16.1-bullseye &
          docker pull mongo:7.0-jammy &
          docker pull mariadb:lts &
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.2.1-sqs &
          docker pull minio/minio &
          docker pull redis &

          wait $(jobs -p)

      - run: yarn --frozen-lockfile

      - name: Test server
        env:
          DD_CIVISIBILITY_AGENTLESS_ENABLED: true
          DD_API_KEY: "${{ secrets.DATADOG_API_KEY }}"
          DD_SITE: "datadoghq.eu"
          NODE_OPTIONS: "-r dd-trace/ci/init"
          DD_ENV: "ci"
          DD_SERVICE: "budibase/packages/server"
        run: |
          if ${{ env.USE_NX_AFFECTED }}; then
            yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test --scope=@budibase/server
          fi

  check-pro-submodule:
    runs-on: ubuntu-latest
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo and submodules
        uses: actions/checkout@v4
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Check pro commit
        id: get_pro_commits
        run: |
          cd packages/pro
          pro_commit=$(git rev-parse HEAD)

          branch="${{ github.base_ref || github.ref_name  }}"
          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"

          base_commit=$(git rev-parse origin/master)

          if [[ ! -z $base_commit ]]; then
            echo "target_branch=$branch"
            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
            echo "pro_commit=$pro_commit"
            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
            echo "base_commit=$base_commit"

            base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit)
            echo "base_commit_excluding_merges=$base_commit_excluding_merges"
            echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing to do - branch to branch merge."
          fi

      - name: Check submodule merged and latest on base branch
        if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }}
        run: |
          cd packages/pro
          base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}'
          pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}'

          any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit)

          if [ -n "$any_commit" ]; then
            echo $any_commit

            echo "An error occurred: <error_message>"
            echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'
            echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md'

            exit 1
          else
            echo 'All good, the submodule had been merged and setup correctly!'
          fi

  check-accountportal-submodule:
    runs-on: ubuntu-latest
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo and submodules
        uses: actions/checkout@v4
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            src:
              - packages/account-portal/**

      - if: steps.changes.outputs.src == 'true'
        name: Check account portal commit
        id: get_accountportal_commits
        run: |
          cd packages/account-portal
          accountportal_commit=$(git rev-parse HEAD)

          branch="${{ github.base_ref || github.ref_name  }}"
          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"

          base_commit=$(git rev-parse origin/master)

          if [[ ! -z $base_commit ]]; then
            echo "target_branch=$branch"
            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
            echo "accountportal_commit=$accountportal_commit"
            echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
            echo "base_commit=$base_commit"
            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing to do - branch to branch merge."
          fi

      - name: Check submodule merged to base branch
        if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
            const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';

            if (submoduleCommit !== baseCommit) {
              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
              console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
              process.exit(1);
            } else {
              console.log('All good, the submodule had been merged and setup correctly!')
            }