budibase/.github/workflows/budibase_ci.yml

name: Budibase CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  # Trigger the workflow on push or pull request,
  # but only for the master branch
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
  workflow_call:
    inputs:
      run_as_oss:
        type: boolean
        required: false
        description: Force running checks as if it was an OSS contributor
        default: false

env:
  BRANCH: ${{ github.event.pull_request.head.ref }}
  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
  NX_BASE_BRANCH: origin/${{ github.base_ref }}
  ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }}
  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - run: yarn lint

  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile

      # Run build all the projects
      - name: Build OSS
        run: yarn build:oss
      - name: Build account portal
        run: yarn build:account-portal
        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
      # Check the types of the projects built via esbuild
      - name: Check types
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
          else
            yarn check:types --ignore @budibase/account-portal-server
          fi          

  helm-lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: azure/setup-helm@v3
      - run: cd charts/budibase && helm lint .

  test-libraries:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - name: Pull testcontainers images
        run: |
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
          docker pull redis &

          wait $(jobs -p)          

      - run: yarn --frozen-lockfile
      - name: Test
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test --ignore=@budibase/worker --ignore=@budibase/server
          fi          

  test-worker:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn --frozen-lockfile
      - name: Test worker
        run: |
          if ${{ env.ONLY_AFFECTED_TASKS }}; then
            node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
          else
            yarn test --scope=@budibase/worker
          fi          

  test-server-with-datasource:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        datasource: [mssql, mysql, postgres, mongo, mariadb, oracle]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn

      - name: Load dotenv
        id: dotenv
        uses: falti/dotenv-action@v1.1.3
        with:
          path: ./packages/server/datasource-sha.env

      - name: Pull testcontainers images
        run: |
          if [ "${{ matrix.datasource }}" == "mssql" ]; then
            docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
          elif [ "${{ matrix.datasource }}" == "mysql" ]; then
            docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
          elif [ "${{ matrix.datasource }}" == "postgres" ]; then
            docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
          elif [ "${{ matrix.datasource }}" == "mongo" ]; then
            docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
          elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
            docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
          elif [ "${{ matrix.datasource }}" == "oracle" ]; then
            docker pull budibase/oracle-database:23.2-slim-faststart
          fi
          docker pull minio/minio &
          docker pull redis &
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &

          wait $(jobs -p)          

      - run: yarn --frozen-lockfile

      - name: Test server
        env:
          DATASOURCE: ${{ matrix.datasource }}
        run: |
          AFFECTED=yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server
          if [ -n "$AFFECTED" ]; then
            cd packages/server
            yarn test --filter ./src/tests/filters/datasource-tests.js
          else
            echo "No affected tests to run"
          fi          

  test-server-without-datasource:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn

      - name: Pull testcontainers images
        run: |
          docker pull minio/minio &
          docker pull redis &
          docker pull testcontainers/ryuk:0.5.1 &
          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &

          wait $(jobs -p)          

      - run: yarn --frozen-lockfile

      - name: Test server
        env:
          DATASOURCE: none
        run: |
          AFFECTED=yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server
          if [ -n "$AFFECTED" ]; then
            cd packages/server
            yarn test --filter ./src/tests/filters/non-datasource-tests.js
          else
            echo "No affected tests to run"
          fi          

  check-pro-submodule:
    runs-on: ubuntu-latest
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo and submodules
        uses: actions/checkout@v4
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - name: Check pro commit
        id: get_pro_commits
        run: |
          cd packages/pro
          pro_commit=$(git rev-parse HEAD)

          branch="${{ github.base_ref || github.ref_name  }}"
          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"

          base_commit=$(git rev-parse origin/master)

          if [[ ! -z $base_commit ]]; then
            echo "target_branch=$branch"
            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
            echo "pro_commit=$pro_commit"
            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
            echo "base_commit=$base_commit"
            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"

            base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit)
            echo "base_commit_excluding_merges=$base_commit_excluding_merges"
            echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing to do - branch to branch merge."
          fi          

      - name: Check submodule merged and latest on base branch
        if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }}
        run: |
          cd packages/pro
          base_commit='${{ steps.get_pro_commits.outputs.base_commit }}'
          base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}'
          pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}'

          any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit)

          if [ -n "$any_commit" ] && [ "$base_commit" != "$pro_commit" ]; then
            echo $any_commit

            echo "An error occurred: <error_message>"
            echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'
            echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md'

            exit 1
          else
            echo 'All good, the submodule had been merged and setup correctly!'
          fi          

  check-accountportal-submodule:
    runs-on: ubuntu-latest
    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
    steps:
      - name: Checkout repo and submodules
        uses: actions/checkout@v4
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
          fetch-depth: 0

      - uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            src:
              - packages/account-portal/**            

      - if: steps.changes.outputs.src == 'true'
        name: Check account portal commit
        id: get_accountportal_commits
        run: |
          cd packages/account-portal
          accountportal_commit=$(git rev-parse HEAD)

          branch="${{ github.base_ref || github.ref_name  }}"
          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"

          base_commit=$(git rev-parse origin/master)

          if [[ ! -z $base_commit ]]; then
            echo "target_branch=$branch"
            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
            echo "accountportal_commit=$accountportal_commit"
            echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
            echo "base_commit=$base_commit"
            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing to do - branch to branch merge."
          fi          

      - name: Check submodule merged to base branch
        if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
            const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';

            if (submoduleCommit !== baseCommit) {
              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
              console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
              process.exit(1);
            } else {
              console.log('All good, the submodule had been merged and setup correctly!')
            }            

  check-lockfile:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}

      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          cache: yarn
      - run: yarn install
      - name: Check for yarn.lock changes
        run: |
          if [[ $(git status --porcelain) == *"yarn.lock"* ]]; then
            echo "yarn.lock file needs to be modified. Please update it locally and commit the changes."
            exit 1
          else
            echo "yarn.lock file is unchanged."
          fi