From caa3be8af79d694918fc72473a5bc9d2c3b1505a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 11:15:43 +0100 Subject: [PATCH 1/7] fix qa nightly build --- packages/builder/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/package.json b/packages/builder/package.json index 569a6c1614..82b06224ed 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -17,7 +17,7 @@ "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record", "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run", "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci", - "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record && npm run cy:ci:report", + "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record; npm run cy:ci:report", "cy:ci:report": "mochawesome-merge cypress/reports/*.json > cypress/reports/testReport.json && marge cypress/reports/testReport.json --reportDir cypress/reports --inline", "cy:ci:notify": "node scripts/cypressResultsWebhook", "cy:debug": "start-server-and-test cy:setup http://localhost:4100/builder cy:open", @@ -121,4 +121,4 @@ "vite": "^2.1.5" }, "gitHead": "115189f72a850bfb52b65ec61d932531bf327072" -} +} \ No newline at end of file From 99d4f0da03ae898680443a32cb23330024e0b7a0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 11:18:24 +0100 Subject: [PATCH 2/7] generate report even if tests fail --- packages/builder/cypress/integration/createApp.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/cypress/integration/createApp.spec.js b/packages/builder/cypress/integration/createApp.spec.js index 179741e21a..544824a963 100644 --- a/packages/builder/cypress/integration/createApp.spec.js +++ b/packages/builder/cypress/integration/createApp.spec.js @@ -34,7 +34,7 @@ filterTests(['smoke', 'all'], () => { } }) - cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist") + cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("not.exist") cy.get(interact.TEMPLATE_CATEGORY).should("exist") cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).its('length').should('be.gt', 1) From 327f78ea4fff0b9f1697ae57a89d1d4bc1cbc96e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 12:01:50 +0100 Subject: [PATCH 3/7] enforce fail --- .github/workflows/smoke_test.yaml | 16 ++++++++++++++-- .../cypress/integration/createApp.spec.js | 3 ++- packages/builder/package.json | 4 ++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 7002c8335b..cffb914aaf 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -1,4 +1,4 @@ -name: Budibase Smoke Test +name: Budibase Nightly Tests on: workflow_dispatch: @@ -6,7 +6,7 @@ on: - cron: "0 5 * * *" # every day at 5AM jobs: - release: + nightly: runs-on: ubuntu-latest steps: @@ -43,6 +43,18 @@ jobs: name: Test Reports path: packages/builder/cypress/reports/testReport.html + # TODO: enable once running in QA test env + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-1 + + # - name: Upload test results HTML + # uses: aws-actions/configure-aws-credentials@v1 + # run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html + - name: Cypress Discord Notify run: yarn test:e2e:ci:notify env: diff --git a/packages/builder/cypress/integration/createApp.spec.js b/packages/builder/cypress/integration/createApp.spec.js index 544824a963..7198088fb4 100644 --- a/packages/builder/cypress/integration/createApp.spec.js +++ b/packages/builder/cypress/integration/createApp.spec.js @@ -34,7 +34,8 @@ filterTests(['smoke', 'all'], () => { } }) - cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("not.exist") + cy.get(".thisisnothere").should("exist") + cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist") cy.get(interact.TEMPLATE_CATEGORY).should("exist") cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).its('length').should('be.gt', 1) diff --git a/packages/builder/package.json b/packages/builder/package.json index 82b06224ed..4f29ce36a4 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -13,8 +13,8 @@ "cy:setup:ci": "node ./cypress/setup.js", "cy:open": "cypress open", "cy:run": "cypress run", - "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js", - "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record", + "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js", + "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record --spec cypress/integration/createApp.spec.js", "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run", "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci", "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record; npm run cy:ci:report", From bfae9f8d516cec67d2a58e6fc15aa656d41c8cbd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 12:34:40 +0100 Subject: [PATCH 4/7] more debug info --- packages/builder/scripts/cypressResultsWebhook.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/builder/scripts/cypressResultsWebhook.js b/packages/builder/scripts/cypressResultsWebhook.js index 457093e013..c22c46207d 100644 --- a/packages/builder/scripts/cypressResultsWebhook.js +++ b/packages/builder/scripts/cypressResultsWebhook.js @@ -114,7 +114,7 @@ async function discordCypressResultsNotification(report) { } const response = await fetch(WEBHOOK_URL, options) - if (response.status >= 400) { + if (response.status >= 201) { const text = await response.text() console.error( `Error sending discord webhook. \nStatus: ${response.status}. \nResponse Body: ${text}. \nRequest Body: ${options.body}` @@ -123,8 +123,12 @@ async function discordCypressResultsNotification(report) { } async function run() { - const report = await generateReport() - await discordCypressResultsNotification(report) + try { + const report = await generateReport() + await discordCypressResultsNotification(report) + } catch (err) { + console.error(err) + } } run() From 8219a37ebe58a59cb12683e75fbb2de687460665 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 13:15:03 +0100 Subject: [PATCH 5/7] post failure notification --- packages/builder/scripts/cypressResultsWebhook.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/builder/scripts/cypressResultsWebhook.js b/packages/builder/scripts/cypressResultsWebhook.js index c22c46207d..4de4c01cc7 100644 --- a/packages/builder/scripts/cypressResultsWebhook.js +++ b/packages/builder/scripts/cypressResultsWebhook.js @@ -5,7 +5,6 @@ const path = require("path") const fs = require("fs") const WEBHOOK_URL = process.env.CYPRESS_WEBHOOK_URL -const OUTCOME = process.env.CYPRESS_OUTCOME const DASHBOARD_URL = process.env.CYPRESS_DASHBOARD_URL const GIT_SHA = process.env.GITHUB_SHA const GITHUB_ACTIONS_RUN_URL = process.env.GITHUB_ACTIONS_RUN_URL @@ -35,6 +34,8 @@ async function discordCypressResultsNotification(report) { skipped, } = report.stats + const OUTCOME = failures > 0 ? "failure" : "success" + const options = { method: "POST", headers: { @@ -123,12 +124,8 @@ async function discordCypressResultsNotification(report) { } async function run() { - try { - const report = await generateReport() - await discordCypressResultsNotification(report) - } catch (err) { - console.error(err) - } + const report = await generateReport() + await discordCypressResultsNotification(report) } run() From 1d152b26a89ebb47d032fb750ea6e410e7b85dcd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 13:45:52 +0100 Subject: [PATCH 6/7] revert back to original cypress command --- packages/builder/cypress/integration/createApp.spec.js | 1 - packages/builder/package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/cypress/integration/createApp.spec.js b/packages/builder/cypress/integration/createApp.spec.js index 7198088fb4..179741e21a 100644 --- a/packages/builder/cypress/integration/createApp.spec.js +++ b/packages/builder/cypress/integration/createApp.spec.js @@ -34,7 +34,6 @@ filterTests(['smoke', 'all'], () => { } }) - cy.get(".thisisnothere").should("exist") cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist") cy.get(interact.TEMPLATE_CATEGORY).should("exist") diff --git a/packages/builder/package.json b/packages/builder/package.json index 4f29ce36a4..42a62399b8 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -14,7 +14,7 @@ "cy:open": "cypress open", "cy:run": "cypress run", "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js", - "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record --spec cypress/integration/createApp.spec.js", + "cy:run:ci:record": "cypress run --browser chrome --record", "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run", "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci", "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record; npm run cy:ci:report", From fc2c62e350d4e881f47681b8ed6e14aee3a9fc4a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Aug 2022 15:25:58 +0100 Subject: [PATCH 7/7] revert back to headed --- packages/builder/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/package.json b/packages/builder/package.json index 42a62399b8..53e0286ed1 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -14,7 +14,7 @@ "cy:open": "cypress open", "cy:run": "cypress run", "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js", - "cy:run:ci:record": "cypress run --browser chrome --record", + "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record", "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run", "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci", "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record; npm run cy:ci:report",