diff --git a/.Rbuildignore b/.Rbuildignore index 01c7d2805..17262dafc 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -25,3 +25,4 @@ ^\.github$ ^REFACTOR_GUIDE\.md$ ^man-roxygen$ +^codecov\.yml$ diff --git a/.github/actions/setup-armadillo-with-dsbase/action.yaml b/.github/actions/setup-armadillo-with-dsbase/action.yaml new file mode 100644 index 000000000..da6972c23 --- /dev/null +++ b/.github/actions/setup-armadillo-with-dsbase/action.yaml @@ -0,0 +1,199 @@ +name: Setup Armadillo with dsBase +description: > + Starts a fresh Armadillo instance (self-managed Rock container), installs R + and system dependencies, waits for readiness, uploads test datasets, and + installs dsBase. Assumes the caller has already checked out dsBaseClient at + ./dsBaseClient. Shared by the Armadillo dsbase shard workflow and the + Armadillo dsdanger job, so this setup only needs fixing in one place. + +inputs: + dsbase-tarball: + description: dsBase tarball filename to install + required: false + default: dsBase_7.0.0-permissive.tar.gz + +runs: + using: composite + steps: + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Download and start Armadillo (jar) + shell: bash + run: | + mkdir -p armadillo_home/config armadillo_home/storage armadillo_home/logs + + cat > armadillo_home/config/application.yml < armadillo_home/logs/stdout.log 2>&1 & + echo $! > armadillo_home/armadillo.pid + docker pull datashield/rock_citest-permissive:latest + working-directory: dsBaseClient + + - name: Uninstall default MySQL + shell: bash + run: | + sudo service mysql stop || true + sudo apt-get update + sudo apt-get remove --purge mysql-client mysql-server mysql-common -y + sudo apt-get autoremove -y + sudo apt-get autoclean -y + sudo rm -rf /var/lib/mysql/ + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: dsBaseClient + dependencies: 'c("Depends", "Imports", "LinkingTo")' + extra-packages: | + cran::devtools + cran::covr + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::opalr + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + github::datashield/dsDangerClient + needs: check + + - name: Wait for Armadillo to be ready + shell: bash + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:8080/actuator/health 2>/dev/null | grep -q '"status":"UP"'; then + echo "Armadillo is ready" + exit 0 + fi + sleep 5 + done + echo "Armadillo did not become ready in time" + cat armadillo_home/logs/stdout.log || true + exit 1 + working-directory: dsBaseClient + + - name: Install test datasets to Armadillo + shell: bash + run: R -q -f "molgenis_armadillo-upload_testing_datasets.R" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsBase to Armadillo + shell: bash + run: | + curl -sf -u admin:admin -X POST http://localhost:8080/ds-profiles/default/start + + profile_status="" + for i in $(seq 1 30); do + profile_status=$(curl -sf -u admin:admin http://localhost:8080/ds-profiles/status | jq -r '.[] | select(.name=="default") | .status') + [ "$profile_status" == "RUNNING" ] && break + sleep 10 + done + if [ "$profile_status" != "RUNNING" ]; then + echo "default profile container did not become RUNNING in time (status: $profile_status)" + exit 1 + fi + + install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ inputs.dsbase-tarball }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) + if [ "$install_status" != "200" ]; then + echo "dsBase install request failed with HTTP status $install_status" + exit 1 + fi + + expected_version=$(tar -xOzf ${{ inputs.dsbase-tarball }} dsBase/DESCRIPTION | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from ${{ inputs.dsbase-tarball }}" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + + actual_version="" + for i in $(seq 1 30); do + packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) + actual_version=$(echo "$packages_json" | jq -r '.[] | select(.name == "dsBase") | .version' 2>/dev/null || true) + if [ "$actual_version" == "$expected_version" ]; then + break + fi + sleep 10 + done + echo "Installed dsBase version: $actual_version" + if [ "$actual_version" != "$expected_version" ]; then + echo "dsBase version mismatch: expected $expected_version, found '$actual_version'" + exit 1 + fi + echo "$actual_version" > dsbase_version.txt + + curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase + working-directory: dsBaseClient + + - name: Dump Armadillo server log + if: failure() + shell: bash + run: grep "Caused by:" armadillo_home/logs/stdout.log | sort -u || true + working-directory: dsBaseClient diff --git a/.github/actions/setup-opal-with-dsbase/action.yaml b/.github/actions/setup-opal-with-dsbase/action.yaml new file mode 100644 index 000000000..e60bec0db --- /dev/null +++ b/.github/actions/setup-opal-with-dsbase/action.yaml @@ -0,0 +1,122 @@ +name: Setup Opal with dsBase +description: > + Starts a fresh Opal instance via docker-compose, installs R and system + dependencies, uploads test datasets, and installs dsBase with disclosure + test options. Assumes the caller has already checked out dsBaseClient at + ./dsBaseClient. Shared by the Opal dsbase shard workflow and the Opal + dsdanger job, so this setup only needs fixing in one place. + +inputs: + dsbase-ref: + description: dsBase GitHub ref to install + required: false + default: v7.0-dev + +runs: + using: composite + steps: + - name: Start Opal docker-compose + shell: bash + run: docker compose -f docker-compose_opal.yml up -d --build + working-directory: dsBaseClient + + - name: Uninstall default MySQL + shell: bash + run: | + sudo service mysql stop || true + sudo apt-get update + sudo apt-get remove --purge mysql-client mysql-server mysql-common -y + sudo apt-get autoremove -y + sudo apt-get autoclean -y + sudo rm -rf /var/lib/mysql/ + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: dsBaseClient + dependencies: 'c("Depends", "Imports", "LinkingTo")' + extra-packages: | + cran::devtools + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::opalr + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + github::datashield/dsDangerClient + needs: check + + - name: Install test datasets to Opal + shell: bash + run: | + for i in $(seq 1 18); do + if curl -sf -o /dev/null http://localhost:8080/; then + echo "Opal is responding" + break + fi + sleep 5 + done + R -q -f "obiba_opal-upload_testing_datasets.R" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsBase to Opal, set disclosure test options + shell: bash + run: | + R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080/'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = '${{ inputs.dsbase-ref }}'); opal.logout(opal)" + + expected_version=$(curl -sf "https://raw.githubusercontent.com/datashield/dsBase/${{ inputs.dsbase-ref }}/DESCRIPTION" | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from GitHub" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + echo "$expected_version" > dsbase_version.txt + + ok=false + for i in $(seq 1 18); do + if Rscript -e " + library(opalr) + opal <- opal.login('administrator', 'datashield_test&', url = 'http://localhost:8080/') + desc <- dsadmin.package_description(opal, 'dsBase') + opal.logout(opal) + installed <- desc[['Version']] + cat('Installed dsBase version:', if (is.null(installed)) '(none yet)' else installed, '\n') + quit(status = if (!is.null(installed) && installed == '$expected_version') 0 else 1) + "; then + ok=true + break + fi + sleep 10 + done + if [ "$ok" != "true" ]; then + echo "dsBase version mismatch or install not detected after polling: expected $expected_version" + exit 1 + fi + + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.profile_init(opal, name = 'default', packages = c('dsBase', 'dsTidyverse', 'resourcer')); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.set_option(opal, 'default.datashield.privacyControlLevel', 'permissive'); opal.logout(opal)" + working-directory: dsBaseClient/tests/testthat/data_files diff --git a/.github/scripts/post-ci-comment.js b/.github/scripts/post-ci-comment.js new file mode 100644 index 000000000..4a6abf442 --- /dev/null +++ b/.github/scripts/post-ci-comment.js @@ -0,0 +1,114 @@ +// Shared by check.yaml / lint.yaml / dsBaseClient_test_suite.yaml's "Post PR +// comment" steps. Each workflow owns a subset of the markers below and only +// ever replaces its own, so the three runs (independent workflows, no +// ordering guarantee) converge on one combined PR comment regardless of +// which finishes first. The headline is recomputed from the row markers' +// current state on every update. + +const ROW_KEYS = ['row:check', 'row:lint', 'row:tests-armadillo', 'row:tests-opal', 'row:coverage']; +const TOP_MARKER = ''; + +const SKELETON = [ + TOP_MARKER, + '⏳ Running checks...', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '
CheckResult
Devtools checks⏳ pending
Code quality⏳ pending
Armadillo unit tests⏳ pending
Opal unit tests⏳ pending
Test coverage⏳ pending
', + '', + 'Tested against dsBase versions:', + 'Armadillo: _pending_', + 'Opal: _pending_', + '', + 'Logs: _pending_ · _pending_ · _pending_ · _pending_ · _pending_' +].join('\n'); + +function replaceMarker(body, key, content) { + const re = new RegExp(`[\\s\\S]*?`); + const replacement = `${content}`; + return re.test(body) ? body.replace(re, replacement) : body; +} + +function computeHeadline(body) { + let pass = 0, fail = 0, pending = 0; + for (const key of ROW_KEYS) { + const re = new RegExp(`([\\s\\S]*?)`); + const m = body.match(re); + const text = m ? m[1] : ''; + if (text.includes('❌')) fail++; + else if (text.includes('✅')) pass++; + else pending++; + } + if (fail > 0) return `❌ ${fail} of ${ROW_KEYS.length} checks failed`; + if (pending > 0) return `⏳ ${pass} of ${ROW_KEYS.length} checks reported so far`; + return `✅ All ${ROW_KEYS.length} checks passed`; +} + +const MAX_ATTEMPTS = 3; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +module.exports = async function postCiComment({ github, context, updates }) { + let prNumber = context.payload.pull_request?.number; + if (!prNumber) { + const branch = context.ref.replace('refs/heads/', ''); + const prs = await github.rest.pulls.list({ + owner: context.repo.owner, repo: context.repo.repo, + head: `${context.repo.owner}:${branch}`, state: 'open' + }); + prNumber = prs.data[0]?.number; + } + if (!prNumber) return; + + // Two workflows (e.g. opal-report/armadillo-report) can finish within + // moments of each other and both read-modify-write this same comment - + // there's no atomic compare-and-swap in the Issues API, so after writing + // we re-read and confirm OUR content actually landed. If a third write + // slipped in between our write and this check, retry the whole cycle + // against the latest body rather than silently losing the update. + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber + }); + const existing = comments.data.find(c => c.body.includes(TOP_MARKER)); + + // A comment from before this skeleton's format changed won't contain our + // current markers - patching it would silently no-op every replace below. + // Reset it to a fresh skeleton (still updating the same comment in place, + // not creating a new one) rather than leaving stale content untouched. + const isCompatible = existing && ROW_KEYS.some(key => existing.body.includes(``)); + let body = isCompatible ? existing.body : SKELETON; + for (const [key, content] of Object.entries(updates)) { + body = replaceMarker(body, key, content); + } + body = replaceMarker(body, 'headline', computeHeadline(body)); + + let commentId; + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body + }); + commentId = existing.id; + } else { + const created = await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body + }); + commentId = created.data.id; + } + + const verify = await github.rest.issues.getComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: commentId + }); + const landed = Object.entries(updates).every(([key, content]) => + verify.data.body.includes(`${content}`) + ); + if (landed) return; + if (attempt < MAX_ATTEMPTS) await sleep(1000 * attempt); + } +}; diff --git a/.github/scripts/summarise-junit.R b/.github/scripts/summarise-junit.R new file mode 100644 index 000000000..b7eb420c1 --- /dev/null +++ b/.github/scripts/summarise-junit.R @@ -0,0 +1,43 @@ +# Shared by opal-report/armadillo-report's "Compute results & write summary" +# steps (dsBaseClient_test_suite.yaml) - parses one backend's merged JUnit +# XML into a pass/fail tally and, if any failures/errors, a testthat-style +# failure block. The two report jobs are otherwise near-identical, so this +# is the one piece that was previously duplicated between them. + +summarise_junit <- function(xml_path, label) { + doc <- xml2::read_xml(xml_path) + suites <- xml2::xml_find_all(doc, ".//testsuite") + n_tests <- sum(as.integer(xml2::xml_attr(suites, "tests")), na.rm = TRUE) + n_failures <- sum(as.integer(xml2::xml_attr(suites, "failures")), na.rm = TRUE) + n_errors <- sum(as.integer(xml2::xml_attr(suites, "errors")), na.rm = TRUE) + n_skipped <- sum(as.integer(xml2::xml_attr(suites, "skipped")), na.rm = TRUE) + n_pass <- n_tests - n_failures - n_errors - n_skipped + tally <- sprintf("[ FAIL %d | WARN 0 | SKIP %d | PASS %d ]", n_failures + n_errors, n_skipped, n_pass) + + failed <- xml2::xml_find_all(doc, ".//testcase[failure or error]") + fail_block <- character(0) + if (length(failed) > 0) { + msgs <- vapply(failed, function(tc) { + node <- xml2::xml_find_first(tc, "failure|error") + m <- xml2::xml_attr(node, "message") + if (is.na(m) || !nzchar(m)) m <- trimws(xml2::xml_text(node)) + m + }, character(1)) + labels <- paste0(xml2::xml_attr(failed, "classname"), "::", xml2::xml_attr(failed, "name")) + fail_block <- unlist(lapply(seq_along(failed), function(i) { + c(sprintf("-- Failure (%s) %s", labels[i], strrep("-", max(1, 60 - nchar(labels[i])))), msgs[i], "") + })) + } + + list( + ok = (n_failures + n_errors) == 0, + tally = tally, + summary = c(sprintf("## %s unit tests", label), "", "```", fail_block, tally, "```") + ) +} + +find_dsbase_version <- function(artifact_dir) { + files <- list.files(artifact_dir, pattern = "dsbase_version\\.txt$", recursive = TRUE, full.names = TRUE) + if (length(files) == 0) return("unknown") + trimws(readLines(files[1], warn = FALSE)[1]) +} diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 000000000..dbe043284 --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,83 @@ +name: Check + +on: + push: + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # Weekly + - cron: '0 1 * * *' # Nightly + +# A new push to the same ref supersedes any run still in progress for it, so +# we don't burn compute on stale commits. Scoped by event_name too, so a +# schedule/workflow_dispatch run is never auto-cancelled by an unrelated push. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + check: + name: Package checks (doc sync, R CMD check) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Depends", "Imports", "LinkingTo")' + extra-packages: | + any::rcmdcheck + cran::devtools + needs: check + + - name: Check manual updated + id: docsync + run: | + orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) + R -q -e "devtools::document()" + new_sum=$(find man -type f | sort -u | xargs cat | md5sum) + if [ "$orig_sum" != "$new_sum" ]; then + echo "Your committed man/*.Rd files are out of sync with the R headers." + exit 1 + fi + + - name: Devtools checks + id: rcmdcheck + if: always() + run: | + R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout + grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout + + - name: Post PR comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const ok = '${{ steps.docsync.outcome }}' === 'success' && '${{ steps.rcmdcheck.outcome }}' === 'success'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const postCiComment = require('${{ github.workspace }}/.github/scripts/post-ci-comment.js'); + await postCiComment({ github, context, updates: { + 'row:check': `Devtools checks${ok ? '✅ passed' : '❌ failed'}`, + 'log:check': `Devtools checks` + }}); diff --git a/.github/workflows/dsBaseClient_test_suite.yaml b/.github/workflows/dsBaseClient_test_suite.yaml index b8a6f3ccf..5ebaa9ae7 100644 --- a/.github/workflows/dsBaseClient_test_suite.yaml +++ b/.github/workflows/dsBaseClient_test_suite.yaml @@ -1,244 +1,643 @@ ################################################################################ +# trigger CI # DataSHIELD GHA test suite - dsBaseClient -# Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz +# Replaces azure-pipelines.yml / opal_azure-pipelines.yml / armadillo_azure-pipelines.yml. # -# Inside the root directory $(Pipeline.Workspace) will be a file tree like: -# /dsBaseClient <- Checked out version of datashield/dsBaseClient -# /dsBaseClient/logs <- Where results of tests and logs are collated -# /testStatus <- Checked out version of datashield/testStatus +# This is one of three separate workflow files, each its own named check on a +# PR/run: this one (all the actual test execution), check.yaml (doc-sync + +# R CMD check), lint.yaml (lintr). Split so each shows as its own category +# rather than one graph mixing test execution with static checks. # -# As of Sept. 2025 this takes ~ 95 mins to run. +# Structure (all jobs below run in parallel except where "needs" says otherwise): +# opal-dsbase (matrix x8) - dsBase suite against Opal, one shard per +# category, plus a dsdanger entry, all grouped under +# one summary box. +# opal-report - needs opal-dsbase; merges results, computes its own +# pass/fail, posts its own row into the shared PR +# comment (see .github/scripts/post-ci-comment.js). +# armadillo-dsbase (matrix x8) - dsBase suite against Armadillo, same split. +# armadillo-report - needs armadillo-dsbase; same as opal-report, plus +# coverage (computed here only - see comment at that +# step for why one backend's figure is sufficient). +# +# There is no separate combining/summary job: dsBaseClient's own R/ source +# never branches on backend, so Armadillo and Opal results are independently +# meaningful and each report job stands alone (gate on both being required +# checks in branch protection, rather than one job that waits on both). +# +# The dsBase suite (matching TEST_FILTER_DSBASE - same 292 files the Azure +# pipelines run) is split into 7 shards - smk (116 files, 2 shards), perf (51 +# fixed 30s-loop benchmarks, 3 shards - by far the slowest per-file), arg (1 +# shard), and misc (1 shard) - each shard with its own testthat filter +# substring. smk/perf are split by the first letter of the function name +# (after any "ds." prefix) rather than an enumerated file list, so newly +# added test files fall into a bucket automatically. The small dsDanger suite +# is folded in as an 8th matrix entry (steps gated on matrix.category == +# 'dsdanger') rather than a standalone job, so it groups under the same +# summary box instead of its own. This is one job with a matrix (not split +# into separate job definitions per category) so all entries share one +# summary box in the run graph, and so they don't share a reusable workflow +# name - GitHub's default same-name concurrency cap of 2 would otherwise +# throttle them to 2-at-a-time. Setup (checkout through installing dsBase) +# lives in a shared composite action (setup-armadillo-with-dsbase / +# setup-opal-with-dsbase), used by every matrix entry including dsdanger, so +# that part stays DRY. +# +# Each dsbase/dsdanger job spins up its OWN backend instance (isolated - no +# shared server state / concurrency risk between categories running at once). +# +# Opal runs via docker-compose (docker-compose_opal.yml). +# Armadillo runs as a plain `java -jar` process (not docker-compose): Armadillo +# self-manages its Rock container over the host Docker socket +# (docker-management-enabled: true / docker-run-in-container: false), which skips +# building/pulling the old custom armadillo_citest image. See +# molgenis-service-armadillo's application.template.yml for that flag pairing. +# +# Every job starts its backend as the very first step so it boots in the +# background while R dependencies install, instead of paying for both serially. +# +# As of Sept. 2025 the single-backend, dsBase-only, unsharded version of this +# took ~ 95 mins; the full (Opal+Armadillo, dsBase+dsDanger) unsharded run is +# well over an hour. ################################################################################ name: dsBaseClient tests' suite on: push: + workflow_dispatch: schedule: - cron: '0 0 * * 0' # Weekly - cron: '0 1 * * *' # Nightly +# A new push to the same ref supersedes any run still in progress for it, so +# we don't burn compute on stale commits. Scoped by event_name too, so a +# schedule/workflow_dispatch run is never auto-cancelled by an unrelated push. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + _r_check_system_clock_: 0 + PROJECT_NAME: dsBaseClient + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + R_KEEP_PKG_SOURCE: yes + # Selects perf_files/__perf-profile.csv as the perf + # test reference rates/tolerances (see tests/testthat/perf_tests/perf_rate.R). + # Reuses the existing azure-pipeline reference files rather than adding new + # ones, matching what the old Azure pipelines set via perf.profile. + PERF_PROFILE: azure-pipeline + # dsBase shards get their filter from matrix.filter per entry instead. + TEST_FILTER_DSDANGER: '__dgr-|datachk_dgr-|smk_dgr-|arg_dgr-|disc_dgr-|smk_expt_dgr-|expt_dgr-|math_dgr-' + jobs: - dsBaseClient_test_suite: - runs-on: ubuntu-latest - timeout-minutes: 120 - permissions: - contents: read - # These should all be constant, except TEST_FILTER. This can be used to test - # subsets of test files in the testthat directory. Options are like: - # '*' <- Run all tests. - # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests. - # '*_smk_*' <- Run all the smoke tests for all functions. + ################################################################################ + # Opal - dsBase suite, sharded, plus dsDanger folded in as an extra matrix + # entry. Each entry is a fully isolated job with its own Opal instance. + ################################################################################ + # One job, 8-entry matrix, so all entries group under a single summary box + # in the run graph (previously split into 3 category-group jobs calling a + # reusable workflow - reverted since that lost the combined summary and, + # worse, made all calls share the reusable workflow's name, which triggers + # GitHub's default same-name concurrency cap of 2 and throttled the shards + # to 2-at-a-time instead of running in parallel). + opal-dsbase: + name: Opal tests (${{ matrix.category }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + # smk and perf are split by the first letter of the function name + # (after any "ds." prefix) rather than an enumerated file list, so + # newly added test files fall into a bucket automatically. dsdanger is + # a separate, smaller suite (steps below are gated on + # matrix.category == 'dsdanger') folded in here so it groups under this + # job's summary box instead of a standalone job. + matrix: + include: + - category: smk-1 + filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)' + - category: smk-2 + filter: 'smk-ds.[m-vM-V]' + - category: arg + filter: 'arg-' + - category: perf-1 + filter: 'perf-ds.[a-cA-C]' + - category: perf-2 + filter: 'perf-ds.[d-mD-M]' + - category: perf-3 + filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)' + - category: misc + filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-' + - category: dsdanger env: - TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-' - _r_check_system_clock_: 0 - WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }} PROJECT_NAME: dsBaseClient - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - REPO_OWNER: ${{ github.repository_owner }} - R_KEEP_PKG_SOURCE: yes - GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }} - + DS_DRIVER: OpalDriver + DSDANGER_REF: '6.3.4' steps: - name: Checkout dsBaseClient uses: actions/checkout@v4 with: path: dsBaseClient - - name: Checkout testStatus - if: ${{ github.actor != 'nektos/act' }} # for local deployment only - uses: actions/checkout@v4 - with: - repository: ${{ env.REPO_OWNER }}/testStatus - ref: master - path: testStatus - persist-credentials: false - token: ${{ env.GITHUB_TOKEN }} + - uses: ./dsBaseClient/.github/actions/setup-opal-with-dsbase - - name: Uninstall default MySQL + - name: Install dsDangerClient + if: matrix.category == 'dsdanger' run: | - curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - - sudo service mysql stop || true - sudo apt-get update - sudo apt-get remove --purge mysql-client mysql-server mysql-common -y - sudo apt-get autoremove -y - sudo apt-get autoclean -y - sudo rm -rf /var/lib/mysql/ + R -q -e " + ref <- Sys.getenv('BRANCH_NAME') + ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE) + if (!ok) pak::pkg_install('github::datashield/dsDangerClient')" + + - name: Install dsDanger package on Opal server + if: matrix.category == 'dsdanger' + run: | + R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsDanger', username = 'datashield', ref = '${{ env.DSDANGER_REF }}'); opal.logout(opal)" + working-directory: dsBaseClient + + - name: Run dsBase tests with JUnit report + if: matrix.category != 'dsdanger' + run: | + R -q -e ' + devtools::load_all(quiet = TRUE); + library(testthat); + output_file <- file("test_console_output_dsbase.txt"); + sink(output_file, split = TRUE); + junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); + progress_rep <- ProgressReporter$new(max_failures = 999999); + multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + test_dir("tests/testthat", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? + cat test_console_output_dsbase.txt + n_tests=$(grep -c ' entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass." + R_EXIT=1 + fi + exit "${R_EXIT:-0}" + working-directory: dsBaseClient + + - name: Run dsDanger tests with JUnit report + if: matrix.category == 'dsdanger' + run: | + R -q -e ' + devtools::load_all(quiet = TRUE); + library(testthat); + output_file <- file("test_console_output_dsdanger.txt"); + sink(output_file, split = TRUE); + junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml")); + progress_rep <- ProgressReporter$new(max_failures = 999999); + multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? + cat test_console_output_dsdanger.txt + n_tests=$(grep -c ' entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass." + R_EXIT=1 + fi + exit "${R_EXIT:-0}" + working-directory: dsBaseClient + + - name: Upload shard results + if: matrix.category != 'dsdanger' + uses: actions/upload-artifact@v4 + with: + name: opal-dsbase-${{ matrix.category }} + path: | + dsBaseClient/test_results_dsbase.xml + dsBaseClient/test_console_output_dsbase.txt + dsBaseClient/tests/testthat/data_files/dsbase_version.txt - - uses: r-lib/actions/setup-pandoc@v2 + - name: Upload dsDanger results + if: matrix.category == 'dsdanger' + uses: actions/upload-artifact@v4 + with: + name: opal-dsdanger + path: | + dsBaseClient/test_results_dsdanger.xml + dsBaseClient/test_console_output_dsdanger.txt + + + ################################################################################ + # Opal - merge all matrix entry results and publish the report. + ################################################################################ + opal-report: + name: Opal report + needs: [opal-dsbase] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient - uses: r-lib/actions/setup-r@v2 with: r-version: release - http-user-agent: release use-public-rspm: true - - name: Install R and dependencies - run: | - sudo apt-get install --no-install-recommends software-properties-common dirmngr -y - wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc - sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" - sudo apt-get update -qq - sudo apt-get upgrade -y - sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y - sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y - sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')" - sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))" - - uses: r-lib/actions/setup-r-dependencies@v2 with: - dependencies: 'c("Imports")' + working-directory: dsBaseClient + packages: 'any::sessioninfo' extra-packages: | - any::rcmdcheck - cran::devtools - cran::git2r - cran::RCurl - cran::readr - cran::magrittr cran::xml2 - cran::purrr - cran::dplyr - cran::stringr - cran::tidyr - cran::quarto - cran::knitr - cran::kableExtra - cran::rmarkdown - cran::downlit - needs: check - - - name: Check manual updated + + - name: Download shard/dsdanger results + uses: actions/download-artifact@v4 + with: + pattern: 'opal-*' + path: dsBaseClient/artifacts + + - name: Merge JUnit results run: | - orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) - R -q -e "devtools::document()" - new_sum=$(find man -type f | sort -u | xargs cat | md5sum) - if [ "$orig_sum" != "$new_sum" ]; then - echo "Your committed man/*.Rd files are out of sync with the R headers." - exit 1 - fi + mkdir -p logs + cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt + + Rscript -e ' + xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE) + docs <- lapply(xml_files, xml2::read_xml) + root <- xml2::xml_new_root("testsuites") + for (doc in docs) { + for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s) + } + xml2::write_xml(root, "logs/test_results.xml") + ' working-directory: dsBaseClient - continue-on-error: true - - name: Devtools checks + - name: Upload merged results + uses: actions/upload-artifact@v4 + with: + name: opal-report-results + path: | + dsBaseClient/logs/test_results.xml + dsBaseClient/logs/test_console_output.txt + + - name: Compute results & write summary + id: results + env: + OPAL_DSBASE_RESULT: ${{ needs.opal-dsbase.result }} run: | - R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout - grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout - working-directory: dsBaseClient - continue-on-error: true + Rscript -e ' + source(".github/scripts/summarise-junit.R") + res <- summarise_junit("logs/test_results.xml", "Opal") + writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY")) + + version <- find_dsbase_version("artifacts") + + # needs.opal-dsbase.result is "failure" if ANY matrix shard did not + # succeed (even if the shards that DID upload results show 0 + # failures) - a shard that never reported must not look like a pass. + shard_ok <- Sys.getenv("OPAL_DSBASE_RESULT") == "success" + ok <- res$ok && shard_ok + if (!shard_ok) message("One or more Opal dsbase/dsdanger matrix entries did not succeed.") - - name: Start Armadillo docker-compose - run: docker compose -f docker-compose_armadillo.yml up -d --build + out <- Sys.getenv("GITHUB_OUTPUT") + cat( + sprintf("ok=%s\n", tolower(ok)), + sprintf("tally=%s\n", res$tally), + sprintf("version=%s\n", version), + file = out, append = TRUE, sep = "" + ) + + if (!ok) message("Opal tests failed - see the PR comment / job summary for details.") + quit(save = "no", status = if (ok) 0 else 1) + ' working-directory: dsBaseClient - - name: Install test datasets + - name: Post PR comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const ok = '${{ steps.results.outputs.ok }}' === 'true'; + const tally = '${{ steps.results.outputs.tally }}'; + const version = '${{ steps.results.outputs.version }}'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js'); + await postCiComment({ github, context, updates: { + 'row:tests-opal': `Opal unit tests${ok ? '✅' : '❌'} ${tally}`, + 'ver:opal': version, + 'log:tests-opal': `Opal unit tests` + }}); + + + ################################################################################ + # Armadillo - dsBase suite, sharded 4 ways. Each shard downloads and runs its + # own Armadillo jar instance. + # + # Runs as a plain `java -jar` process instead of docker-compose: Armadillo + # self-manages its own Rock container over the host Docker socket + # (docker-management-enabled: true / docker-run-in-container: false), which + # avoids building/pulling the old custom armadillo_citest image and its + # dockerised Armadillo layer. The latest GitHub release jar is downloaded at + # run time. No process is restarted after installing dsBase - install then + # whitelist directly. + ################################################################################ + # One job, 8-entry matrix, so all entries group under a single summary box + # in the run graph (previously split into 3 category-group jobs calling a + # reusable workflow - reverted since that lost the combined summary and, + # worse, made all calls share the reusable workflow's name, which triggers + # GitHub's default same-name concurrency cap of 2 and throttled the shards + # to 2-at-a-time instead of running in parallel). + armadillo-dsbase: + name: Armadillo tests (${{ matrix.category }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + # smk and perf are split by the first letter of the function name + # (after any "ds." prefix) rather than an enumerated file list, so + # newly added test files fall into a bucket automatically. dsdanger is + # a separate, smaller suite (steps below are gated on + # matrix.category == 'dsdanger') folded in here so it groups under this + # job's summary box instead of a standalone job. + matrix: + include: + - category: smk-1 + filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)' + - category: smk-2 + filter: 'smk-ds.[m-vM-V]' + - category: arg + filter: 'arg-' + - category: perf-1 + filter: 'perf-ds.[a-cA-C]' + - category: perf-2 + filter: 'perf-ds.[d-mD-M]' + - category: perf-3 + filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)' + - category: misc + filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-' + - category: dsdanger + env: + PROJECT_NAME: dsBaseClient + DS_DRIVER: ArmadilloDriver + DSDANGER_TARBALL: dsDanger_6.3.4.tar.gz + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - uses: ./dsBaseClient/.github/actions/setup-armadillo-with-dsbase + + - name: Install dsDangerClient + if: matrix.category == 'dsdanger' run: | - sleep 60 - R -q -f "molgenis_armadillo-upload_testing_datasets.R" - working-directory: dsBaseClient/tests/testthat/data_files + R -q -e " + ref <- Sys.getenv('BRANCH_NAME') + ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE) + if (!ok) pak::pkg_install('github::datashield/dsDangerClient')" - - name: Install dsBase to Armadillo + - name: Install dsDanger package on Armadillo server + if: matrix.category == 'dsdanger' run: | - curl -u admin:admin -X GET http://localhost:8080/packages - curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@dsBase_7.0.0-permissive.tar.gz" -X POST http://localhost:8080/install-package - sleep 60 - docker restart dsbaseclient-armadillo-1 - sleep 30 - curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase + curl -u admin:admin http://localhost:8080/whitelist + install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSDANGER_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) + if [ "$install_status" != "200" ]; then + echo "dsDanger install request failed with HTTP status $install_status" + exit 1 + fi + + for i in $(seq 1 30); do + packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) + if echo "$packages_json" | jq -e '.[] | select(.name == "dsDanger")' >/dev/null 2>&1; then + break + fi + sleep 10 + done + + curl -u admin:admin -X POST http://localhost:8080/whitelist/dsDanger + curl -u admin:admin http://localhost:8080/whitelist working-directory: dsBaseClient - - name: Run tests with coverage & JUnit report + - name: Run dsBase tests with coverage & JUnit report + if: matrix.category != 'dsdanger' run: | - mkdir -p logs - R -q -e "devtools::reload();" + R -q -e "devtools::load_all();" R -q -e ' - write.csv( - covr::coverage_to_list( - covr::package_coverage( - type = c("none"), - code = c('"'"' - output_file <- file("test_console_output.txt"); - sink(output_file); - sink(output_file, type = "message"); - junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml")); - progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); - multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); - options("datashield.return_errors" = FALSE, "default_driver" = "ArmadilloDriver"); - testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' - ) - ) - ), - "coveragelist.csv" - )' - - mv coveragelist.csv logs/ - mv test_* logs/ + cov <- covr::package_coverage( + type = c("none"), + code = c('"'"' + output_file <- file("test_console_output_dsbase.txt"); + sink(output_file, split = TRUE); + junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); + progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); + multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' + ) + ) + saveRDS(cov, "coverage.rds") + write.csv(covr::coverage_to_list(cov), "coveragelist.csv") + covr::to_cobertura(cov, "cobertura.xml")' || R_EXIT=$? + cat test_console_output_dsbase.txt + n_tests=$(grep -c ' entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass." + R_EXIT=1 + fi + exit "${R_EXIT:-0}" working-directory: dsBaseClient - - name: Check for JUnit errors - run: | - issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true) - echo "Number of testsuites with issues: $issue_count" - sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true - cat issues.log || true - # continue with workflow even when some tests fail - exit 0 - working-directory: dsBaseClient/logs - - - name: Write versions to file + - name: Run dsDanger tests with JUnit report + if: matrix.category == 'dsdanger' run: | - echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt - echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt - echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt - working-directory: dsBaseClient/logs + R -q -e ' + devtools::load_all(quiet = TRUE); + library(testthat); + output_file <- file("test_console_output_dsdanger.txt"); + sink(output_file, split = TRUE); + junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml")); + progress_rep <- ProgressReporter$new(max_failures = 999999); + multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? + cat test_console_output_dsdanger.txt + n_tests=$(grep -c ' entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass." + R_EXIT=1 + fi + exit "${R_EXIT:-0}" + working-directory: dsBaseClient + + - name: Upload shard results + if: matrix.category != 'dsdanger' + uses: actions/upload-artifact@v4 + with: + name: armadillo-dsbase-${{ matrix.category }} + path: | + dsBaseClient/test_results_dsbase.xml + dsBaseClient/test_console_output_dsbase.txt + dsBaseClient/coveragelist.csv + dsBaseClient/coverage.rds + dsBaseClient/dsbase_version.txt - - name: Parse results from testthat and covr + - name: Upload dsDanger results + if: matrix.category == 'dsdanger' + uses: actions/upload-artifact@v4 + with: + name: armadillo-dsdanger + path: | + dsBaseClient/test_results_dsdanger.xml + dsBaseClient/test_console_output_dsdanger.txt + + - name: Upload coverage to Codecov + if: matrix.category != 'dsdanger' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: dsBaseClient/cobertura.xml + flags: armadillo-${{ matrix.category }} + fail_ci_if_error: false + + + ################################################################################ + # Armadillo - merge all matrix entry results and publish the report. + ################################################################################ + armadillo-report: + name: Armadillo report + needs: [armadillo-dsbase] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: dsBaseClient + packages: 'any::sessioninfo' + extra-packages: | + cran::xml2 + cran::covr + + - name: Download shard/dsdanger results + uses: actions/download-artifact@v4 + with: + pattern: 'armadillo-*' + path: dsBaseClient/artifacts + + - name: Merge JUnit results run: | - Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ + mkdir -p logs + cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt + + Rscript -e ' + xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE) + docs <- lapply(xml_files, xml2::read_xml) + root <- xml2::xml_new_root("testsuites") + for (doc in docs) { + for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s) + } + xml2::write_xml(root, "logs/test_results.xml") + ' working-directory: dsBaseClient - - name: Render report + - name: Upload merged results + uses: actions/upload-artifact@v4 + with: + name: armadillo-report-results + path: | + dsBaseClient/logs/test_results.xml + dsBaseClient/logs/test_console_output.txt + + - name: Compute results & write summary + id: results + env: + ARMADILLO_DSBASE_RESULT: ${{ needs.armadillo-dsbase.result }} run: | - cd testStatus + Rscript -e ' + source(".github/scripts/summarise-junit.R") + res <- summarise_junit("logs/test_results.xml", "Armadillo") + writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY")) - mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ + version <- find_dsbase_version("artifacts") - # Copy logs to new logs directory location - cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + # Coverage is only computed here (not for Opal): dsBaseClients own + # R/ source never branches on backend (verified - the only + # Opal/Armadillo-specific references in R/ are inside roxygen + # @examples comments, not executable code), so a single backends + # coverage figure is equivalent to a combined one. + coverage_threshold <- 80 + rds_files <- list.files("artifacts", pattern = "coverage\\.rds$", recursive = TRUE, full.names = TRUE) + if (length(rds_files) > 0) { + tallies <- lapply(rds_files, function(f) covr::tally_coverage(readRDS(f))) + agg <- aggregate(value ~ filename + line, data = do.call(rbind, tallies), FUN = sum) + totalcoverage <- round(sum(agg$value > 0) / nrow(agg) * 100, 1) + coverage_ok <- totalcoverage >= coverage_threshold + coverage_text <- sprintf("%.1f%% vs %d%% target", totalcoverage, coverage_threshold) + } else { + coverage_ok <- NA + coverage_text <- "no coverage data found" + } + coverage_icon <- if (isTRUE(coverage_ok)) "✅" else if (isFALSE(coverage_ok)) "❌" else "❓" - R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' - mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html - cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest + # needs.armadillo-dsbase.result is "failure" if ANY matrix shard did + # not succeed (even if the shards that DID upload results show 0 + # failures) - a shard that never reported must not look like a pass. + shard_ok <- Sys.getenv("ARMADILLO_DSBASE_RESULT") == "success" + ok <- res$ok && shard_ok + if (!shard_ok) message("One or more Armadillo dsbase/dsdanger matrix entries did not succeed.") - env: - PROJECT_NAME: ${{ env.PROJECT_NAME }} - BRANCH_NAME: ${{ env.BRANCH_NAME }} - WORKFLOW_ID: ${{ env.WORKFLOW_ID }} + out <- Sys.getenv("GITHUB_OUTPUT") + cat( + sprintf("ok=%s\n", tolower(ok)), + sprintf("tally=%s\n", res$tally), + sprintf("version=%s\n", version), + sprintf("coverage_icon=%s\n", coverage_icon), + sprintf("coverage_text=%s\n", coverage_text), + file = out, append = TRUE, sep = "" + ) - - name: Upload test logs - uses: actions/upload-artifact@v4 + if (!ok) message("Armadillo tests failed - see the PR comment / job summary for details.") + quit(save = "no", status = if (ok) 0 else 1) + ' + working-directory: dsBaseClient + + - name: Post PR comment + if: always() + uses: actions/github-script@v7 with: - name: dsbaseclient-logs - path: testStatus/new + script: | + const ok = '${{ steps.results.outputs.ok }}' === 'true'; + const tally = '${{ steps.results.outputs.tally }}'; + const version = '${{ steps.results.outputs.version }}'; + const coverageIcon = '${{ steps.results.outputs.coverage_icon }}'; + const coverageText = '${{ steps.results.outputs.coverage_text }}'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const codecovUrl = `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`; + + const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js'); + await postCiComment({ github, context, updates: { + 'row:tests-armadillo': `Armadillo unit tests${ok ? '✅' : '❌'} ${tally}`, + 'row:coverage': `Test coverage${coverageIcon} ${coverageText}`, + 'ver:armadillo': version, + 'log:tests-armadillo': `Armadillo unit tests`, + 'log:coverage': `Codecov` + }}); - - name: Dump environment info - run: | - echo -e "\n#############################" - echo -e "ls /: ######################" - ls -al . - echo -e "\n#############################" - echo -e "lscpu: ######################" - lscpu - echo -e "\n#############################" - echo -e "memory: #####################" - free -m - echo -e "\n#############################" - echo -e "env: ########################" - env - echo -e "\n#############################" - echo -e "R sessionInfo(): ############" - R -e 'sessionInfo()' - sudo apt install tree -y - tree . diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..a0615746c --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,104 @@ +name: Lint + +on: + push: + branches: [main, master] + pull_request: + +# A new push to the same ref supersedes any run still in progress for it, so +# we don't burn compute on stale commits. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + lint: + name: R lint (lintr) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + security-events: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Depends", "Imports", "LinkingTo")' + extra-packages: | + cran::lintr + cran::jsonlite + + - name: Get PR changed files + if: github.event_name == 'pull_request' + id: changed + run: | + files=$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" -- '*.R' '*.Rmd') + echo "files<> "$GITHUB_ENV" + echo "$files" >> "$GITHUB_ENV" + echo "CHANGED_FILES_EOF" >> "$GITHUB_ENV" + + - name: Lint + id: lint + env: + IS_PR: ${{ github.event_name == 'pull_request' }} + CHANGED_FILES: ${{ env.files }} + run: | + Rscript -e ' + al <- lintr::available_linters() + keep <- vapply(al$tags, function(t) any(t %in% c("correctness", "common_mistakes", "robustness")), logical(1)) + sel <- al$linter[keep] + linter_funs <- lapply(sel, function(n) get(n, envir = asNamespace("lintr"))()) + names(linter_funs) <- sel + lints <- lintr::lint_package(linters = linter_funs) + cat("n lints:", length(lints), "\n") + print(lints) # emits ::warning file=...,line=...:: annotations (auto-detects GitHub Actions) + lintr::sarif_output(lints, "lintr_results.sarif") + + is_pr <- Sys.getenv("IS_PR") == "true" + changed_files <- strsplit(Sys.getenv("CHANGED_FILES"), "\n")[[1]] + changed_files <- changed_files[nzchar(changed_files)] + + lint_files <- vapply(lints, function(l) l$filename, character(1)) + is_new <- if (is_pr) lint_files %in% changed_files else rep(FALSE, length(lints)) + n_new <- sum(is_new) + + cat(sprintf("n_new=%d\n", n_new), file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) + + if (n_new > 0) { + message(sprintf("Lint found %d issue(s) in files changed by this PR - see the annotations above or the SARIF upload for details.", n_new)) + } + quit(save = "no", status = if (n_new > 0) 1 else 0) + ' + + - name: Upload lint results + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: lintr_results.sarif + + - name: Post PR comment + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const nNew = parseInt('${{ steps.lint.outputs.n_new }}' || '0', 10); + const ok = nNew === 0; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const postCiComment = require('${{ github.workspace }}/.github/scripts/post-ci-comment.js'); + await postCiComment({ github, context, updates: { + 'row:lint': `Code quality${ok ? '✅ 0 findings' : `❌ ${nNew} finding${nNew === 1 ? '' : 's'}`}`, + 'log:lint': `Code quality` + }}); diff --git a/.gitignore b/.gitignore index 60d56797e..93decc0f6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ azure-pipelines.Rout tests/testthat/connection_to_datasets/local_settings.csv tests/docker/armadillo/standard/logs/ tests/docker/armadillo/standard/data/ +lintr_results.sarif diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..8455adf15 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,15 @@ +coverage: + status: + # Whole-repo coverage trend - informational only, never fails a PR. The + # combined figure is still shown in the GitHub Actions job summary + # (test-summary job) alongside a link to the full Codecov report. + project: + default: + target: auto + informational: true + # Coverage of lines actually changed by the PR - this is the enforced + # gate, so new/changed code is held to a bar without blocking on the + # pre-existing coverage backlog elsewhere in the repo. + patch: + default: + target: 80%