Add verified tooling and a CM4 development environment

.gitattributes

@@ -0,0 +1,2 @@
+# Unified patches preserve upstream indentation and blank context lines.
+*.patch whitespace=-blank-at-eol,-space-before-tab

.github/workflows/emulator.yml

@@ -0,0 +1,48 @@
+name: Host CI
+
+on:
+  workflow_call:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    name: Linux tests and GUI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install test dependencies
+        run: sudo apt-get update && sudo apt-get install -y shellcheck xvfb python3-tk qemu-utils dosfstools e2fsprogs libasound2-dev
+      - name: Run complete tests with GUI and shell checks
+        timeout-minutes: 20
+        run: xvfb-run -a make check PYTHON=/usr/bin/python3
+
+  macos:
+    name: macOS complete tests and GUI
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install native test dependencies
+        run: brew install shellcheck python-tk@3.12 qemu dosfstools e2fsprogs
+      - name: Run complete tests and shell checks
+        timeout-minutes: 20
+        run: make check PYTHON="$(brew --prefix)/bin/python3.12"
+
+  windows:
+    name: Windows host-tool portability
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m unittest discover -s tests -p test_emulator.py -v

.github/workflows/publish-wiki.yml

@@ -12,6 +12,9 @@ permissions:
   contents: write
 jobs:
   publish-wiki:
+    # Forks do not inherit the upstream wiki repository. Publish only after
+    # these files reach the canonical repository's default branch.
+    if: github.repository == 'clockworkpi/uConsole'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/release.yml

@@ -0,0 +1,199 @@
+name: Release packages
+
+on:
+  workflow_dispatch:
+    inputs:
+      capture_public_desktop:
+        description: 'Record the public factory guest desktop on Linux x86_64 (not an automatic desktop pass)'
+        type: boolean
+        default: false
+      desktop_scenario:
+        description: 'Public disposable guest recording recipe; screenshots require review'
+        type: choice
+        options:
+          - capture
+          - onboarding
+        default: capture
+  pull_request:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+permissions:
+  contents: read
+
+jobs:
+  host-tests:
+    name: Host and GUI tests
+    uses: ./.github/workflows/emulator.yml
+
+  firmware:
+    name: Build keyboard firmware
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-arduino-cli@v2
+      - name: Install STM32F1 core
+        run: |
+          arduino-cli config init
+          arduino-cli config add board_manager.additional_urls https://dan.drown.org/stm32duino/package_STM32duino_index.json
+          arduino-cli core update-index
+          arduino-cli core install stm32duino:STM32F1@2021.2.22
+      - name: Build firmware
+        run: make firmware
+      - uses: actions/upload-artifact@v4
+        with:
+          name: keyboard-firmware
+          path: |
+            build/firmware/uconsole_keyboard.ino.bin
+            build/firmware/provenance.json
+          if-no-files-found: error
+
+  packages:
+    name: Package ${{ matrix.target }}
+    needs: firmware
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: linux-x86_64
+            runner: ubuntu-24.04
+          - target: linux-aarch64
+            runner: ubuntu-24.04-arm
+          - target: macos-arm64
+            runner: macos-15
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: actions/download-artifact@v4
+        with:
+          name: keyboard-firmware
+          path: build/firmware
+      - name: Install native package qualification dependencies
+        shell: bash
+        run: |
+          if [[ "$RUNNER_OS" == Linux ]]; then
+            sudo apt-get update
+            sudo apt-get install -y python3-tk xvfb ninja-build pkg-config libglib2.0-dev libpixman-1-dev libslirp-dev libgtk-3-dev qemu-system-arm qemu-utils dosfstools e2fsprogs
+          else
+            brew install python-tk ninja pkg-config glib pixman libslirp qemu dosfstools e2fsprogs
+          fi
+      - name: Build and package uConsole Workbench
+        run: |
+          make package
+      - name: Test native helper and IDE package
+        run: |
+          if build/upload-reset.elf >/dev/null 2>&1; then
+            echo 'upload-reset unexpectedly accepted missing arguments' >&2
+            exit 1
+          fi
+          python3 -m unittest tests.test_keyboard_flash -v
+          test "$(find build -maxdepth 1 -name 'uconsole-workbench-*.tar.gz' | wc -l | tr -d ' ')" = 1
+      - name: Qualify extracted native archive and GUI
+        shell: bash
+        run: |
+          archive=build/uconsole-workbench-${{ matrix.target }}.tar.gz
+          if [[ "$RUNNER_OS" == Linux ]]; then
+            xvfb-run -a python3 tools/validate_native_archive.py --archive "$archive" --output build/native-archive-validation --gui --preferred-python /usr/bin/python3
+          else
+            python3 tools/validate_native_archive.py --archive "$archive" --output build/native-archive-validation --gui --preferred-python /opt/homebrew/bin/python3
+          fi
+      - name: Build and test QEMU using only extracted package resources
+        shell: bash
+        run: |
+          resources="$GITHUB_WORKSPACE/build/native-archive-validation/uconsole-workbench/libexec/uconsole-workbench"
+          evidence="$GITHUB_WORKSPACE/build/native-archive-validation"
+          export UCONSOLE_BUILD_DIR="$RUNNER_TEMP/uconsole-package-qemu"
+          unset UCONSOLE_ROOT PYTHONPATH
+          cd "$RUNNER_TEMP"
+          python3 "$resources/tools/build_emulator_qemu.py" --jobs 4 > "$evidence/qemu-build.log" 2>&1
+          python3 "$resources/tools/build_emulator_qemu.py" --jobs 4 > "$evidence/qemu-rebuild.log" 2>&1
+          python3 "$resources/tools/test_emulator_watchdog.py" > "$evidence/watchdog.log" 2>&1
+          python3 "$resources/tools/test_emulator_pmic.py" > "$evidence/pmic.log" 2>&1
+          python3 "$resources/tools/test_emulator_adc_migration.py" > "$evidence/adc-migration.log" 2>&1
+      - name: Record public factory desktop on native x86_64
+        if: github.event_name == 'workflow_dispatch' && inputs.capture_public_desktop && matrix.target == 'linux-x86_64'
+        timeout-minutes: 60
+        shell: bash
+        run: |
+          set -euo pipefail
+          resources="$GITHUB_WORKSPACE/build/native-archive-validation/uconsole-workbench/libexec/uconsole-workbench"
+          export UCONSOLE_BUILD_DIR="$RUNNER_TEMP/uconsole-package-qemu"
+          unset UCONSOLE_ROOT PYTHONPATH
+          factory="$RUNNER_TEMP/uConsole_CM4_v3.1_64bit.img.bz2"
+          workspace="$RUNNER_TEMP/public-factory-desktop"
+          evidence="$GITHUB_WORKSPACE/build/public-desktop-evidence"
+          mkdir -m 700 "$evidence"
+          # Diskless real-UART reproduction: synchronous QMP stalls when the
+          # host stops draining serial; the recorder worker must keep Tk live.
+          xvfb-run -a /usr/bin/python3 "$resources/tools/validate_desktop_serial_backpressure.py" --output "$evidence/serial-backpressure" > "$evidence/serial-backpressure.log" 2>&1
+          df -h "$RUNNER_TEMP"
+          # This manufacturer mirror is HTTP. The independently pinned SHA256
+          # is mandatory before any decompression or boot. Never use user images.
+          curl --fail --location --retry 2 --max-time 900 --output "$factory" http://dl.clockworkpi.com/uConsole_CM4_v3.1_64bit.img.bz2
+          printf '%s  %s\n' ef95242cdb0125e8ed08157400a26d4665acddd083481ec2314acd6e073b74ab "$factory" | sha256sum --check
+          cd "$RUNNER_TEMP"
+          /usr/bin/python3 "$resources/tools/uconsole_emulator.py" --workspace "$workspace" prepare "$factory" > "$evidence/prepare.log" 2>&1
+          /usr/bin/python3 "$resources/tools/uconsole_emulator.py" --workspace "$workspace" configure-display --timeout 600 > "$evidence/configure-display.log" 2>&1
+          case '${{ inputs.desktop_scenario }}' in
+            capture|'') recipe=factory-desktop-capture.json ;;
+            onboarding) recipe=factory-desktop-onboarding.json ;;
+            *) echo 'Unknown desktop scenario' >&2; exit 1 ;;
+          esac
+          scenario="$GITHUB_WORKSPACE/build/native-archive-validation/uconsole-workbench/share/doc/uconsole-workbench/scenarios/$recipe"
+          xvfb-run -a /usr/bin/python3 "$resources/tools/drive_forge_desktop.py" --workspace "$workspace" --scenario "$scenario" --output "$evidence/driver"
+      - name: Retain public desktop observations only
+        if: always() && github.event_name == 'workflow_dispatch' && inputs.capture_public_desktop && matrix.target == 'linux-x86_64'
+        uses: actions/upload-artifact@v4
+        with:
+          name: public-desktop-linux-x86_64
+          # Explicit allowlist: no disks, account password fixture or user data.
+          path: |
+            build/public-desktop-evidence/**/*.log
+            build/public-desktop-evidence/driver/driver.json
+            build/public-desktop-evidence/serial-backpressure/acceptance.json
+            ${{ runner.temp }}/public-factory-desktop/desktop-record-*/screen-*.png
+            ${{ runner.temp }}/public-factory-desktop/desktop-record-*/record.json
+            ${{ runner.temp }}/public-factory-desktop/desktop-record-*/console.log
+            ${{ runner.temp }}/public-factory-desktop/serial.log
+            ${{ runner.temp }}/public-factory-desktop/workbench-qemu.log
+            ${{ runner.temp }}/public-factory-desktop/machine.json
+          if-no-files-found: error
+      - name: Retain package qualification evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: qualification-${{ matrix.target }}
+          path: |
+            build/native-archive-validation/native-archive-acceptance.json
+            build/native-archive-validation/*.log
+          if-no-files-found: warn
+      - uses: actions/upload-artifact@v4
+        with:
+          name: uconsole-workbench-${{ matrix.target }}
+          path: build/uconsole-workbench-*.tar.gz
+          if-no-files-found: error
+
+  publish:
+    name: Publish GitHub release
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    needs: [packages, host-tests]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: uconsole-workbench-*
+          path: dist
+          merge-multiple: true
+      - name: Create checksums
+        working-directory: dist
+        run: sha256sum uconsole-workbench-*.tar.gz > SHA256SUMS
+      - name: Publish release and assets
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create '${{ github.ref_name }}' dist/* --repo '${{ github.repository }}' --verify-tag --generate-notes --title '${{ github.ref_name }}'

.gitignore

@@ -0,0 +1,3 @@
+/build/
+__pycache__/
+.forge.lock

Bin/uconsole_keyboard_flash/README.md

@@ -5,4 +5,44 @@ the bin file must be generated by selecting the upload method to be stm32duino-b
 
 so that the start address will be at 0x08002000 
 
+## Build and flash from a checkout
 
+At the repository root, run `make firmware flash-tool` after installing the
+toolchain described in `Code/uconsole_keyboard/README.md`. This builds a native
+reset helper without replacing the bundled binaries for other architectures.
+
+`make flash-bundle` packages the current scripts, newly compiled firmware,
+and native reset helper into
+`build/uconsole_keyboard_flash-<os>-<architecture>.tar.gz`. The archive target
+is detected from the helper's ELF or Mach-O header; extract it on that target
+and run its `flash.sh`. Other architectures' precompiled reset helpers are not
+included. The tracked legacy tarball is not overwritten by this build.
+
+When ready to flash a connected keyboard, run from the repository root:
+
+```sh
+sudo env UPLOAD_RESET="$PWD/build/upload-reset.elf" \
+  Bin/uconsole_keyboard_flash/maple_upload ttyACM0 2 1EAF:0003 \
+  "$PWD/build/firmware/uconsole_keyboard.ino.bin"
+```
+
+Install `dfu-util` first (`apt install dfu-util` on Debian/Ubuntu or
+`brew install dfu-util` on macOS). If it is not on `PATH`, set `DFU_UTIL` to
+its absolute path in the same `env` command. Check which serial port belongs
+to your keyboard and replace `ttyACM0` if necessary; `/dev/ttyACM0` also works.
+On macOS, `flash.sh` selects the first `/dev/cu.usbmodem*` device or requires
+the device path as its second argument when none can be detected.
+
+For the prebuilt bundle, `flash.sh` works from any current directory and
+accepts optional firmware and serial-port arguments:
+
+```sh
+/path/to/uconsole_keyboard_flash/flash.sh /path/to/firmware.bin ttyACM0
+```
+
+The reset may fail when the keyboard is already in bootloader mode, so the
+script still attempts DFU. A failed DFU operation returns a nonzero status
+and never reports success. A successful transfer followed by a serial-device
+timeout is reported separately and also returns nonzero. Neither a completed
+transfer nor serial enumeration proves that every keyboard function works;
+test keys, trackball, and recovery access after flashing.

Bin/uconsole_keyboard_flash/flash.sh

@@ -1,3 +1,27 @@
 #!/bin/bash
-
-sudo  ./maple_upload ttyACM0 2 1EAF:0003  uconsole_keyboard.ino.bin
+set -euo pipefail
+DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+if (( $# > 2 )); then
+    echo "Usage: $0 [firmware.bin] [serial_port]" >&2
+    exit 2
+fi
+firmware=${1:-$DIR/uconsole_keyboard.ino.bin}
+serial_port=${2:-}
+if [[ -z $serial_port ]]; then
+    if [[ $(uname -s) == Darwin ]]; then
+        ports=(/dev/cu.usbmodem*)
+        if [[ -e ${ports[0]} ]]; then
+            serial_port=${ports[0]}
+        else
+            echo "No /dev/cu.usbmodem device found; pass the serial device as the second argument." >&2
+            exit 2
+        fi
+    else
+        serial_port=ttyACM0
+    fi
+fi
+if (( EUID == 0 )); then
+    exec "$DIR/maple_upload" "$serial_port" 2 1EAF:0003 "$firmware"
+else
+    exec sudo "$DIR/maple_upload" "$serial_port" 2 1EAF:0003 "$firmware"
+fi