Chainweb Revolution

.dockerignore

@@ -0,0 +1,28 @@
+# .git
+**.hi
+**.o
+*.eventlog
+*.eventlog
+*.hp
+*.prof
+*.ps
+*.tags
+.DS_Store
+.ghc.environment*
+.stack-work
+Dockerfile
+Tmp*
+chainweb-node
+chainweb-node*
+chainweb-node-prof*
+chainweb-tests*
+ci-logs
+ci-logs*
+dist
+dist-newstyle
+hie.log
+out*
+prof-data
+tags
+tmp
+tmp*

.github/actions/install-chainweb-from-artifacts/action.yml

@@ -25,8 +25,11 @@ runs:
         ubuntu-22.04)
           sudo apt-get install -u libgflags2.2
           ;;
+        ubuntu-24.04)
+          sudo apt-get install -u libgflags2.2
+          ;;
         mac*)
-          brew update && brew install gnu-tar || true
+          brew update && brew install gflags llvm snappy || true
           ;;
         *)
           echo "::error::unsupported operating system ${{ inputs.os }}"

.github/workflows/applications.yml

@@ -88,12 +88,12 @@ jobs:
         DEBUG_INFO=${{ github.event.inputs.debugInfo }}
         if [[ -n "$DEBUG_INFO" && "${DEBUG_INFO^^}" = "TRUE" ]] ; then
           echo "building binaries with dwarf debug symbols"
-          echo "debug-info=True" >> $GITHUB_OUTPUT
+          echo "debug-info=2" >> $GITHUB_OUTPUT
           echo "stripping=False" >> $GITHUB_OUTPUT
           TAG_SUFFIX+=-dwarf
         else
           echo "building binaries without dwarf debug symbols"
-          echo "debug-info=False" >> $GITHUB_OUTPUT
+          echo "debug-info=0" >> $GITHUB_OUTPUT
           echo "stripping=True" >> $GITHUB_OUTPUT
         fi
 
@@ -140,16 +140,16 @@ jobs:
         if "${{ github.event_name == 'schedule' }}" == "true"; then
         MATRIX="$(jq -c '.' <<EOF
         {
-
+          # The first version in this list is the version to be used for release
           "ghc": ["9.10.3"],
-          "cabal": ["3.14"],
+          "cabal": ["latest"],
           "os": ["ubuntu-22.04"],
           "use-freeze-file": ["false"],
           "include" : [
             {
               # These are the build values that will be used for release scripts
               "ghc": "9.10.3",
-              "cabal": "3.14",
+              "cabal": "latest",
               "os": "ubuntu-22.04",
               "use-freeze-file": "true"
             }
@@ -161,10 +161,22 @@ jobs:
         MATRIX="$(jq -c '.' <<EOF
         {
           "ghc": ["9.10.3"],
-          "cabal": ["3.14"],
-          "os": ["ubuntu-22.04"],
+          "cabal": ["latest"],
+          "os": ["ubuntu-22.04", "ubuntu-24.04"],
           "use-freeze-file": ["false"],
           "include" : [
+            {
+              "ghc": "9.12.2",
+              "cabal": "latest",
+              "os": "ubuntu-24.04",
+              "use-freeze-file": "false"
+            },
+            {
+              "ghc": "9.12.2",
+              "cabal": "latest",
+              "os": "macos-latest",
+              "use-freeze-file": "false"
+            },
             {
               "ghc": "9.10.3",
               "cabal": "3.14",
@@ -256,6 +268,9 @@ jobs:
     - name: Install non-Haskell dependencies (ubuntu)
       if: contains(matrix.os, 'ubuntu')
       run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev libmpfr-dev
+    - name: Install non-Haskell dependencies (macos)
+      if: contains(matrix.os, 'macos')
+      run: brew install -q gflags llvm snappy || true
     # Project Configuration
     - name: Create cabal.project.local
       run: |
@@ -336,6 +351,7 @@ jobs:
         diff -w <(git show HEAD:cabal.project.freeze) cabal.project.freeze || true
     - name: Sync from cabal cache
       if: env.USE_CABAL_CACHE == 'true'
+      continue-on-error: true
       uses: larskuhtz/cabal-cache-action@4b537195b33898fcd9adc62cee2a44986fd7b1b6
       with:
         bucket: "community-kda-cabal-cache"
@@ -347,28 +363,33 @@ jobs:
     - name: Install build dependencies
       run: cabal build chainweb --only-dependencies
     - name: Build chainweb library
-      run: cabal build --ghc-options=-j2 lib:chainweb
+      run: cabal build lib:chainweb
+    # # Temporarily disabled on lars/pp/* branches
     - name: Build chainweb applications
       run: |
-        cabal build -j2 --ghc-options=-j2 \
-          chainweb:bench:bench \
+        cabal build \
           exe:b64 \
           exe:compact \
-          exe:db-checksum \
           exe:ea \
           exe:genconf \
-          exe:header-dump \
           exe:known-graphs \
+          exe:standalone-pruner \
           exe:pact-diff \
-          exe:run-nodes \
+          exe:evm-genesis \
           exe:tx-list \
+          exe:run-nodes \
           test:chainweb-tests \
-          test:compaction-tests \
           test:multi-node-network-tests \
           test:remote-tests \
-          test:chainweb-storage-tests
+          test:chainweb-storage-tests \
+          test:blockhistory-migration-tests \
+          chainweb:bench:bench
+      # TODO: PP
+      # exe:header-dump \
+      # exe:db-checksum \
+      # test:compaction-tests \
     - name: Build chainweb-node application
-      run: cabal build -j2 --ghc-options=-j2 chainweb-node:exe:chainweb-node
+      run: cabal build chainweb-node:exe:chainweb-node
 
     # Checks
     - name: Check that working directory tree is clean
@@ -377,6 +398,7 @@ jobs:
           echo "Git working tree is not clean. The build changed some file that is checked into git." 1>&2
           exit 1
         fi
+
     - name: Run ea and verify consistency of genesis headers
       run: |
         cabal run ea
@@ -389,30 +411,32 @@ jobs:
     - name: Prepare artifacts
       run: |
         mkdir -p artifacts/chainweb
-        cp $(cabal list-bin b64) artifacts/chainweb
-        cp $(cabal list-bin bench) artifacts/chainweb
         cp $(cabal list-bin chainweb-node) artifacts/chainweb
-        cp $(cabal list-bin chainweb-storage-tests) artifacts/chainweb
-        cp $(cabal list-bin chainweb-tests) artifacts/chainweb
+        cp $(cabal list-bin test:chainweb-storage-tests) artifacts/chainweb
+        cp $(cabal list-bin test:chainweb-tests) artifacts/chainweb
+        cp $(cabal list-bin test:remote-tests) artifacts/chainweb
+        cp $(cabal list-bin test:blockhistory-migration-tests) artifacts/chainweb
+        cp $(cabal list-bin test:multi-node-network-tests) artifacts/chainweb
+        cp $(cabal list-bin bench) artifacts/chainweb
+        cp $(cabal list-bin evm-genesis) artifacts/chainweb
         cp $(cabal list-bin compact) artifacts/chainweb
-        cp $(cabal list-bin compaction-tests) artifacts/chainweb
-        cp $(cabal list-bin db-checksum) artifacts/chainweb
         cp $(cabal list-bin ea) artifacts/chainweb
         cp $(cabal list-bin genconf) artifacts/chainweb
-        cp $(cabal list-bin header-dump) artifacts/chainweb
         cp $(cabal list-bin known-graphs) artifacts/chainweb
-        cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb
         cp $(cabal list-bin pact-diff) artifacts/chainweb
-        cp $(cabal list-bin remote-tests) artifacts/chainweb
-        cp $(cabal list-bin run-nodes) artifacts/chainweb
+        cp $(cabal list-bin standalone-pruner) artifacts/chainweb
         cp $(cabal list-bin tx-list) artifacts/chainweb
+        cp $(cabal list-bin run-nodes) artifacts/chainweb
         cp README.md artifacts/chainweb
         cp CHANGELOG.md artifacts/chainweb
         cp LICENSE artifacts/chainweb
         cp chainweb.cabal artifacts/chainweb
         cp cabal.project artifacts/chainweb
         cp cabal.project.local artifacts/chainweb
         cp cabal.project.freeze artifacts/chainweb
+      # cp $(cabal list-bin header-dump) artifacts/chainweb
+      # cp $(cabal list-bin compaction-tests) artifacts/chainweb
+      # cp $(cabal list-bin db-checksum) artifacts/chainweb
     - name: Create artifacts archive
       run: |
         echo "tar -C ./artifacts/ -czf $ARTIFACTS_ARCHIVE chainweb"
@@ -455,6 +479,7 @@ jobs:
       run: remote-tests
 
   compaction-tests:
+    if: false # diabled on the evm branches
     name: Compaction tests
     needs: [config, build]
     runs-on: ${{ matrix.os }}
@@ -483,6 +508,8 @@ jobs:
           compaction-tests
 
   multi-node-network-tests:
+    # disabled for now on the evm branches. 10 nodes with 100 chains probably overwhelms the runners.
+    if: false
     name: Multi-node network tests
     needs: [config, build]
     runs-on: ${{ matrix.os }}
@@ -536,10 +563,41 @@ jobs:
       timeout-minutes: 2
       run: chainweb-storage-tests
 
+  blockhistory-migration-tests:
+    if: false # disabled because it needs databases as input
+    name: RocksDB library tests
+    needs: [config, build]
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download build artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACTS_NAME }}
+        path: .
+    - name: Install chainweb from artifacts
+      uses: ./.github/actions/install-chainweb-from-artifacts
+      with:
+        os: ${{ matrix.os }}
+        ghc: ${{ matrix.ghc }}
+        use-freeze-file: ${{ matrix.use-freeze-file }}
+    - name: Run blockhistory-migration-tests tests
+      timeout-minutes: 2
+      run: blockhistory-migration-tests
+
   # ########################################################################## #
   # Run Benchmarks
 
+  # Benchmarks take a long time to run and should run in controlled environment.
+  # Github runners tent to be noisy. So, we only build but don't run the
+  # benchmarks on push events.
   benchmarks:
+    if: false
     name: Benchmarks
     needs: [config, build]
     strategy:
@@ -606,7 +664,12 @@ jobs:
       timeout-minutes: 10
       run: |
           ulimit -n 10000
-          chainweb-tests --hide-successes --results-json test-results.json
+          grep -q -e '&&' FAILING_TESTS && echo "warning: FAILING_TESTS file is not empty" 1>&2
+          chainweb-tests \
+              --hide-successes \
+              --results-json test-results.json \
+              -p "$(tr '\n' ' ' < FAILING_TESTS)"
+
 
     # Publish test results to S3
     - name: Upload test results
@@ -665,6 +728,9 @@ jobs:
         - ghc: "9.10.3"
           os: "ubuntu-22.04"
           use-freeze-file: "true"
+        - ghc: "9.10.3"
+          os: "ubuntu-22.04"
+          use-freeze-file: "false"
     env:
       OS: ${{ matrix.os }}
       ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}