WIP PoC: migrate Cloudserver to Yarn 4 (Berry)

.github/actions/setup-ci/action.yaml

@@ -25,10 +25,17 @@ runs:
       run: |-
         set -exu;
         mkdir -p /tmp/coverage/${JOB_NAME}/;
+    - name: Enable Corepack
+      shell: bash
+      run: corepack enable
     - uses: actions/setup-node@v4
       id: node
       with:
         node-version: '22.23.1'
+    # setup-node installs its own Node, so re-enable Corepack against it.
+    - name: Enable Corepack (post setup-node)
+      shell: bash
+      run: corepack enable
     # Cache node_modules directly (not the yarn download cache) so the git
     # dependencies' TypeScript compilation and relink are skipped on a hit.
     # Key includes runner.os and the resolved Node version because node_modules
@@ -43,11 +50,11 @@ runs:
     - name: install typescript
       shell: bash
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
-      run: yarn global add typescript@4.9.5
+      run: npm install -g typescript@4.9.5
     - name: install dependencies
       shell: bash
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
-      run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+      run: yarn install --immutable
     - uses: actions/cache@v6
       with:
         path: ~/.cache/pip

.github/workflows/lint.yaml

@@ -17,16 +17,18 @@ jobs:
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v6
         with:
           node-version: '22.23.1'
           cache: yarn
       - name: install dependencies
-        run: yarn install --frozen-lockfile --network-concurrency 1
+        run: yarn install --immutable
       - name: Prettier (changed files)
         shell: bash
         env:
           BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: |
           MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}")
-          yarn run --silent prettier:diff --check "${MERGE_BASE}..HEAD"
+          yarn prettier:diff --check "${MERGE_BASE}..HEAD"

.github/workflows/tests.yaml

@@ -83,6 +83,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v4
         id: node
         with:
@@ -96,10 +98,10 @@ jobs:
       - name: install typescript
         shell: bash
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn global add typescript@4.9.5
+        run: npm install -g typescript@4.9.5
       - name: install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn install --frozen-lockfile --network-concurrency 1
+        run: yarn install --immutable
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
@@ -108,9 +110,9 @@ jobs:
         run: pip install flake8
       - name: Lint Javascript (strict, excluding async migration rules)
         run: |
-          yarn run --silent lint -- --max-warnings 0 --rule "promise/prefer-await-to-then: off" --rule "n/callback-return: off"
+          yarn lint --max-warnings 0 --rule "promise/prefer-await-to-then: off" --rule "n/callback-return: off"
       - name: Lint Markdown
-        run: yarn run --silent lint_md
+        run: yarn lint_md
       - name: Lint python
         run: flake8 $(git ls-files "*.py")
       - name: Lint Yaml
@@ -132,6 +134,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v4
         id: node
         with:
@@ -144,7 +148,7 @@ jobs:
           key: ${{ runner.os }}-node-${{ steps.node.outputs.node-version }}-modules-${{ hashFiles('yarn.lock') }}
       - name: install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn install --frozen-lockfile --network-concurrency 1
+        run: yarn install --immutable
       - name: Count async/await migration progress
         run: yarn run count-async
 
@@ -153,6 +157,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v4
         id: node
         with:
@@ -166,15 +172,18 @@ jobs:
       - name: install typescript
         shell: bash
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn global add typescript@4.9.5
+        run: npm install -g typescript@4.9.5
       - name: install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn install --frozen-lockfile --network-concurrency 1
+        run: yarn install --immutable
       - name: Unit Coverage
         run: |
           set -ex
           yarn run cover test
           yarn run cover test_legacy_location
+          # Berry does not run post* lifecycle scripts, so postcover
+          # (which writes the lcov report) has to be invoked explicitly.
+          yarn run postcover
         env:
           S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
           NO_PROXY: 'test.scality.com,scality.com'
@@ -1171,6 +1180,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v4
         id: node
         with:
@@ -1183,7 +1194,7 @@ jobs:
           key: ${{ runner.os }}-node-${{ steps.node.outputs.node-version }}-modules-${{ hashFiles('yarn.lock') }}
       - name: Install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: yarn install --frozen-lockfile --network-concurrency 1
+        run: yarn install --immutable
       - name: Delete stale GCP CI buckets
         run: yarn run cleanup_gcp_buckets
         env:

.gitignore

@@ -39,3 +39,11 @@ junit
 
 # config file build using jq to merge configs json
 tests/functional/sse-kms-migration/config.json
+
+# Yarn (Berry) — see .yarnrc.yml
+.yarn/*
+!.yarn/patches
+!.yarn/releases
+!.yarn/plugins
+!.yarn/sdks
+!.yarn/versions

.yarnrc.yml

@@ -0,0 +1,21 @@
+---
+nodeLinker: node-modules
+
+# Required, not optional: Yarn's default is an empty allowlist, which blocks
+# every Git fetch, so a repo with Git dependencies cannot install without
+# this. Scoped to Scality rather than the "**" Yarn writes during migration.
+# Every Git resolution here uses the .git clone form, which is the code path
+# that actually enforces this (hosted github: shorthand would not be).
+# npmMinimalAgeGate is deliberately left at Yarn's 1-day default, and
+# enableScripts at its secure default — the packages that genuinely need
+# compiling are allowlisted individually in package.json dependenciesMeta.
+approvedGitRepositories:
+  - 'https://github.com/scality/*'
+  - 'ssh://git@github.com/scality/*'
+
+# Several dependencies are Git repos that still carry Yarn 1 lockfiles, so Yarn
+# bootstraps them with Yarn Classic when repacking. Run those serially: in
+# parallel they race on Classic's shared cache and packing fails with ENOENT
+# (YN0058). This replaces the --network-concurrency 1 flag Yarn 1 used, which
+# Berry rejects on the CLI but still honours as a setting.
+networkConcurrency: 1

Dockerfile

@@ -25,13 +25,29 @@ ENV PYTHON=python3
 RUN npm install -g \
     node-gyp \
     typescript@4.9.5
-COPY package.json yarn.lock /usr/src/app/
+COPY package.json yarn.lock .yarnrc.yml /usr/src/app/
 
-RUN yarn install --production --frozen-lockfile --ignore-engines --network-concurrency 1
+# `workspaces focus` cannot enforce lockfile immutability (it silently
+# re-resolves), so validate the lockfile first with a cheap build-less
+# install. Together these preserve what --frozen-lockfile used to give us.
+RUN corepack enable \
+    && yarn install --immutable --mode=skip-build \
+    && yarn workspaces focus --production
 
 ################################################################################
 FROM node:${NODE_VERSION} AS production
 
+# The production stage runs `yarn start`, and package.json pins Yarn 4 via
+# packageManager, so the image's bundled Yarn 1 refuses to run at all. Install
+# Yarn into the image rather than only enabling Corepack: a bare `corepack
+# enable` leaves the CLI to be downloaded on every container start, which
+# breaks air-gapped deployments. COREPACK_HOME is shared and world-readable so
+# images that drop privileges (images/federation runs as `scality`) can use it.
+ENV COREPACK_HOME=/usr/local/corepack
+RUN corepack enable \
+    && corepack install -g yarn@4.18.0 \
+    && chmod -R a+rX ${COREPACK_HOME}
+
 ENV NO_PROXY=localhost,127.0.0.1
 ENV no_proxy=localhost,127.0.0.1
 
@@ -61,6 +77,6 @@ CMD [ "yarn", "start" ]
 ################################################################################
 FROM production AS testcoverage
 
-RUN yarn global add nyc
+RUN npm install -g nyc
 
 CMD [ "./docker-test-with-coverage.sh" ]

package.json

@@ -34,6 +34,7 @@
         "@opentelemetry/instrumentation-http": "~0.218.0",
         "@opentelemetry/instrumentation-ioredis": "~0.64.0",
         "@opentelemetry/instrumentation-mongodb": "~0.69.0",
+        "@scality/cloudserverclient": "1.0.12",
         "@smithy/node-http-handler": "^3.0.0",
         "arsenal": "git+https://github.com/scality/arsenal#8.5.15",
         "async": "2.6.4",
@@ -57,14 +58,13 @@
         "request": "^2.88.2",
         "scubaclient": "git+https://github.com/scality/scubaclient.git#1.1.4",
         "sql-where-parser": "^2.2.1",
-        "utapi": "github:scality/utapi#8.2.4",
+        "utapi": "github:scality/utapi#8.2.7",
         "utf-8-validate": "^6.0.5",
         "utf8": "^3.0.0",
         "uuid": "^11.0.3",
         "vaultclient": "scality/vaultclient#8.5.8",
         "werelogs": "scality/werelogs#semver:^8.2.4",
         "ws": "^8.18.0",
-        "@scality/cloudserverclient": "1.0.12",
         "xml2js": "^0.6.2"
     },
     "devDependencies": {
@@ -95,9 +95,7 @@
     "resolutions": {
         "jsonwebtoken": "^9.0.0",
         "nan": "v2.22.0",
-        "fast-xml-parser": "^5.5.6",
-        "ts-morph/**/brace-expansion": "^5.0.5",
-        "ts-morph/**/picomatch": "^4.0.4"
+        "fast-xml-parser": "^5.5.6"
     },
     "countAsyncSourcePaths": [
         "lib/**/*.js",
@@ -123,14 +121,14 @@
         "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/multipleBackend --exit",
         "ft_checksums_disabled": "cd tests/functional/checksumsDisabled && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 120000 *.js --exit",
         "ft_mixed_bucket_format_version": "cd tests/functional/metadata && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json MixedVersionFormat.js --exit",
-        "ft_management": "cd tests/functional/report && yarn test",
+        "ft_management": "cd tests/functional/report && mocha -t 40000 *.js",
         "ft_backbeat": "cd tests/functional/backbeat && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
-        "ft_node": "cd tests/functional/raw-node && yarn test",
-        "ft_node_routes": "cd tests/functional/raw-node && yarn run test-routes",
+        "ft_node": "cd tests/functional/raw-node && mocha -t 40000 test/ --exit",
+        "ft_node_routes": "cd tests/functional/raw-node && mocha -t 40000 test/routes/ --exit",
         "ft_route_backbeat": "cd tests/multipleBackend/routes && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 routeBackbeat.js routeBackbeatForReplication.js --exit",
-        "ft_gcp": "cd tests/functional/raw-node && yarn run test-gcp",
+        "ft_gcp": "cd tests/functional/raw-node && mocha -t 40000 test/GCP/ --exit",
         "cleanup_gcp_buckets": "node .github/scripts/cleanupOldGCPBuckets.js",
-        "ft_healthchecks": "cd tests/functional/healthchecks && yarn test",
+        "ft_healthchecks": "cd tests/functional/healthchecks && mocha -t 40000 test/ --exit",
         "ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
         "ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
         "ft_scripts": "cd tests/functional/scripts && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
@@ -171,5 +169,32 @@
         "check-diff-async": "node .github/scripts/check-diff-async.mjs",
         "prettier": "prettier",
         "prettier:diff": "bash scripts/prettier-diff.sh"
+    },
+    "packageManager": "yarn@4.18.0+sha512.fcb8716fe7cd0eece141ffc18b92193a9df9204c1ba83189c288835223fc0bbe64af473bab0d5e9927a7daeb5caf2bb07eb2787cc9338ca040ea125f2a1f2f7e",
+    "dependenciesMeta": {
+        "aws-crt": {
+            "built": true
+        },
+        "bufferutil": {
+            "built": true
+        },
+        "diskusage": {
+            "built": true
+        },
+        "fcntl": {
+            "built": true
+        },
+        "ioctl": {
+            "built": true
+        },
+        "leveldown": {
+            "built": true
+        },
+        "scubaclient": {
+            "built": true
+        },
+        "utf-8-validate": {
+            "built": true
+        }
     }
 }