Implement restricted-hash semantics

docs/about/changelog.md

@@ -6,6 +6,9 @@ priorities and future plans.
 
 ## Work in progress
 
+- Snapshot mutable array entries while marking package roots and during reachability walks so concurrent thread teardown cannot fail with `ConcurrentModificationException`.
+
+- Enforce restricted-hash key access and implement `Hash::Util` key, value, and hash locking (#1502, #1172).
 - Report the caller's scalar context correctly when inspecting frames in overloaded exceptions (#1859).
 - Avoid heap exhaustion when compiling very large generated subroutines, and preserve heredoc statement boundaries with `\&name` fallbacks (#1252).
 - Cover nested require caller packages and imported `CLASS` constants, and match defer ordering and caller frames around loop `continue` blocks (#1601, #1405, #1383).

docs/reference/bundled-modules.md

@@ -242,7 +242,7 @@ These are loaded automatically or via `use`:
 | `Scalar::Readonly` | Java + Perl | Scalar read-only query, enable, and disable operations replacing the CPAN XS implementation |
 | `Scalar::Type` | Java + Perl | Scalar storage-type introspection without native XS |
 | `List::Util` | Java | `reduce`, `first`, `min`, `max`, `sum`, `mesh`/`zip`, etc. |
-| `Hash::Util` | Java | `lock_keys`, `lock_hash`, etc. |
+| `Hash::Util` | Java | Restricted-key enforcement and key/value/hash locks; `lock_keys`, `lock_keys_plus`, inspection helpers, and `lock_hash` are implemented. |
 
 ### File & I/O
 

docs/reference/feature-matrix.md

@@ -376,8 +376,8 @@ my @copy = @{$z};         # ERROR
 - ✅  **Array delete**: `delete` for array indexes is implemented.
 - ✅  **Tied Arrays**: Tied arrays are implemented. See also [Tied Scalars](#scalars), [Tied Hashes](#arrays-hashes-and-lists), [Tied Handles](#io-operations).
 - ✅  **Tied Hashes**: Tied hashes are implemented. See also [Tied Scalars](#scalars), [Tied Arrays](#arrays-hashes-and-lists), [Tied Handles](#io-operations).
-- 🟡 **Restricted hashes**: `Hash::Util` supplies compatibility entry points,
-  but lock/unlock helpers do not enforce native Perl hash restrictions.
+- ✅ **Restricted hashes**: `Hash::Util` enforces allowed-key access and
+  supports key/value/hash locking, unlocking, and restriction inspection.
 
 ---
 

src/main/java/org/perlonjava/frontend/parser/PrototypeArgs.java

@@ -1008,6 +1008,21 @@ private static void handleListOrHashArgument(Parser parser, ListNode args, boole
         }
 
         ListNode argList = ListParser.parseZeroOrMoreList(parser, 0, false, false, false, false);
+        if ("all_keys".equals(operatorName) || (operatorName != null && operatorName.endsWith("::all_keys"))) {
+            // Hash::Util::all_keys has two output arrays after its hash
+            // argument. Preserve those lvalues as references for its XS-like
+            // Java implementation instead of flattening their current values.
+            for (Node node : argList.elements) {
+                if (node instanceof OperatorNode sigil && sigil.operator.equals("@")) {
+                    Node reference = new OperatorNode("\\", node, node.getIndex());
+                    reference.setAnnotation("context", "SCALAR");
+                    args.elements.add(reference);
+                } else {
+                    args.elements.add(node);
+                }
+            }
+            return;
+        }
         // @ and % consume remaining arguments in LIST context
 //        for (Node element : argList.elements) {
 //            element.setAnnotation("context", "LIST");

src/main/java/org/perlonjava/runtime/perlmodule/HashUtil.java

@@ -36,6 +36,16 @@ public static void initialize() {
             hashUtil.registerMethod("unlock_keys", "unlock_keys", "\\%@");
             hashUtil.registerMethod("lock_hash", "lock_hash", "\\%");
             hashUtil.registerMethod("unlock_hash", "unlock_hash", "\\%");
+            hashUtil.registerMethod("lock_value", "lock_value", "\\%$");
+            hashUtil.registerMethod("unlock_value", "unlock_value", "\\%$");
+            hashUtil.registerMethod("lock_keys_plus", "lock_keys_plus", "\\%;@");
+            hashUtil.registerMethod("hashref_locked", "hashref_locked", null);
+            hashUtil.registerMethod("hash_locked", "hash_locked", "\\%");
+            hashUtil.registerMethod("hash_unlocked", "hash_unlocked", "\\%");
+            hashUtil.registerMethod("legal_keys", "legal_keys", "\\%");
+            hashUtil.registerMethod("all_keys", "all_keys", "\\%@@");
+            hashUtil.registerMethod("hidden_keys", "hidden_keys", "\\%");
+            hashUtil.registerMethod("legal_ref_keys", "legal_ref_keys", "$");
             hashUtil.registerMethod("hash_seed", "hash_seed", null);
             hashUtil.registerMethod("num_buckets", "num_buckets", "\\%");
 
@@ -185,28 +195,82 @@ private static int estimateUsedBuckets(Map<String, RuntimeScalar> map, int capac
     // Placeholder implementations for other Hash::Util functions
 
     public static RuntimeList lock_keys(RuntimeArray args, int ctx) {
-        // TODO: Implement hash key locking
-        if (args.size() > 0) {
-            return args.get(0).getList();  // Return the hash reference
-        }
-        return RuntimeScalarCache.scalarUndef.getList();
+        if (args.size() == 0) return RuntimeScalarCache.scalarUndef.getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        lockKeys(hash, args, 1);
+        return args.get(0).getList();
     }
 
     public static RuntimeList unlock_keys(RuntimeArray args, int ctx) {
-        // TODO: Implement hash key unlocking
-        if (args.size() > 0) {
-            return args.get(0).getList();  // Return the hash reference
+        if (args.size() == 0) return RuntimeScalarCache.scalarUndef.getList();
+        hashArgument(args.get(0)).legalHashKeys = null;
+        return args.get(0).getList();
+    }
+
+    private static void lockKeys(RuntimeHash hash, RuntimeArray args, int firstKey) {
+        java.util.LinkedHashSet<String> allowed = new java.util.LinkedHashSet<>();
+        if (args.size() == firstKey) {
+            allowed.addAll(hash.elements.keySet());
+        } else {
+            for (int i = firstKey; i < args.size(); i++) allowed.add(args.get(i).toString());
+            for (String existing : hash.elements.keySet()) {
+                if (!allowed.contains(existing)) {
+                    throw new PerlCompilerException("Hash has key '" + existing + "' which is not in the new key set");
+                }
+            }
+        }
+        hash.legalHashKeys = allowed;
+    }
+
+    public static RuntimeList lock_keys_plus(RuntimeArray args, int ctx) {
+        if (args.size() == 0) return RuntimeScalarCache.scalarUndef.getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        java.util.LinkedHashSet<String> allowed = new java.util.LinkedHashSet<>(hash.elements.keySet());
+        for (int i = 1; i < args.size(); i++) allowed.add(args.get(i).toString());
+        hash.legalHashKeys = allowed;
+        return args.get(0).getList();
+    }
+
+    public static RuntimeList lock_value(RuntimeArray args, int ctx) {
+        if (args.size() < 2) return RuntimeScalarCache.scalarUndef.getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        if (!hash.isHashRestricted()) {
+            throw new PerlCompilerException("Cannot usefully lock values in an unlocked hash");
+        }
+        String key = args.get(1).toString();
+        hash.validateHashKey(key, "read");
+        hash.lockedHashValues = hash.lockedHashValues == null ? new java.util.HashSet<>() : hash.lockedHashValues;
+        hash.lockedHashValues.add(key);
+        RuntimeScalar value = hash.elements.get(key);
+        if (value != null) makeScalarReadOnly(value);
+        return hash.get(key).getList();
+    }
+
+    public static RuntimeList unlock_value(RuntimeArray args, int ctx) {
+        if (args.size() < 2) return RuntimeScalarCache.scalarUndef.getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        String key = args.get(1).toString();
+        if (hash.lockedHashValues != null) hash.lockedHashValues.remove(key);
+        RuntimeScalar value = hash.elements.get(key);
+        if (value != null) makeScalarWritable(value);
+        return hash.get(key).getList();
+    }
+
+    private static void makeScalarWritable(RuntimeScalar scalar) {
+        if (scalar.type == RuntimeScalarType.READONLY_SCALAR && scalar.value instanceof RuntimeScalar inner) {
+            scalar.type = inner.type;
+            scalar.value = inner.value;
         }
-        return RuntimeScalarCache.scalarUndef.getList();
     }
 
     public static RuntimeList lock_hash(RuntimeArray args, int ctx) {
         if (args.size() > 0) {
             RuntimeHash hash = hashArgument(args.get(0));
+            hash.legalHashKeys = new java.util.LinkedHashSet<>(hash.elements.keySet());
+            hash.allHashValuesLocked = true;
             for (RuntimeScalar scalar : hash.elements.values()) {
                 makeScalarReadOnly(scalar);
             }
-            hash.type = RuntimeHash.READONLY_HASH;
             return args.get(0).getList();  // Return the hash reference
         }
         return RuntimeScalarCache.scalarUndef.getList();
@@ -215,14 +279,16 @@ public static RuntimeList lock_hash(RuntimeArray args, int ctx) {
     public static RuntimeList unlock_hash(RuntimeArray args, int ctx) {
         if (args.size() > 0) {
             RuntimeHash hash = hashArgument(args.get(0));
+            hash.legalHashKeys = null;
+            hash.lockedHashValues = null;
+            hash.allHashValuesLocked = false;
             for (RuntimeScalar scalar : hash.elements.values()) {
                 if (scalar.type == RuntimeScalarType.READONLY_SCALAR
                         && scalar.value instanceof RuntimeScalar inner) {
                     scalar.type = inner.type;
                     scalar.value = inner.value;
                 }
             }
-            hash.type = RuntimeHash.PLAIN_HASH;
             return args.get(0).getList();  // Return the hash reference
         }
         return RuntimeScalarCache.scalarUndef.getList();
@@ -235,6 +301,69 @@ private static RuntimeHash hashArgument(RuntimeScalar arg) {
         return arg.hashDeref();
     }
 
+    public static RuntimeList hashref_locked(RuntimeArray args, int ctx) {
+        return new RuntimeScalar(args.size() > 0 && hashArgument(args.get(0)).isHashRestricted()).getList();
+    }
+
+    public static RuntimeList hash_locked(RuntimeArray args, int ctx) {
+        if (args.size() == 0) return new RuntimeScalar(false).getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        return new RuntimeScalar(hash.isHashRestricted() || hash.allHashValuesLocked
+                || (hash.lockedHashValues != null && !hash.lockedHashValues.isEmpty())).getList();
+    }
+
+    public static RuntimeList hash_unlocked(RuntimeArray args, int ctx) {
+        if (args.size() == 0) return new RuntimeScalar(true).getList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        boolean locked = hash.isHashRestricted() || hash.allHashValuesLocked
+                || (hash.lockedHashValues != null && !hash.lockedHashValues.isEmpty());
+        return new RuntimeScalar(!locked).getList();
+    }
+
+    public static RuntimeList legal_keys(RuntimeArray args, int ctx) { return keyList(args, true); }
+    public static RuntimeList all_keys(RuntimeArray args, int ctx) {
+        if (args.size() == 0) return new RuntimeList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        RuntimeArray visible = outputArray(args, 1);
+        RuntimeArray hidden = outputArray(args, 2);
+        if (visible != null) visible.elements.clear();
+        if (hidden != null) hidden.elements.clear();
+        for (String key : hash.elements.keySet()) {
+            if (visible != null) visible.elements.add(new RuntimeScalar(key));
+        }
+        if (hash.legalHashKeys != null) for (String key : hash.legalHashKeys) {
+            if (!hash.elements.containsKey(key) && hidden != null) hidden.elements.add(new RuntimeScalar(key));
+        }
+        return args.get(0).getList();
+    }
+    public static RuntimeList hidden_keys(RuntimeArray args, int ctx) {
+        if (args.size() == 0) return new RuntimeList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        RuntimeArray result = new RuntimeArray();
+        if (hash.legalHashKeys != null) for (String key : hash.legalHashKeys) {
+            if (!hash.elements.containsKey(key)) result.elements.add(new RuntimeScalar(key));
+        }
+        return result.getList();
+    }
+    public static RuntimeList legal_ref_keys(RuntimeArray args, int ctx) { return keyList(args, true); }
+    private static RuntimeArray outputArray(RuntimeArray args, int index) {
+        if (index >= args.size()) return null;
+        RuntimeScalar scalar = args.get(index);
+        if (scalar.type == RuntimeScalarType.ARRAYREFERENCE && scalar.value instanceof RuntimeArray array) return array;
+        return null;
+    }
+    private static RuntimeList keyList(RuntimeArray args, boolean legalOnly) {
+        if (args.size() == 0) return new RuntimeList();
+        RuntimeHash hash = hashArgument(args.get(0));
+        RuntimeArray result = new RuntimeArray();
+        if (legalOnly && hash.legalHashKeys != null) {
+            for (String key : hash.legalHashKeys) result.elements.add(new RuntimeScalar(key));
+        } else {
+            for (String key : hash.elements.keySet()) result.elements.add(new RuntimeScalar(key));
+        }
+        return result.getList();
+    }
+
     private static void makeScalarReadOnly(RuntimeScalar scalar) {
         if (scalar.type == RuntimeScalarType.READONLY_SCALAR
                 || scalar instanceof RuntimeScalarReadOnly) {

src/main/java/org/perlonjava/runtime/perlmodule/Internals.java

@@ -32,11 +32,9 @@ public static void initialize() {
         try {
             internals.registerMethod("SvREADONLY", "svReadonly", "\\[$@%];$");
             internals.registerMethod("SvREFCNT", "svRefcount", "$;$");
-            // Clear placeholder slots in a restricted hash. PerlOnJava doesn't
-            // implement Perl's restricted-hash placeholder machinery (used by
-            // Hash::Util / fields), so this is a safe no-op. Modules like
-            // Const::Fast call it when sealing a hash readonly; failing here
-            // breaks loading the whole module.
+            // Clear placeholder slots in a restricted hash. PerlOnJava tracks
+            // allowed-but-absent keys separately, so there are no physical
+            // placeholder entries to remove.
             internals.registerMethod("hv_clear_placeholders", "hvClearPlaceholders", "\\%");
             // Phase 0 diagnostic: expose PerlOnJava-internal refcount state
             // (refCount, flags, tracking mode) for differential testing
@@ -980,7 +978,7 @@ else if (scalar.type != RuntimeScalarType.READONLY_SCALAR
                 // Check hash reference
                 if (effectiveScalar.type == RuntimeScalarType.HASHREFERENCE
                         && effectiveScalar.value instanceof RuntimeHash hash) {
-                    return new RuntimeScalar(hash.type == RuntimeHash.READONLY_HASH).getList();
+                    return new RuntimeScalar(hash.type == RuntimeHash.READONLY_HASH || hash.isHashRestricted()).getList();
                 }
                 // Check scalar reference: SvREADONLY checks whether SvRV(arg) has readonly flag
                 if (effectiveScalar.type == RuntimeScalarType.REFERENCE

src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java

@@ -750,7 +750,11 @@ static <T extends RuntimeBase> T markPackageGlobalRoot(T root) {
                 hash.markPackageRootedValue(value);
             }
         } else if (root instanceof RuntimeArray array) {
-            for (RuntimeScalar value : array.elements) {
+            // Package-root marking may race with another thread mutating a
+            // shared array. This is bookkeeping, so mark the values from a
+            // best-effort snapshot instead of leaking ArrayList's
+            // fail-fast ConcurrentModificationException into Perl code.
+            for (RuntimeScalar value : array.elements.toArray(RuntimeScalar[]::new)) {
                 array.markPackageRootedValue(value);
             }
         }

src/main/java/org/perlonjava/runtime/runtimetypes/ReachabilityWalker.java

@@ -324,7 +324,13 @@ private void bfsEndBlockRoots(java.util.ArrayDeque<RuntimeBase> todo) {
                     visitScalar(value, todo);
                 }
             } else if (cur instanceof RuntimeArray array) {
-                for (RuntimeScalar value : array.elements) {
+                // A child thread can run this lifecycle walk while another
+                // thread mutates a shared array. Iterating the live ArrayList
+                // leaks its fail-fast ConcurrentModificationException into
+                // thread teardown (seen in threads/t/free2.t). Traverse a
+                // point-in-time copy instead; the graph walk is advisory and
+                // must tolerate concurrent updates to shared Perl values.
+                for (RuntimeScalar value : array.elements.toArray(RuntimeScalar[]::new)) {
                     addReachable(value, todo);
                     visitScalar(value, todo);
                 }
@@ -373,7 +379,11 @@ private void bfs(java.util.ArrayDeque<RuntimeBase> todo, boolean walkCaptures) {
                 if (a.elements instanceof TieArray tieArray) {
                     visitScalar(tieArray.getSelf(), todo);
                 }
-                for (RuntimeScalar v : a.elements) {
+                // This broader weak-reference walk can also run during a
+                // child thread's teardown while another thread mutates a
+                // shared array. Never expose ArrayList's fail-fast iterator
+                // to concurrent Perl updates.
+                for (RuntimeScalar v : a.elements.toArray(RuntimeScalar[]::new)) {
                     recordWeakArraySlotWitness(a, v);
                     addReachable(v, todo);
                     visitScalar(v, todo);