diff --git a/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl new file mode 100644 index 00000000..9fbcea68 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl @@ -0,0 +1,204 @@ +{"index":0,"relation":"APIContainRelation","description":"Adadelta.step() always calls Adadelta._init_group()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._accelerator_graph_capture_health_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._optimizer_step_code()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"APIContainRelation","description":"Adadelta.step() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros_like()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"APIContainRelation","description":"Adadelta.step() always calls accelerator.current_accelerator()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta.adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"APIContainRelation","description":"Adadelta.step() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._get_scalar_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"APIContainRelation","description":"Adadelta.step() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"APIContainRelation","description":"Conv2d.forward() always calls Conv2d._conv_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"APIContainRelation","description":"Conv2d.forward() always calls _VariableFunctionsClass.conv2d()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with ndim=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"DistinctArgumentRelation","description":"Conv2d.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":85,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"DistinctArgumentRelation","description":"Dropout.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":196,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls compiler.is_exporting()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl new file mode 100644 index 00000000..b5594e51 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl @@ -0,0 +1,564 @@ +{"index":0,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"APIContainRelation","description":"Embedding.forward() always calls _VariableFunctionsClass.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"APIContainRelation","description":"Embedding.forward() always calls functional.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":85,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"DistinctArgumentRelation","description":"LayerNorm.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":196,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":204,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":205,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":206,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":207,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":208,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":209,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":210,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":211,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":212,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":213,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":214,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":215,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":216,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":217,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":218,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":219,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":220,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":221,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":222,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":223,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":224,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":225,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":226,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":227,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":228,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":229,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":230,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":231,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":232,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":233,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":234,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":235,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":236,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":237,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":238,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":239,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":240,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":241,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":242,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":243,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":244,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":245,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":246,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":247,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":248,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":249,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":250,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":251,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":252,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":253,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":254,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":255,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":256,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":257,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":258,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":259,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":260,"relation":"DistinctArgumentRelation","description":"MultiheadAttention.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":261,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":262,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":263,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":264,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":265,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":266,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":267,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":268,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":269,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":270,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":271,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":272,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":273,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":274,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":275,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":276,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":277,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":278,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":279,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":280,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":281,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":282,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":283,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":284,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":285,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":286,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":287,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":288,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":289,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":290,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":291,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":292,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":293,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":294,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":295,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":296,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":297,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":298,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":299,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":300,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":301,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":302,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":303,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":304,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":305,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":306,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":307,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":308,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":309,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":310,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":311,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":312,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":313,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":314,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":315,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":316,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":317,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":318,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":319,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":320,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":321,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":322,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":323,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":324,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":325,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":326,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":327,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":328,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":329,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":330,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":331,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":332,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":333,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":334,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":335,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":336,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":337,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":338,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls ModuleList._get_abs_string_index()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":339,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":340,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":341,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":342,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":343,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":344,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.full()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":345,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":346,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":347,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":348,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.triu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":349,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":350,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":351,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":352,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":353,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":354,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":355,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":356,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":357,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._detect_is_causal_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":358,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._generate_square_subsequent_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":359,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._get_seq_len()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":360,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":361,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":362,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":363,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":364,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":365,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":366,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":367,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":368,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":369,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":370,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":371,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":372,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":373,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":374,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":375,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":376,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":377,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":378,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":379,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":380,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":381,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":382,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":383,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":384,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":385,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":386,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":387,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":388,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":389,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":390,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":391,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":392,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":393,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":394,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":395,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":396,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":397,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":398,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":399,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":400,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":401,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":402,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":403,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":404,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":405,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":406,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":407,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":408,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":409,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":410,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":411,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":412,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":413,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":414,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":415,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":416,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":417,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":418,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":419,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":420,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":421,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":422,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":423,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":424,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":425,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":426,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":427,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":428,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":429,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":430,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":431,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":432,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":433,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":434,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":435,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":436,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":437,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":438,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":439,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":440,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":441,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":442,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":443,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":444,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":445,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":446,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":447,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":448,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":449,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":450,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":451,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":452,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":453,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":454,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":455,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":456,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":457,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":458,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":459,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":460,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":461,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":462,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":463,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":464,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":465,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":466,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":467,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":468,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":469,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":470,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":471,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":472,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":473,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":474,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":475,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":476,"relation":"DistinctArgumentRelation","description":"TransformerEncoderLayer.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":477,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":478,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":479,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":480,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":481,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":482,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":483,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":484,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":485,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":486,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":487,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":488,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":489,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":490,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":491,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":492,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":493,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":494,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":495,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":496,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":497,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":498,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":499,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":500,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":501,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":502,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":503,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":504,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":505,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":506,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":507,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":508,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":509,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":510,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":511,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":512,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":513,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":514,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":515,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":516,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":517,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":518,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":519,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":520,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":521,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":522,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":523,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module._named_members()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":524,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_modules()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":525,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_parameters()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":526,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_mul_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":527,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":528,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.clamp()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":529,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.stack()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":530,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._device_has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":531,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":532,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._group_tensors_by_device_and_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":533,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":534,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._clip_grads_with_norm_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":535,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._get_total_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":536,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":537,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":538,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":539,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":540,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_fn=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":541,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":542,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":543,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":544,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_leaf=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":545,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":546,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":547,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":548,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":549,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":550,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":551,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":552,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":553,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":554,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":555,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":556,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":557,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":558,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":559,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with nbytes=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":560,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with ndim=0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":561,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with requires_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":562,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":563,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with shape=()","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation.json b/docs/measurements/data/fresh-run-invariant-validation.json new file mode 100644 index 00000000..95340944 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation.json @@ -0,0 +1,1894 @@ +{ + "study_date": "2026-10-25", + "harness_commit": "cad62cc98ef5fb7b313325443caa33f715ed141e", + "source_composition": { + "base": "99d16905296d83495570fdcafa91370b242d2528", + "trace_integrity": "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4", + "torchscript": "9ac61d1140cf8679c24442a174ec12f4de069980", + "state_fix_hunks_from": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "changed_python_files": [ + "tests/test_dynamic_typename.py", + "tests/test_precondition_state_isolation.py", + "tests/test_torchscript_compat.py", + "tests/test_trace_integrity.py", + "traincheck/instrumentor/dumper.py", + "traincheck/instrumentor/torchscript_compat.py", + "traincheck/instrumentor/tracer.py", + "traincheck/invariant/precondition.py", + "traincheck/utils.py" + ], + "dynamic_typename_fix": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "source_sha256": { + "tests/test_utils.py": "cc63afa128223d1a7c518becf2c215ffd0ab91891a01b23e0bbeba2e9726e56c", + "tests/test_dynamic_policy.py": "ac82536add642da8179acd54b07d86cff81b5856d90fc41896d18a57334ccdec", + "tests/test_inference_sampling.py": "41a632102208e53091615d21714b7dc1d6840465fd550efa33caf8eeca3fe361", + "tests/test_torchscript_compat.py": "653be1032409579030ebd3ec5694cc3e43f34df8e4f5750a76a9ffc64dd0a2e2", + "tests/test_trace_integrity.py": "0d12f81b39c4772e9d6743605c4a627078b4a9f5e85b254d694ad999d3496f08", + "tests/test_loop_injection.py": "c9fda263c5b014f2a56030d070f72144e5aea3be50c431d2a67a910e5978e11d", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/test_display_names.py": "bf85fee31e1bb8384468df76bf30d4209aa8e14d0b9b822a2af9860ee6c624cd", + "tests/test_dynamic_typename.py": "fc86e92919f07a82f5f06e2d9ebcd52431ec2ea175ceb5d1c2417aab45924ba8", + "tests/test_call_index.py": "979c4b4489b5ca8c2468db7ee5d7b39bb9fe8b3c193f7b43f1627ada17c7b541", + "tests/test_example_reservoir.py": "4805845c77e5bdeaf48793dd8b8cc1ed42126e9c2ea5154c073e9fa2cc075b90", + "tests/test_violation_summary.py": "fa5bd46bbc5592867eb7d018a494b81aa5d59f451fd282a147ed56aefd7e3619", + "tests/test_precondition_state_isolation.py": "d56e2e0e3ca5d410ccef5cb1de350a21f37a6be679efd302b523aae1f7a1d919", + "tests/test_identity_cache.py": "733575f48f690cfc2913169bff2a8656787fe94f37e4658850e29d2f4656ac50", + "tests/test_context_index.py": "7702f13e695f430e2aa1e302e3719ce9d5e8bd74a2d65cac5b7fbfa2178159dc", + "tests/test_policy_injection.py": "9409b8f693da8b05e556f658486346422c6e9b853f1a991cad28572dfd977848", + "tests/test_variable_tracking_policy.py": "059178ac644d6d9305dd10ffcea75dc99e5fd5019b02f09f37a8a93728f5d989", + "tests/fixtures/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/fixtures/legacy_trace_pandas.py": "342bef3c1de26d77cc8b1e34f94dc470f45d929e038e458ac00eff11eebc1860", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/utils.py": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/precondition.py": "dfd02e6c68a6c3f2d06d6a9beaa109dcf99ef099efeda75252fd8c559fdbb5da", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/dumper.py": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/instrumentor/torchscript_compat.py": "0d0c348fa02d5f93bb74e874829e5fb4c57365b82ce973ae8ff1f21a42c2d64e", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/tracer.py": "ad57320f0a3764c02c73cf4bfa623f1ef73b9835dfa6cb1365dc2988551de428", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0", + "device": "cpu", + "native_threads": 1 + }, + "protocol": { + "reference_seed": 42, + "reference_steps": 10, + "control_seed": 42, + "control_steps": 10, + "holdout_seed": 43, + "holdout_steps": 20, + "batch_size": 2, + "prepared_data_steps": 20, + "check_relation_first": false, + "inference_workers": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "each_phase_fresh_OS_process": true, + "holdout_inputs_overlap_reference": true, + "API_instrumentation": true, + "proxy_scanning": false + }, + "tests": { + "local_counter_and_mutation": 13, + "matching_linux_counter_and_mutation": 13, + "matching_linux_composed_production_stack": 29 + }, + "archives_sha256": { + "harness-cad62cc-complete.tar.gz": "b9ed7f3401363e73d8b69c9b12f183429365df18c28a61c1969e3be90285cb5a", + "integration-v3-source.tar.gz": "4b19005ae6ecad19e48b531185a7552cdfd0d08b6d23c907460ac33adb6498c3", + "final-mnist-74b-diagnostic.tar.gz": "0612f10428e1f22ebc831e6e63f990a91c917aa6128044e43a86c3790ba61195", + "final-mnist-cad62cc-evidence.tar.gz": "6a9a6866c4ac78d6050527fd6dac1f7d9586427b2cd607cbe182e86bd8135e6d", + "final-transformer-cad62cc-evidence.tar.gz": "f3b5d6b4c9065362d3d59cfe2339363700137a2aec1cbe865217a3e5f6120acc" + }, + "workloads": { + "mnist": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "control": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 2840, + "precondition_group_passes": 2840, + "actual_decisions": 2840, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 3880, + "precondition_group_passes": 3880, + "actual_decisions": 3760, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 1, + "opportunities": 2840, + "precondition_group_passes": 2801, + "actual_decisions": 2801, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 2, + "opportunities": 3880, + "precondition_group_passes": 3802, + "actual_decisions": 3682, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347106 + ], + "thread": [ + 140498110309440 + ] + }, + "control": { + "process": [ + 1347133 + ], + "thread": [ + 140011360134208 + ] + }, + "holdout": { + "process": [ + 1347155 + ], + "thread": [ + 140714782577728 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 21, + "healthy_decisions": 40, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "architecture": "exact Net class" + }, + "parameter_count": 1199882, + "precision": "float32", + "extracted_symbols": [ + "Net" + ], + "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", + "extracted_definitions_sha256": "f839142a57ce449054e05ffa79d746e5282e2599e101810e6ccf6e7a9dd33ce5", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "count": 40, + "files": [ + { + "path": "train-images-idx3-ubyte", + "sha256": "ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db" + }, + { + "path": "train-labels-idx1-ubyte", + "sha256": "65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5" + } + ], + "kind": "cached_mnist_training_prefix", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + "ordering": "first samples in IDX order; no shuffle" + }, + "trace_rows": { + "reference": 1584, + "control": 1584, + "holdout": 3104, + "fault": 3104 + }, + "failed_inference_hypotheses": 20, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "e513882d9150e0f71fd1389dc3a2c21901a3098c44475d207856fb0f0c0020d3", + "baseline-holdout/metrics.json": "b6fed469dd317b29f34b26a0be43035ce046483277b4b2ec0786deed72a4a445", + "baseline-holdout/phase-result.json": "57a7f3f7877aec96c5a7581b1bc3934e3aca82521719c35646b95033d56b0f34", + "check-control/coverage.json": "f07b558285dbc44eafccaea64f6f812f9b3ed3f52a8155b22e0c8fdc649a4a6b", + "check-control/phase-result.json": "3f19692915fe7560759742d1bc0e50d40a74f7c0eed628e36621e83b06dcdf3b", + "check-fault/coverage.json": "d3cc7954b4b07506c319bfb36ef8b60205392a02ceaaa8b4e5a17b2a8077097e", + "check-fault/phase-result.json": "d382c3a28c73aec5594cb5e998b58c1a32f858ba11b6126fd6479919acb3fbc4", + "check-holdout/coverage.json": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "check-holdout/phase-result.json": "87586fb7106d5425b88dce9d696c661773b29bcf5daa123fefb57d17176f6cf7", + "check-holdout-direct/coverage.json": "29d316bf84d3de6dec6d2441c8a4f52e6a03a953e1f16182cf1255ae31a5fde0", + "check-holdout-direct/phase-result.json": "4c978a879f1af1e88595bbb7126ed8da69dc566598fe0a8958aa149d9cb24523", + "check-reference/coverage.json": "b30b981c7a88357ab883d3cde688e88d40c784204b474418d10d42a8643f99b8", + "check-reference/phase-result.json": "de96730dcb4d33a6760189b98824f0e4d5728bb9488984f7a2d69e36920c8d15", + "comparison.json": "e9feb79f1954ac3f5dc7a75e703a6bd00b29f4b59a6519d268f801e8a9a0697e", + "control/adaptation.json": "5abdf5a1f238cbd0e8a83ed88a908698e8b8cdb460a466f043f558950e7532af", + "control/metrics.json": "61e93a78e4612d6201decf311dde5b726d62405c5772f541c3ced16102226788", + "control/phase-result.json": "8c7a42816ea67cf150fe037715e14a64437f7519c803460183e8a26909f26c3c", + "fault/mutation.json": "c4b7db0e8d68453fb5cb89611573db5aec1773ebaca0ada67f866938e0da1a0a", + "holdout/adaptation.json": "2d4c4079c071a0500d8f08e734096b767c7da1998ebd667f4ace6b1c97d14f6d", + "holdout/metrics.json": "30e4e104d56b2a9bb5218e3b891dbab483362a435b1f3f3455fb3aba33bd6218", + "holdout/phase-result.json": "b00787f9a5b65f2b8b724ef6e38b71b126ec0bb0c19897806e8ae9fd24e4c754", + "inference/phase-result.json": "cec09cc23d492a1a5fa910ee69412d210b0cc47ca5b6dfe6ac878570f6043ad3", + "prepare-fault/phase-result.json": "46590662108731e2fac6096653c72c9e2dc6ddb51355c82bae69d5119993dd56", + "reference/adaptation.json": "8204e4744995631ee4982033a3949d1b59c99fa58cdf624031c250610dac8603", + "reference/metrics.json": "31a6c6e58a0ead22b57ad69584672e956491dc54872f14b8a20c08968df444b0", + "reference/phase-result.json": "d511dfe65964b24f114852e0ad8acfe36a016dc43aa4833e8c537f82fe23cb02", + "study-plan.json": "bd5d1b4ec4ffa8ef68813ef0f781de6b3cfa9790b323e4d98447cc8de501af87" + }, + "invariants_jsonl_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "trace_sha256": { + "control/traces/trace_API_1347133_140011360134208.log": "e68a120d1713eeec0fb916a7938ee8b2ee2088ff16f9e2c41e89f1291cbee8d2", + "fault/traces/trace_API_1347155_140714782577728.log": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa", + "holdout/traces/trace_API_1347155_140714782577728.log": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "reference/traces/trace_API_1347106_140498110309440.log": "c4800bbf95b3ef0fa367e107ca621beaba9768dc6a6c07fcc4e9439246da20be" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "dtype", + "invariant_index": 21, + "invariant": { + "text_description": "Conv2d.forward() consistently returns tensors with dtype=torch.float32", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.conv.Conv2d.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "dtype", + "const_value": "torch.float32" + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 20, + "num_negative_examples": 0 + }, + "func_call_id": "9d110c7f11d845f9ad86b2a2e542b000_23960437373776510", + "edits": [ + { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + ], + "traces": { + "trace_API_1347155_140714782577728.log": { + "original_sha256": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "fault_sha256": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa" + } + }, + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "healthy_coverage_sha256": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-mnist.jsonl", + "sha256": "59a8cf816bd4474ed4505d23959d52a7d050ef01ea222471f8fc3f1020d27bbd", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + } + }, + "transformer": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 0, + "opportunities": 1100, + "precondition_group_passes": 1100, + "actual_decisions": 1100, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "control": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 1100, + "precondition_group_passes": 1082, + "actual_decisions": 1082, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 12020, + "precondition_group_passes": 12020, + "actual_decisions": 12020, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 16900, + "precondition_group_passes": 16900, + "actual_decisions": 16900, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 1, + "opportunities": 12020, + "precondition_group_passes": 12001, + "actual_decisions": 12001, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 1, + "opportunities": 16900, + "precondition_group_passes": 16881, + "actual_decisions": 16881, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347485 + ], + "thread": [ + 140490739766336 + ] + }, + "control": { + "process": [ + 1347544 + ], + "thread": [ + 140262258558016 + ] + }, + "holdout": { + "process": [ + 1347647 + ], + "thread": [ + 139745137321024 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 84, + "healthy_decisions": 20, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "clip": 0.25, + "dropout_argument": 0.2, + "lr": 1.0, + "nhead": 2, + "nhid": 64, + "ninp": 32, + "nlayers": 2, + "ntoken": 293, + "sequence_length": 16 + }, + "parameter_count": 36197, + "precision": "float32", + "extracted_symbols": [ + "PositionalEncoding", + "TransformerModel" + ], + "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", + "extracted_definitions_sha256": "1f2db95454690be6a91aaa141e3a5a246308f9abca11444c9113e4adfb78b95d", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": "train.txt", + "prefix_sha256": "457b31f0b69fdfbf48c919d7fa9a7b073c2b03b8adfa51de3543e109b0e01903", + "sha256": "9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f", + "token_count": 642, + "vocabulary_size": 293 + }, + "trace_rows": { + "reference": 4446, + "control": 4446, + "holdout": 8886, + "fault": 8886 + }, + "failed_inference_hypotheses": 11, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "d73ea9cff0d15217963372d4c6b18eba13e0eaa4e03d45aa627ff9c753597a32", + "baseline-holdout/metrics.json": "f9d111863e26ff2023405c41cf862513d8ea167464ad4f7c10989abb257d669d", + "baseline-holdout/phase-result.json": "02e5f2576d7223c059e159f624d6e5030f3d08c9bab8391ee2e9bd051d1af4ef", + "check-control/coverage.json": "2fcdad5051bf4679d3663a0aaf86048a236e971a80517f33890a7f56b17f64c0", + "check-control/phase-result.json": "58d12a56e1be8fb59638f8ab13cb6257958be989bda2e0028d900a28dc3e64c6", + "check-fault/coverage.json": "a89b68a24e46228872553b8ee6fed51984e172e52651583be2ac4f23330d6d6d", + "check-fault/phase-result.json": "e063ef35ebd7eea7accff6e046e5fc2ca7650e4c94e3c3c2af7f69f8c99b114e", + "check-holdout/coverage.json": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "check-holdout/phase-result.json": "4807c3c7edd8e49d87150c29a3bd8aeee71992d21e08718c68976da3967c6d89", + "check-holdout-direct/coverage.json": "c060b0f9ef7da9002198017625d56876950518c5f4072d39d75e3982cc5019dd", + "check-holdout-direct/phase-result.json": "86455b01be33c9798433c7be940fc8bad767789e95852a0014fdabdbce1b34a9", + "check-reference/coverage.json": "771e09f0692dee2230d8e073d515945ebc74292b61baa04d830facb049284af2", + "check-reference/phase-result.json": "fde3c128ccc040852965261555e0d65b543a9360268f46a443cc2c8382a06eb7", + "comparison.json": "ce41f5fa74255424fb7c87344fbf3ba540629004f6e37662e2b5509444fed201", + "control/adaptation.json": "f1f9d3cebb92dc9a53a1810fd7ac2b9784f826359010fe8669209f17a2d6afef", + "control/metrics.json": "c91ee13d398cad807b7a62841f47e82d37492d49c23b961dcdb28a68ca7503f9", + "control/phase-result.json": "4d8a83a3b84ed1cc11c745392b8c3cdd5d34b395a99a4fb732c7449582de372a", + "fault/mutation.json": "cadfcc8f26752d903f5d05170cb38ff320b66f431e46518b190c8e3f0d7cd444", + "holdout/adaptation.json": "d9162c4392cf9a64111ed97779344ceda32f74e02bcf4c1274b2e14e2d7ec6da", + "holdout/metrics.json": "17a4d6ff99936f05e49552b26569ae8309955ff7088d69c3130327bed2b61d6c", + "holdout/phase-result.json": "2bb9f6cc8ffda717bb89688697da7889ce5855965c9e7f3b5087000b4c06e8d8", + "inference/phase-result.json": "15c4fbd12e3cf9609dd9be61491fd8a9f04e5b9813f55eed0656766b1a538d7c", + "prepare-fault/phase-result.json": "9b90cb87a59c3e12fb99dd819276f8b85231c431d0566a18ba88559c51b30cfb", + "reference/adaptation.json": "6078c8beafec0759c2742020cf94899ded01b5ebcbfb633d34ed6bb982d79c5f", + "reference/metrics.json": "710ef8a517ffd905f03efb88a59e35f2c7353b20d512d75e48f331738cd87b57", + "reference/phase-result.json": "25adbbf0eb426c585ced6ef4605d2d227bcda0b5c7079b9ceafa4908d1a89780", + "study-plan.json": "9d065aae89605f2fe63b299f15164f07275980c775bfe993c7baa12a6a0a9534" + }, + "invariants_jsonl_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "trace_sha256": { + "control/traces/trace_API_1347544_140262258558016.log": "440ad458b125167c19146e5043ab355e236067dae7df623a8abd992e16b3df96", + "fault/traces/trace_API_1347647_139745137321024.log": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55", + "holdout/traces/trace_API_1347647_139745137321024.log": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "reference/traces/trace_API_1347485_140490739766336.log": "4184c9abeacc8c3c49a583038f507a3b0c3fb7218733230f34530df54003f60a" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "shape", + "invariant_index": 84, + "invariant": { + "text_description": "Embedding.forward() consistently returns tensors with shape=(16, 2, 32)", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.sparse.Embedding.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "shape", + "const_value": [ + 16, + 2, + 32 + ] + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "func_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502411558452", + "edits": [ + { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + ], + "traces": { + "trace_API_1347647_139745137321024.log": { + "original_sha256": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "fault_sha256": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55" + } + }, + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "healthy_coverage_sha256": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "healthy_false_alarm_evidence": [ + { + "index": 523, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module._named_members()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module._named_members", + "arguments": { + "args": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "get_members_fn": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "func_name": "torch.nn.modules.module.Module._named_members" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884819012", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769353813", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502638163694", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + } + } + }, + { + "index": 525, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module.named_parameters()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module.named_parameters", + "arguments": { + "args": { + "recurse": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "func_name": "torch.nn.modules.module.Module.named_parameters" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884631538", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769166145", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637977940", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + } + } + } + ], + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-transformer.jsonl", + "sha256": "2e7ec6dbc35408bb1a39bff7be96943e00a64cf9d464c6091d54f275d06534f9", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + }, + "literal_object_ID_argument_false_alarm_rules": 2 + } + }, + "limitations": [ + "One reference/control/holdout per workload; no population generalization estimate.", + "Fresh initialization and training RNG with longer horizon; not disjoint-dataset validation.", + "Offline checking; no online, distributed, performance, or inference-memory claim.", + "Faults alter recorded metadata only, not real training execution.", + "Active means at least one actual predicate decision; it does not mean every opportunity was evaluated.", + "Checker stops at first violation per rule; violation counts are failing invariants, not all bad events.", + "No raw process/thread clauses were inferred here; original TorchTitan literal-thread-ID rules remain untested.", + "The 74b3adf MNIST attempt stopped after healthy phases because its shape-only fault selector had no eligible rule; preserved separately." + ], + "metric_semantics": { + "identity_bearing_invariants": "Frozen checker counter means raw process/thread-ID PRECONDITION clauses only; excludes object IDs in API argument selectors.", + "identity_bearing_active": "Active rules among that same narrow PID/TID-precondition group.", + "fully_validated_active": "At least one completed decision and no requested proxy-check skip; does not mean all opportunities were evaluated." + } +} diff --git a/docs/measurements/data/fresh-validation-contracts-before-after.json b/docs/measurements/data/fresh-validation-contracts-before-after.json new file mode 100644 index 00000000..f5120925 --- /dev/null +++ b/docs/measurements/data/fresh-validation-contracts-before-after.json @@ -0,0 +1,35 @@ +[ + { + "fixture": "renamed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "malformed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "catching_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + } +] diff --git a/docs/measurements/data/fresh-validation-hardening-integration.json b/docs/measurements/data/fresh-validation-hardening-integration.json new file mode 100644 index 00000000..9b2147cf --- /dev/null +++ b/docs/measurements/data/fresh-validation-hardening-integration.json @@ -0,0 +1,186 @@ +{ + "scope": "Frozen saved-study observer verification; no new training or performance claim", + "harness_source_commit": "f3804c0386454ddd1f97bf17b7a1ab9287cd23d2", + "source_composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + }, + "git_history": "No local merge of #144 into #148; updated production source validated in an isolated snapshot", + "schema_version": 2, + "comparison": "Every original per-invariant field, summary field, raw trace PID/TID, input hash, row count and checking mode preserved; additive schema fields allowed. JSON NaN tokens compare as retained NaN, without deleting/coercing fields. Only operational PID/runtime-source and harness-provenance fields are excluded from semantic equality. Runtime source path verified explicitly.", + "tests": { + "composed_production_linux": 140, + "harness_linux": 28, + "stronger_mypy_1_9_files": 9 + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "per_rule_reports": 3072, + "runs": [ + { + "workload": "mnist", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 3, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 4, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + } + ] +} diff --git a/docs/measurements/fresh-run-invariant-validation.md b/docs/measurements/fresh-run-invariant-validation.md new file mode 100644 index 00000000..80b02f75 --- /dev/null +++ b/docs/measurements/fresh-run-invariant-validation.md @@ -0,0 +1,144 @@ +# Fresh-process invariant validation + +Fresh-process checking works for the tested MNIST rules, but exposes two object-ID +false alarms in the small Transformer rules. This is a reproducible offline +validation harness and a bounded result, not a generalization or performance claim. +No production code changes are included. + +| Workload | Learned rules | Reference active / violations | Fresh same-seed control active / violations | Longer changed-seed holdout active / violations | Dormant rules | Requested proxy checks skipped | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| MNIST, 1,199,882 parameters | 204 | 204 / 0 | 204 / 0 | 204 / 0 | 0 | 0 | +| Transformer, 36,197 parameters | 564 | 564 / 0 | 564 / 2 | 564 / 2 | 0 | 0 | + +Every active rule completed at least one actual relation decision. That does not +mean every candidate opportunity was evaluated: for example, MNIST holdout IO +rules have 3,880 opportunities, 3,880 precondition passes, and 3,760 comparisons +because missing values skip 120 comparisons. Rules stop on their first violation. +There were no checker errors or unsupported-coverage results. The direct checker +without observers produced identical CheckerResult structures on both holdouts. + +For both workloads, the same-seed fresh control's complete loss trajectory and +initial/final canonical parameter hashes exactly match the reference. The +instrumented 20-step holdout likewise exactly matches its uninstrumented baseline. +These comparisons apply to these CPU runs and are not a claim of universal +instrumentation transparency. + +The two Transformer alarms are unconditional APIContain rules asserting that +`clip_grad_norm_` contains `Module.named_parameters` / `Module._named_members` with +particular arguments. Both child calls are present in the failed parent's strict +time interval, on the same process/thread. Their recorded model arguments differ +only in `_TRAINCHECK_src_mask_ID`: the reference's memory-object ID was embedded +literally into both learned child selectors. Fresh processes have different IDs. +Thus both rules fail even on the same-seed control despite identical losses and +parameters. The evidence retains the frozen selectors and raw child arguments; +no IDs or learned rules were rewritten. This is an object-identity portability +failure, distinct from thread-ID precondition portability. + +The post-hoc single-field controls activate a passing frozen rule and violate it +at the edited call. MNIST changes one recorded output dtype from `torch.float32` +to `torch.float64`: one output rule plus two IO rules fail (3 new violations). +Transformer changes one output shape from `[16, 2, 32]` to `[17, 2, 32]`: two rules +newly fail, in addition to the 2 existing healthy alarms (4 total). These are +trace-metadata sensitivity controls, not injected training bugs. They establish +neither broad defect coverage nor the consistency of an alternative tensor execution. + +The [compact evidence](data/fresh-run-invariant-validation.json) records source +commits/hashes, data hashes, actual runtime, paired parameter hashes/losses, raw +process/thread IDs, fault edits, family totals, and explicit failed-attempt history. +The [MNIST ledger](data/fresh-run-invariant-validation-mnist.jsonl) and +[Transformer ledger](data/fresh-run-invariant-validation-transformer.jsonl) retain +every rule's opportunities, group-level precondition counts, completed decisions, +visited/unvisited counts, violations, and zeros for all four checked traces. +Their run arrays use the evidence JSON's `coverage_columns`; null means not +applicable/unobserved. `fully_validated_active` means an active rule without a +requested proxy-check skip; it does not promise all candidates were checked. + +Final runs use signed harness `cad62cc98ef5fb7b313325443caa33f715ed141e`, Python +3.11.4, PyTorch `2.16.0.dev20261012+cpu`, pandas 2.3.3, and NumPy 2.1.0. Thirteen +focused counter/mutation tests pass locally and on that Linux runtime; 29 composed +production-stack tests also pass there. Both complete study commands exited 0, +with all input/source immutability and comparison gates satisfied. The +`study_completed` status means those gates passed; it does not erase the two +healthy Transformer violations. + +The initial workloads are benchmark-derived MNIST and a small Transformer. +`benchmark_workload.py` extracts the named model definitions from a supplied +TrainCheck-Benchmarks checkout; it never executes the benchmark's top-level CLI +or downloads data. MNIST requires cached real IDX data. The Transformer uses the +checked-in WikiText prefix. See `tools/fresh_validation/ADAPTERS.md` for the original +adaptations. The fixed `data_steps=20` prepares the same corpus/vocabulary/model +shape for every run: reference/control train 10 steps with seed 42; holdout and its +uninstrumented baseline train 20 steps with seed 43. The first half of holdout inputs +overlaps reference inputs; this is not disjoint-dataset validation. + +`prepare_source.py` creates an isolated snapshot from signed source revisions: +identity/cache stack 99d1690, tracing integrity 7d6c683, TorchScript 9ac61d1, +precondition-state fix 1b5aedf (only its two production hunks), and dynamic type-name +fix 2f034ae. The existing deterministic sampling/reservoir changes are retained. +The harness PR contains no production-code change. Earlier failed setup and +historical-compatibility-patch pilots remain separate from this final recipe. + +Example from a checkout containing those commits, on Linux with an existing +compatible CPU PyTorch runtime and benchmark/data files: + +```sh +python tools/fresh_validation/prepare_source.py --repo "$PWD" --output /path/to/new-source +python tools/fresh_validation/run_study.py \ + --source /path/to/new-source --python /path/to/runtime/bin/python \ + --benchmark-root /path/to/TrainCheck-Benchmarks --data-root /path/to/MNIST-cache \ + --output /path/to/new-study --workload mnist +# Use --workload transformer and omit --data-root for the language-model adapter. +``` + +Each phase starts a new process, with native threads limited to one. A monitor +polls every 250 ms and stops the process group after a 600-second timeout, +8 GiB process-group RSS threshold, or 256 MiB API-trace threshold. These are +sampled monitor thresholds, not OS hard resource limits. Inference is +single-worker and disables FunctionLeadRelation/FunctionCoverRelation. These +bounds and observed phase times are operational controls, not performance data. + +Coverage calls the real `inv.check(trace, check_relation_first=False)`. It counts +precondition group verifications separately from actual relation decisions using +AST-located line observations of the installed checker. API opportunities are +calls; DistinctArgument opportunities are within-step argument pairs. The checker +stops at the first violation per invariant, so unvisited remainders are explicit. +A `triggered` flag or zero violations alone does not establish active checking. +Missing tensor values, dormant rules, checker errors, unsupported coverage, and +requested-but-skipped proxy-variable checks are distinguished. Direct checking +without observers must produce exactly the same CheckerResult structures. + +Raw process/thread IDs and frozen predicates are preserved. ID-bearing clause +kinds/values and activation are reported; their mere presence does not establish +that an ID caused dormancy. A workload with no ID-bearing rules leaves the +original TorchTitan rule set's portability unresolved. Neither new workload has +any process/thread-ID-bearing inferred clause. The frozen `identity_bearing_*` +counters measure only PID/TID precondition clauses, not object IDs in API arguments. In particular, these results do not +validate the original 836-rule corpus's three literal-thread-ID APIContain rules +or its fourth nonliteral thread-inequality rule. + +A post-hoc sensitivity control selects a healthy, actually evaluated output-shape +rule, or an output-dtype rule when no eligible shape rule exists. It changes one +recorded shape dimension or float32/float64 dtype value, retaining its JSON type, +IDs and precondition inputs. Every other semantic field and every other raw line +must remain identical. The frozen checker must activate and violate the selected +rule. This is trace-level detector-path validation, not a reproduced training bug +or a general defect-coverage result. Dtype mutation changes recorded metadata only; +the tensor type-name key remains unchanged, so it does not represent a complete +alternative tensor execution. The original 74b3adf MNIST run completed its healthy +phases but stopped at the shape-only selector; that diagnostic remains separate +from fresh runs with the revised selector. + +`compare_study.py` verifies source/input immutability, fresh producer PIDs, exact +same-seed control losses and canonical parameter hashes, instrumented-versus-plain +holdout losses/hashes, observer/direct equality, and the targeted fault response. +Its `study_completed` status certifies these evidence gates. It does not assert +that every invariant passed the healthy runs; those violations are retained for +analysis. Per-family reports distinguish all active decisions from rules whose +requested checks were fully executed. + +Focused counter tests: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + python -m unittest tests.test_fresh_validation_coverage tests.test_fresh_validation_fault -v +``` diff --git a/docs/measurements/fresh-validation-contract-hardening.md b/docs/measurements/fresh-validation-contract-hardening.md new file mode 100644 index 00000000..850bba25 --- /dev/null +++ b/docs/measurements/fresh-validation-contract-hardening.md @@ -0,0 +1,120 @@ +# Fresh validation: typing and report contracts + +This hardens the reusable harness introduced in PR #148. The comparison base is +`27e67aafc1a526ed8e69e0a8e59ff30b8ff827cb`. Frozen predicates, training workloads, +and the original measured results are unchanged by this pass. + +## Problem and change + +All 36 original helper functions had incomplete type signatures. Worker/CLI +inputs, model factories, phase results, mutation reports, checker observation +state, and study comparisons now have explicit contracts. The eight helper +modules now contain 61 fully annotated function definitions, including boundary +validators. The new contract tests are annotated too. + +JSON enters through `object` and is narrowed at required-field boundaries. +Missing fields, wrong types, out-of-range invariant indexes, incomplete phases, +and loss rows inconsistent with the claimed step count fail explicitly. The +single `cast` documents a checked string-keyed, read-only mapping boundary; +there are no new `Any` annotations or type-checker ignores. + +The observer now requires each family's exact identity field. API containment +requires `parent_func_call_id`; the three single-call consistency/threshold +families require `func_call_id`; distinct-argument decisions require both event +records and their string call IDs. It never substitutes another family's field +or `None` when checker source changes. The proxy-skip flag is also required when +that precondition is requested. + +`ObservationReport` uses optional detail fields while the mutable report is being +constructed. `finalize_report` enforces the conditional schema before returning +or aggregating it: completed checks require their checker payload and boolean +outcome; observed coverage requires counters, groups, decisions, source metadata, +and the family-specific call/pair fields. A captured observer failure remains +primary even if the checker catches it and later returns or raises a different +exception. The latter exception remains available as secondary error evidence. + +The benchmark adapter validates dynamically loaded model classes and initializer +callbacks. Its helper-import bootstrap appends the harness path, preserving the +separately composed TrainCheck source selected by `PYTHONPATH`. + +## Report compatibility + +Coverage output now declares `schema_version: 2`. Existing completed-check fields, +checker payloads, raw identities, and decision records keep their meaning. +`coverage_status` distinguishes `observed`, `disabled`, `unsupported`, and `error`. +`observer_error` is separate from `checker_error`; either causes a nonzero CLI +result rather than a successful validation. + +Aggregate active/dormant/identity-active totals are `null` when any coverage is +unobserved. `coverage_unobserved_invariants` and `observed_*` subset counts explain +that limit. Likewise, unknown checker outcomes produce a `null` violation total, +with `check_outcome_unknown_invariants` and `observed_violated_invariants` retained. +A direct checker run can have a known outcome while its coverage remains unknown. +These are deliberate schema corrections, not new performance measurements. + +## Evidence + +The [before/after fixtures](data/fresh-validation-contracts-before-after.json) run +both original and hardened observers against three controlled checker-source +changes: a renamed identity local, a `None` identity, and a checker that catches +the observer exception. Originally all three were labeled `completed` and emitted +one `func_call_id: null` decision. After hardening, all three report +`observer_error` and emit no fabricated decisions: 3/3 false completions become +0/3. A fourth regression covers a subsequent different checker exception. + +Validation on Python 3.10.16, Torch 2.2.2, and Pandas 2.2.3: + +- All 28 targeted tests pass: the 13 original checker/fault tests and 15 contract + tests, including observer/direct parity, dormant rules, early failure, + required identities, malformed reports, and composed-source import precedence. +- Pinned Mypy 1.9.0 checks all eight helper modules and the new test module with + `--check-untyped-defs --disallow-untyped-defs --disallow-any-generics + --warn-return-any --explicit-package-bases --follow-imports=silent + --ignore-missing-imports`. Its `--python-executable` points at the installed + TrainCheck runtime, so Torch annotations resolve. This is a stronger scoped + check, not a claim of repository-wide `--strict` compliance. +- Configured Ruff 0.3.3, Black 24.3.0, and isort 5.10.1 hooks pass on the changed + paths. The configured mypy hook does not select `tools/`; the explicit command + above covers that gap. `git diff --check` passes. + +Run the focused tests with: + +```sh +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 \ +python -m unittest tests.test_fresh_validation_coverage \ + tests.test_fresh_validation_fault tests.test_fresh_validation_contracts -q +``` + +The source-builder recipe remains pinned to its original, reproducible composition. +Full saved-trace replay and the newly composed PR stack require separate integration +validation. This change makes no new training-overhead or generalization claim. + +## Updated production-stack integration + +An isolated snapshot combines signed inference `ce25c12`, tracing `b64023b`, +and only the #145 production state-isolation patch from `aec748e`, plus the exact +signed harness source `f3804c0`. It does not change this PR's Git history or its +reproducible original source-builder recipe. The combined production passes 140 +Linux regressions; all 28 harness regressions also pass against that source. +Stronger Mypy 1.9 checks pass all nine harness/test files against the updated +production interfaces with actual Torch type resolution. + +All eight frozen saved-study checker runs (reference, same-seed control, +holdout, and controlled fault for each workload) complete successfully with +unchanged sources/inputs and the expected production module path. Every original +semantic field in **3,072 per-rule reports** matches, including decisions, +coverage, raw trace IDs, checker payloads, and summary counts. Additive schema-2 +fields are allowed; meaningful fields and array order are never dropped. Retained +JSON NaN tokens compare as NaN rather than using Python's non-reflexive float +equality. Operational process IDs and changed harness/source provenance are +separate from semantic equality. + +| Workload | Active rules in each run | Reference violations | Control | Holdout | Fault | +|---|---:|---:|---:|---:|---:| +| MNIST | 204 | 0 | 0 | 0 | 3 | +| Transformer | 564 | 0 | 2 | 2 | 4 | + +The Transformer's two documented healthy false alarms remain visible. These are +saved-input checker replays, not new model training or a generalization claim. +[Compact integration evidence](data/fresh-validation-hardening-integration.json) +records exact source composition, runtime, counts, and comparison rules. diff --git a/tests/test_fresh_validation_contracts.py b/tests/test_fresh_validation_contracts.py new file mode 100644 index 00000000..1379c61d --- /dev/null +++ b/tests/test_fresh_validation_contracts.py @@ -0,0 +1,277 @@ +"""Negative-path contracts for emitted reports and observed call identities.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import ( + ObserverContractError, + decision_record, + finalize_report, + observe, + summarize, +) +from tools.fresh_validation.compare_study import compare, losses +from tools.fresh_validation.contracts import boolean, integer, json_object, required +from tools.fresh_validation.fault_control import digest, mutate +from traincheck.invariant.base_cls import CheckerResult, Invariant +from traincheck.invariant.consistency_transient_vars import ConsistentOutputRelation +from traincheck.trace.trace import Trace + + +def renamed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + # This still has the exact output-predicate AST but its local identity was + # renamed. The previous observer silently emitted {func_call_id: None}. + for renamed_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def malformed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for real_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(real_call) + func_call_id = None + returned_tensors = [func_call_event, func_call_id] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def catching_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + # The observer must retain failure even when the checker catches it. + continue + return CheckerResult(None, inv, True, True) + + +def catching_then_failing_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + pass + raise RuntimeError("checker failed after catching observer error") + + +class UnsupportedOutput(ConsistentOutputRelation): + pass + + +class ValidationContractTests(unittest.TestCase): + def test_required_family_identity_does_not_use_another_family_fallback( + self, + ) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "ConsistentOutputRelation", {"parent_func_call_id": "parent"} + ) + with self.assertRaisesRegex(ObserverContractError, "parent_func_call_id"): + decision_record("APIContainRelation", {"func_call_id": "child"}) + + def test_pair_requires_both_record_identities(self) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "DistinctArgumentRelation", + {"event1": {"func_call_id": "a"}, "event2": {}}, + ) + + def test_identity_wrong_types_fail_explicitly(self) -> None: + for identity in (None, False, 42, ["call"]): + with ( + self.subTest(identity=identity), + self.assertRaises(ObserverContractError), + ): + decision_record("ConsistentOutputRelation", {"func_call_id": identity}) + + def test_checker_source_drift_is_observer_error_not_successful_null_identity( + self, + ) -> None: + for checker in ( + renamed_local_checker, + malformed_local_checker, + catching_local_checker, + ): + with ( + self.subTest(checker=checker.__name__), + patch.object(ConsistentOutputRelation, "static_check_all", checker), + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertIsNone(report["check_passed"]) + self.assertIsNone(report["active"]) + self.assertFalse(report["coverage_observed"]) + self.assertEqual(report["decision_records"], []) + + def test_observer_failure_remains_primary_if_checker_then_raises(self) -> None: + with patch.object( + ConsistentOutputRelation, "static_check_all", catching_then_failing_checker + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertFalse(report["coverage_observed"]) + self.assertIsNone(report["active"]) + self.assertEqual(report["error"]["type"], "ObserverContractError") + self.assertEqual(report["checker_error_after_observer"]["type"], "RuntimeError") + + def test_disabled_and_unsupported_counts_are_unknown_not_zero(self) -> None: + trace = frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]) + direct = observe(trace, output_rule(), enabled=False) + rule = output_rule() + rule.relation = UnsupportedOutput + unsupported = observe(trace, rule) + for report in (direct, unsupported): + summary = summarize([report]) + self.assertIsNone(summary["active_invariants"]) + self.assertIsNone(summary["dormant_invariants"]) + self.assertEqual(summary["coverage_unobserved_invariants"], 1) + self.assertEqual(summary["observed_active_invariants"], 0) + self.assertEqual(direct["coverage_status"], "disabled") + self.assertEqual(unsupported["coverage_status"], "unsupported") + self.assertIsNone(summarize([unsupported])["violated_invariants"]) + self.assertEqual(summarize([direct])["violated_invariants"], 0) + + def test_completed_report_requires_checker_payload(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["checker_result"] + with self.assertRaisesRegex(ObserverContractError, "checker_result"): + finalize_report(report) + + def test_observed_report_requires_decision_counters(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["decisions_completed"] + with self.assertRaisesRegex(ObserverContractError, "decisions_completed"): + finalize_report(report) + + def test_checker_error_does_not_imply_zero_violations(self) -> None: + rule = output_rule() + with patch.object( + rule, "check", side_effect=ValueError("fixture checker error") + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + rule, + enabled=False, + ) + self.assertEqual(report["status"], "checker_error") + self.assertIsNone(summarize([report])["violated_invariants"]) + self.assertEqual(summarize([report])["observed_violated_invariants"], 0) + + def test_cli_bootstrap_preserves_composed_traincheck_source(self) -> None: + adapter = ( + Path(__file__).resolve().parents[1] + / "tools/fresh_validation/benchmark_workload.py" + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + composed = root / "composed" + (composed / "traincheck").mkdir(parents=True) + module = composed / "traincheck/__init__.py" + module.write_text("SOURCE = 'fixture'\n") + code = ( + "import runpy; runpy.run_path(" + + repr(str(adapter)) + + "); import traincheck; print(traincheck.__file__)" + ) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=root, + env={ + **os.environ, + "PYTHONPATH": str(composed), + "PYTHONDONTWRITEBYTECODE": "1", + }, + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(Path(result.stdout.strip()), module) + + def test_json_required_fields_and_wrong_types_are_not_coerced(self) -> None: + with self.assertRaisesRegex(ValueError, "required"): + required({}, "required") + with self.assertRaises(ValueError): + boolean("false", "active") + with self.assertRaises(ValueError): + integer(True, "count") + with self.assertRaises(ValueError): + json_object({"invalid": object()}) + + def test_empty_loss_records_do_not_establish_equality(self) -> None: + for steps in ([], [{"step": 1}]): + with self.subTest(steps=steps), self.assertRaisesRegex(ValueError, "loss"): + losses({"steps": steps}) + + def test_truncated_loss_rows_do_not_match_claimed_completed_steps(self) -> None: + with self.assertRaisesRegex(ValueError, "completed_steps"): + losses({"steps": [{"step": 1, "loss": 1.0}], "completed_steps": 2}) + + def test_incomplete_phase_set_cannot_report_study_completed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(ValueError, "Missing completed phases"): + compare(Path(temporary)) + + def test_fault_index_cannot_silently_select_last_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, rules, coverage = ( + root / name for name in ("trace.jsonl", "rules.jsonl", "coverage.json") + ) + trace.write_text( + json.dumps({"type": "function_call (post)", "func_call_id": "a"}) + "\n" + ) + rules.write_text(json.dumps(output_rule().to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": {str(p): digest(p) for p in (trace, rules)}, + "per_invariant": [{"invariant_index": -1}], + } + ) + ) + with self.assertRaisesRegex(ValueError, "out of range"): + mutate([trace], rules, coverage, root / "fault") + self.assertFalse((root / "fault").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_validation_coverage.py b/tests/test_fresh_validation_coverage.py new file mode 100644 index 00000000..a0519e42 --- /dev/null +++ b/tests/test_fresh_validation_coverage.py @@ -0,0 +1,278 @@ +"""Small real-checker counter oracles, including dormant and early-exit cases.""" + +import copy +import logging +import unittest + +import pandas as pd + +from tools.fresh_validation.check_coverage import observe +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.invariant.base_cls import ( + PT, + APIParam, + GroupedPreconditions, + InputOutputParam, + Invariant, + Precondition, + PreconditionClause, + Preconditions, + UnconditionalPrecondition, + VarTypeParam, +) +from traincheck.invariant.consistency_transient_vars import ( + ConsistentInputOutputRelation, + ConsistentOutputRelation, + ThresholdRelation, +) +from traincheck.invariant.contain_relation import ( + APIContainRelation, + cache_events_scanner, +) +from traincheck.invariant.DistinctArgumentRelation import ( + EXP_GROUP_NAME, + DistinctArgumentRelation, +) +from traincheck.trace.trace_pandas import TracePandas + + +def frame(outputs, *, steps=None, inputs=None): + rows = [] + for i, output in enumerate(outputs): + tensor = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + common = { + "func_call_id": f"call-{i}", + "function": "torch.fake", + "process_id": 1, + "thread_id": 2, + "is_bound_method": False, + "obj_id": None, + "meta_vars.step": steps[i] if steps else i, + "args": {"0": copy.deepcopy(tensor if inputs is None else inputs[i])}, + "kwargs": {}, + "return_values": copy.deepcopy(output), + } + rows += [ + {**common, "type": TraceLineType.FUNC_CALL_PRE, "time": i * 3}, + {**common, "type": TraceLineType.FUNC_CALL_POST, "time": i * 3 + 1}, + ] + return TracePandas(pd.DataFrame(rows), truncate_incomplete_func_calls=False) + + +def precondition(group="pre_event", enabled=True): + return GroupedPreconditions( + {group: Preconditions([UnconditionalPrecondition()] if enabled else [])} + ) + + +def output_rule(enabled=True): + return Invariant( + ConsistentOutputRelation, + [ + APIParam("torch.fake"), + VarTypeParam("torch.Tensor", "dtype", const_value="torch.float32"), + ], + precondition(enabled=enabled), + "fixture output dtype", + ) + + +class CoverageTests(unittest.TestCase): + def setUp(self): + logging.disable(logging.CRITICAL) + self.good = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + + def assert_same_result(self, make_trace, inv): + cache_events_scanner.clear() + observed = observe(make_trace(), inv) + cache_events_scanner.clear() + plain = observe(make_trace(), inv, enabled=False) + self.assertEqual(observed.get("checker_result"), plain.get("checker_result")) + self.assertIsNone(plain["active"]) + self.assertIsNone(plain["dormant"]) + self.assertEqual(observed["status"], "completed", observed) + return observed + + def test_healthy_real_decisions(self): + got = self.assert_same_result(lambda: frame([self.good] * 3), output_rule()) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (3, 3, 3), + ) + self.assertTrue(got["active"]) + + def test_all_preconditions_false_are_dormant(self): + got = self.assert_same_result( + lambda: frame([self.good] * 3), output_rule(False) + ) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_failures"], + got["decisions_completed"], + ), + (3, 3, 0), + ) + self.assertTrue(got["dormant"]) + + def test_missing_io_value_passes_precondition_but_runs_no_predicate(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("missing",), None, False), + ], + precondition(), + "missing value", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual(got["precondition_group_passes"], 1) + self.assertEqual(got["decisions_completed"], 0) + self.assertTrue(got["dormant"]) + + def test_distinct_single_call_has_zero_pairs_despite_triggered(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (0, 1, 0), + ) + self.assertTrue(got["checker_triggered"]) + self.assertTrue(got["dormant"]) + + def test_first_failure_leaves_unvisited_remainder(self): + wrong = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float64"}} + got = self.assert_same_result( + lambda: frame([wrong, self.good, self.good]), output_rule() + ) + self.assertEqual( + ( + got["opportunities"], + got["decisions_completed"], + got["unvisited_calls_after_stop"], + ), + (3, 1, 2), + ) + self.assertFalse(got["check_passed"]) + + def test_missing_tensor_is_completed_output_violation(self): + got = self.assert_same_result(lambda: frame([{"int": 1}]), output_rule()) + self.assertEqual(got["decisions_completed"], 1) + self.assertFalse(got["check_passed"]) + + def test_distinct_pair_count_and_first_violation(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result( + lambda: frame([self.good] * 3, steps=[1, 1, 1]), inv + ) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + self.assertIsNone(got["unvisited_calls_after_stop"]) + self.assertEqual(got["unvisited_pairs_after_stop"], 2) + + def test_io_actual_comparison_and_failure(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, False), + ], + precondition(), + "shape agreement", + ) + for shape, passed in (([2], True), ([9], False)): + got = self.assert_same_result( + lambda: frame([{"torch.FloatTensor": {"shape": shape}}]), inv + ) + self.assertEqual( + (got["decisions_completed"], got["check_passed"]), (1, passed) + ) + + def test_containment_counts_parent_decisions_even_when_empty(self): + cache_events_scanner.clear() + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), APIParam("torch.child")], + precondition("parent_func_call_pre"), + "requires child", + ) + got = self.assert_same_result(lambda: frame([self.good] * 3), inv) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + + def test_requested_variable_precondition_skipped_without_proxy_data(self): + conditions = precondition("parent_func_call_pre") + conditions.grouped_preconditions["var_events"] = Preconditions( + [ + Precondition( + [ + PreconditionClause( + "meta_vars.phase", str, PT.CONSTANT, None, {"train"} + ) + ] + ) + ] + ) + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), VarTypeParam("torch.Tensor", "shape")], + conditions, + "requires variable checking", + ) + + def bound_trace(): + trace = frame([self.good]) + trace.events["is_bound_method"] = True + trace.events["obj_id"] = 123 + return trace + + got = self.assert_same_result(bound_trace, inv) + self.assertTrue(got["variable_precondition_requested"]) + self.assertTrue(got["variable_check_skipped"]) + self.assertFalse(got["fully_validated_active"]) + + def test_threshold_comparison_after_successful_extraction(self): + # torch.clamp has a real signature; max=3 is the checker threshold. + def make_trace(): + trace = frame([{"torch.FloatTensor": {"max": 2}}]) + trace.events["function"] = "torch.clamp" + trace.events["kwargs"] = [{"max": {"int": 3}}] * 2 + return trace + + inv = Invariant( + ThresholdRelation, + [ + InputOutputParam("max", None, "int", None, "torch.clamp", True), + APIParam("torch.clamp"), + InputOutputParam(None, 0, "torch.Tensor", ("max",), None, False), + ], + precondition(), + "max threshold", + ) + got = self.assert_same_result(make_trace, inv) + self.assertEqual(got["decisions_completed"], 1) + self.assertTrue(got["check_passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_validation_fault.py b/tests/test_fresh_validation_fault.py new file mode 100644 index 00000000..a308506d --- /dev/null +++ b/tests/test_fresh_validation_fault.py @@ -0,0 +1,81 @@ +"""The fallback must activate the real frozen checker, without other edits.""" + +import json +import tempfile +import unittest +from pathlib import Path + +import pandas as pd + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import observe +from tools.fresh_validation.fault_control import differences, digest, mutate +from traincheck.trace.trace_pandas import TracePandas + + +class FaultControlTests(unittest.TestCase): + def run_mutation(self, attribute, shape): + rule = output_rule() + if attribute == "shape": + rule.params[1].attr_name = "shape" + rule.params[1].const_value = tuple(shape) + good = {"torch.FloatTensor": {"shape": shape, "dtype": "torch.float32"}} + traced = frame([good, good]) + healthy = observe(traced, rule) + healthy["invariant_index"] = 0 + before = traced.events.to_dict("records") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, invariants, coverage = ( + root / x for x in ("trace.log", "rules.jsonl", "coverage.json") + ) + trace.write_text("".join(json.dumps(row) + "\n" for row in before)) + invariants.write_text(json.dumps(rule.to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": { + str(p): digest(p) for p in (trace, invariants) + }, + "per_invariant": [healthy], + } + ) + ) + manifest = mutate([trace], invariants, coverage, root / "fault") + after = [ + json.loads(line) + for line in (root / "fault/traces/trace.log").read_text().splitlines() + ] + self.assertEqual(len(differences(before, after)), 1) + self.assertEqual(before[0], after[0]) + self.assertEqual(before[2:], after[2:]) + self.assertEqual(manifest["attribute"], attribute) + self.assertEqual( + manifest["edits"][0]["path"][-1], 0 if attribute == "shape" else "dtype" + ) + checked = observe( + TracePandas(pd.DataFrame(after), truncate_incomplete_func_calls=False), + rule, + ) + self.assertTrue(healthy["check_passed"]) + self.assertEqual(checked["status"], "completed") + self.assertFalse(checked["check_passed"]) + self.assertEqual(checked["precondition_group_passes"], 1) + self.assertEqual(checked["decisions_completed"], 1) + self.assertEqual( + checked["decision_records"][0]["func_call_id"], manifest["func_call_id"] + ) + self.assertTrue( + any( + row["func_call_id"] == manifest["func_call_id"] + for row in checked["checker_result"]["trace"] + ) + ) + + def test_no_shape_rule_uses_live_dtype_and_detects_exact_edited_call(self): + self.run_mutation("dtype", []) + + def test_mutable_shape_keeps_original_one_dimension_control(self): + self.run_mutation("shape", [2, 3]) diff --git a/tools/fresh_validation/ADAPTERS.md b/tools/fresh_validation/ADAPTERS.md new file mode 100644 index 00000000..c3f84528 --- /dev/null +++ b/tools/fresh_validation/ADAPTERS.md @@ -0,0 +1,29 @@ +# Bounded trace-impact workloads + +`benchmark_workload.py` reads TrainCheck-Benchmarks without modifying it. Run each invocation in a fresh process with the chosen TrainCheck source tree in `PYTHONPATH`: + +```sh +PYTHONPATH=/path/to/traincheck-variant python benchmark_workload.py \ + --benchmark-root /path/to/clean/TrainCheck-Benchmarks \ + --workload mnist --steps 5 --seed 42 --batch-size 2 \ + --device cuda --data-root /path/to/MNIST-cache \ + --output /path/to/new/result --mode instrumented --save-checkpoint +``` + +Use `--mode baseline` for uninstrumented execution. That mode does not import TrainCheck. Modes share source, batch preparation, model initialization, explicit training RNG reset, CUDA synchronization, and training math. The output directory must not already contain a run. No datasets or packages are downloaded. + +These are **model micro-workloads adapted from the benchmark**, not full end-to-end benchmark reproductions: + +| Name | Exact benchmark definitions | Training/data adaptation | +| --- | --- | --- | +| `mnist` | `instr-overhead/overhead-e2e/mnist/main.py:Net` | Original forward, NLL, Adadelta(lr=1.0). First cached MNIST training samples if `--data-root` is supplied; otherwise explicitly synthetic uniform images and random labels. Original normalization preserved. No epoch scheduler or evaluation. | +| `dcgan` | `instr-overhead/overhead-e2e/dcgan/main.py:Generator,Discriminator,weights_init` | Original discriminator real/fake BCE then generator BCE updates and Adam hyperparameters. ngf/ndf reduced from 64 to 8. Synthetic uniform RGB images and precomputed Gaussian noise; not torchvision FakeData's exact generation. No image visualization. | +| `transformer` | `false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py:PositionalEncoding,TransformerModel` | Two 32-dimensional encoder layers, 64-dimensional feedforward, 16-token sequences, vocabulary from checked-in WikiText training prefix. NLL/backward/clipped manual SGD. This repo's main loop is RNN-specific; this is an explicit Transformer training adaptation. | + +`adaptation.json` identifies exact source paths/SHA256, extracted class source SHA256, dataset hashes and adaptation details, version, model configuration, parameter count, and initial state shapes/hash. Full class/function source spans are saved in `benchmark_definitions.py`; top-level source CLIs are never executed. Original benchmark files retain their original license/copyright. + +Instrumented mode calls the public `Instrumentor` programmatically over `torch`, with API-only tracing, normal API coverage (`use_full_instr=False`), no selected API list, no proxy/model tracking, interval 1, and no warmup. It enables tracing only during training bodies via explicit `start_step`; initialization, dataset preparation, metric extraction, timing synchronization, and checkpoint writes are excluded. In both modes the training RNG is reset to `seed + 1` after setup. `TC_ORIGIN_AUDIT=1` enables a separately supplied `origin_audit.install()` before instrumentation, for a distinct diagnostic run. + +`steps.jsonl` preserves completed step timing/losses even if a later step fails. `metrics.json` includes completion, error, canonical initial/final model-state hashes and state shapes. `--save-checkpoint` writes the final model state dictionaries after successful training; it does not save optimizer/RNG state and therefore is not by itself a resume-equivalence evaluation. Canonical state hashes cover sorted names, dtypes, shapes, and tensor bytes, avoiding `torch.save` archive metadata differences. + +For fresh validation, `--data-steps 20` fixes corpus preparation and model vocabulary across runs that execute different numbers of training steps. `adaptation.json` records both the preparation horizon and actual steps, together with the producer PID, runtime and imported tracing-module path. diff --git a/tools/fresh_validation/benchmark_workload.py b/tools/fresh_validation/benchmark_workload.py new file mode 100644 index 00000000..50379d24 --- /dev/null +++ b/tools/fresh_validation/benchmark_workload.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +"""Bounded, source-identified model micro-workloads for the trace bug study. + +Run each invocation in a fresh process with the desired TrainCheck variant on +PYTHONPATH. The benchmark checkout is only read. These are explicit adaptations, +not claims to reproduce the complete benchmark's data or end-to-end training. +""" + +from __future__ import annotations + +import argparse +import ast +import copy +import gzip +import hashlib +import json +import os +import random +import struct +import subprocess +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Protocol, TypedDict + +if TYPE_CHECKING: + import torch + import torch.nn as nn + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import JSONObject, json_value # noqa: E402 + + +class TracingConfig(Protocol): + DISABLE_WRAPPER: bool + MODEL_TRACKER_STYLE: str | None + INSTR_DESCRIPTORS: bool + INSTRUMENTATION_POLICY: dict[str, int] + + +def configure_tracing(config: TracingConfig) -> None: + config.DISABLE_WRAPPER = True + config.MODEL_TRACKER_STYLE = None + config.INSTR_DESCRIPTORS = False + config.INSTRUMENTATION_POLICY.update(interval=1, warm_up=0) + + +class WorkloadOptions(argparse.Namespace): + benchmark_root: Path + workload: Literal["mnist", "dcgan", "transformer"] + steps: int + data_steps: int | None + seed: int + output: Path + mode: Literal["baseline", "instrumented"] + device: Literal["cpu", "cuda"] + batch_size: int + cpu_threads: int + data_root: Path | None + save_checkpoint: bool + + +class TensorMetadata(TypedDict): + shape: list[int] + dtype: str + + +class WorkloadMetrics(TypedDict): + success: bool + error: dict[str, str] | None + completed_steps: int + steps: list[dict[str, int | float]] + initial_model_state_sha256: str + final_model_state_sha256: str + state_shapes_unchanged: bool + state_shapes: dict[str, TensorMetadata] + timing: str + + +SOURCES = { + "mnist": "instr-overhead/overhead-e2e/mnist/main.py", + "dcgan": "instr-overhead/overhead-e2e/dcgan/main.py", + "transformer": "false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", +} + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def definitions(source: Path, names: Sequence[str]) -> str: + """Extract exact source spans, never execute the benchmark's top-level CLI.""" + text = source.read_text() + nodes = { + node.name: node + for node in ast.parse(text).body + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + missing = set(names) - nodes.keys() + if missing: + raise ValueError("Missing benchmark definitions: " + str(sorted(missing))) + segments: list[str] = [] + for name in names: + segment = ast.get_source_segment(text, nodes[name]) + if segment is None: + raise ValueError(f"Benchmark definition {name} has no source span") + segments.append(segment) + return "\n\n".join(segments) + "\n" + + +def mnist_raw(data_root: Path, count: int) -> tuple[bytes, bytes, JSONObject]: + """Read local torchvision-style IDX cache without network or torchvision.""" + candidates = [data_root / "MNIST" / "raw", data_root / "raw", data_root] + for root in candidates: + image = next( + ( + root / name + for name in ("train-images-idx3-ubyte", "train-images-idx3-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + labels = next( + ( + root / name + for name in ("train-labels-idx1-ubyte", "train-labels-idx1-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + if image and labels: + break + else: + raise FileNotFoundError( + "--data-root must contain the cached MNIST training IDX images and labels" + ) + + def prefix(path: Path, length: int) -> bytes: + if path.suffix == ".gz": + with gzip.open(path, "rb") as handle: + return handle.read(length) + with path.open("rb") as raw_handle: + return raw_handle.read(length) + + images = prefix(image, 16 + count * 28 * 28) + targets = prefix(labels, 8 + count) + magic, available, rows, cols = struct.unpack(">IIII", images[:16]) + label_magic, available_labels = struct.unpack(">II", targets[:8]) + if (magic, rows, cols, label_magic) != (2051, 28, 28, 2049): + raise ValueError("Unexpected MNIST IDX header") + if ( + min(available, available_labels) < count + or len(images) != 16 + count * 784 + or len(targets) != 8 + count + ): + raise ValueError("MNIST cache has fewer requested samples than needed") + return ( + images[16:], + targets[8:], + { + "kind": "cached_mnist_training_prefix", + "count": count, + "files": [{"path": str(p), "sha256": sha256(p)} for p in (image, labels)], + "ordering": "first samples in IDX order; no shuffle", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + }, + ) + + +def text_prefix(path: Path, count: int) -> tuple[list[int], int, JSONObject]: + words = [] + with path.open() as handle: + for line in handle: + words.extend(line.split() + [""]) + if len(words) >= count: + break + if len(words) < count: + raise ValueError("WikiText source is shorter than the bounded workload") + words = words[:count] + vocab: dict[str, int] = {} + ids = [vocab.setdefault(word, len(vocab)) for word in words] + return ( + ids, + len(vocab), + { + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": str(path), + "sha256": sha256(path), + "token_count": len(ids), + "vocabulary_size": len(vocab), + "prefix_sha256": hashlib.sha256("\n".join(words).encode()).hexdigest(), + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + }, + ) + + +def module_class(namespace: Mapping[str, object], name: str) -> type[nn.Module]: + import torch.nn as nn + + value = namespace[name] + if not isinstance(value, type) or not issubclass(value, nn.Module): + raise ValueError(f"Benchmark definition {name} must be an nn.Module class") + return value + + +def prepare( + args: WorkloadOptions, +) -> tuple[dict[str, nn.Module], Callable[[int], dict[str, torch.Tensor]], JSONObject]: + import math + + import torch + import torch.nn as nn + import torch.nn.functional as F + + source = args.benchmark_root / SOURCES[args.workload] + symbols = { + "mnist": ["Net"], + "dcgan": ["weights_init", "Generator", "Discriminator"], + "transformer": ["PositionalEncoding", "TransformerModel"], + }[args.workload] + extracted = definitions(source, symbols) + (args.output / "benchmark_definitions.py").write_text(extracted) + ns: dict[str, object] = { + "__name__": "trace_impact_benchmark", + "torch": torch, + "nn": nn, + "F": F, + "math": math, + "nc": 3, + "nz": 100, + "ngf": 8, + "ndf": 8, + } + exec(compile(extracted, str(args.output / "benchmark_definitions.py"), "exec"), ns) + manifest: JSONObject = { + "workload_kind": "bounded_model_micro_workload", + "benchmark_source": str(source), + "benchmark_source_sha256": sha256(source), + "extracted_symbols": [symbol for symbol in symbols], + "extracted_definitions_sha256": hashlib.sha256(extracted.encode()).hexdigest(), + "steps": args.steps, + "seed": args.seed, + "batch_size": args.batch_size, + "device": args.device, + "cpu_threads": args.cpu_threads, + "precision": "float32", + "deterministic_algorithms": True, + "training_only_trace": True, + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing", + ], + } + try: + manifest["benchmark_git_head"] = subprocess.check_output( + ["git", "-C", str(args.benchmark_root), "rev-parse", "HEAD"], text=True + ).strip() + except (OSError, subprocess.CalledProcessError): + manifest["benchmark_git_head"] = None + batch = args.batch_size + count = args.steps * batch + device = torch.device(args.device) + if args.workload == "mnist": + model = module_class(ns, "Net")().to(device) + optimizer = torch.optim.Adadelta(model.parameters(), lr=1.0) + if args.data_root: + raw_images, raw_labels, provenance = mnist_raw(args.data_root, count) + x = ( + torch.tensor(list(raw_images), dtype=torch.float32).reshape( + count, 1, 28, 28 + ) + / 255 + ) + y = torch.tensor(list(raw_labels), dtype=torch.int64) + else: + x = torch.rand(count, 1, 28, 28) + y = torch.randint(0, 10, (count,)) + provenance = { + "kind": "synthetic_uniform_images_and_random_labels", + "count": count, + "is_original_mnist": False, + "normalization": "(uniform[0,1] - 0.1307) / 0.3081", + } + x = ((x - 0.1307) / 0.3081).to(device) + y = y.to(device) + model.train() + + def step(index: int) -> dict[str, torch.Tensor]: + optimizer.zero_grad() + output = model(x[index * batch : (index + 1) * batch]) + loss = F.nll_loss(output, y[index * batch : (index + 1) * batch]) + loss.backward() + optimizer.step() + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={"architecture": "exact Net class"}, + update="Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", + ) + elif args.workload == "dcgan": + generator = module_class(ns, "Generator")(1).to(device) + initializer = ns["weights_init"] + if not callable(initializer): + raise ValueError("Benchmark weights_init must be callable") + + def initialize(module: nn.Module) -> None: + result: object = initializer(module) + if result is not None: + raise ValueError("Benchmark weights_init must return None") + + generator.apply(initialize) + discriminator = module_class(ns, "Discriminator")(1).to(device) + discriminator.apply(initialize) + optimizer_d = torch.optim.Adam( + discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + optimizer_g = torch.optim.Adam( + generator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + criterion = nn.BCELoss() + images = torch.rand(count, 3, 64, 64).to(device) + # FakeData/ToTensor in the benchmark also supplies [0,1] images, but its + # per-index generation is not identical to these explicitly synthetic inputs. + noise_batches = torch.randn(args.steps, batch, 100, 1, 1).to(device) + generator.train() + discriminator.train() + + def step(index: int) -> dict[str, torch.Tensor]: + discriminator.zero_grad() + real = images[index * batch : (index + 1) * batch] + label = torch.full((batch,), 1, dtype=real.dtype, device=device) + output = discriminator(real) + error_real = criterion(output, label) + error_real.backward() + fake = generator(noise_batches[index]) + label.fill_(0) + output = discriminator(fake.detach()) + error_fake = criterion(output, label) + error_fake.backward() + loss_d = error_real + error_fake + optimizer_d.step() + generator.zero_grad() + label.fill_(1) + output = discriminator(fake) + loss_g = criterion(output, label) + loss_g.backward() + optimizer_g.step() + return {"loss_discriminator": loss_d, "loss_generator": loss_g} + + models = {"generator": generator, "discriminator": discriminator} + manifest.update( + data={ + "kind": "synthetic_uniform_rgb_images_and_fixed_gaussian_noise", + "count": count, + "is_original_dataset": False, + "noise_precomputed_before_trace": True, + }, + model_config={ + "ngpu": 1, + "nz": 100, + "ngf": 8, + "ndf": 8, + "nc": 3, + "image_size": 64, + }, + update="Benchmark BCE discriminator real/fake updates then generator update; Adam lr=0.0002 betas=(0.5,0.999). Channels reduced from defaults 64 to 8. Image visualization and logging reductions omitted.", + ) + else: + seq = 16 + corpus = source.parent / "data/wikitext-2/train.txt" + ids, vocab, provenance = text_prefix(corpus, (args.steps * seq + 1) * batch) + data = ( + torch.tensor(ids, dtype=torch.int64) + .view(batch, -1) + .t() + .contiguous() + .to(device) + ) + model = module_class(ns, "TransformerModel")(vocab, 32, 2, 64, 2, 0.2).to( + device + ) + model.train() + + def step(index: int) -> dict[str, torch.Tensor]: + start = index * seq + inputs = data[start : start + seq] + targets = data[start + 1 : start + seq + 1].reshape(-1) + model.zero_grad() + output = model(inputs) + loss = F.nll_loss(output.reshape(-1, vocab), targets) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25) + for parameter in model.parameters(): + if parameter.grad is not None: + parameter.data.add_(parameter.grad, alpha=-1.0) + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={ + "ntoken": vocab, + "ninp": 32, + "nhead": 2, + "nhid": 64, + "nlayers": 2, + "dropout_argument": 0.2, + "sequence_length": seq, + "lr": 1.0, + "clip": 0.25, + }, + update="Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", + ) + return models, step, manifest + + +def model_digest( + models: Mapping[str, nn.Module] +) -> tuple[str, dict[str, TensorMetadata]]: + """Hash tensor contents, names, dtype and shape; independent of torch.save metadata.""" + import torch + + digest = hashlib.sha256() + shapes: dict[str, TensorMetadata] = {} + for model_name, model in sorted(models.items()): + for name, value in sorted(model.state_dict().items()): + key = model_name + "." + name + if not isinstance(value, torch.Tensor): + raise ValueError(f"Benchmark state {key} is not a tensor") + tensor = value.detach().cpu().contiguous() + metadata: TensorMetadata = { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + } + shapes[key] = metadata + digest.update( + key.encode() + + b"\0" + + json.dumps(metadata, sort_keys=True).encode() + + b"\0" + ) + digest.update(tensor.numpy().tobytes()) + return digest.hexdigest(), shapes + + +def run(args: WorkloadOptions) -> None: + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ["TRAINCHECK_OUTPUT_DIR"] = str(args.output / "traces") + import torch + + torch.set_num_threads(args.cpu_threads) + torch.set_num_interop_threads(1) + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.use_deterministic_algorithms(True) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + prepare_args = copy.copy(args) + prepare_args.steps = args.data_steps or args.steps + if prepare_args.steps < args.steps: + raise ValueError("data_steps must cover every training step") + models, step, manifest = prepare(prepare_args) + manifest.update( + steps=args.steps, + prepared_data_steps=prepare_args.steps, + process_id=os.getpid(), + source_root=os.environ.get("PYTHONPATH"), + ) + initial_hash, shapes = model_digest(models) + manifest["adapter_sha256"] = sha256(Path(__file__)) + manifest["mode"] = args.mode + manifest["torch_version"] = str(torch.__version__) + manifest["parameter_count"] = sum( + p.numel() for model in models.values() for p in model.parameters() + ) + manifest["initial_model_state_sha256"] = initial_hash + manifest["state_shapes"] = json_value(shapes) + manifest["instrumentation"] = { + "target": "torch", + "models_to_track": [], + "use_full_instr": False, + "scan_proxy_in_args": False, + "funcs_to_instr": None, + "sampling_interval": 1, + "warm_up_steps": 0, + } + manifest["python_version"] = __import__("sys").version + write_json(args.output / "adaptation.json", manifest) + tc_config: TracingConfig | None = None + start_step: Callable[[], object] | None = None + if args.mode == "instrumented": + (args.output / "traces").mkdir() + import traincheck.instrumentor.tracer as tracer_module + from traincheck.config import config as active_config + from traincheck.instrumentor.control import start_step as active_start_step + from traincheck.instrumentor.tracer import Instrumentor + + tc_config = active_config + start_step = active_start_step + manifest["tracer_module"] = str(Path(tracer_module.__file__).resolve()) + configure_tracing(tc_config) + if os.environ.get("TC_ORIGIN_AUDIT"): + from origin_audit import install + + install() + count = Instrumentor( + torch, + scan_proxy_in_args=False, + use_full_instr=False, + funcs_to_instr=None, + API_dump_stack_trace=False, + sampling_interval=1, + warm_up_steps=0, + ).instrument() + manifest["instrumented_function_count"] = count + write_json(args.output / "adaptation.json", manifest) + # Imports/instrumentation are setup, and must not alter training RNG state. + random.seed(args.seed + 1) + torch.manual_seed(args.seed + 1) + if args.device == "cuda": + torch.cuda.manual_seed_all(args.seed + 1) + manifest["training_rng_seed"] = args.seed + 1 + write_json(args.output / "adaptation.json", manifest) + records: list[dict[str, int | float]] = [] + success = False + error: dict[str, str] | None = None + try: + for index in range(args.steps): + if args.device == "cuda": + torch.cuda.synchronize() + started = time.perf_counter() + if start_step: + start_step() + try: + losses = step(index) + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + if args.device == "cuda": + torch.cuda.synchronize() + elapsed = time.perf_counter() - started + record = { + "step": index + 1, + "seconds": elapsed, + **{name: float(loss.detach().item()) for name, loss in losses.items()}, + } + records.append(record) + with (args.output / "steps.jsonl").open("a") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + print(json.dumps(record, sort_keys=True), flush=True) + success = True + except BaseException as exc: + error = {"type": type(exc).__name__, "message": str(exc)} + raise + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + final_hash, final_shapes = model_digest(models) + result: WorkloadMetrics = { + "success": success, + "error": error, + "completed_steps": len(records), + "steps": records, + "initial_model_state_sha256": initial_hash, + "final_model_state_sha256": final_hash, + "state_shapes_unchanged": shapes == final_shapes, + "state_shapes": final_shapes, + "timing": "synchronized training-body wall seconds; includes start_step and wrapper-disable assignment; excludes loss extraction/data preparation/checkpoint", + } + write_json(args.output / "metrics.json", result) + if success and args.save_checkpoint: + torch.save( + {name: model.state_dict() for name, model in models.items()}, + args.output / "checkpoint.pt", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--workload", choices=sorted(SOURCES), required=True) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument( + "--data-steps", + type=int, + help="Fixed corpus/model preparation horizon shared by reference and holdout", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=("baseline", "instrumented"), required=True) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--cpu-threads", type=int, default=1) + parser.add_argument( + "--data-root", type=Path, help="Optional local MNIST IDX cache; never downloads" + ) + parser.add_argument("--save-checkpoint", action="store_true") + args = parser.parse_args(namespace=WorkloadOptions()) + if args.steps < 1 or args.batch_size < 1 or args.cpu_threads < 1: + parser.error("steps, batch-size, cpu-threads must be positive") + if args.data_root and args.workload != "mnist": + parser.error("--data-root currently applies only to MNIST") + args.benchmark_root = args.benchmark_root.resolve() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + if (args.output / "adaptation.json").exists() or ( + args.output / "steps.jsonl" + ).exists(): + parser.error("Output already contains a run; choose a new directory") + run(args) + + +if __name__ == "__main__": + main() diff --git a/tools/fresh_validation/check_coverage.py b/tools/fresh_validation/check_coverage.py new file mode 100644 index 00000000..f0db31de --- /dev/null +++ b/tools/fresh_validation/check_coverage.py @@ -0,0 +1,782 @@ +"""Observe the real offline checker; never replace its predicate or learned rules. + +Counters have explicit units. API candidates are calls, DistinctArgument candidates +are within-step argument pairs, and precondition counters are group-verification +calls. Each checker stops at its first violation. No wall-time claims are made. +""" + +import argparse +import ast +import hashlib +import importlib.metadata +import inspect +import json +import logging +import math +import os +import sys +from collections import Counter +from collections.abc import Callable, Mapping +from pathlib import Path +from types import FrameType +from typing import Literal, TypeAlias, TypedDict +from unittest.mock import patch + +from traincheck.invariant import read_inv_file +from traincheck.invariant.base_cls import APIParam, GroupedPreconditions, Invariant +from traincheck.invariant.DistinctArgumentRelation import ( + get_event_data_per_function_per_step, +) +from traincheck.trace.trace import Trace +from traincheck.trace.trace_pandas import TracePandas, read_trace_file_Pandas +from traincheck.trace.types import ( + FuncCallExceptionEvent, + IncompleteFuncCallEvent, + MDNONEJSONEncoder, +) + +# Direct CLI execution must resolve this checkout's harness package even when +# PYTHONPATH intentionally points to a separately composed TrainCheck source. +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_list, + json_object, + json_value, + mapping, + required, + sequence, + string, +) + + +class ObserverContractError(ValueError): + """Checker internals no longer satisfy the observer's verified contract.""" + + +class ObserverSites(TypedDict): + starts: dict[int, str] + ends: dict[int, int] + visits: set[int] + method_source_sha256: str + + +class CallDecision(TypedDict): + func_call_id: str + + +class PairDecision(TypedDict): + pair_func_call_ids: list[str] + + +Decision: TypeAlias = CallDecision | PairDecision + + +class PendingDecision(TypedDict): + start: int + end: int + record: Decision + + +class GroupCounts(TypedDict): + attempts: int + passes: int + failures: int + errors: int + + +class CandidateCounts(TypedDict): + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + + +class OptionalReportFields(TypedDict, total=False): + invariant_index: int + checker_result: JSONObject + error: dict[str, str] + checker_error_after_observer: dict[str, str] + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + method_source_sha256: str + decisions_started: int + decisions_completed: int + decisions_not_completed: int + visited_calls: int | None + precondition_group_attempts: int + precondition_group_passes: int + precondition_group_failures: int + precondition_group_errors: int + precondition_groups: dict[str, GroupCounts] + decision_records: list[Decision] + unvisited_calls_after_stop: int | None + unvisited_pairs_after_stop: int + variable_precondition_requested: bool + variable_check_skipped: bool + fully_validated_active: bool + + +class ObservationReport(OptionalReportFields): + # Optional detail fields are present only for the matching status. Core + # outcome/observation fields are always present; unknown is never false. + relation: str + description: str | None + identity_clauses: list[JSONObject] + observer_supported: bool + status: Literal[ + "completed", "checker_error", "observer_error", "unsupported_coverage" + ] + coverage_status: Literal["observed", "disabled", "unsupported", "error"] + coverage_observed: bool + active: bool | None + dormant: bool | None + check_passed: bool | None + checker_triggered: bool | None + + +class RuntimeMetadata(TypedDict): + python: str + executable: str + trace_module: str + packages: dict[str, str] + + +class CoverageOutput(TypedDict): + schema_version: Literal[2] + pid: int + check_relation_first: Literal[False] + observer_enabled: bool + runtime: RuntimeMetadata + harness_sha256: str + raw_process_ids: list[JSONValue] + raw_thread_ids: list[JSONValue] + trace_rows: int + input_sha256: dict[str, str] + inputs_unchanged: bool + summary: dict[str, int | None] + per_invariant: list[ObservationReport] + + +TraceCallback: TypeAlias = Callable[[FrameType, str, object], "TraceCallback | None"] + + +def decision_record(family: str, local_values: Mapping[str, object]) -> Decision: + """Read the exact family-specific identity fields, failing closed on drift.""" + try: + if family == "DistinctArgumentRelation": + ids = [] + for name in ("event1", "event2"): + event = mapping(required(local_values, name), name) + ids.append( + string(required(event, "func_call_id"), f"{name}.func_call_id") + ) + return {"pair_func_call_ids": ids} + name = ( + "parent_func_call_id" if family == "APIContainRelation" else "func_call_id" + ) + if family not in SUPPORTED: + raise ValueError(f"Unsupported observer family: {family}") + return {"func_call_id": string(required(local_values, name), name)} + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + + +def finalize_report(report: ObservationReport) -> ObservationReport: + """Validate the conditional schema before exposing a mutable report. + + OptionalReportFields is an internal construction accumulator, not permission + to omit completed-result or observed-coverage fields in emitted reports. + """ + status = report["status"] + coverage_status = report["coverage_status"] + if status not in { + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + }: + raise ObserverContractError(f"Unknown checker status: {status}") + if coverage_status not in {"observed", "disabled", "unsupported", "error"}: + raise ObserverContractError(f"Unknown coverage status: {coverage_status}") + try: + if status == "completed": + checker = mapping(required(report, "checker_result"), "checker_result") + passed = boolean(report["check_passed"], "check_passed") + triggered = boolean(report["checker_triggered"], "checker_triggered") + if ( + boolean( + required(checker, "check_passed"), "checker_result.check_passed" + ) + != passed + or boolean(required(checker, "triggered"), "checker_result.triggered") + != triggered + ): + raise ValueError("Checker outcome and report outcome disagree") + mapping(required(checker, "invariant"), "checker_result.invariant") + else: + if ( + report["check_passed"] is not None + or report["checker_triggered"] is not None + ): + raise ValueError("Incomplete checking must have unknown outcome") + if status in {"checker_error", "observer_error"}: + error = mapping(required(report, "error"), "error") + string(required(error, "type"), "error.type") + string(required(error, "message"), "error.message") + if coverage_status == "observed": + if report["coverage_observed"] is not True or status not in { + "completed", + "checker_error", + }: + raise ValueError("Observed coverage has incompatible checker status") + boolean(report["active"], "active") + boolean(report["dormant"], "dormant") + for key in ( + "candidate_calls", + "opportunities", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + ): + if integer(required(report, key), key) < 0: + raise ValueError(f"{key} cannot be negative") + for key in ( + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active", + ): + boolean(required(report, key), key) + for key in ("opportunity_unit", "method_source_sha256"): + string(required(report, key), key) + groups = mapping( + required(report, "precondition_groups"), "precondition_groups" + ) + for name, raw_group in groups.items(): + group = mapping(raw_group, name) + for key in ("attempts", "passes", "failures", "errors"): + if integer(required(group, key), f"{name}.{key}") < 0: + raise ValueError(f"{name}.{key} cannot be negative") + if report["relation"] == "DistinctArgumentRelation": + if ( + required(report, "visited_calls") is not None + or required(report, "unvisited_calls_after_stop") is not None + ): + raise ValueError("Pair coverage must not claim call visit counts") + integer( + required(report, "unvisited_pairs_after_stop"), + "unvisited_pairs_after_stop", + ) + else: + integer(required(report, "visited_calls"), "visited_calls") + integer( + required(report, "unvisited_calls_after_stop"), + "unvisited_calls_after_stop", + ) + for raw in sequence( + required(report, "decision_records"), "decision_records" + ): + decision = mapping(raw, "decision") + if report["relation"] == "DistinctArgumentRelation": + pair = sequence( + required(decision, "pair_func_call_ids"), "pair_func_call_ids" + ) + if len(pair) != 2: + raise ValueError("Pair decisions require exactly two call IDs") + for identity in pair: + string(identity, "pair call ID") + else: + string(required(decision, "func_call_id"), "decision.func_call_id") + elif ( + report["coverage_observed"] is not False + or report["active"] is not None + or report["dormant"] is not None + ): + raise ValueError( + "Unobserved coverage must have unknown active/dormant states" + ) + if status == "unsupported_coverage" and coverage_status != "unsupported": + raise ValueError("Unsupported checker coverage must be marked unsupported") + if status == "observer_error" and coverage_status != "error": + raise ValueError("Observer failure must mark coverage as an error") + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + return report + + +def summarize(results: list[ObservationReport]) -> dict[str, int | None]: + counts: Counter[str] = Counter() + unobserved = 0 + unknown_outcomes = 0 + for result in results: + finalize_report(result) + counts[result["status"]] += 1 + observed = result["coverage_status"] == "observed" + if observed: + active, dormant = result["active"], result["dormant"] + assert active is not None and dormant is not None + counts["active_invariants"] += active + counts["dormant_invariants"] += dormant + counts["identity_bearing_active"] += ( + bool(result["identity_clauses"]) and active + ) + else: + unobserved += 1 + unknown_outcomes += result["check_passed"] is None + counts["violated_invariants"] += result["check_passed"] is False + counts["identity_bearing_invariants"] += bool(result["identity_clauses"]) + summary: dict[str, int | None] = dict(counts) + summary["coverage_unobserved_invariants"] = unobserved + summary["check_outcome_unknown_invariants"] = unknown_outcomes + summary["observed_violated_invariants"] = counts["violated_invariants"] + summary["violated_invariants"] = ( + None if unknown_outcomes else counts["violated_invariants"] + ) + # A disabled/unsupported/error observer cannot establish the total active or + # dormant population. Keep the observed subset explicitly separate. + for name in ("active_invariants", "dormant_invariants", "identity_bearing_active"): + summary["observed_" + name] = counts[name] + summary[name] = None if unobserved else counts[name] + return summary + + +SUPPORTED = { + "APIContainRelation", + "ConsistentOutputRelation", + "ConsistentInputOutputRelation", + "ThresholdRelation", + "DistinctArgumentRelation", +} + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observer_sites(function: Callable[..., object], family: str) -> ObserverSites: + """Locate exact predicate ASTs in the installed method, failing closed on drift.""" + lines, first = inspect.getsourcelines(function) + # Dedent code by its method indentation. A zero-indented line inside a + # multiline diagnostic string defeats textwrap.dedent's common prefix. + indent = len(lines[0]) - len(lines[0].lstrip()) + source = "".join( + line[indent:] if line.startswith(" " * indent) else line for line in lines + ) + tree = ast.parse(source) + starts: dict[int, str] = {} + ends: dict[int, int] = {} + visits: set[int] = set() + predicates: dict[str, Callable[[str], bool]] = { + "APIContainRelation": lambda s: "skip_var_unchanged_check" in s + and "found_expected_child_event" in s, + "ConsistentInputOutputRelation": lambda s: s == "input_value != output_value", + "ConsistentOutputRelation": lambda s: s == "len(returned_tensors) == 0", + "ThresholdRelation": lambda s: s + in ("output_value >= threshold_value", "output_value <= threshold_value"), + "DistinctArgumentRelation": lambda s: s + == "is_arguments_list_same(event1['args'], event2['args'])", + } + for node in ast.walk(tree): + if isinstance(node, ast.If) and predicates[family](ast.unparse(node.test)): + line = first + node.lineno - 1 + starts[line] = ast.unparse(node.test) + if node.test.end_lineno is None: + raise ObserverContractError("Predicate has no ending source line") + ends[line] = first + node.test.end_lineno - 1 + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id in {"func_call_event", "parent_event_type"} + for t in node.targets + ): + visits.add(first + node.lineno - 1) + expected = 2 if family in {"ThresholdRelation", "DistinctArgumentRelation"} else 1 + if len(starts) != expected: + raise ObserverContractError( + f"Unrecognized checker source for {family}: {starts}" + ) + return { + "starts": starts, + "ends": ends, + "visits": visits, + "method_source_sha256": hashlib.sha256(source.encode()).hexdigest(), + } + + +def candidate_counts(trace: TracePandas, inv: Invariant) -> CandidateCounts: + family = inv.relation.__name__ + api = ( + inv.params[1] + if family in {"ConsistentInputOutputRelation", "ThresholdRelation"} + else inv.params[0] + ) + if not isinstance(api, APIParam): + raise ObserverContractError(f"{family} expected an APIParam") + calls = trace.get_func_call_ids(api.api_full_name) + if family == "DistinctArgumentRelation": + _, groups = get_event_data_per_function_per_step(trace, {api.api_full_name}) + pairs = sum( + math.comb(sum(map(len, records.values())), 2) + for records in groups.get(api.api_full_name, {}).values() + ) + return { + "candidate_calls": len(calls), + "opportunities": pairs, + "opportunity_unit": "within_step_argument_pair", + "incomplete_or_exception_calls": None, + } + if family == "APIContainRelation": + eligible = len(calls) + else: + eligible = sum( + not isinstance( + trace.query_func_call_event(call), + (FuncCallExceptionEvent, IncompleteFuncCallEvent), + ) + for call in calls + ) + return { + "candidate_calls": len(calls), + "opportunities": eligible, + "opportunity_unit": ( + "parent_call" + if family == "APIContainRelation" + else "complete_function_call" + ), + "incomplete_or_exception_calls": len(calls) - eligible, + } + + +def id_clauses(value: JSONValue) -> list[JSONObject]: + found: list[JSONObject] = [] + if isinstance(value, dict): + if isinstance(value.get("prop_name"), str) and value.get("prop_name") in { + "process_id", + "thread_id", + "meta_vars.process_id", + "meta_vars.thread_id", + }: + found.append(value) + for child in value.values(): + found.extend(id_clauses(child)) + elif isinstance(value, list): + for child in value: + found.extend(id_clauses(child)) + return found + + +def observe( + trace: TracePandas, inv: Invariant, enabled: bool = True +) -> ObservationReport: + """Run one unchanged inv.check(False), counting only observed completed decisions.""" + family = inv.relation.__name__ + assert inv.precondition is not None, "Invariant precondition must be set" + report: ObservationReport = { + "relation": family, + "description": inv.text_description, + "identity_clauses": id_clauses( + json_value( + json.loads( + json.dumps(inv.precondition.to_dict(), cls=MDNONEJSONEncoder) + ) + ) + ), + "observer_supported": family in SUPPORTED, + "status": "unsupported_coverage", + "coverage_status": "disabled" if not enabled else "unsupported", + "coverage_observed": False, + "active": None, + "dormant": None, + "check_passed": None, + "checker_triggered": None, + } + if not enabled: + # Direct oracle: no opportunity pre-scan, source parsing, or monkeypatches. + try: + result = inv.check(trace, check_relation_first=False) + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } + ) + except Exception as exc: + report.update( + { + "status": "checker_error", + "error": {"type": type(exc).__name__, "message": str(exc)}, + "check_passed": None, + } + ) + report.update({"coverage_observed": False, "active": None, "dormant": None}) + return finalize_report(report) + if family not in SUPPORTED: + return finalize_report(report) + candidates = candidate_counts(trace, inv) + report.update(candidates) + sites = observer_sites(inv.relation.static_check_all, family) + report["method_source_sha256"] = sites["method_source_sha256"] + counts: Counter[str] = Counter( + decisions_started=0, + decisions_completed=0, + visited_calls=0, + precondition_group_attempts=0, + precondition_group_passes=0, + precondition_group_failures=0, + precondition_group_errors=0, + ) + groups: dict[str, GroupCounts] = {} + decision_records: list[Decision] = [] + variable_precondition_requested = ( + family == "APIContainRelation" + and "var_events" in inv.precondition.get_group_names() + and len(inv.precondition.get_group("var_events")) > 0 + and not inv.precondition.is_group_unconditional("var_events") + ) + variable_check_skipped = False + method_code = inv.relation.static_check_all.__code__ + pending: PendingDecision | None = None + observer_failure: ObserverContractError | None = None + original_verify = GroupedPreconditions.verify + + def verify( + precondition: GroupedPreconditions, + records: list[object], + group_name: str, + trace_for_metavars: Trace | None = None, + ) -> bool: + group = groups.setdefault( + group_name, {"attempts": 0, "passes": 0, "failures": 0, "errors": 0} + ) + counts["precondition_group_attempts"] += 1 + group["attempts"] += 1 + try: + result = original_verify( + precondition, records, group_name, trace_for_metavars + ) + except Exception: + counts["precondition_group_errors"] += 1 + group["errors"] += 1 + raise + label: Literal["passes", "failures"] = "passes" if result else "failures" + group[label] += 1 + counts["precondition_group_" + label] += 1 + return result + + def trace_event_impl( + frame: FrameType, event: str, arg: object + ) -> TraceCallback | None: + nonlocal pending, variable_check_skipped + if frame.f_code is not method_code: + return None + if event == "exception": + # An extraction error can be caught by the real method; it is not a decision. + pending = None + if pending is not None: + done = event == "return" + if event == "line": + if family == "ConsistentOutputRelation": + done = frame.f_lineno in sites["visits"] + else: + done = not pending["start"] <= frame.f_lineno <= pending["end"] + if done: + counts["decisions_completed"] += 1 + decision_records.append(pending["record"]) + pending = None + if event == "line": + if frame.f_lineno in sites["visits"]: + counts["visited_calls"] += 1 + if frame.f_lineno in sites["starts"] and pending is None: + if variable_precondition_requested: + try: + skipped = boolean( + required(frame.f_locals, "skip_var_unchanged_check"), + "skip_var_unchanged_check", + ) + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + if skipped: + variable_check_skipped = True + record = decision_record(family, frame.f_locals) + counts["decisions_started"] += 1 + pending = { + "start": frame.f_lineno, + "end": sites["ends"][frame.f_lineno], + "record": record, + } + return trace_event + + def trace_event(frame: FrameType, event: str, arg: object) -> TraceCallback | None: + nonlocal observer_failure + try: + return trace_event_impl(frame, event, arg) + except ObserverContractError as exc: + # A checker may catch an exception raised by sys.settrace. Retain it + # independently so source drift cannot become a completed check. + observer_failure = exc + raise + + previous_trace = sys.gettrace() + if enabled and previous_trace is not None: + raise RuntimeError("A pre-existing Python trace observer would be overwritten") + try: + with patch.object(GroupedPreconditions, "verify", verify): + sys.settrace(trace_event) + result = inv.check(trace, check_relation_first=False) + if observer_failure is not None: + raise observer_failure + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } + ) + except Exception as exc: + failure = observer_failure if observer_failure is not None else exc + report.update( + { + "status": ( + "observer_error" + if isinstance(failure, ObserverContractError) + else "checker_error" + ), + "error": {"type": type(failure).__name__, "message": str(failure)}, + "check_passed": None, + } + ) + if observer_failure is not None and exc is not observer_failure: + report["checker_error_after_observer"] = { + "type": type(exc).__name__, + "message": str(exc), + } + finally: + if enabled: + sys.settrace(previous_trace) + report["decisions_started"] = counts["decisions_started"] + report["decisions_completed"] = counts["decisions_completed"] + report["visited_calls"] = counts["visited_calls"] + report["precondition_group_attempts"] = counts["precondition_group_attempts"] + report["precondition_group_passes"] = counts["precondition_group_passes"] + report["precondition_group_failures"] = counts["precondition_group_failures"] + report["precondition_group_errors"] = counts["precondition_group_errors"] + report["precondition_groups"] = groups + report["decision_records"] = decision_records + report["decisions_not_completed"] = ( + counts["decisions_started"] - counts["decisions_completed"] + ) + stopped = report["status"] != "completed" or report["check_passed"] is False + if family == "DistinctArgumentRelation": + report["visited_calls"] = None + report["unvisited_calls_after_stop"] = None + report["unvisited_pairs_after_stop"] = ( + max(0, candidates["opportunities"] - counts["decisions_started"]) + if stopped + else 0 + ) + else: + report["unvisited_calls_after_stop"] = ( + max(0, candidates["candidate_calls"] - counts["visited_calls"]) + if stopped + else 0 + ) + report["active"] = counts["decisions_completed"] > 0 + report["dormant"] = report["status"] == "completed" and not report["active"] + report["coverage_observed"] = report["status"] != "observer_error" + report["coverage_status"] = "observed" if report["coverage_observed"] else "error" + if not report["coverage_observed"]: + report["active"] = None + report["dormant"] = None + report["variable_precondition_requested"] = variable_precondition_requested + report["variable_check_skipped"] = variable_check_skipped + report["fully_validated_active"] = ( + report["active"] is True + and not variable_check_skipped + and report["status"] == "completed" + ) + return finalize_report(report) + + +class CoverageOptions(argparse.Namespace): + traces: list[Path] + invariants: Path + output: Path + observer_off: bool + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--observer-off", action="store_true") + args = parser.parse_args(namespace=CoverageOptions()) + logging.basicConfig(level=logging.ERROR) + import traincheck.utils + + traincheck.utils._suppress_inner_progress = True + before = {str(p): sha256(p) for p in [*args.traces, args.invariants]} + trace = read_trace_file_Pandas(list(map(str, args.traces))) + invariants = read_inv_file(str(args.invariants)) + results: list[ObservationReport] = [] + for index, inv in enumerate(invariants): + result = observe(trace, inv, enabled=not args.observer_off) + result["invariant_index"] = index + results.append(result) + summary = summarize(results) + import traincheck.trace.trace_pandas as trace_module + + output: CoverageOutput = { + "schema_version": 2, + "pid": os.getpid(), + "check_relation_first": False, + "observer_enabled": not args.observer_off, + "runtime": { + "python": sys.version, + "executable": sys.executable, + "trace_module": str(Path(trace_module.__file__).resolve()), + "packages": { + p: importlib.metadata.version(p) for p in ("torch", "pandas", "numpy") + }, + }, + "harness_sha256": sha256(Path(__file__)), + "raw_process_ids": json_list(trace.get_process_ids(), "raw_process_ids"), + "raw_thread_ids": json_list(trace.get_thread_ids(), "raw_thread_ids"), + "trace_rows": len(trace.events), + "input_sha256": before, + "inputs_unchanged": all(sha256(Path(p)) == h for p, h in before.items()), + "summary": dict(summary), + "per_invariant": results, + } + args.output.write_text(json.dumps(output, cls=MDNONEJSONEncoder, indent=2) + "\n") + print(json.dumps(summary), flush=True) + return int(any(result["status"] != "completed" for result in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/fresh_validation/compare_study.py b/tools/fresh_validation/compare_study.py new file mode 100644 index 00000000..1822556c --- /dev/null +++ b/tools/fresh_validation/compare_study.py @@ -0,0 +1,369 @@ +"""Verify actual paired-study outputs; missing observation never becomes a pass.""" + +import argparse +import hashlib +import json +import math +import sys +from collections import Counter +from collections.abc import Mapping +from pathlib import Path +from typing import TypedDict + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + boolean, + integer, + json_object, + json_value, + mapping, + number, + read_object, + required, + sequence, + string, +) + + +class TrainingComparison(TypedDict): + loss_trajectory_equal: bool + initial_state_sha256_equal: bool + final_state_sha256_equal: bool + shapes_equal: bool + + +class StudyComparison(TypedDict): + status: str + invariants_sha256: str + training_comparisons: dict[str, TrainingComparison] + benchmark_data_runtime_source_and_harness_matched: bool + observer_matches_direct_checker: bool + per_run_summary: dict[str, JSONObject] + per_relation: dict[str, dict[str, dict[str, int]]] + raw_ids: dict[str, JSONObject] + fault: JSONObject + limitations: list[str] + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def canonical(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def object_field(data: Mapping[str, object], key: str) -> Mapping[str, object]: + return mapping(required(data, key), key) + + +def objects_field(data: Mapping[str, object], key: str) -> list[Mapping[str, object]]: + return [mapping(item, key) for item in sequence(required(data, key), key)] + + +def bool_field(data: Mapping[str, object], key: str) -> bool: + return boolean(required(data, key), key) + + +def int_field(data: Mapping[str, object], key: str) -> int: + return integer(required(data, key), key) + + +def str_field(data: Mapping[str, object], key: str) -> str: + return string(required(data, key), key) + + +def losses(metric: Mapping[str, object]) -> list[dict[str, int | float]]: + rows = [ + { + key: number(value, key) + for key, value in step.items() + if key.startswith("loss") + } + for step in objects_field(metric, "steps") + ] + require( + bool(rows) and all(row for row in rows), + "Every training step must contain a loss", + ) + require( + all(math.isfinite(value) for row in rows for value in row.values()), + "Non-finite loss", + ) + require( + int_field(metric, "completed_steps") == len(rows), + "Loss row count does not match completed_steps", + ) + return rows + + +def compare(root: Path) -> StudyComparison: + def read(name: str) -> JSONObject: + return read_object(root / name) + + phases = {p.parent.name: read_object(p) for p in root.glob("*/phase-result.json")} + expected_phases = { + "reference", + "control", + "holdout", + "baseline-holdout", + "inference", + "check-reference", + "check-control", + "check-holdout", + "check-holdout-direct", + "prepare-fault", + "check-fault", + } + require( + expected_phases <= phases.keys(), + f"Missing completed phases: {sorted(expected_phases - phases.keys())}", + ) + for name, phase in phases.items(): + require( + int_field(phase, "exit_code") == 0 + and required(phase, "stop_reason") is None + and bool_field(phase, "source_unchanged"), + f"Phase {name} did not complete with unchanged source", + ) + roles = ("reference", "control", "holdout", "baseline-holdout") + metrics = {role: read(role + "/metrics.json") for role in roles} + adaptations = {role: read(role + "/adaptation.json") for role in roles} + require( + all( + bool_field(m, "success") and bool_field(m, "state_shapes_unchanged") + for m in metrics.values() + ), + "Training failed or state shapes changed", + ) + workload_keys = ( + "benchmark_source_sha256", + "extracted_definitions_sha256", + "adapter_sha256", + "data", + "prepared_data_steps", + "model_config", + "batch_size", + "device", + "cpu_threads", + "precision", + "deterministic_algorithms", + "torch_version", + "python_version", + ) + for role in roles: + for key in workload_keys: + require( + required(adaptations[role], key) + == required(adaptations["reference"], key), + f"Mismatched {role} adaptation field: {key}", + ) + invocations = [read(name + "/invocation.json") for name in phases] + for key in ("source_sha256", "harness_sha256"): + require( + all( + object_field(i, key) == object_field(invocations[0], key) + for i in invocations + ), + f"Mismatched phase {key}", + ) + require( + len({int_field(adaptations[role], "process_id") for role in roles}) == 4, + "Training phases must use distinct processes", + ) + for role in roles: + require( + int_field(metrics[role], "completed_steps") + == (10 if role in ("reference", "control") else 20), + f"Incorrect completed step count for {role}", + ) + if role != "baseline-holdout": + require( + str_field(adaptations[role], "tracer_module") + == str( + Path(str_field(adaptations[role], "source_root")) + / "traincheck/instrumentor/tracer.py" + ), + f"Wrong tracer source for {role}", + ) + require( + not str_field(object_field(adaptations[role], "data"), "kind").startswith( + "synthetic" + ), + "Synthetic data is outside this study protocol", + ) + + paired: dict[str, TrainingComparison] = {} + for name, left, right in ( + ("same_seed_fresh_control", "reference", "control"), + ("holdout_instrumentation_comparison", "holdout", "baseline-holdout"), + ): + m1, m2 = metrics[left], metrics[right] + checks: TrainingComparison = { + "loss_trajectory_equal": losses(m1) == losses(m2), + "initial_state_sha256_equal": str_field(m1, "initial_model_state_sha256") + == str_field(m2, "initial_model_state_sha256"), + "final_state_sha256_equal": str_field(m1, "final_model_state_sha256") + == str_field(m2, "final_model_state_sha256"), + "shapes_equal": object_field(m1, "state_shapes") + == object_field(m2, "state_shapes"), + } + require(all(checks.values()), f"Training comparison failed: {name}: {checks}") + paired[name] = checks + coverage = { + role: read("check-" + role + "/coverage.json") + for role in ("reference", "control", "holdout", "holdout-direct", "fault") + } + require( + all(bool_field(c, "inputs_unchanged") for c in coverage.values()), + "Coverage inputs changed", + ) + frozen_path = root / "inference/invariants.jsonl" + frozen_hash = hashlib.sha256(frozen_path.read_bytes()).hexdigest() + for checked in coverage.values(): + require( + str_field(object_field(checked, "input_sha256"), str(frozen_path)) + == frozen_hash, + "Frozen invariants changed", + ) + observed = objects_field(coverage["holdout"], "per_invariant") + direct = objects_field(coverage["holdout-direct"], "per_invariant") + require(len(observed) == len(direct), "Observer/direct result counts differ") + for a, b in zip(observed, direct): + require( + str_field(a, "status") == str_field(b, "status") == "completed", + "Checker did not complete", + ) + require( + canonical(object_field(a, "checker_result")) + == canonical(object_field(b, "checker_result")), + "Observer changed checker result", + ) + require( + required(b, "active") is None + and required(b, "dormant") is None + and not bool_field(b, "coverage_observed"), + "Disabled observation must remain unknown", + ) + + mutation = read("fault/mutation.json") + selected = int_field(mutation, "invariant_index") + fault_results = objects_field(coverage["fault"], "per_invariant") + require( + 0 <= selected < len(observed) and selected < len(fault_results), + "Fault invariant index is out of range", + ) + healthy, faulty = observed[selected], fault_results[selected] + require( + bool_field(healthy, "check_passed") and bool_field(healthy, "active"), + "Fault target was not healthy and active", + ) + require( + str_field(faulty, "status") == "completed" + and not bool_field(faulty, "check_passed"), + "Fault did not cause a completed violation", + ) + require( + int_field(faulty, "decisions_completed") > 0 + and int_field(faulty, "precondition_group_passes") > 0, + "Fault target did not activate", + ) + fault_identity = str_field(mutation, "func_call_id") + # This selected rule is ConsistentOutputRelation; both records require a + # single call identity. A pair record or missing ID is a malformed report. + require( + any( + str_field(d, "func_call_id") == fault_identity + for d in objects_field(faulty, "decision_records") + ), + "Fault decision identity not observed", + ) + require( + any( + str_field(event, "func_call_id") == fault_identity + for event in objects_field(object_field(faulty, "checker_result"), "trace") + ), + "Fault identity absent from checker violation", + ) + edits = objects_field(mutation, "edits") + require( + bool_field(mutation, "ids_unchanged") + and bool_field(mutation, "precondition_inputs_unchanged") + and len(edits) == 1, + "Fault changed more than one permitted field", + ) + for key in ("raw_process_ids", "raw_thread_ids"): + require( + required(coverage["fault"], key) == required(coverage["holdout"], key), + f"Fault altered {key}", + ) + families: dict[str, dict[str, dict[str, int]]] = {} + for role in ("reference", "control", "holdout", "fault"): + family_counts: dict[str, Counter[str]] = {} + for item in objects_field(coverage[role], "per_invariant"): + require( + str_field(item, "status") == "completed" + and bool_field(item, "coverage_observed"), + "Incomplete or unobserved coverage", + ) + counts = family_counts.setdefault(str_field(item, "relation"), Counter()) + counts.update( + invariants=1, + active=int(bool_field(item, "active")), + dormant=int(bool_field(item, "dormant")), + violations=int(not bool_field(item, "check_passed")), + opportunities=int_field(item, "opportunities"), + precondition_group_passes=int_field(item, "precondition_group_passes"), + actual_decisions=int_field(item, "decisions_completed"), + proxy_checks_skipped=int(bool_field(item, "variable_check_skipped")), + fully_validated_active=int(bool_field(item, "fully_validated_active")), + ) + families[role] = {name: dict(counts) for name, counts in family_counts.items()} + output: StudyComparison = { + "status": "study_completed", + "invariants_sha256": frozen_hash, + "training_comparisons": paired, + "benchmark_data_runtime_source_and_harness_matched": True, + "observer_matches_direct_checker": True, + "per_run_summary": { + r: json_object(required(coverage[r], "summary")) + for r in ("reference", "control", "holdout", "fault") + }, + "per_relation": families, + "raw_ids": { + r: { + "process": json_value(required(coverage[r], "raw_process_ids")), + "thread": json_value(required(coverage[r], "raw_thread_ids")), + } + for r in ("reference", "control", "holdout") + }, + "fault": { + "kind": str_field(mutation, "kind"), + "invariant_index": selected, + "healthy_decisions": int_field(healthy, "decisions_completed"), + "fault_decisions": int_field(faulty, "decisions_completed"), + "fault_precondition_group_passes": int_field( + faulty, "precondition_group_passes" + ), + "violation": True, + "edit": json_object(edits[0]), + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant.", + ], + } + (root / "comparison.json").write_text(json.dumps(output, indent=2) + "\n") + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + print(json.dumps(compare(parser.parse_args().root.resolve()), indent=2)) diff --git a/tools/fresh_validation/contracts.py b/tools/fresh_validation/contracts.py new file mode 100644 index 00000000..831ebe17 --- /dev/null +++ b/tools/fresh_validation/contracts.py @@ -0,0 +1,91 @@ +"""Validated JSON boundaries shared by the correctness-study tools. + +Unknown JSON and frame metadata enter as ``object``. These helpers never replace +required fields with defaults, and distinguish booleans from numeric counts. +""" + +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import TypeAlias, cast + +JSONValue: TypeAlias = ( + None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] +) +JSONObject: TypeAlias = dict[str, JSONValue] + + +def mapping(value: object, context: str) -> Mapping[str, object]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{context} must be an object with string keys") + # The check establishes the key type; read-only Mapping does not promise + # anything about external values before the individual field validators run. + return cast(Mapping[str, object], value) + + +def sequence(value: object, context: str) -> Sequence[object]: + if not isinstance(value, list): + raise ValueError(f"{context} must be a JSON array") + return value + + +def string(value: object, context: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string") + return value + + +def integer(value: object, context: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{context} must be an integer") + return value + + +def boolean(value: object, context: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{context} must be a boolean") + return value + + +def number(value: object, context: str) -> int | float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{context} must be a number") + return value + + +def required(data: Mapping[str, object], key: str) -> object: + if key not in data: + raise ValueError(f"Missing required field: {key}") + return data[key] + + +def json_value(value: object, context: str = "JSON") -> JSONValue: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list): + return [json_value(child, context) for child in value] + if isinstance(value, Mapping): + return { + key: json_value(child, f"{context}.{key}") + for key, child in mapping(value, context).items() + } + raise ValueError(f"{context} has a non-JSON value of type {type(value).__name__}") + + +def json_object(value: object, context: str = "JSON") -> JSONObject: + result = json_value(value, context) + if not isinstance(result, dict): + raise ValueError(f"{context} must be an object") + return result + + +def json_list(value: object, context: str) -> list[JSONValue]: + result = json_value(value, context) + if not isinstance(result, list): + raise ValueError(f"{context} must be an array") + return result + + +def read_object(path: Path) -> JSONObject: + value: object = json.loads(path.read_text()) + return json_object(value, str(path)) diff --git a/tools/fresh_validation/fault_control.py b/tools/fresh_validation/fault_control.py new file mode 100644 index 00000000..66e64e34 --- /dev/null +++ b/tools/fresh_validation/fault_control.py @@ -0,0 +1,305 @@ +"""Post-hoc, one-field trace sensitivity control for a live frozen output invariant. + +This does not inject a training bug. It changes one recorded tensor shape or dtype after +healthy training, preserves IDs and precondition inputs, and checks the same rules. +""" + +import argparse +import copy +import hashlib +import json +import sys +from pathlib import Path +from typing import TypedDict + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_object, + mapping, + read_object, + required, + sequence, + string, +) + + +class Edit(TypedDict): + file: str + zero_based_line: int + path: list[str | int] + attribute: str + before_value: JSONValue + after_value: JSONValue + raw_process_id: JSONValue + raw_thread_id: JSONValue + + +class Fingerprints(TypedDict): + original_sha256: str + fault_sha256: str + + +class MutationReport(TypedDict): + kind: str + selection: str + attribute: str + invariant_index: int + invariant: JSONObject + func_call_id: str + edits: list[Edit] + traces: dict[str, Fingerprints] + invariants_sha256: str + healthy_coverage_sha256: str + precondition_inputs_unchanged: bool + ids_unchanged: bool + invariants_not_retrained_or_rewritten: bool + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def differences( + before: object, after: object, path: tuple[str | int, ...] = () +) -> list[tuple[str | int, ...]]: + if type(before) is not type(after): + return [path] + if isinstance(before, dict): + before_dict = mapping(before, "before") + after_dict = mapping(after, "after") + if before_dict.keys() != after_dict.keys(): + return [path] + return [ + p + for k in before_dict + for p in differences(before_dict[k], after_dict[k], (*path, k)) + ] + if isinstance(before, list): + assert isinstance(after, list) + if len(before) != len(after): + return [path] + return [ + p + for i, (a, b) in enumerate(zip(before, after)) + for p in differences(a, b, (*path, i)) + ] + if before == after or ( + isinstance(before, float) and before != before and after != after + ): + return [] + return [path] + + +def replacement(attributes: JSONValue, attribute: str) -> JSONValue: + """Return one valid same-type unequal metadata value, or no eligible edit.""" + if not isinstance(attributes, dict): + return None + value = attributes.get(attribute) + if attribute == "shape": + if ( + isinstance(value, list) + and value + and isinstance(value[0], int) + and not isinstance(value[0], bool) + ): + return [value[0] + 1, *value[1:]] + elif attribute == "dtype" and value in ("torch.float32", "torch.float64"): + return "torch.float64" if value == "torch.float32" else "torch.float32" + return None + + +def mutate( + traces: list[Path], invariants: Path, healthy: Path, output: Path +) -> MutationReport: + rules = [ + json_object(json.loads(line), "invariant") + for line in invariants.read_text().splitlines() + ] + coverage = read_object(healthy) + if not boolean( + required(coverage, "inputs_unchanged"), "inputs_unchanged" + ) or not boolean(required(coverage, "observer_enabled"), "observer_enabled"): + raise ValueError( + "Fault selection requires unchanged inputs and enabled observation" + ) + input_hashes = mapping(required(coverage, "input_sha256"), "input_sha256") + + def verify_hash(path: Path) -> None: + if string(required(input_hashes, str(path)), str(path)) != digest(path): + raise ValueError(f"Input changed since coverage collection: {path}") + + verify_hash(invariants) + chosen: JSONObject | None = None + call_id: str | None = None + selected_index: int | None = None + post_events: dict[str, list[JSONObject]] = {} + for trace in traces: + verify_hash(trace) + for line in trace.read_bytes().splitlines(): + event = json_object(json.loads(line), "trace event") + if string(required(event, "type"), "event.type") == "function_call (post)": + identity = string(required(event, "func_call_id"), "event.func_call_id") + post_events.setdefault(identity, []).append(event) + # Prefer shape; dtype is a prospective fallback when no live mutable shape + # rule exists. Selection remains post hoc and does not change learned rules. + for attribute in ("shape", "dtype"): + for raw_item in sequence(required(coverage, "per_invariant"), "per_invariant"): + item = json_object(raw_item, "per_invariant item") + index = integer(required(item, "invariant_index"), "invariant_index") + if not 0 <= index < len(rules): + raise ValueError(f"Invariant index out of range: {index}") + rule = rules[index] + if ( + string(required(rule, "relation"), "relation") + != "ConsistentOutputRelation" + ): + continue + params = sequence(required(rule, "params"), "params") + if len(params) != 2: + raise ValueError("ConsistentOutputRelation requires two parameters") + output_param = mapping(params[1], "output parameter") + if not ( + string(required(output_param, "attr_name"), "attr_name") == attribute + and boolean(required(item, "active"), "active") + and boolean(required(item, "check_passed"), "check_passed") + and integer( + required(item, "decisions_completed"), "decisions_completed" + ) + > 0 + ): + continue + for raw_decision in sequence( + required(item, "decision_records"), "decision_records" + ): + decision = mapping(raw_decision, "decision") + identity = string( + required(decision, "func_call_id"), "decision.func_call_id" + ) + events = post_events.get(identity, []) + if len(events) != 1: + continue + values = events[0]["return_values"] + tensors = values if isinstance(values, list) else [values] + eligible = any( + "Tensor" in name and replacement(attributes, attribute) is not None + for tensor in tensors + if isinstance(tensor, dict) + for name, attributes in tensor.items() + ) + if eligible: + chosen, call_id, selected_index = item, identity, index + break + if chosen is not None: + break + if chosen is not None: + break + if chosen is None: + raise ValueError("No healthy, actually evaluated mutable shape or dtype rule") + assert call_id is not None and selected_index is not None + rule = rules[selected_index] + output.mkdir(parents=True, exist_ok=False) + (output / "traces").mkdir() + edits: list[Edit] = [] + fingerprints: dict[str, Fingerprints] = {} + for trace in traces: + verify_hash(trace) + raw_lines = trace.read_bytes().splitlines(keepends=True) + transformed = list(raw_lines) + for index, raw in enumerate(raw_lines): + event = json_object(json.loads(raw), "trace event") + if ( + string(required(event, "type"), "event.type") != "function_call (post)" + or string(required(event, "func_call_id"), "event.func_call_id") + != call_id + ): + continue + before = copy.deepcopy(event) + values = event["return_values"] + tensor_values = values if isinstance(values, list) else [values] + for value in tensor_values: + if not isinstance(value, dict): + continue + for typename, attributes in value.items(): + if "Tensor" not in typename or not isinstance(attributes, dict): + continue + new_value = replacement(attributes, attribute) + if new_value is None: + continue + old = copy.deepcopy(attributes[attribute]) + attributes[attribute] = new_value + diff = differences(before, event) + assert len(diff) == 1 + assert diff[0][:1] == ("return_values",) + assert ( + before["process_id"] == event["process_id"] + and before["thread_id"] == event["thread_id"] + ) + edits.append( + { + "file": trace.name, + "zero_based_line": index, + "path": list(diff[0]), + "attribute": attribute, + "before_value": old, + "after_value": attributes[attribute], + "raw_process_id": event["process_id"], + "raw_thread_id": event["thread_id"], + } + ) + transformed[index] = ( + json.dumps(event, separators=(",", ":")).encode() + b"\n" + ) + break + if edits: + break + target = output / "traces" / trace.name + target.write_bytes(b"".join(transformed)) + assert all( + a == b + for i, (a, b) in enumerate(zip(raw_lines, transformed)) + if not any( + edit["file"] == trace.name and edit["zero_based_line"] == i + for edit in edits + ) + ) + fingerprints[trace.name] = { + "original_sha256": digest(trace), + "fault_sha256": digest(target), + } + assert len(edits) == 1, edits + manifest: MutationReport = { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": attribute, + "invariant_index": selected_index, + "invariant": rule, + "func_call_id": call_id, + "edits": edits, + "traces": fingerprints, + "invariants_sha256": digest(invariants), + "healthy_coverage_sha256": digest(healthy), + "precondition_inputs_unchanged": True, + "ids_unchanged": True, + "invariants_not_retrained_or_rewritten": True, + } + (output / "mutation.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--healthy-coverage", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print( + json.dumps( + mutate(args.traces, args.invariants, args.healthy_coverage, args.output) + ) + ) diff --git a/tools/fresh_validation/prepare_source.py b/tools/fresh_validation/prepare_source.py new file mode 100644 index 00000000..39eedf6d --- /dev/null +++ b/tools/fresh_validation/prepare_source.py @@ -0,0 +1,120 @@ +"""Build the explicitly composed, disposable validation source without changing Git.""" + +import argparse +import hashlib +import io +import json +import subprocess +import tarfile +from pathlib import Path +from typing import TypedDict + + +class SourceManifest(TypedDict): + base: str + trace_integrity: str + torchscript: str + state_fix_hunks_from: str + changed_python_files: list[str] + dynamic_typename_fix: str + source_sha256: dict[str, str] + + +BASE = "99d16905296d83495570fdcafa91370b242d2528" +INTEGRITY = "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4" +TORCHSCRIPT = "9ac61d1140cf8679c24442a174ec12f4de069980" +STATE = "1b5aedffbeefd4cdd9d30036dbb0a833521526cf" +TYPENAME = "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c" + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def build(repo: Path, destination: Path) -> SourceManifest: + def git(*args: str) -> bytes: + return subprocess.check_output(["git", "-C", str(repo), *args]) + + destination.mkdir(parents=True, exist_ok=False) + with tarfile.open( + fileobj=io.BytesIO(git("archive", BASE, "traincheck", "tests")) + ) as tar: + for member in tar: + if member.isfile(): + member_path = destination / member.name + member_path.parent.mkdir(parents=True, exist_ok=True) + extracted = tar.extractfile(member) + assert ( + extracted is not None + ), "Regular archive member must have contents" + member_path.write_bytes(extracted.read()) + old_hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + # These instrumentor files have no changes on the inference/cache stack. + for path in ( + "traincheck/instrumentor/tracer.py", + "traincheck/instrumentor/dumper.py", + ): + assert git("show", f"{INTEGRITY}^:{path}") == (destination / path).read_bytes() + assert ( + git("show", f"{TYPENAME}^:traincheck/utils.py") + == (destination / "traincheck/utils.py").read_bytes() + ) + replacements = { + "traincheck/instrumentor/tracer.py": TORCHSCRIPT, + "traincheck/instrumentor/dumper.py": INTEGRITY, + "traincheck/instrumentor/torchscript_compat.py": TORCHSCRIPT, + "tests/test_trace_integrity.py": INTEGRITY, + "tests/test_torchscript_compat.py": TORCHSCRIPT, + "tests/test_precondition_state_isolation.py": STATE, + "traincheck/utils.py": TYPENAME, + "tests/test_dynamic_typename.py": TYPENAME, + } + for path, commit in replacements.items(): + (destination / path).write_bytes(git("show", f"{commit}:{path}")) + source_path = destination / "traincheck/invariant/precondition.py" + source = source_path.read_text() + old = "_pruned_clauses: set[PreconditionClause] = set()," + new = "_pruned_clauses: set[PreconditionClause] | None = None," + assert source.count(old) == 1 + source = source.replace(old, new) + # Copy only the two approved state hunks, preserving the base RNG isolation. + state_source = git("show", f"{STATE}:traincheck/invariant/precondition.py").decode() + comment = " # Recursive children share pruning state; independent solves start fresh.\n" + insertion = ( + comment + " if _pruned_clauses is None:\n _pruned_clauses = set()\n" + ) + assert insertion in state_source + anchor = state_source.split(insertion)[1][:100] + assert source.count(anchor) == 1 + source = source.replace(anchor, insertion + anchor, 1) + assert "group_rng = random.Random(42)" in source + source_path.write_text(source) + hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + changed = sorted(p for p, h in hashes.items() if h != old_hashes.get(p)) + manifest: SourceManifest = { + "base": BASE, + "trace_integrity": INTEGRITY, + "torchscript": TORCHSCRIPT, + "state_fix_hunks_from": STATE, + "changed_python_files": changed, + "dynamic_typename_fix": TYPENAME, + "source_sha256": hashes, + } + (destination / "source-manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n" + ) + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(build(args.repo.resolve(), args.output.resolve()), indent=2)) diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py new file mode 100644 index 00000000..9e52a598 --- /dev/null +++ b/tools/fresh_validation/run_phase.py @@ -0,0 +1,159 @@ +"""Run one bounded correctness phase in a fresh OS process (Linux only).""" + +import argparse +import hashlib +import json +import os +import signal +import subprocess +import time +from pathlib import Path +from typing import Literal, TypedDict + + +class PhaseOptions(argparse.Namespace): + source: Path + output: Path + timeout: float + memory_gib: float + trace_mib: float + command: list[str] + + +StopReason = Literal["process_group_rss_budget", "timeout", "trace_size_budget"] + + +class PhaseResult(TypedDict): + exit_code: int + stop_reason: StopReason | None + source_unchanged: bool + pid: int + elapsed_seconds_operational_only: float + peak_process_group_rss_bytes_operational_only: int + + +def group_memory(group: int) -> tuple[int, list[int]]: + total = 0 + processes: list[int] = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + stat = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(stat[2]) != group: + continue + rss = next( + int(line.split()[1]) * 1024 + for line in (entry / "status").read_text().splitlines() + if line.startswith("VmRSS:") + ) + total += rss + processes.append(int(entry.name)) + except (FileNotFoundError, ProcessLookupError, StopIteration): + continue + return total, processes + + +def run(args: PhaseOptions) -> int: + args.output.mkdir(parents=True, exist_ok=False) + source_hashes = { + str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in args.source.rglob("*.py") + } + env = os.environ.copy() + env.update( + PYTHONPATH=str(args.source.resolve()), + PYTHONHASHSEED="0", + PYTHONUNBUFFERED="1", + PYTHONDONTWRITEBYTECODE="1", + CUDA_VISIBLE_DEVICES="", + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + NUMEXPR_NUM_THREADS="1", + VECLIB_MAXIMUM_THREADS="1", + ) + command = args.command[1:] if args.command[0] == "--" else args.command + started, peak = time.monotonic(), 0 + reason: StopReason | None = None + with (args.output / "stdout.log").open("w") as log: + proc = subprocess.Popen( + ["nice", "-n", "10", *command], + cwd=args.output, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + invocation = { + "command": command, + "pid": proc.pid, + "native_threads": 1, + "timeout_seconds": args.timeout, + "memory_limit_gib": args.memory_gib, + "trace_limit_mib": args.trace_mib, + "source_sha256": source_hashes, + "harness_sha256": { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in Path(__file__).resolve().parent.glob("*.py") + }, + } + (args.output / "invocation.json").write_text( + json.dumps(invocation, indent=2) + "\n" + ) + try: + while proc.poll() is None: + memory, _ = group_memory(proc.pid) + peak = max(peak, memory) + if memory > args.memory_gib * 1024**3: + reason = "process_group_rss_budget" + if time.monotonic() - started > args.timeout: + reason = "timeout" + if ( + sum(p.stat().st_size for p in args.output.rglob("trace_API*.log")) + > args.trace_mib * 1024**2 + ): + reason = "trace_size_budget" + if reason: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + break + time.sleep(0.25) + except BaseException: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=10) + raise + unchanged = all( + hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h + for p, h in source_hashes.items() + ) + assert proc.returncode is not None, "Supervisor must reap its child" + result: PhaseResult = { + "exit_code": proc.returncode, + "stop_reason": reason, + "source_unchanged": unchanged, + "pid": proc.pid, + "elapsed_seconds_operational_only": time.monotonic() - started, + "peak_process_group_rss_bytes_operational_only": peak, + } + (args.output / "phase-result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + return int(proc.returncode != 0 or reason is not None or not unchanged) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--memory-gib", type=float, default=8) + parser.add_argument("--trace-mib", type=float, default=256) + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args(namespace=PhaseOptions()) + if not options.command: + parser.error("A command is required") + raise SystemExit(run(options)) diff --git a/tools/fresh_validation/run_study.py b/tools/fresh_validation/run_study.py new file mode 100644 index 00000000..0cbd0f48 --- /dev/null +++ b/tools/fresh_validation/run_study.py @@ -0,0 +1,197 @@ +"""Collect reference/control/holdout traces and check frozen same-workload rules. + +Every training, inference, and checking phase starts a separate bounded process. +Only the reference is used to infer rules. This is a correctness/coverage study; +the supervisor's time and memory observations are operational limits only. +""" + +import argparse +import hashlib +import json +import subprocess +from collections.abc import Sequence +from pathlib import Path +from typing import Literal + + +class StudyOptions(argparse.Namespace): + source: Path + python: Path + benchmark_root: Path + data_root: Path | None + output: Path + workload: Literal["mnist", "transformer"] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--data-root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--workload", choices=("mnist", "transformer"), required=True) + args = parser.parse_args(namespace=StudyOptions()) + if args.workload == "mnist" and args.data_root is None: + parser.error("MNIST requires an existing real IDX data cache") + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=False) + harness = Path(__file__).resolve().parent + hashes = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in harness.glob("*.py") + } + (args.output / "study-plan.json").write_text( + json.dumps( + { + "workload": args.workload, + "reference_steps": 10, + "holdout_steps": 20, + "reference_seed": 42, + "holdout_seed": 43, + "data_steps": 20, + "reference_inputs_overlap_first_half_of_holdout": True, + "disabled_relations": ["FunctionLeadRelation", "FunctionCoverRelation"], + "harness_sha256": hashes, + }, + indent=2, + ) + + "\n" + ) + + def phase(name: str, command: Sequence[str | Path | int]) -> None: + subprocess.run( + [ + str(args.python), + str(harness / "run_phase.py"), + "--source", + str(args.source.resolve()), + "--output", + str(args.output / name), + "--", + str(args.python), + *map(str, command), + ], + check=True, + ) + + for role, steps, seed, mode in ( + ("reference", 10, 42, "instrumented"), + ("control", 10, 42, "instrumented"), + ("holdout", 20, 43, "instrumented"), + ("baseline-holdout", 20, 43, "baseline"), + ): + command: list[str | Path | int] = [ + harness / "benchmark_workload.py", + "--benchmark-root", + args.benchmark_root.resolve(), + "--workload", + args.workload, + "--steps", + steps, + "--data-steps", + 20, + "--seed", + seed, + "--batch-size", + 2, + "--device", + "cpu", + "--mode", + mode, + "--output", + args.output / role, + ] + if args.data_root is not None: + command.extend(["--data-root", args.data_root.resolve()]) + phase(role, command) + invariants = args.output / "inference/invariants.jsonl" + traces = sorted((args.output / "reference/traces").glob("trace_API*.log")) + if not traces: + raise RuntimeError("No reference API traces") + phase( + "inference", + [ + "-m", + "traincheck.infer_engine", + "-t", + *traces, + "-o", + invariants, + "--backend", + "pandas", + "-j", + 1, + "--disable-relation", + "FunctionLeadRelation", + "FunctionCoverRelation", + ], + ) + for role in ("reference", "control", "holdout"): + traces = sorted((args.output / role / "traces").glob("trace_API*.log")) + phase( + "check-" + role, + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--output", + args.output / ("check-" + role) / "coverage.json", + ], + ) + traces = sorted((args.output / "holdout/traces").glob("trace_API*.log")) + phase( + "check-holdout-direct", + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--observer-off", + "--output", + args.output / "check-holdout-direct/coverage.json", + ], + ) + + phase( + "prepare-fault", + [ + harness / "fault_control.py", + "--traces", + *traces, + "--invariants", + invariants, + "--healthy-coverage", + args.output / "check-holdout/coverage.json", + "--output", + args.output / "fault", + ], + ) + faulty_traces = sorted((args.output / "fault/traces").glob("trace_API*.log")) + phase( + "check-fault", + [ + harness / "check_coverage.py", + "--traces", + *faulty_traces, + "--invariants", + invariants, + "--output", + args.output / "check-fault/coverage.json", + ], + ) + subprocess.run( + [ + str(args.python), + str(harness / "compare_study.py"), + "--root", + str(args.output), + ], + check=True, + ) + + +if __name__ == "__main__": + main()