[ocaml5-issue] Sys stress test may fail or timeout

[jmid]

The revision in #304 of the current Sys tests can either fail or timeout, with the latter being most likely.
This has been observed on both macOS (Intel and ARM64) with 5.3/5.4/trunk and on Linux ARM64 and MSVC with 5.3.

Here's an example of macOS Intel with trunk timing out after 3h:
https://github.com/ocaml-multicore/multicoretests/actions/runs/15207484605/job/42773854908?pr=304

Starting 2-th run

random seed: 531872336
generated error fail pass / total     time test name

[ ]    0    0    0    0 / 1000     0.0s STM Sys test sequential
[ ]    0    0    0    0 / 1000     0.0s STM Sys test sequential (generating)
[✓] 1000    0    0 1000 / 1000     8.8s STM Sys test sequential

[ ]    0    0    0    0 / 1000     0.0s STM Sys stress test parallel
[ ]  142    0    0  142 / 1000    51.6s STM Sys stress test parallel
[ ]  273    0    0  273 / 1000   111.7s STM Sys stress test parallel
[ ]  497    0    0  497 / 1000   171.8s STM Sys stress test parallel
Error: The operation was canceled.

Here's an example of a strange Linux ARM64 failure with 5.3 after 54m (way below the workflow timeout) causing a cancellation,
with little useful info in the CI log:
https://github.com/ocaml-multicore/multicoretests/actions/runs/15207485548/job/42773857490?pr=304

Starting 5-th run

random seed: 28017438
generated error fail pass / total     time test name

[ ]    0    0    0    0 / 1000     0.0s STM Sys test sequential
[ ]    0    0    0    0 / 1000     0.0s STM Sys test sequential (generating)
[✓] 1000    0    0 1000 / 1000     1.5s STM Sys test sequential

[jmid]

I've worked on reproducing this and after some manual shrinking I am now down to the following which triggers consistently on macOS with either an Intel or ARM64 CPU. I was also able to trigger it locally on an Intel-based MacBook Pro:

                                       |                   
                                Mkdir ([], hhh)            
                              Mkfile ([hhh], iii)          
                              Mkdir ([hhh], hhh)           
                                       |                   
                    .------------------------------------.
                    |                                    |                   
         Mkdir ([hhh; iii], eee)          Rename ([hhh; iii], [iii; ccc])    
         Rename ([hhh; hhh], [])                  Rmdir ([], hhh)            
           Rename ([bbb], [])                   Mkdir ([hhh], iii)           
                                                  Rmdir ([], hhh)          

When run, the symptom is that the system locks up and becomes non-responsive - likely a deadlock.
Locally, I've thus not been able to interact with an lldb debugger to understand better what is going on.

Removing Sys.rename generation, this failed to trigger with random test inputs, which indicates that the function is (partly) responsible. I've therefore studied caml_sys_rename which doesn't register its arguments with the GC. AFAICS it should be safe not to do so, as no GC can happen before both potential GC roots are dead (unused).

Interestingly, all the cmds in the parallel legs will raise an exception, which makes me wonder if this is exception related.
The issue could also be timing or scheduling-related, explaining why it primarily shows up on macOS.
This reproducer does not trigger on Linux ARM64 (as mentioned above), though. 🤔

[jmid]

Below follows my current reproducer. It consistently locks up the CI in the first iteration (also confirmed on my old macbook so I stopped that):

let sandbox_root = "_sandbox"

let init_sut () =
  try Sys.mkdir sandbox_root 0o700
  with Sys_error msg when msg = sandbox_root ^ ": File exists" -> ()

let cleanup _ =
  match Sys.os_type with
  | "Cygwin"
  | "Unix"  -> ignore (Sys.command ("rm -r " ^ Filename.quote sandbox_root))
  | "Win32" -> ignore (Sys.command ("rd /s /q " ^ Filename.quote sandbox_root))
  | v -> failwith ("Sys tests not working with " ^ v)

let mkfile filepath =
  let flags = [Open_wronly; Open_creat; Open_excl] in
  Out_channel.with_open_gen flags 0o666 filepath (fun _ -> ())

let stress_prop_par () =
  let sut = init_sut () in

  Sys.mkdir "_sandbox/hhh" 0o755;
  mkfile "_sandbox/hhh/iii";
  Sys.mkdir "_sandbox/hhh/hhh" 0o755;

  let barrier = Atomic.make 2 in
  let dom1 () =
    Atomic.decr barrier;
    while Atomic.get barrier <> 0 do Domain.cpu_relax() done;
    (try Sys.rename "_sandbox/hhh/hhh" "_sandbox"; assert false with _ -> ());
    (try Sys.rename "_sandbox/bbb" "_sandbox"; assert false with _ -> ());
    (try Sys.mkdir "_sandbox/hhh/iii/eee" 0o755; assert false with _ -> ());
  in
  let dom2 () =
    Atomic.decr barrier;
    while Atomic.get barrier <> 0 do Domain.cpu_relax() done;
    (try Sys.rename "_sandbox/hhh/iii" "_sandbox/iii/ccc"; assert false with _ -> ());
    (try Sys.mkdir "_sandbox/hhh/iii" 0o755; assert false with _ -> ());
    (try Sys.rmdir "_sandbox/hhh"; assert false with _ -> ());
    (try Sys.rmdir "_sandbox/hhh"; assert false with _ -> ());
  in
  let dom1 = Domain.spawn dom1 in
  let dom2 = Domain.spawn dom2 in
  Domain.join dom1;
  Domain.join dom2;
  cleanup sut

let rec repeat n prop input =
  if n=0 then () else (prop input; Printf.printf "#%!"; repeat (n-1) prop input)

let _ =
  let rep_count = 50 in (* No. of inner repetitions of the non-deterministic property *)
  for i=1 to 1000 do
    Printf.printf "\nIteration %i %!" i;
    repeat rep_count stress_prop_par () (* 50 times each *)
  done

[jmid]

I've now reported this upstream as ocaml/ocaml#14278.