ocaml-multicore · nikochiko · Mar 1, 2023 · Mar 1, 2023 · Mar 10, 2023 · Mar 16, 2023
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -8,17 +8,11 @@ on:
     - cron: 0 1 * * MON
 
 jobs:
-  build:
-    strategy:
-      matrix:
-        os:
-          - ubuntu-latest
-          - macos-latest
-        ocaml-compiler:
-          - ocaml-base-compiler.5.0.0~alpha0
-          - ocaml-variants.5.1.0+trunk
-
-    runs-on: ${{ matrix.os }}
+  windows:
+    runs-on: windows-latest
+
+    env:
+      QCHECK_MSG_INTERVAL: '60'
 
     steps:
       - name: Checkout code
@@ -31,18 +25,21 @@ jobs:
           curl -sH "Accept: application/vnd.github.v3+json" \
           https://api.github.com/repos/ocaml/ocaml/commits/trunk \
           | jq .commit.tree.sha | xargs printf '::set-output name=commit::%s'
+
       - name: Use OCaml ${{ matrix.ocaml-compiler }}
         uses: ocaml/setup-ocaml@v2
         with:
-          ocaml-compiler: ${{ matrix.ocaml-compiler }}
+          opam-pin: false
+          opam-depext: false
+          ocaml-compiler: ocaml.5.0.0,ocaml-option-mingw
           opam-repositories: |
-            default: https://github.com/ocaml/opam-repository.git
-            alpha: https://github.com/kit-ty-kate/opam-alpha-repository.git
+            dra27: https://github.com/dra27/opam-repository.git#windows-5.0
+            default: https://github.com/ocaml-opam/opam-repository-mingw.git#sunset
+            upstream: https://github.com/ocaml/opam-repository.git
           cache-prefix: ${{ steps.multicore_hash.outputs.commit }}
-          opam-depext: false
 
       - run: opam install . --deps-only --with-test
 
       - run: opam exec -- dune build
 
-      - run: opam exec -- dune runtest
+      - run: opam exec -- dune runtest
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,6 @@ tmp
 *.install
 *.native
 *.byte
-*.merlin
+*.merlin
+*.json
+node_modules
diff --git a/.merlin b/.merlin
diff --git a/.ocamlformat b/.ocamlformat
@@ -1,2 +1,2 @@
 profile = default
-version = 0.23.0
+version = 0.25.1
diff --git a/.ocp-indent b/.ocp-indent
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,14 @@
+{
+  "arrowParens": "avoid",
+  "bracketSpacing": false,
+  "printWidth": 80,
+  "semi": false,
+  "singleQuote": true,
+  "proseWrap": "always",
+  "overrides": [
+    {
+      "files": ["*.md"],
+      "excludeFiles": "_build/*"
+    }
+  ]
+}
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,22 +1,28 @@
-# Release notes
+## 0.4.0
 
-All notable changes to this project will be documented in this file.
+- Add docs and rename/refactor to add a lockfree package (@lyrm)
+- CI clean up and set up Windows CI (@lyrm)
+- Adopt OCaml Code of Conduct (@Sudha247)
+- Mark alcotest as a test dependency (@Khady)
+- Set QCHECK_MSG_INTERVAL to avoid clutter in CI logs (@jmid)
+- Fix space leaks in MS Queue (@polytypic, @lyrm)
+- Add STM tests for current data structures (@lyrm, @jmid)
 
-## 0.3.1 
+## 0.3.1
 
-* Rework dscheck integration to work with utop (@lyrm)
-* Add OCaml 4 compatability (@sudha247) 
-* Add STM ws_deque tests (@jmid, @lyrm)
+- Rework dscheck integration to work with utop (@lyrm)
+- Add OCaml 4 compatability (@sudha247)
+- Add STM ws_deque tests (@jmid, @lyrm)
 
 ## 0.3.0
 
-* Add MPSC queue (@lyrm)
-* Add SPSC queue (@bartoszmodelski)
-* Add MPMC relaxed queue (@bartoszmodelski, @lyrm)
-* Add Michael-Scott Queue (@tmcgilchrist, @bartoszmodelski, @lyrm)
-* Add Treiber Stack (@tmcgilchrist , @bartoszmodelski, @lyrm)
-* Integrate model-checker (DSCheck) (@bartoszmodelski)
+- Add MPSC queue (@lyrm)
+- Add SPSC queue (@bartoszmodelski)
+- Add MPMC relaxed queue (@bartoszmodelski, @lyrm)
+- Add Michael-Scott Queue (@tmcgilchrist, @bartoszmodelski, @lyrm)
+- Add Treiber Stack (@tmcgilchrist , @bartoszmodelski, @lyrm)
+- Integrate model-checker (DSCheck) (@bartoszmodelski)
 
 ## v0.2.0
 
-* Add Chase-Lev Work-stealing deque `Ws_deque`. (@ctk21)
+- Add Chase-Lev Work-stealing deque `Ws_deque`. (@ctk21)
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
@@ -0,0 +1,14 @@
+# Code of Conduct
+
+This project has adopted the [OCaml Code of Conduct](https://github.com/ocaml/code-of-conduct/blob/main/CODE_OF_CONDUCT.md).
+
+# Enforcement
+
+This project follows the OCaml Code of Conduct
+[enforcement policy](https://github.com/ocaml/code-of-conduct/blob/main/CODE_OF_CONDUCT.md#enforcement).
+
+To report any violations, please contact:
+
+* Bartosz Modelski <bartosz [at] tarides [dot] com>
+* Carine Morel <carine [at] tarides [dot] com>
+* Sudha Parimala <sudha [at] tarides [dot] com>
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,30 @@
+## Contributing
+
+Any contributions are appreciated! Please create issues/PRs to this repo.
+
+### Maintainers
+
+The current list of maintainers is as follows:
+
+- @kayceesrk KC Sivaramakrishnan
+- @lyrm Carine Morel
+- @Sudha247 Sudha Parimala
+
+### Guidelines for new data structures implementation
+
+Reviewing most implementation takes times. Here are a few guidelines to make it
+easier for the reviewers :
+
+- the issue tracker has a good list of data structures to choose from
+- implement a well know algorithm (there are a lot !)
+  - from a _reviewed_ paper, ideally with proof of main claimed properties (like
+    lock-freedom, deadlock freedom etc..)
+  - from a well known and used concurrent library (like `java.util.concurrent`)
+- write tests with **multiple** domains. All the following tests are expected to
+  be provided before a proper review is done, especially for implementations
+  that do not come from a well-know algorithm :
+  - unitary tests and `qcheck tests` : with one and multiple domains. If domains
+    have specific roles (producer, consumer, stealer, etc..), it should appear
+    in the tests.
+  - tests using `STM` from `multicoretest`
+  - (_optional_) `dscheck` tests (for non-blocking implementation)
diff --git a/README.md b/README.md
@@ -1,27 +1,32 @@
-# `lockfree` — Lock-free data structures for Multicore OCaml
---------------------------------------------------------
+# `Saturn` — parallelism-safe data structures for Multicore OCaml
 
-A collection of Concurrent Lockfree Data Structures for OCaml 5. It contains:
+---
 
-* [Treiber Stack](src/treiber_stack.mli) A classic multi-producer multi-consumer stack, robust and flexible. Recommended starting point when needing LIFO structure. 
-
-* [Michael-Scott Queue](src/michael_scott_queue.mli) A classic multi-producer multi-consumer queue, robust and flexible. Recommended starting point when needing FIFO structure. It is based on [Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms](https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf).
+This repository is a collection of parallelism-safe data structures for OCaml 5.
+They are contained in two packages:
 
-* [Chase-Lev Work-Stealing Deque](src/ws_deque.mli) Single-producer, multi-consumer dynamic-size double-ended queue (deque) (see [Dynamic circular work-stealing deque](https://dl.acm.org/doi/10.1145/1073970.1073974) and [Correct and efficient work-stealing for weak memory models](https://dl.acm.org/doi/abs/10.1145/2442516.2442524)). Ideal for throughput-focused scheduling using per-core work distribution. Note, [pop] and [steal] follow different ordering (respectively LIFO and FIFO) and have different linearization contraints.
+- `Saturn` that includes every data structures and should be used by default if
+  you just want parallelism-safe data structures..
+- `Saturn_lockfree` that includes only lock-free data structures.
 
-* [SPSC Queue](src/spsc_queue.mli) Simple single-producer single-consumer fixed-size queue. Thread-safe as long as at most one thread acts as producer and at most one as consumer at any single point in time.
+The available data structures are :
 
-* [MPMC Relaxed Queue](src/mpmc_relaxed_queue.mli) Multi-producer, multi-consumer, fixed-size relaxed queue. Optimised for high number of threads. Not strictly FIFO. Note, it exposes two interfaces: a lockfree and a non-lockfree (albeit more practical) one. See the mli for details. 
-
-* [MPSC Queue](src/mpsc_queue.mli) A multi-producer, single-consumer, thread-safe queue without support for cancellation. This makes a good data structure for a scheduler's run queue. It is used in [Eio](https://github.com/ocaml-multicore/eio). It is a single consumer version of the queue described in [Implementing lock-free queues](https://people.cs.pitt.edu/~jacklange/teaching/cs2510-f12/papers/implementing_lock_free.pdf).
+| Names                                                        | Names in `Saturn` <br> (in `Saturn_lockfree`) | What is it ?                                                                                                                                                                                                                                                                   | Sources                                                                                                                                                                                                      |
+| ------------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [Treiber stack](src_lockfree/treiber_stack.mli)              | `Stack` (same)                                | A classic multi-producer multi-consumer stack, robust and flexible. Recommended starting point when needing a LIFO structure                                                                                                                                                   |                                                                                                                                                                                                              |
+| [Michael-Scott queue](src_lockfree/michael_scott_queue.mli)  | `Queue` (same)                                | A classic multi-producer multi-consumer queue, robust and flexible. Recommended starting point when needing a FIFO structure.                                                                                                                                                  | [Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms](https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf)                                                         |
+| [Chase-Lev Work-Stealing Dequeue](src_lockfree/ws_deque.mli) | `Work_stealing_deque` (same)                  | Single-producer, multi-consumer dynamic-size double-ended queue (deque). Ideal for throughput-focused scheduling using per-core work distribution. Note, `pop` and `steal` follow different ordering (respectively LIFO and FIFO) and have different linearization contraints. | [Dynamic circular work-stealing deque](https://dl.acm.org/doi/10.1145/1073970.1073974) and [Correct and efficient work-stealing for weak memory models](https://dl.acm.org/doi/abs/10.1145/2442516.2442524)) |
+| [SPSC Queue](src_lockfree/spsc_queue.mli)                    | `Single_prod_single_`<br>`cons_queue` (same)  | Simple single-producer single-consumer fixed-size queue. Thread-safe as long as at most one thread acts as producer and at most one as consumer at any single point in time.                                                                                                   |                                                                                                                                                                                                              |
+| [MPMC bounded relaxed queue](src/mpmc_relaxed_queue.mli)             | `Relaxed queue` (same)                | Multi-producer, multi-consumer, fixed-size relaxed queue. Optimised for high number of threads. Not strictly FIFO. Note, it exposes two interfaces: a lockfree and a non-lockfree (albeit more practical) one. See the mli for details.                                        |                                                                                                                                                                                                              |
+| [MPSC Queue](src_lockfree/mpsc_queue.mli)                    | `Single_consumer_queue` (same)                | A multi-producer, single-consumer, thread-safe queue without support for cancellation. This makes a gooddata structure for a scheduler's run queue. It is used in [Eio](https://github.com/ocaml-multicore/eio).                                                               | It is a single consumer version of the queue described in [Implementing lock-free queues](https://people.cs.pitt.edu/~jacklange/teaching/cs2510-f12/papers/implementing_lock_free.pdf).                      |
 
 ## Usage
 
-lockfree can be installed from `opam`: `opam install lockfree`. Sample usage of
-`Ws_deque` is illustrated below.
+`Saturn` can be installed from `opam`: `opam install saturn`. Sample usage of
+`Work_stealing_deque` is illustrated below.
 
 ```ocaml
-module Ws_deque = Ws_deque.M
+module Ws_deque = Work_stealing_deque.M
 
 let q = Ws_deque.create ()
 
@@ -30,11 +35,26 @@ let () = Ws_deque.push q 100
 let () = assert (Ws_deque.pop q = 100)
 ```
 
+## Tests
+
+Several kind of tests are provided for each data structure:
+
+- unitary tests and `qcheck` tests: check semantics and expected behaviors with
+  one and more domains;
+- `STM` tests: check _linearizability_ for two domains (see
+  [multicoretests library](https://github.com/ocaml-multicore/multicoretests));
+- `dscheck`: checks _non-blocking_ property for as many domains as wanted (for
+  two domains most of the time). See
+  [dscheck](https://github.com/ocaml-multicore/dscheck).
+
+See [test/README.md](test/README.md) for more details.
+
 ## Benchmarks
 
-There is a number of benchmarks in `bench/` directory. You can run them with `make bench`. See [bench/README.md](bench/README.md) for more details.
+There is a number of benchmarks in `bench` directory. You can run them with
+`make bench`. See [bench/README.md](bench/README.md) for more details.
 
 ## Contributing
 
-Contributions of more lockfree data structures appreciated! Please create
-issues/PRs to this repo.
+Contributions are appreciated! If you intend to add a new data structure, please
+read [this](CONTRIBUTING.md) before.
diff --git a/bench/README.md b/bench/README.md
@@ -1,11 +1,13 @@
-Benchmarks for lockfree
+Benchmarks for Saturn
 
-# General usage 
+# General usage
 
-Execute `make bench` from root of the repository to run the standard set of benchmarks. The output is in JSON, as it is intended to be consumed by ocaml-benchmark CI (in progress). 
+Execute `make bench` from root of the repository to run the standard set of
+benchmarks. The output is in JSON, as it is intended to be consumed by
+ocaml-benchmark CI (in progress).
 
-# Specific structures 
+# Specific structures
 
 Some benchmarks expose commandline interface targeting particular structures:
 
-* [mpmc_queue.exe](mpmc_queue_cmd.ml)
+- [mpmc_queue.exe](mpmc_queue_cmd.ml)
diff --git a/bench/backoff.ml b/bench/backoff.ml
@@ -5,7 +5,7 @@ type 'a t = { value : 'a; next : 'a t option Atomic.t }
 let empty () = { value = Obj.magic (); next = Atomic.make None }
 
 let push ~backoff_once t value =
-  let b = Lockfree.Backoff.create () in
+  let b = Saturn.Backoff.create () in
   let new_head = ({ value; next = Atomic.make None } : 'a t) in
   let rec push_f () =
     let head = Atomic.get t.next in
@@ -18,7 +18,7 @@ let push ~backoff_once t value =
   push_f ()
 
 let rec pop ?min_wait ~backoff_once t =
-  let b = Lockfree.Backoff.create ?min_wait () in
+  let b = Saturn.Backoff.create ?min_wait () in
   let head = Atomic.get t.next in
   match head with
   | None -> None
@@ -95,8 +95,8 @@ let run_artificial ~backoff_once () =
 
 let bench ~run_type ~with_backoff () =
   let backoff_once =
-    if with_backoff then Lockfree.Backoff.once
-    else fun (_ : Lockfree.Backoff.t) -> ()
+    if with_backoff then Saturn.Backoff.once
+    else fun (_ : Saturn.Backoff.t) -> ()
   in
   let results = ref [] in
   let run =

diff --git a/bench/bench_spsc_queue.ml b/bench/bench_spsc_queue.ml
@@ -1,4 +1,4 @@
-open Lockfree
+module Spsc_queue = Saturn.Single_prod_single_cons_queue
 
 let item_count = 2_000_000
 

diff --git a/bench/dune b/bench/dune
@@ -1,3 +1,3 @@
 (executables
  (names main mpmc_queue_cmd)
- (libraries lockfree unix yojson))
+ (libraries saturn unix yojson))
diff --git a/bench/main.ml b/bench/main.ml
@@ -27,7 +27,7 @@ let () =
     |> String.concat ", "
   in
   let output =
-    Printf.sprintf {| {"name": "lockfree", "results": [%s]}|} results
+    Printf.sprintf {| {"name": "saturn", "results": [%s]}|} results
     (* Cannot use Yojson rewriters as of today none works on OCaml 5.1.0.
        This at least verifies that the manually crafted JSON is well-formed.
 

diff --git a/bench/mpmc_queue.ml b/bench/mpmc_queue.ml
@@ -1,4 +1,4 @@
-open Lockfree.Mpmc_relaxed_queue
+open Saturn.Relaxed_queue
 
 let num_of_elements = ref 500_000
 let num_of_pushers = ref 4
@@ -57,8 +57,8 @@ let create_output ~time_median ~throughput_median ~throughput_stddev =
 
 let run_bench () =
   if !use_cas_intf then (
-    push := Lockfree.Mpmc_relaxed_queue.Not_lockfree.CAS_interface.push;
-    pop := Lockfree.Mpmc_relaxed_queue.Not_lockfree.CAS_interface.pop);
+    push := Saturn.Relaxed_queue.Not_lockfree.CAS_interface.push;
+    pop := Saturn.Relaxed_queue.Not_lockfree.CAS_interface.pop);
   let queue = create ~size_exponent:10 () in
   let orchestrator =
     Orchestrator.init

diff --git a/bench/orchestrator.mli b/bench/orchestrator.mli
@@ -1,8 +1,37 @@
-(** Helper library that ensures all workers have started before any 
+(** Helper library that ensures all workers have started before any
   starts making progress on the benchmark. *)
 
 type t
+(** An orchestrator is similar to a counter that ensures each domain
+    has started and complete each round simultanously. All domains
+    wait for the other before beginning the next round. *)
 
 val init : total_domains:int -> rounds:int -> t
+(** [init ~total_domains:nd ~rounds:nr] create an orchestrator that
+    will run [nr] rounds for a test that uses exactly [nd] worker
+    domains *)
+
 val worker : t -> (unit -> unit) -> unit
+(** [worker t f] builds the function to pass to [Domain.spawn] while
+    using the orchestrator [t]. Doing [Domain.spawn (fun () -> worker
+    t f)] is similar to [Domain.spawn f] except that the orchestrator
+    is used to synchronize all domains progress.
+ *)
+
 val run : ?drop_first:bool -> t -> float List.t
+(** [run t] is launching the benchmark by enabling domains to progress. Benchmarks code should have the following structure :
+
+{[
+   (* Initialize the orchestrator, with [nd] the number of domains we want. *)
+   let orchestrator = init ~total_domain:nd ~round:100 in
+   (* Spawn domains with [worker] *)
+   let domains =
+         List.init nd (fun _ ->
+               Domain.spawn (fun () ->
+                      worker orchestrator (fun () -> some_function ()))) in
+   (* Run the benchmarks by freeing domains round by round. *)
+   let times = run orchestrator in
+   ...
+]}
+
+*)
diff --git a/dune-project b/dune-project
@@ -1,2 +1,8 @@
 (lang dune 3.0)
-(name lockfree)
+(name saturn)
+(package
+ (name saturn)
+ (synopsis "Collection of parallelism-safe data structures for Multicore OCaml"))
+(package
+ (name saturn_lockfree)
+ (synopsis "Collection of lock-free data structures for Multicore OCaml"))
diff --git a/examples/dune b/examples/dune
@@ -0,0 +1,3 @@
+(executables
+ (names michael_scott_queue)
+ (libraries saturn_lockfree))