Merge pull request #4 from glotzerlab/include-any

Evaluate any/all group include conditions.

Author: joaander

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,15 +13,9 @@ Resolves #???
 
 <!--- Please describe how you tested your changes. -->
 
-## Change log
-
-<!-- Propose a change log entry. -->
-```
-
-```
-
 ## Checklist:
 
 - [ ] I have reviewed the [**Contributor Guidelines**](https://github.com/glotzerlab/row/blob/trunk/doc/src/developers/contributing.md).
 - [ ] I agree with the terms of the [**Row Contributor Agreement**](https://github.com/glotzerlab/row/blob/trunk/ContributorAgreement.md).
 - [ ] My name is on the list of contributors (`doc/src/contributors.md`) in the pull request source branch.
+- [ ] I have added a change log entry to `doc/src/release-notes.md`.

.github/workflows/pre-commit.yaml

@@ -8,7 +8,7 @@ on:
   pull_request:
   push:
     branches:
-    - "*"
+    - trunk
 
   workflow_dispatch:
 

.github/workflows/release.yaml

@@ -12,7 +12,7 @@ on:
 
   push:
     branches:
-      - "*"
+    - trunk
     tags:
       - "*"
 

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ on:
   pull_request:
   push:
     branches:
-    - "*"
+    - trunk
 
   workflow_dispatch:
 

DESIGN.md

@@ -295,4 +295,3 @@ status may take a long time, so it should display a progress bar.
 - **workspace**: The location on the file system that contains **directories**.
 
 # TODO: logo
-# TODO: Expand include to apply any to the array and allow condition or all elements.

doc/src/developers/contributing.md

@@ -88,6 +88,6 @@ When applicable, update or write a new tutorial or how-to guide.
 Update the contributors documentation to name each developer that has contributed to the
 code.
 
-### Propose a change log entry
+### Add a change log entry
 
-Propose a short concise entry describing the change in the pull request description.
+Add a short concise entry describing the change in `doc/src/release-notes.md`.

doc/src/guide/howto/same.md

@@ -2,9 +2,8 @@
 
 You can submit the same action to different groups and resources. To do so,
 create multiple elements in the action array *with the same name*. Each must use
-[`group.include`](../../workflow/action/group.md#include) to select *non-overlapping
-subsets*. You can use [`action.from`](../../workflow/action/index.md#from) to copy all
-fields from one action and selectively override others.
+[`action.group.include`] to select *non-overlapping subsets*. You can use
+[`action.from`] to copy all fields from one action and selectively override others.
 
 For example, this `workflow.toml` uses 4 processors on directories with small *N* and 8
 those with a large *N*.
@@ -20,11 +19,16 @@ products = ["results.out"]
 walltime.per_submission = "12:00:00"
 processes.per_directory = 4
 [action.group]
-include = [["/N", "<=", "4096"]]
 maximum_size = 32
+[[action.group.include]]
+condition = ["/N", "<=", "4096"]
 
 [[action]]
 from = "compute"
 resources.processes.per_directory = 8
-group.include = [["/N", ">", "4096"]]
+[[action.group.include]]
+condition = ["/N", ">", "4096"]
 ```
+
+[`action.group.include`]: ../../workflow/action/group.md#include
+[`action.from`]: ../../workflow/action/index.md#from

doc/src/guide/tutorial/group-workflow2.toml

@@ -4,11 +4,11 @@ value_file = "value.json"
 [[action]]
 name = "process_point"
 command = "echo {directory}"
-[action.group]
-include = [["/type", "==", "point"]]
+[[action.group.include]]
+condition = ["/type", "==", "point"]
 
 [[action]]
 name = "process_letter"
 command = "echo {directory}"
-[action.group]
-include = [["/type", "==", "letter"]]
+[[action.group.include]]
+condition = ["/type", "==", "letter"]

doc/src/guide/tutorial/group-workflow3.toml

@@ -5,11 +5,12 @@ value_file = "value.json"
 name = "process_point"
 command = "echo {directory}"
 [action.group]
-include = [["/type", "==", "point"]]
 sort_by = ["/x"]
+[[action.group.include]]
+condition = ["/type", "==", "point"]
 
 [[action]]
 name = "process_letter"
 command = "echo {directory}"
-[action.group]
-include = [["/type", "==", "letter"]]
+[[action.group.include]]
+condition = ["/type", "==", "letter"]

doc/src/guide/tutorial/group-workflow4.toml

@@ -5,12 +5,13 @@ value_file = "value.json"
 name = "process_point"
 command = "echo {directory}"
 [action.group]
-include = [["/type", "==", "point"]]
 sort_by = ["/x"]
 split_by_sort_key = true
+[[action.group.include]]
+condition = ["/type", "==", "point"]
 
 [[action]]
 name = "process_letter"
 command = "echo {directory}"
-[action.group]
-include = [["/type", "==", "letter"]]
+[[action.group.include]]
+condition = ["/type", "==", "letter"]

doc/src/guide/tutorial/group-workflow5.toml

@@ -5,12 +5,13 @@ value_file = "value.json"
 name = "process_point"
 command = "echo {directory}"
 [action.group]
-include = [["/type", "==", "point"]]
 sort_by = ["/x"]
 maximum_size = 4
+[[action.group.include]]
+condition = ["/type", "==", "point"]
 
 [[action]]
 name = "process_letter"
 command = "echo {directory}"
-[action.group]
-include = [["/type", "==", "letter"]]
+[[action.group.include]]
+condition = ["/type", "==", "letter"]

doc/src/guide/tutorial/group.md

@@ -53,15 +53,15 @@ This workflow will apply the `process_point` action to the directories where
 `value/type == "point"` and the `process_letter` action to the directories where
 `value/type == "letter"`.
 
-`include` is an array. Each element is a length 3 array with the contents: `[JSON
-pointer, operator, operand]`. Think of each element as an expression. The [*JSON
-pointer*](../concepts/json-pointers.md) is a string that reads a particular value
+`condition` is a length 3 array with the contents: `[JSON pointer, operator, operand]`.
+Think of each element as an expression. The
+[*JSON pointer*](../concepts/json-pointers.md) is a string that reads a particular value
 from the directory's **value**. The *operator* is a comparison operator: `"<"`, `"<="`,
 `"=="`, `">="`, or `">"`. The *operand* is the value to compare to. Together, these 3
 elements make a *condition*.
 
-**Row** applies these *conditions* to all directories in the workspace. When all
-*conditions* are true, the directory is included in the action's **groups**.
+**Row** applies the *condition* to all directories in the workspace. When the
+*condition* is true, the directory is included in the action's **groups**.
 
 > Note: This implies that every JSON pointer used in an `include` condition **MUST**
 > be present in every value file.

doc/src/workflow/action/group.md

@@ -6,12 +6,13 @@ that it submits.
 Example:
 ```toml
 [action.group]
-include = [["/subproject", "==", "project_one"]]
 sort_by = ["/value"]
 split_by_sort_key = true
 maximum_size = 16
 submit_whole = true
 reverse_sort = true
+[[action.group.include]]
+condition = ["/subproject", "==", "project_one"]
 ```
 
 > Note: You may omit `[action.group]` entirely.
@@ -21,27 +22,36 @@ groups of directories included in a given action.
 
 ## include
 
-`action.group.include`: **array** of **arrays** - Define a set of conditions that must
-all be true for a directory to be included in this group. Each condition is an **array**
-of three elements: The *JSON pointer*, *the operator*, and the *operand*. The [JSON
-pointer](../../guide/concepts/json-pointers.md) points to a specific element
-from the directory's value. The operator may be `"<"`, `"<="`, `"=="`, `">="`, or `">"`.
+`action.group.include`: **array** of **tables** - Define a set of selectors, *any* of
+which may be true for a directory to be included in this group.
+
+Each selector is a **table** with only one of the following keys:
+* `condition`: An array of three elements: The *JSON pointer*, *the operator*, and the
+  *operand*. The [JSON pointer](../../guide/concepts/json-pointers.md) points to a
+  specific element from the directory's value. The operator may be `"<"`, `"<="`,
+  `"=="`, `">="`, or `">"`.
+* `all`: Array of conditions (see above). All conditions must be true for this selector
+  to be true.
 
 For example, select all directories where a value is in the given range:
 ```toml
-include = [["/value", ">", 0.2], ["/value", "<", 0.9]]
+[[action.group.include]]
+all = [["/value", ">", 0.2], ["/value", "<", 0.9]]
 ```
 Choose directories where an array element is equal to a specific value:
 ```toml
-include = [["/array/1", "==", 12]]
+[[action.group.include]]
+condition = ["/array/1", "==", 12]
 ```
 Match against strings:
 ```toml
-include = [["/map/name", "==", "string"]]
+[[action.group.include]]
+condition = ["/map/name", "==", "string"]
 ```
 Compare by array:
 ```toml
-include = [["/array", "==", [1, "string", 14.0]]]
+[[action.group.include]]
+condition = ["/array", "==", [1, "string", 14.0]
 ```
 
 Both operands **must** have the same data type. The JSON pointer must be present in the

doc/src/workflow/action/index.md

@@ -32,7 +32,7 @@ action. The name may be set by [from](#from).
 > the same name. All elements with the same name **must** have identical
 > [`products`](#products) and [`previous_actions`](#previous_actions). All elements
 > with the same name **must also** select non-intersecting subsets of directories with
-> [`group.include`](group.md#include).
+> [`action.group.include`](group.md#include).
 
 ## command
 

src/project.rs

@@ -17,7 +17,7 @@ use crate::scheduler::bash::Bash;
 use crate::scheduler::slurm::Slurm;
 use crate::scheduler::Scheduler;
 use crate::state::State;
-use crate::workflow::{Action, Workflow};
+use crate::workflow::{Action, Selector, Workflow};
 use crate::{Error, MultiProgressContainer};
 
 /// Encapsulate the workflow, state, and scheduler into a project.
@@ -184,23 +184,55 @@ impl Project {
 
         'outer: for name in directories {
             if let Some(value) = self.state.values().get(&name) {
-                for (include, comparison, expected) in action.group.include() {
-                    let actual = value
-                        .pointer(include)
-                        .ok_or_else(|| Error::JSONPointerNotFound(name.clone(), include.clone()))?;
-                    if !expr::evaluate_json_comparison(comparison, actual, expected).ok_or_else(
-                        || {
-                            Error::CannotCompareInclude(
-                                actual.clone(),
-                                expected.clone(),
-                                name.clone(),
-                            )
-                        },
-                    )? {
-                        continue 'outer;
+                if action.group.include().is_empty() {
+                    matching_directories.push(name);
+                } else {
+                    for selector in action.group.include() {
+                        let result = match selector {
+                            Selector::Condition((include, comparison, expected)) => {
+                                let actual = value.pointer(include).ok_or_else(|| {
+                                    Error::JSONPointerNotFound(name.clone(), include.clone())
+                                })?;
+
+                                expr::evaluate_json_comparison(comparison, actual, expected)
+                                    .ok_or_else(|| {
+                                        Error::CannotCompareInclude(
+                                            actual.clone(),
+                                            expected.clone(),
+                                            name.clone(),
+                                        )
+                                    })
+                            }
+
+                            Selector::All(conditions) => {
+                                let mut matches = 0;
+                                for (include, comparison, expected) in conditions {
+                                    let actual = value.pointer(include).ok_or_else(|| {
+                                        Error::JSONPointerNotFound(name.clone(), include.clone())
+                                    })?;
+
+                                    if expr::evaluate_json_comparison(comparison, actual, expected)
+                                        .ok_or_else(|| {
+                                            Error::CannotCompareInclude(
+                                                actual.clone(),
+                                                expected.clone(),
+                                                name.clone(),
+                                            )
+                                        })?
+                                    {
+                                        matches += 1;
+                                    }
+                                }
+                                Ok(matches == conditions.len())
+                            }
+                        };
+
+                        if result? {
+                            matching_directories.push(name);
+                            continue 'outer;
+                        }
                     }
                 }
-                matching_directories.push(name);
             } else {
                 warn!("Directory '{}' not found in workspace.", name.display());
             }
@@ -424,7 +456,8 @@ products = ["one"]
 name = "two"
 command = "c"
 products = ["two"]
-group.include = [["/i", "<", {}]]
+[[action.group.include]]
+condition = ["/i", "<", {}]
 
 [[action]]
 name = "three"
@@ -464,15 +497,43 @@ previous_actions = ["two"]
             all_directories[0..6]
         );
 
+        // Check all conditions.
         let mut action = project.workflow.action[1].clone();
         let include = action.group.include.as_mut().unwrap();
-        include.push(("/i".into(), Comparison::GreaterThan, Value::from(4)));
+        include.clear();
+        include.push(Selector::All(vec![
+            ("/i".into(), Comparison::GreaterThan, Value::from(4)),
+            ("/i".into(), Comparison::LessThan, Value::from(6)),
+        ]));
         assert_eq!(
             project
                 .find_matching_directories(&action, all_directories.clone())
                 .unwrap(),
             vec![PathBuf::from("dir5")]
         );
+
+        // Check any conditions.
+        let mut action = project.workflow.action[1].clone();
+        let include = action.group.include.as_mut().unwrap();
+        include.clear();
+        include.push(Selector::Condition((
+            "/i".into(),
+            Comparison::LessThan,
+            Value::from(1),
+        )));
+
+        include.push(Selector::Condition((
+            "/i".into(),
+            Comparison::GreaterThan,
+            Value::from(6),
+        )));
+
+        assert_eq!(
+            project
+                .find_matching_directories(&action, all_directories.clone())
+                .unwrap(),
+            vec![PathBuf::from("dir0"), PathBuf::from("dir7")]
+        );
     }
 
     #[test]