Merge branch 'main' into ED-1-formats

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.17.dev
+current_version = 3.18.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?

CHANGELOG.rst

@@ -1,3 +1,67 @@
+dxtbx 3.17.0 (2023-11-03)
+=========================
+
+Features
+--------
+
+- Add ``nxmx_writer``, a tool for converting any data dxtbx can read to NeXus data. (`#615 <https://github.com/cctbx/dxtbx/issues/615>`_)
+- Remove circular dependencies between dxtbx and ``cctbx.xfel``, by using the new ``serialtbx``. (`#627 <https://github.com/cctbx/dxtbx/issues/627>`_)
+- Set the beam probe to ``electron`` in both ``FormatNXmxED`` and ``FormatSER``. (`#661 <https://github.com/cctbx/dxtbx/issues/661>`_)
+
+
+Bugfixes
+--------
+
+- ``dxtbx.image_average``: Better handle detector gain and pixel data type. (`#660 <https://github.com/cctbx/dxtbx/issues/660>`_)
+- ``Beam.probe`` is no longer reset if any geometrical override is provided at import. (`#661 <https://github.com/cctbx/dxtbx/issues/661>`_)
+- Pilatus 4: Do not invert module size that is correctly written in master file. (`#663 <https://github.com/cctbx/dxtbx/issues/663>`_)
+- ``dxtbx.plot_detector_models``: Use noninteractive matpotlib backend, if using the ``pdf_file=`` option. (`#664 <https://github.com/cctbx/dxtbx/issues/664>`_)
+
+
+Deprecations and Removals
+-------------------------
+
+- Legacy ``Datablock`` support has been removed, after being deprecated for several years. If you have any experiments that use these, they will need to be re-imported. (`#504 <https://github.com/cctbx/dxtbx/issues/504>`_)
+
+
+Misc
+----
+
+- `#622 <https://github.com/cctbx/dxtbx/issues/622>`_
+
+
+Dxtbx 3.17 (2023-11-03)
+=======================
+
+Features
+--------
+
+- Add nxmx_writer, a tool for converting any data dxtbx can read to NeXus data (`#615 <https://github.com/cctbx/dxtbx/issues/615>`_)
+- Remove circular dependencies between dxtbx and ``cctbx.xfel`` by using the new ``serialtbx``. (`#627 <https://github.com/cctbx/dxtbx/issues/627>`_)
+- Set the beam probe to ``electron`` in both ``FormatNXmxED`` and ``FormatSER``. (`#661 <https://github.com/cctbx/dxtbx/issues/661>`_)
+
+
+Bugfixes
+--------
+
+- Bugfix for dxtbx.image_average: handle detector gain and pixel data type better (`#660 <https://github.com/cctbx/dxtbx/issues/660>`_)
+- The beam probe is no longer reset if any geometrical override is provided at import. (`#661 <https://github.com/cctbx/dxtbx/issues/661>`_)
+- Pilatus 4: do not invert module size (is written correctly in master file) (`#663 <https://github.com/cctbx/dxtbx/issues/663>`_)
+- ``dxtbx.plot_detector_models``: use noninteractive matpotlib backend if using the pdf_file option (`#664 <https://github.com/cctbx/dxtbx/issues/664>`_)
+
+
+Deprecations and Removals
+-------------------------
+
+- dxtbx: remove legacy datablock object (obsolete for several years) (`#504 <https://github.com/cctbx/dxtbx/issues/504>`_)
+
+
+Misc
+----
+
+- `#622 <https://github.com/cctbx/dxtbx/issues/622>`_
+
+
 DIALS 3.16.1 (2023-09-05)
 =========================
 

SConscript

@@ -176,7 +176,8 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
         LIBS=env_etc.libs_python
         + env_etc.libm
         + env_etc.dxtbx_libs
-        + env_etc.dxtbx_hdf5_libs,
+        + env_etc.dxtbx_hdf5_libs
+        + env["LIBS"],
     )
 
     dxtbx_format_image_ext_sources = [

newsfragments/620.misc

@@ -0,0 +1 @@
+Add properties table to Scan

newsfragments/665.bugfix

@@ -0,0 +1 @@
+Importing the (deprecated and removed) ``dxtbx.datablock`` module failed to display warning properly.

newsfragments/667.misc

@@ -0,0 +1 @@
+Fix oscillation not being set correctly in ``Scan.set_image_range``.

newsfragments/669.bugfix

@@ -0,0 +1 @@
+Fix scan comparison for scan properties changes

newsfragments/670.misc

@@ -0,0 +1 @@
+Fix regression test after scan changes

setup.py

@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.17.dev"
+__version_tag__ = "3.18.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

src/dxtbx/CMakeLists.txt

@@ -11,7 +11,7 @@ Python_add_library( dxtbx_format_nexus_ext MODULE format/boost_python/nexus_ext.
 target_link_libraries( dxtbx_format_nexus_ext PRIVATE  Boost::python CCTBX::scitbx hdf5::hdf5 )
 
 Python_add_library( dxtbx_imageset_ext MODULE boost_python/imageset_ext.cc )
-target_link_libraries( dxtbx_imageset_ext PUBLIC Boost::python CCTBX::scitbx )
+target_link_libraries( dxtbx_imageset_ext PUBLIC Boost::python CCTBX::scitbx CCTBX::scitbx::boost_python )
 
 Python_add_library( dxtbx_format_image_ext
     MODULE

src/dxtbx/array_family/flex_table_suite.h

@@ -330,6 +330,80 @@ namespace dxtbx { namespace af { namespace flex_table_suite {
     }
   };
 
+  struct column_range_to_string_visitor : public boost::static_visitor<std::string> {
+    template <typename U>
+    std::string operator()(const U &column) const {
+      std::ostringstream os;
+      os << column[0] << " - " << column[column.size() - 1] << "\n";
+      return os.str();
+    }
+  };
+
+  template <typename T>
+  struct compare_column_visitor : public boost::static_visitor<bool> {
+    T &self;
+    typename T::key_type key;
+    compare_column_visitor(T &self_, typename T::key_type key_)
+        : self(self_), key(key_) {}
+
+    template <typename U>
+    bool operator()(const U &other_column) const {
+      U self_column = self[key];
+      DXTBX_ASSERT(self_column.size() == other_column.size());
+      for (std::size_t i = 0; i < self_column.size(); ++i) {
+        if (self_column[i] != other_column[i]) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    bool operator()(const scitbx::af::shared<double> &other_column) const {
+      scitbx::af::shared<double> self_column = self[key];
+      double eps = 1e-7;
+      DXTBX_ASSERT(self_column.size() == other_column.size());
+      for (std::size_t i = 0; i < self_column.size(); ++i) {
+        if (std::abs(self_column[i] - other_column[i]) > eps) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    bool operator()(const scitbx::af::shared<vec2<double> > &other_column) const {
+      scitbx::af::shared<vec2<double> > self_column = self[key];
+      double eps = 1e-7;
+      DXTBX_ASSERT(self_column.size() == other_column.size());
+      for (std::size_t i = 0; i < self_column.size(); ++i) {
+        if (std::abs(self_column[i][0] - other_column[i][0]) > eps) {
+          return false;
+        }
+        if (std::abs(self_column[i][1] - other_column[i][1]) > eps) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    bool operator()(const scitbx::af::shared<vec3<double> > &other_column) const {
+      scitbx::af::shared<vec3<double> > self_column = self[key];
+      double eps = 1e-7;
+      DXTBX_ASSERT(self_column.size() == other_column.size());
+      for (std::size_t i = 0; i < self_column.size(); ++i) {
+        if (std::abs(self_column[i][0] - other_column[i][0]) > eps) {
+          return false;
+        }
+        if (std::abs(self_column[i][1] - other_column[i][1]) > eps) {
+          return false;
+        }
+        if (std::abs(self_column[i][2] - other_column[i][2]) > eps) {
+          return false;
+        }
+      }
+      return true;
+    }
+  };
+
   /**
    * Initialise the column table from a list of (key, column) pairs
    * @param columns The list of columns
@@ -359,6 +433,22 @@ namespace dxtbx { namespace af { namespace flex_table_suite {
     return column.apply_visitor(visitor);
   }
 
+  template <typename T>
+  bool compare_columns(T &self, T &other) {
+    typedef typename T::const_iterator iterator;
+    // Implicitly assumed that self and other are the same size
+    DXTBX_ASSERT(self.nrows() == other.nrows());
+    bool same_column;
+    for (iterator it = other.begin(); it != other.end(); ++it) {
+      compare_column_visitor<T> visitor(self, it->first);
+      same_column = it->second.apply_visitor(visitor);
+      if (!same_column) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   /**
    * Delete a column of data
    * @param self The column table

src/dxtbx/command_line/plot_detector_models.py

@@ -6,7 +6,6 @@
 import sys
 from collections.abc import Sequence
 
-import matplotlib.pyplot as plt
 import numpy as np
 from matplotlib.backends.backend_pdf import PdfPages
 from matplotlib.patches import FancyArrowPatch
@@ -170,6 +169,12 @@ def run(args=None):
             except Exception:
                 raise Sorry("Unrecognized argument %s" % arg)
     params = phil_scope.fetch(sources=user_phil).extract()
+    if params.pdf_file:
+        import matplotlib
+
+        matplotlib.use("Agg")
+
+    import matplotlib.pyplot as plt
 
     fig = plt.figure()
     colormap = plt.cm.gist_ncar

src/dxtbx/datablock.py

@@ -2,7 +2,7 @@
 
 import warnings
 
-from dxtbx.experiment_list import (
+from dxtbx.model.experiment_list import (
     BeamComparison,
     DetectorComparison,
     FormatChecker,

src/dxtbx/dxtbx_model_ext.pyi

@@ -36,6 +36,14 @@ Vec6Float = Tuple[float, float, float, float, float, float]
 Vec9Float = Tuple[float, float, float, float, float, float, float, float, float]
 Vec2Int = Tuple[int, int]
 Vec4Int = Tuple[int, int, int, int]
+ScanPropertyTypes = Union[
+    flex.int,
+    flex.double,
+    flex.bool,
+    flex.std_string,
+    flex.vec2_double,
+    flex.vec3_double,
+]
 
 class BeamBase:
     @property
@@ -745,6 +753,14 @@ class Scan(ScanBase):
         batch_offset: int,
         deg: bool = ...,
     ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        image_range: Vec2Int,
+        properties_table: Dict,
+        batch_offset: int,
+        deg: bool = ...,
+    ) -> None: ...
     def append(self, other: Scan, scan_tolerance: float) -> None: ...
     @staticmethod
     def from_dict(data: Dict) -> Scan: ...
@@ -788,6 +804,22 @@ class Scan(ScanBase):
     def get_oscillation_range(self, deg: bool = ...) -> Vec2Float: ...
     def get_valid_image_ranges(self, i: str) -> List[Vec2Int]: ...
     def set_valid_image_ranges(self, i: str, ranges: List[Vec2Int]) -> None: ...
+    def get_properties(self) -> dict: ...
+    def set_properties(self, properties: dict) -> None: ...
+    def has_property(self, key: str) -> bool: ...
+    def get_property(self, key: str) -> ScanPropertyTypes: ...
+    @overload
+    def set_property(self, key: str, value: flex.double) -> None: ...
+    @overload
+    def set_property(self, key: str, value: flex.bool) -> None: ...
+    @overload
+    def set_property(self, key: str, value: flex.int) -> None: ...
+    @overload
+    def set_property(self, key: str, value: flex.std_string) -> None: ...
+    @overload
+    def set_property(self, key: str, value: flex.vec2_double) -> None: ...
+    @overload
+    def set_property(self, key: str, value: flex.vec3_double) -> None: ...
     @overload
     def is_angle_valid(self, angle: float, deg: bool = ...) -> bool: ...
     @overload

src/dxtbx/format/FormatNXmxEigerFilewriter.py

@@ -52,8 +52,12 @@ def _get_nxmx(self, fh: h5py.File):
         # data_size is reversed - we should probably be more specific in when
         # we do this, i.e. check data_size is in a list of known reversed
         # values
+        known_safe = [
+            (1082, 1035),
+        ]
         for module in nxdetector.modules:
-            module.data_size = module.data_size[::-1]
+            if not tuple(module.data_size) in known_safe:
+                module.data_size = module.data_size[::-1]
         return nxmx_obj
 
     def get_raw_data(self, index):

src/dxtbx/imageset.h

@@ -41,6 +41,7 @@ using model::Goniometer;
 using model::Panel;
 using model::Scan;
 using scitbx::rad_as_deg;
+using scitbx::vec2;
 using scitbx::af::int2;
 
 namespace detail {