From 05b78060a61ae8e6f1eaa5a1936b86aa25240eec Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Thu, 2 Nov 2023 19:02:23 +0000 Subject: [PATCH 01/10] Do not invert for Pilatus 4 (#663) * Do not invert for Pilatus 4 * Rename newsfragments/XXX.bugfix to newsfragments/663.bugfix --------- Co-authored-by: DiamondLightSource-build-server --- newsfragments/663.bugfix | 2 ++ src/dxtbx/format/FormatNXmxEigerFilewriter.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/663.bugfix diff --git a/newsfragments/663.bugfix b/newsfragments/663.bugfix new file mode 100644 index 000000000..c632cc50a --- /dev/null +++ b/newsfragments/663.bugfix @@ -0,0 +1,2 @@ +Pilatus 4: do not invert module size (is written correctly in master file) + diff --git a/src/dxtbx/format/FormatNXmxEigerFilewriter.py b/src/dxtbx/format/FormatNXmxEigerFilewriter.py index 7e8eb2924..a3e73c145 100644 --- a/src/dxtbx/format/FormatNXmxEigerFilewriter.py +++ b/src/dxtbx/format/FormatNXmxEigerFilewriter.py @@ -52,8 +52,10 @@ def _get_nxmx(self, fh: h5py.File): # data_size is reversed - we should probably be more specific in when # we do this, i.e. check data_size is in a list of known reversed # values + known_safe = [(1082,1035),] for module in nxdetector.modules: - module.data_size = module.data_size[::-1] + if not tuple(module.data_size) in known_safe: + module.data_size = module.data_size[::-1] return nxmx_obj def get_raw_data(self, index): From 40327dd773270ae10d3197897271e8579395cc6d Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 3 Nov 2023 12:57:16 +0000 Subject: [PATCH 02/10] Allow use of noninteractive backend for dxtbx.plot_detector_models (#664) * Allow use of noninteractive backend for plot_detector_models. Add test * Rename newsfragments/XXX.bugfix to newsfragments/664.bugfix * Don't fail on warnings --------- Co-authored-by: DiamondLightSource-build-server --- newsfragments/664.bugfix | 1 + src/dxtbx/command_line/plot_detector_models.py | 7 ++++++- tests/command_line/test_plot_detector_models.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 newsfragments/664.bugfix create mode 100644 tests/command_line/test_plot_detector_models.py diff --git a/newsfragments/664.bugfix b/newsfragments/664.bugfix new file mode 100644 index 000000000..d1c5abf15 --- /dev/null +++ b/newsfragments/664.bugfix @@ -0,0 +1 @@ +``dxtbx.plot_detector_models``: use noninteractive matpotlib backend if using the pdf_file option diff --git a/src/dxtbx/command_line/plot_detector_models.py b/src/dxtbx/command_line/plot_detector_models.py index 4034e7417..b9f228280 100644 --- a/src/dxtbx/command_line/plot_detector_models.py +++ b/src/dxtbx/command_line/plot_detector_models.py @@ -6,7 +6,6 @@ import sys from collections.abc import Sequence -import matplotlib.pyplot as plt import numpy as np from matplotlib.backends.backend_pdf import PdfPages from matplotlib.patches import FancyArrowPatch @@ -170,6 +169,12 @@ def run(args=None): except Exception: raise Sorry("Unrecognized argument %s" % arg) params = phil_scope.fetch(sources=user_phil).extract() + if params.pdf_file: + import matplotlib + + matplotlib.use("Agg") + + import matplotlib.pyplot as plt fig = plt.figure() colormap = plt.cm.gist_ncar diff --git a/tests/command_line/test_plot_detector_models.py b/tests/command_line/test_plot_detector_models.py new file mode 100644 index 000000000..160e37bb3 --- /dev/null +++ b/tests/command_line/test_plot_detector_models.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import os.path +import shutil +import subprocess + + +def test_plot_models_noninteractive_to_pdf(dials_data, tmp_path): + ssx = dials_data("cunir_serial_processed", pathlib=True) + expts = str(ssx / "imported_no_ref_5.expt") + subprocess.run( + [shutil.which("dxtbx.plot_detector_models"), expts, "pdf_file=plot_test.pdf"], + cwd=tmp_path, + capture_output=True, + ) + assert os.path.exists(tmp_path / "plot_test.pdf") From a33c84fb6e7aac992401d0d1f6b7b65c9596b8c9 Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Fri, 3 Nov 2023 13:23:46 +0000 Subject: [PATCH 03/10] dxtbx 3.17.0 Changelog towncrier --name=dxtbx --version='3.17.0' --- CHANGELOG.rst | 64 +++++++++++++++++++++++++++++++++++++++ newsfragments/504.removal | 1 - newsfragments/615.feature | 1 - newsfragments/622.misc | 1 - newsfragments/627.feature | 1 - newsfragments/660.bugfix | 1 - newsfragments/661.bugfix | 1 - newsfragments/661.feature | 1 - newsfragments/663.bugfix | 2 -- newsfragments/664.bugfix | 1 - 10 files changed, 64 insertions(+), 10 deletions(-) delete mode 100644 newsfragments/504.removal delete mode 100644 newsfragments/615.feature delete mode 100644 newsfragments/622.misc delete mode 100644 newsfragments/627.feature delete mode 100644 newsfragments/660.bugfix delete mode 100644 newsfragments/661.bugfix delete mode 100644 newsfragments/661.feature delete mode 100644 newsfragments/663.bugfix delete mode 100644 newsfragments/664.bugfix diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 862574949..cb89dc0a5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,67 @@ +dxtbx 3.17.0 (2023-11-03) +========================= + +Features +-------- + +- Add ``nxmx_writer``, a tool for converting any data dxtbx can read to NeXus data. (`#615 `_) +- Remove circular dependencies between dxtbx and ``cctbx.xfel``, by using the new ``serialtbx``. (`#627 `_) +- Set the beam probe to ``electron`` in both ``FormatNXmxED`` and ``FormatSER``. (`#661 `_) + + +Bugfixes +-------- + +- ``dxtbx.image_average``: Better handle detector gain and pixel data type. (`#660 `_) +- ``Beam.probe`` is no longer reset if any geometrical override is provided at import. (`#661 `_) +- Pilatus 4: Do not invert module size that is correctly written in master file. (`#663 `_) +- ``dxtbx.plot_detector_models``: Use noninteractive matpotlib backend, if using the ``pdf_file=`` option. (`#664 `_) + + +Deprecations and Removals +------------------------- + +- Legacy ``Datablock`` support has been removed, after being deprecated for several years. If you have any experiments that use these, they will need to be re-imported. (`#504 `_) + + +Misc +---- + +- `#622 `_ + + +Dxtbx 3.17 (2023-11-03) +======================= + +Features +-------- + +- Add nxmx_writer, a tool for converting any data dxtbx can read to NeXus data (`#615 `_) +- Remove circular dependencies between dxtbx and ``cctbx.xfel`` by using the new ``serialtbx``. (`#627 `_) +- Set the beam probe to ``electron`` in both ``FormatNXmxED`` and ``FormatSER``. (`#661 `_) + + +Bugfixes +-------- + +- Bugfix for dxtbx.image_average: handle detector gain and pixel data type better (`#660 `_) +- The beam probe is no longer reset if any geometrical override is provided at import. (`#661 `_) +- Pilatus 4: do not invert module size (is written correctly in master file) (`#663 `_) +- ``dxtbx.plot_detector_models``: use noninteractive matpotlib backend if using the pdf_file option (`#664 `_) + + +Deprecations and Removals +------------------------- + +- dxtbx: remove legacy datablock object (obsolete for several years) (`#504 `_) + + +Misc +---- + +- `#622 `_ + + DIALS 3.16.1 (2023-09-05) ========================= diff --git a/newsfragments/504.removal b/newsfragments/504.removal deleted file mode 100644 index 1500b053e..000000000 --- a/newsfragments/504.removal +++ /dev/null @@ -1 +0,0 @@ -dxtbx: remove legacy datablock object (obsolete for several years) diff --git a/newsfragments/615.feature b/newsfragments/615.feature deleted file mode 100644 index b13456787..000000000 --- a/newsfragments/615.feature +++ /dev/null @@ -1 +0,0 @@ -Add nxmx_writer, a tool for converting any data dxtbx can read to NeXus data diff --git a/newsfragments/622.misc b/newsfragments/622.misc deleted file mode 100644 index 033071027..000000000 --- a/newsfragments/622.misc +++ /dev/null @@ -1 +0,0 @@ -Add sample_to_source_distance to Beam diff --git a/newsfragments/627.feature b/newsfragments/627.feature deleted file mode 100644 index 1f2c28178..000000000 --- a/newsfragments/627.feature +++ /dev/null @@ -1 +0,0 @@ -Remove circular dependencies between dxtbx and ``cctbx.xfel`` by using the new ``serialtbx``. diff --git a/newsfragments/660.bugfix b/newsfragments/660.bugfix deleted file mode 100644 index ab953f341..000000000 --- a/newsfragments/660.bugfix +++ /dev/null @@ -1 +0,0 @@ -Bugfix for dxtbx.image_average: handle detector gain and pixel data type better diff --git a/newsfragments/661.bugfix b/newsfragments/661.bugfix deleted file mode 100644 index 2cfc35640..000000000 --- a/newsfragments/661.bugfix +++ /dev/null @@ -1 +0,0 @@ -The beam probe is no longer reset if any geometrical override is provided at import. diff --git a/newsfragments/661.feature b/newsfragments/661.feature deleted file mode 100644 index 19b0f32b5..000000000 --- a/newsfragments/661.feature +++ /dev/null @@ -1 +0,0 @@ -Set the beam probe to ``electron`` in both ``FormatNXmxED`` and ``FormatSER``. diff --git a/newsfragments/663.bugfix b/newsfragments/663.bugfix deleted file mode 100644 index c632cc50a..000000000 --- a/newsfragments/663.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Pilatus 4: do not invert module size (is written correctly in master file) - diff --git a/newsfragments/664.bugfix b/newsfragments/664.bugfix deleted file mode 100644 index d1c5abf15..000000000 --- a/newsfragments/664.bugfix +++ /dev/null @@ -1 +0,0 @@ -``dxtbx.plot_detector_models``: use noninteractive matpotlib backend if using the pdf_file option From 2562aa71b2254c0d8bc0c68a1ce75965892c687f Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Fri, 3 Nov 2023 13:23:47 +0000 Subject: [PATCH 04/10] =?UTF-8?q?Bump=20version:=203.17.dev=20=E2=86=92=20?= =?UTF-8?q?3.18.dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9417f73e2..d6e63b9d0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.17.dev +current_version = 3.18.dev commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z]+)?(?P\d+)? diff --git a/setup.py b/setup.py index 8fbd4a3e7..7fb2e9551 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Static version number which is updated by bump2version # Do not change this manually - use 'bump2version ' -__version_tag__ = "3.17.dev" +__version_tag__ = "3.18.dev" setup_kwargs = { "name": "dxtbx", From 4c59ebc305affd957b91e41acc431304c0c35ee3 Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Sun, 5 Nov 2023 05:11:01 +0000 Subject: [PATCH 05/10] Clean Clutter --- src/dxtbx/format/FormatNXmxEigerFilewriter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dxtbx/format/FormatNXmxEigerFilewriter.py b/src/dxtbx/format/FormatNXmxEigerFilewriter.py index a3e73c145..fbfde2929 100644 --- a/src/dxtbx/format/FormatNXmxEigerFilewriter.py +++ b/src/dxtbx/format/FormatNXmxEigerFilewriter.py @@ -52,7 +52,9 @@ def _get_nxmx(self, fh: h5py.File): # data_size is reversed - we should probably be more specific in when # we do this, i.e. check data_size is in a list of known reversed # values - known_safe = [(1082,1035),] + known_safe = [ + (1082, 1035), + ] for module in nxdetector.modules: if not tuple(module.data_size) in known_safe: module.data_size = module.data_size[::-1] From 27f168c0294518c9b3c2c22a882d656d8b83374a Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Mon, 6 Nov 2023 15:25:34 +0000 Subject: [PATCH 06/10] Update removed datablock module and warning (#665) This attempted to import some code from experiment_list before printing the warning, but used an incorrect import path. This didn't get tested because this was added in the commit otherwise removing datablock completely. --- newsfragments/665.bugfix | 1 + src/dxtbx/datablock.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/665.bugfix diff --git a/newsfragments/665.bugfix b/newsfragments/665.bugfix new file mode 100644 index 000000000..a0a44e441 --- /dev/null +++ b/newsfragments/665.bugfix @@ -0,0 +1 @@ +Importing the (deprecated and removed) ``dxtbx.datablock`` module failed to display warning properly. diff --git a/src/dxtbx/datablock.py b/src/dxtbx/datablock.py index 436f52ea3..ea8d8fc3e 100644 --- a/src/dxtbx/datablock.py +++ b/src/dxtbx/datablock.py @@ -2,7 +2,7 @@ import warnings -from dxtbx.experiment_list import ( +from dxtbx.model.experiment_list import ( BeamComparison, DetectorComparison, FormatChecker, From 6883714d5dc2193df30bc48d777bd0e06409c298 Mon Sep 17 00:00:00 2001 From: David McDonagh <60879630+toastisme@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:05:37 +0000 Subject: [PATCH 07/10] Add properties table to Scan (#620) Added properties to Scan to allow for more diverse experiment types (e.g. 3D time-of-flight data). --- SConscript | 3 +- newsfragments/620.misc | 1 + src/dxtbx/CMakeLists.txt | 2 +- src/dxtbx/array_family/flex_table_suite.h | 90 ++++ src/dxtbx/dxtbx_model_ext.pyi | 32 ++ src/dxtbx/imageset.h | 1 + src/dxtbx/model/boost_python/scan.cc | 373 +++++++++++++--- src/dxtbx/model/experiment_list.py | 2 +- src/dxtbx/model/scan.h | 522 ++++++++++++++++------ src/dxtbx/model/scan.py | 62 ++- src/dxtbx/model/scan_helpers.h | 4 + tests/model/test_scan.py | 440 +++++++++++++++++- tests/model/test_scan_data.py | 10 +- tests/serialize/test_serialize.py | 13 +- 14 files changed, 1321 insertions(+), 234 deletions(-) create mode 100644 newsfragments/620.misc diff --git a/SConscript b/SConscript index fe962f34f..aaafe78f4 100644 --- a/SConscript +++ b/SConscript @@ -176,7 +176,8 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"): LIBS=env_etc.libs_python + env_etc.libm + env_etc.dxtbx_libs - + env_etc.dxtbx_hdf5_libs, + + env_etc.dxtbx_hdf5_libs + + env["LIBS"], ) dxtbx_format_image_ext_sources = [ diff --git a/newsfragments/620.misc b/newsfragments/620.misc new file mode 100644 index 000000000..ba905f660 --- /dev/null +++ b/newsfragments/620.misc @@ -0,0 +1 @@ +Add properties table to Scan diff --git a/src/dxtbx/CMakeLists.txt b/src/dxtbx/CMakeLists.txt index 0c8aea194..fccf3f698 100644 --- a/src/dxtbx/CMakeLists.txt +++ b/src/dxtbx/CMakeLists.txt @@ -11,7 +11,7 @@ Python_add_library( dxtbx_format_nexus_ext MODULE format/boost_python/nexus_ext. target_link_libraries( dxtbx_format_nexus_ext PRIVATE Boost::python CCTBX::scitbx hdf5::hdf5 ) Python_add_library( dxtbx_imageset_ext MODULE boost_python/imageset_ext.cc ) -target_link_libraries( dxtbx_imageset_ext PUBLIC Boost::python CCTBX::scitbx ) +target_link_libraries( dxtbx_imageset_ext PUBLIC Boost::python CCTBX::scitbx CCTBX::scitbx::boost_python ) Python_add_library( dxtbx_format_image_ext MODULE diff --git a/src/dxtbx/array_family/flex_table_suite.h b/src/dxtbx/array_family/flex_table_suite.h index b7132717b..73e366e83 100644 --- a/src/dxtbx/array_family/flex_table_suite.h +++ b/src/dxtbx/array_family/flex_table_suite.h @@ -330,6 +330,80 @@ namespace dxtbx { namespace af { namespace flex_table_suite { } }; + struct column_range_to_string_visitor : public boost::static_visitor { + template + std::string operator()(const U &column) const { + std::ostringstream os; + os << column[0] << " - " << column[column.size() - 1] << "\n"; + return os.str(); + } + }; + + template + struct compare_column_visitor : public boost::static_visitor { + T &self; + typename T::key_type key; + compare_column_visitor(T &self_, typename T::key_type key_) + : self(self_), key(key_) {} + + template + bool operator()(const U &other_column) const { + U self_column = self[key]; + DXTBX_ASSERT(self_column.size() == other_column.size()); + for (std::size_t i = 0; i < self_column.size(); ++i) { + if (self_column[i] != other_column[i]) { + return false; + } + } + return true; + } + + bool operator()(const scitbx::af::shared &other_column) const { + scitbx::af::shared self_column = self[key]; + double eps = 1e-7; + DXTBX_ASSERT(self_column.size() == other_column.size()); + for (std::size_t i = 0; i < self_column.size(); ++i) { + if (std::abs(self_column[i] - other_column[i]) > eps) { + return false; + } + } + return true; + } + + bool operator()(const scitbx::af::shared > &other_column) const { + scitbx::af::shared > self_column = self[key]; + double eps = 1e-7; + DXTBX_ASSERT(self_column.size() == other_column.size()); + for (std::size_t i = 0; i < self_column.size(); ++i) { + if (std::abs(self_column[i][0] - other_column[i][0]) > eps) { + return false; + } + if (std::abs(self_column[i][1] - other_column[i][1]) > eps) { + return false; + } + } + return true; + } + + bool operator()(const scitbx::af::shared > &other_column) const { + scitbx::af::shared > self_column = self[key]; + double eps = 1e-7; + DXTBX_ASSERT(self_column.size() == other_column.size()); + for (std::size_t i = 0; i < self_column.size(); ++i) { + if (std::abs(self_column[i][0] - other_column[i][0]) > eps) { + return false; + } + if (std::abs(self_column[i][1] - other_column[i][1]) > eps) { + return false; + } + if (std::abs(self_column[i][2] - other_column[i][2]) > eps) { + return false; + } + } + return true; + } + }; + /** * Initialise the column table from a list of (key, column) pairs * @param columns The list of columns @@ -359,6 +433,22 @@ namespace dxtbx { namespace af { namespace flex_table_suite { return column.apply_visitor(visitor); } + template + bool compare_columns(T &self, T &other) { + typedef typename T::const_iterator iterator; + // Implicitly assumed that self and other are the same size + DXTBX_ASSERT(self.nrows() == other.nrows()); + bool same_column; + for (iterator it = other.begin(); it != other.end(); ++it) { + compare_column_visitor visitor(self, it->first); + same_column = it->second.apply_visitor(visitor); + if (!same_column) { + return false; + } + } + return true; + } + /** * Delete a column of data * @param self The column table diff --git a/src/dxtbx/dxtbx_model_ext.pyi b/src/dxtbx/dxtbx_model_ext.pyi index 9ca38cffd..0f4619852 100644 --- a/src/dxtbx/dxtbx_model_ext.pyi +++ b/src/dxtbx/dxtbx_model_ext.pyi @@ -36,6 +36,14 @@ Vec6Float = Tuple[float, float, float, float, float, float] Vec9Float = Tuple[float, float, float, float, float, float, float, float, float] Vec2Int = Tuple[int, int] Vec4Int = Tuple[int, int, int, int] +ScanPropertyTypes = Union[ + flex.int, + flex.double, + flex.bool, + flex.std_string, + flex.vec2_double, + flex.vec3_double, +] class BeamBase: @property @@ -745,6 +753,14 @@ class Scan(ScanBase): batch_offset: int, deg: bool = ..., ) -> None: ... + @overload + def __init__( + self, + image_range: Vec2Int, + properties_table: Dict, + batch_offset: int, + deg: bool = ..., + ) -> None: ... def append(self, other: Scan, scan_tolerance: float) -> None: ... @staticmethod def from_dict(data: Dict) -> Scan: ... @@ -788,6 +804,22 @@ class Scan(ScanBase): def get_oscillation_range(self, deg: bool = ...) -> Vec2Float: ... def get_valid_image_ranges(self, i: str) -> List[Vec2Int]: ... def set_valid_image_ranges(self, i: str, ranges: List[Vec2Int]) -> None: ... + def get_properties(self) -> dict: ... + def set_properties(self, properties: dict) -> None: ... + def has_property(self, key: str) -> bool: ... + def get_property(self, key: str) -> ScanPropertyTypes: ... + @overload + def set_property(self, key: str, value: flex.double) -> None: ... + @overload + def set_property(self, key: str, value: flex.bool) -> None: ... + @overload + def set_property(self, key: str, value: flex.int) -> None: ... + @overload + def set_property(self, key: str, value: flex.std_string) -> None: ... + @overload + def set_property(self, key: str, value: flex.vec2_double) -> None: ... + @overload + def set_property(self, key: str, value: flex.vec3_double) -> None: ... @overload def is_angle_valid(self, angle: float, deg: bool = ...) -> bool: ... @overload diff --git a/src/dxtbx/imageset.h b/src/dxtbx/imageset.h index bb04b1f02..bc1d79b9e 100644 --- a/src/dxtbx/imageset.h +++ b/src/dxtbx/imageset.h @@ -41,6 +41,7 @@ using model::Goniometer; using model::Panel; using model::Scan; using scitbx::rad_as_deg; +using scitbx::vec2; using scitbx::af::int2; namespace detail { diff --git a/src/dxtbx/model/boost_python/scan.cc b/src/dxtbx/model/boost_python/scan.cc index dfd5a5336..8a442d9f4 100644 --- a/src/dxtbx/model/boost_python/scan.cc +++ b/src/dxtbx/model/boost_python/scan.cc @@ -18,10 +18,13 @@ #include #include #include +#include +#include +#include namespace dxtbx { namespace model { namespace boost_python { - using namespace boost::python; + using dxtbx::model::scan_property_types; using scitbx::deg_as_rad; using scitbx::rad_as_deg; @@ -37,19 +40,141 @@ namespace dxtbx { namespace model { namespace boost_python { return angles; } + static scitbx::af::shared deg_as_rad( + scitbx::af::shared angles_in_deg) { + scitbx::af::shared angles_in_rad; + angles_in_rad.resize(angles_in_deg.size()); + std::transform(angles_in_deg.begin(), + angles_in_deg.end(), + angles_in_rad.begin(), + scitbx::deg_as_rad); + return angles_in_rad; + } + std::string scan_to_string(const Scan &scan) { std::stringstream ss; ss << scan; return ss.str(); } + flex_table extract_properties_table( + boost::python::dict properties_dict, + int num_images, + bool convert_oscillation_to_rad = false) { + boost::python::list keys = properties_dict.keys(); + boost::python::list values = boost::python::list(properties_dict.values()); + + flex_table properties = + flex_table(num_images); + + // Extract each dictionary value based on Python type + for (int i = 0; i < len(keys); ++i) { + std::string key = boost::python::extract(keys[i]); + boost::python::object value = values[i]; + DXTBX_ASSERT(len(value) == num_images); + std::string obj_type = boost::python::extract( + value[0].attr("__class__").attr("__name__")); + + // Handled explicitly as it is in deg when serialised but rad in code + if (key == "oscillation") { + DXTBX_ASSERT(obj_type == "float"); + scitbx::af::shared osc = + boost::python::extract >(value); + DXTBX_ASSERT(Scan::oscillation_has_constant_width(osc)); + + if (convert_oscillation_to_rad) { + properties[key] = deg_as_rad(osc); + } else { + properties[key] = osc; + } + } else if (obj_type == "int") { + properties[key] = boost::python::extract >(value); + } else if (obj_type == "float") { + properties[key] = boost::python::extract >(value); + } else if (obj_type == "bool") { + properties[key] = boost::python::extract >(value); + } else if (obj_type == "str") { + properties[key] = + boost::python::extract >(value); + + } else if (obj_type == "tuple") { // vec2 or vec3 + std::string element_type = boost::python::extract( + value[0][0].attr("__class__").attr("__name__")); + int tuple_size = len(value[0]); + + if (tuple_size == 2) { // vec2 + if (element_type == "float") { + properties[key] = + boost::python::extract > >(value); + } else { + throw DXTBX_ERROR("Unknown type for column name " + key); + } + } else if (tuple_size == 3) { // vec3 + if (element_type == "float") { + properties[key] = + boost::python::extract > >(value); + } else { + throw DXTBX_ERROR("Unknown type for column name " + key); + } + } else { + throw DXTBX_ERROR("Unknown type for column name " + key); + } + } else { + throw DXTBX_ERROR("Unknown type for column name " + key); + } + } + + return properties; + } + + void set_properties_table_from_dict(Scan &obj, boost::python::dict properties_dict) { + int num_images = len(boost::python::list(properties_dict.values())[0]); + flex_table properties = + extract_properties_table(properties_dict, num_images); + obj.set_properties(properties); + } + struct ScanPickleSuite : boost::python::pickle_suite { + typedef flex_table::const_iterator const_iterator; + static boost::python::tuple getinitargs(const Scan &obj) { - return boost::python::make_tuple(obj.get_image_range(), - rad_as_deg(obj.get_oscillation()), - obj.get_exposure_times(), - obj.get_epochs(), - obj.get_batch_offset()); + return boost::python::make_tuple(obj.get_image_range(), obj.get_batch_offset()); + } + static boost::python::tuple getstate(const Scan &obj) { + flex_table properties = obj.get_properties(); + boost::python::dict properties_dict; + dxtbx::af::flex_table_suite::column_to_object_visitor visitor; + for (const_iterator it = properties.begin(); it != properties.end(); ++it) { + properties_dict[it->first] = it->second.apply_visitor(visitor); + } + + return boost::python::make_tuple( + properties.nrows(), properties.ncols(), properties_dict); + } + + static void setstate(Scan &obj, boost::python::tuple state) { + DXTBX_ASSERT(boost::python::len(state) == 3); + std::size_t nrows = boost::python::extract(state[0]); + std::size_t ncols = boost::python::extract(state[1]); + boost::python::dict properties_dict = + boost::python::extract(state[2]); + + DXTBX_ASSERT(len(properties_dict) == ncols); + flex_table properties = + extract_properties_table(properties_dict, nrows); + obj.set_properties(properties); + } + }; + + template + struct scan_property_table_wrapper + : public dxtbx::af::flex_table_suite::flex_table_wrapper { + typedef dxtbx::af::flex_table_suite::flex_table_wrapper base_type; + typedef typename base_type::flex_table_type flex_table_type; + typedef typename base_type::class_type class_type; + + static class_type wrap(const char *name) { + return base_type::wrap(name); } }; @@ -67,14 +192,40 @@ namespace dxtbx { namespace model { namespace boost_python { return dictionary; } + boost::python::dict get_properties_dict(const Scan &obj) { + boost::python::dict properties_dict; + flex_table properties = obj.get_properties(); + + dxtbx::af::flex_table_suite::column_to_object_visitor visitor; + for (const_iterator it = properties.begin(); it != properties.end(); ++it) { + properties_dict[it->first] = + boost::python::tuple(it->second.apply_visitor(visitor)); + } + return properties_dict; + } + template <> boost::python::dict to_dict(const Scan &obj) { boost::python::dict result; result["image_range"] = obj.get_image_range(); result["batch_offset"] = obj.get_batch_offset(); - result["oscillation"] = rad_as_deg(obj.get_oscillation()); - result["exposure_time"] = boost::python::list(obj.get_exposure_times()); - result["epochs"] = boost::python::list(obj.get_epochs()); + + flex_table properties = obj.get_properties(); + boost::python::dict properties_dict; + dxtbx::af::flex_table_suite::column_to_object_visitor visitor; + + for (const_iterator it = properties.begin(); it != properties.end(); ++it) { + if (it->first == "oscillation") { // Handled explicitly due to unit conversion + properties_dict[it->first] = + boost::python::tuple(obj.get_oscillation_arr_in_deg()); + } else { + properties_dict[it->first] = + boost::python::list(it->second.apply_visitor(visitor)); + } + } + + result["properties"] = properties_dict; + boost::python::dict valid_image_ranges = MaptoPythonDict(obj.get_valid_image_ranges_map()); result["valid_image_ranges"] = valid_image_ranges; @@ -130,36 +281,61 @@ namespace dxtbx { namespace model { namespace boost_python { return result; } + scitbx::af::shared make_oscillation_arr(std::size_t num_images, + vec2 oscillation) { + scitbx::af::shared oscillation_arr((scitbx::af::reserve(num_images))); + for (std::size_t i = 0; i < num_images; ++i) { + oscillation_arr.push_back(oscillation[0] + oscillation[1] * i); + } + return oscillation_arr; + } + template <> Scan *from_dict(boost::python::dict obj) { vec2 ir = boost::python::extract >(obj["image_range"]); int bo = boost::python::extract(obj["batch_offset"]); - vec2 osc = - deg_as_rad(boost::python::extract >(obj["oscillation"])); DXTBX_ASSERT(ir[1] >= ir[0]); - std::size_t num = ir[1] - ir[0] + 1; - Scan *scan = - new Scan(ir, - osc, - make_exposure_times(num, - boost::python::extract( - obj.get("exposure_time", boost::python::list()))), - make_epochs(num, - boost::python::extract( - obj.get("epochs", boost::python::list()))), - bo); + std::size_t num_images = ir[1] - ir[0] + 1; + + Scan *scan = new Scan(ir, bo); + if (!obj.has_key("properties")) { + /* + * Legacy case with no properties table + */ + + if (obj.has_key("oscillation")) { + vec2 osc = + deg_as_rad(boost::python::extract >(obj["oscillation"])); + scan->set_oscillation(osc); + } + if (obj.has_key("exposure_time")) { + scan->set_exposure_times( + make_exposure_times(num_images, + boost::python::extract( + obj.get("exposure_time", boost::python::list())))); + } + if (obj.has_key("epochs")) { + scan->set_epochs(make_epochs(num_images, + boost::python::extract( + obj.get("epochs", boost::python::list())))); + } + } else { + boost::python::dict properties_dict = + boost::python::extract(obj["properties"]); + scan->set_properties(extract_properties_table(properties_dict, num_images, true)); + } + boost::python::dict rangemap = boost::python::extract(obj["valid_image_ranges"]); - boost::python::list keys = rangemap.keys(); - boost::python::list values = rangemap.values(); - for (int i = 0; i < len(keys); ++i) { - boost::python::extract extracted_key(keys[i]); + boost::python::list valid_img_keys = rangemap.keys(); + boost::python::list valid_img_values = rangemap.values(); + for (int i = 0; i < len(valid_img_keys); ++i) { + std::string key = boost::python::extract(valid_img_keys[i]); scitbx::af::shared > result; - int n_tuples = boost::python::len(values[i]); + int n_tuples = boost::python::len(valid_img_values[i]); for (int n = 0; n < n_tuples; ++n) { - result.push_back(boost::python::extract >(values[i][n])); + result.push_back(boost::python::extract >(valid_img_values[i][n])); } - std::string key = extracted_key; scan->set_valid_image_ranges_array(key, result); } return scan; @@ -179,7 +355,7 @@ namespace dxtbx { namespace model { namespace boost_python { int n = boost::python::len(obj); scitbx::af::shared > ranges; for (int k = 0; k < n; ++k) { - ranges.push_back(boost_python::extract >(obj[k])); + ranges.push_back(boost::python::extract >(obj[k])); } scan.set_valid_image_ranges_array(i, ranges); } @@ -231,6 +407,27 @@ namespace dxtbx { namespace model { namespace boost_python { return scan; } + static Scan *make_scan_w_properties(vec2 image_range, + boost::python::dict properties_dict, + int batch_offset, + bool deg) { + int num_images = 1 + image_range[1] - image_range[0]; + flex_table properties = + extract_properties_table(properties_dict, num_images); + + if (deg && properties.contains("oscillation")) { + scitbx::af::shared osc_in_deg = properties.get("oscillation"); + scitbx::af::shared osc = deg_as_rad(osc_in_deg); + dxtbx::af::flex_table_suite::setitem_column( + properties, "oscillation", osc.const_ref()); + } + return new Scan(image_range, properties, batch_offset); + } + + static Scan *make_scan_wo_properties(vec2 image_range, int batch_offset) { + return new Scan(image_range, batch_offset); + } + static vec2 get_oscillation_range(const Scan &scan, bool deg) { vec2 range = scan.get_oscillation_range(); return deg ? rad_as_deg(range) : range; @@ -322,23 +519,23 @@ namespace dxtbx { namespace model { namespace boost_python { return scan[index]; } - static Scan getitem_slice(const Scan &scan, const slice index) { + static Scan getitem_slice(const Scan &scan, const boost::python::slice index) { // Ensure no step - DXTBX_ASSERT(index.step() == object()); + DXTBX_ASSERT(index.step() == boost::python::object()); // Get start index int start = 0, stop = 0; - if (index.start() == object()) { + if (index.start() == boost::python::object()) { start = 0; } else { - start = extract(index.start()); + start = boost::python::extract(index.start()); } // Get stop index - if (index.stop() == object()) { + if (index.stop() == boost::python::object()) { stop = scan.get_num_images(); } else { - stop = extract(index.stop()); + stop = boost::python::extract(index.stop()); } // Check ranges @@ -346,25 +543,14 @@ namespace dxtbx { namespace model { namespace boost_python { DXTBX_ASSERT(stop <= scan.get_num_images()); DXTBX_ASSERT(start < stop); - // Create the new epoch array + flex_table properties = scan.get_properties_slice(index); + int first_image_index = scan.get_image_range()[0] + start; int last_image_index = scan.get_image_range()[0] + stop - 1; - scitbx::af::shared new_epochs(stop - start); - for (std::size_t i = 0; i < new_epochs.size(); ++i) { - new_epochs[i] = scan.get_image_epoch(first_image_index + i); - } - - // Create the new epoch array - scitbx::af::shared new_exposure_times(stop - start); - for (std::size_t i = 0; i < new_exposure_times.size(); ++i) { - new_exposure_times[i] = scan.get_image_exposure_time(first_image_index + i); - } // Create the new scan object return Scan(vec2(first_image_index, last_image_index), - scan.get_image_oscillation(first_image_index), - new_exposure_times, - new_epochs, + properties, scan.get_batch_offset()); } @@ -372,29 +558,63 @@ namespace dxtbx { namespace model { namespace boost_python { std::swap(lhs, rhs); } + template + boost::python::object get_scan_property(const Scan &scan, + const typename T::key_type &key) { + DXTBX_ASSERT(scan.contains(key)); + flex_table properties = scan.get_properties(); + return dxtbx::af::flex_table_suite::getitem_column(properties, key); + } + + template + void set_scan_property(Scan &scan, + const std::string &key, + const scitbx::af::shared &value) { + scan.set_property(key, value.const_ref()); + } + void export_scan() { + scan_property_table_wrapper >::wrap( + "scan_property_table"); + // Export ScanBase - class_("ScanBase"); + boost::python::class_("ScanBase"); + + using boost::python::arg; + using boost::python::self; // Export Scan : ScanBase - class_, bases >("Scan") - .def(init()) + boost::python::class_, boost::python::bases >( + "Scan") + .def(boost::python::init()) + .def("__init__", + boost::python::make_constructor(&make_scan, + boost::python::default_call_policies(), + (arg("image_range"), + arg("oscillation"), + arg("batch_offset") = 0, + arg("deg") = true))) .def("__init__", - make_constructor(&make_scan, - default_call_policies(), - (arg("image_range"), - arg("oscillation"), - arg("batch_offset") = 0, - arg("deg") = true))) + boost::python::make_constructor(&make_scan_w_epoch, + boost::python::default_call_policies(), + (arg("image_range"), + arg("oscillation"), + arg("exposure_times"), + arg("epochs"), + arg("batch_offset") = 0, + arg("deg") = true))) .def("__init__", - make_constructor(&make_scan_w_epoch, - default_call_policies(), - (arg("image_range"), - arg("oscillation"), - arg("exposure_times"), - arg("epochs"), - arg("batch_offset") = 0, - arg("deg") = true))) + boost::python::make_constructor(&make_scan_w_properties, + boost::python::default_call_policies(), + (arg("image_range"), + arg("properties"), + arg("batch_offset") = 0, + arg("deg") = true))) + .def( + "__init__", + boost::python::make_constructor(&make_scan_wo_properties, + boost::python::default_call_policies(), + (arg("image_range"), arg("batch_offset") = 0))) .def("get_image_range", &Scan::get_image_range) .def("get_valid_image_ranges", get_valid_image_ranges) .def("set_valid_image_ranges", set_valid_image_ranges) @@ -451,6 +671,21 @@ namespace dxtbx { namespace model { namespace boost_python { (arg("angle"), arg("deg") = true)) .def("__getitem__", &getitem_single) .def("__getitem__", &getitem_slice) + .def("get_property", + &get_scan_property >, + (arg("key"))) + .def("get_properties", &get_properties_dict) + .def("set_properties", &set_properties_table_from_dict, (arg("properties_dict"))) + .def("set_property", &set_scan_property, (arg("key"), arg("value"))) + .def("set_property", &set_scan_property, (arg("key"), arg("value"))) + .def("set_property", &set_scan_property, (arg("key"), arg("value"))) + .def("set_property", &set_scan_property, (arg("key"), arg("value"))) + .def( + "set_property", &set_scan_property >, (arg("key"), arg("value"))) + .def( + "set_property", &set_scan_property >, (arg("key"), arg("value"))) + .def("set_property", &set_scan_property, (arg("key"), arg("value"))) + .def("has_property", &Scan::contains, (arg("key"))) .def(self == self) .def(self != self) .def(self < self) @@ -464,7 +699,9 @@ namespace dxtbx { namespace model { namespace boost_python { .def("__str__", &scan_to_string) .def("swap", &scan_swap) .def("to_dict", &to_dict) - .def("from_dict", &from_dict, return_value_policy()) + .def("from_dict", + &from_dict, + boost::python::return_value_policy()) .staticmethod("from_dict") .def_pickle(ScanPickleSuite()); } diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 58b447920..6704ebc24 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -426,7 +426,7 @@ def decode(self): # make a new bigger scan o = eobj_scan[imageset_ref].get_oscillation() s = scan.get_oscillation() - assert o[1] == s[1] + assert abs(o[1] - (s[1])) < 1e-7 scan = copy.deepcopy(scan) scan.set_image_range((min(i[0], j[0]), max(i[1], j[1]))) scan.set_oscillation((min(o[0], s[0]), o[1])) diff --git a/src/dxtbx/model/scan.h b/src/dxtbx/model/scan.h index 20adf8d62..3838aa51b 100644 --- a/src/dxtbx/model/scan.h +++ b/src/dxtbx/model/scan.h @@ -20,51 +20,80 @@ #include #include #include "scan_helpers.h" +#include +#include +#include namespace dxtbx { namespace model { + using dxtbx::af::flex_table; + using dxtbx::af::flex_type_generator; using scitbx::rad_as_deg; using scitbx::vec2; + using scitbx::vec3; using scitbx::constants::pi; typedef std::map > > ExpImgRangeMap; - /** A scan base class */ + typedef flex_type_generator, + vec3 >::type scan_property_types; + + typedef dxtbx::af::flex_table::const_iterator const_iterator; + class ScanBase {}; - /** A class to represent a scan */ class Scan : public ScanBase { public: - /** The default constructor */ Scan() : image_range_(0, 0), - oscillation_(0.0, 0.0), num_images_(1), - exposure_times_(1, 0.0), - epochs_(1, 0.0), - batch_offset_(0) {} + batch_offset_(0), + properties_(flex_table(1)) {} + + /** + * @param image_range The range of images covered by the scan + * @param batch_offset An offset to add to the image number (for tracking of + * unique batch numbers for multi-crystal datasets) + */ + Scan(vec2 image_range, int batch_offset = 0) + : image_range_(image_range), + num_images_(1 + image_range_[1] - image_range_[0]), + batch_offset_(batch_offset) { + DXTBX_ASSERT(num_images_ >= 0); + properties_ = flex_table(num_images_); + } /** - * Initialise the class * @param image_range The range of images covered by the scan * @param oscillation A tuple containing the start angle of the first image * and the oscillation range (the angular width) of each * frame - * @param batch_offset A offset to add to the image number (for tracking of + * @param batch_offset An offset to add to the image number (for tracking of * unique batch numbers for multi-crystal datasets) */ Scan(vec2 image_range, vec2 oscillation, int batch_offset = 0) : image_range_(image_range), - oscillation_(oscillation), num_images_(1 + image_range_[1] - image_range_[0]), - batch_offset_(batch_offset), - exposure_times_(num_images_, 0.0), - epochs_(num_images_, 0.0) { + batch_offset_(batch_offset) { DXTBX_ASSERT(num_images_ >= 0); + properties_ = flex_table(num_images_); + + scitbx::af::shared exposure_times = + scitbx::af::shared(num_images_, 0.0); + set_property("exposure_time", exposure_times.const_ref()); + + scitbx::af::shared epochs = scitbx::af::shared(num_images_, 0.0); + set_property("epochs", epochs.const_ref()); + + set_oscillation(oscillation); + DXTBX_ASSERT(properties_.is_consistent()); } /** - * Initialise the class * @param image_range The range of images covered by the scan * @param oscillation A tuple containing the start angle of the first image * and the oscillation range (the angular width) of each @@ -80,51 +109,64 @@ namespace dxtbx { namespace model { const scitbx::af::shared &epochs, int batch_offset = 0) : image_range_(image_range), - oscillation_(oscillation), num_images_(1 + image_range_[1] - image_range_[0]), - batch_offset_(batch_offset), - exposure_times_(exposure_times), - epochs_(epochs) { + batch_offset_(batch_offset) { DXTBX_ASSERT(num_images_ >= 0); - if (exposure_times_.size() == 1 && num_images_ > 1) { + DXTBX_ASSERT(oscillation[1] >= 0); + properties_ = flex_table(num_images_); + + if (exposure_times.size() == 1 && num_images_ > 1) { // assume same exposure time for all images - there is // probably a better way of coding this... scitbx::af::shared expanded_exposure_times; expanded_exposure_times.reserve(num_images_); for (int j = 0; j < num_images_; j++) { expanded_exposure_times.push_back(exposure_times[0]); - exposure_times_ = expanded_exposure_times; } + set_property("exposure_time", expanded_exposure_times.const_ref()); + } else { + set_property("exposure_time", exposure_times.const_ref()); } - DXTBX_ASSERT(exposure_times_.size() == num_images_); - DXTBX_ASSERT(epochs_.size() == num_images_); - DXTBX_ASSERT(oscillation_[1] >= 0.0); + set_property("epochs", epochs.const_ref()); + set_oscillation(oscillation); + DXTBX_ASSERT(properties_.is_consistent()); + } + + /** + * @param image_range The range of images covered by the scan + * @param properties_table Hash table of different properties for each image + * @param batch_offset A offset to add to the image number (for tracking of + * unique batch numbers for multi-crystal datasets) + */ + Scan(vec2 image_range, + flex_table &properties_table, + int batch_offset = 0) + : image_range_(image_range), + num_images_(1 + image_range_[1] - image_range_[0]), + batch_offset_(batch_offset) { + DXTBX_ASSERT(num_images_ >= 0); + DXTBX_ASSERT(properties_table.is_consistent()); + DXTBX_ASSERT(properties_table.size() == num_images_); + properties_ = properties_table; } /** Copy */ Scan(const Scan &rhs) : image_range_(rhs.image_range_), valid_image_ranges_(rhs.valid_image_ranges_), - oscillation_(rhs.oscillation_), num_images_(rhs.num_images_), - batch_offset_(rhs.batch_offset_), - exposure_times_(scitbx::af::reserve(rhs.exposure_times_.size())), - epochs_(scitbx::af::reserve(rhs.epochs_.size())) { - std::copy(rhs.epochs_.begin(), rhs.epochs_.end(), std::back_inserter(epochs_)); - std::copy(rhs.exposure_times_.begin(), - rhs.exposure_times_.end(), - std::back_inserter(exposure_times_)); + batch_offset_(rhs.batch_offset_) { + boost::python::dict d; + set_properties(dxtbx::af::flex_table_suite::deepcopy(rhs.properties_, d)); } - /** Virtual destructor */ virtual ~Scan() {} - /** Get the image range */ vec2 get_image_range() const { return image_range_; } - /** Get the map, not exported to python **/ + /** Not exported to python **/ ExpImgRangeMap get_valid_image_ranges_map() const { return valid_image_ranges_; } @@ -155,144 +197,289 @@ namespace dxtbx { namespace model { valid_image_ranges_[i] = values; } - /** Get the batch offset */ int get_batch_offset() const { return batch_offset_; } - /** Get the still flag */ + bool contains(std::string property_name) const { + return properties_.contains(property_name); + } + + template + scitbx::af::shared get_property(const typename T::key_type &key) const { + DXTBX_ASSERT(properties_.contains(key)); + return properties_.get(key); + } + + template + void set_property(const typename flex_table::key_type &key, + const scitbx::af::const_ref &value) { + DXTBX_ASSERT(value.size() == properties_.size()); + + // Edge case for oscillation where checks are needed + if (key == "oscillation") { + if (!std::is_same::value) { + throw DXTBX_ERROR("Expected oscillation to have type double"); + } + const scitbx::af::const_ref &osc = + reinterpret_cast &>(value); + + if (osc.size() == 1) { + DXTBX_ASSERT(properties_.contains("oscillation_width")); + } else { + DXTBX_ASSERT(Scan::oscillation_has_constant_width(osc)); + } + } + dxtbx::af::flex_table_suite::setitem_column(properties_, key, value); + } + + flex_table get_properties() const { + return properties_; + } + + void set_properties(flex_table new_table) { + DXTBX_ASSERT(new_table.is_consistent()); + DXTBX_ASSERT(new_table.size() == num_images_); + + // Edge case for oscillation to ensure constant oscillation width + if (new_table.contains("oscillation")) { + scitbx::af::shared osc = new_table.get("oscillation"); + DXTBX_ASSERT(Scan::oscillation_has_constant_width(osc)); + } + + properties_ = new_table; + } + bool is_still() const { - return std::abs(oscillation_[1]) < min_oscillation_width_; + if (!properties_.contains("oscillation")) { + return true; + } + if (properties_.size() == 0) { + return true; + } + return std::abs(get_oscillation()[1]) < min_oscillation_width_; } - /** Get the batch number for a given image index */ int get_batch_for_image_index(int index) const { return index + batch_offset_; } - /** Get the batch number for a given array index */ int get_batch_for_array_index(int index) const { return index + batch_offset_ + 1; } - /** Get the batch range */ vec2 get_batch_range() const { return vec2(image_range_[0] + batch_offset_, image_range_[1] + batch_offset_); } - /** Get the array range (zero based) */ + /** (zero based) */ vec2 get_array_range() const { return vec2(image_range_[0] - 1, image_range_[1]); } - /** Get the oscillation */ vec2 get_oscillation() const { - return oscillation_; + DXTBX_ASSERT(properties_.contains("oscillation")); + scitbx::af::shared osc = properties_.get("oscillation"); + + // Edge case for Scans with single image + if (properties_.size() == 1) { + DXTBX_ASSERT(properties_.contains("oscillation_width")); + scitbx::af::shared osc_width = + properties_.get("oscillation_width"); + return vec2(osc[0], osc_width[0]); + } + + DXTBX_ASSERT(properties_.size() > 1); + return vec2(osc[0], osc[1] - osc[0]); + } + + vec2 get_oscillation_in_deg() const { + vec2 osc = get_oscillation(); + return vec2(rad_as_deg(osc[0]), rad_as_deg(osc[1])); + } + + scitbx::af::shared get_oscillation_arr_in_deg() const { + DXTBX_ASSERT(properties_.contains("oscillation")); + scitbx::af::shared osc = properties_.get("oscillation"); + scitbx::af::shared osc_in_deg; + osc_in_deg.resize(osc.size()); + std::transform(osc.begin(), osc.end(), osc_in_deg.begin(), rad_as_deg); + return osc_in_deg; + } + + static bool oscillation_has_constant_width( + const scitbx::af::shared oscillation_arr) { + DXTBX_ASSERT(oscillation_arr.size() > 0); + + if (oscillation_arr.size() == 1) { + return true; + } + double eps = 1e-7; + double expected_width = oscillation_arr[1] - oscillation_arr[0]; + for (std::size_t i = 0; i < oscillation_arr.size() - 1; ++i) { + double width = oscillation_arr[i + 1] - oscillation_arr[i]; + if (std::abs(expected_width - width) > eps) { + return false; + } + } + return true; + } + + static bool oscillation_has_constant_width( + const scitbx::af::const_ref oscillation_arr) { + DXTBX_ASSERT(oscillation_arr.size() > 0); + + if (oscillation_arr.size() == 1) { + return true; + } + double eps = 1e-7; + double expected_width = oscillation_arr[1] - oscillation_arr[0]; + for (std::size_t i = 0; i < oscillation_arr.size() - 1; ++i) { + double width = oscillation_arr[i + 1] - oscillation_arr[i]; + if (std::abs(expected_width - width) > eps) { + return false; + } + } + return true; } - /** Get the number of images */ int get_num_images() const { return num_images_; } - /** Get the exposure time */ scitbx::af::shared get_exposure_times() const { - return exposure_times_; + DXTBX_ASSERT(properties_.contains("exposure_time")); + return properties_.get("exposure_time"); } - /** Get the image epochs */ scitbx::af::shared get_epochs() const { - return epochs_; + DXTBX_ASSERT(properties_.contains("epochs")); + return properties_.get("epochs"); } - /** Set the image range */ void set_image_range(vec2 image_range) { + /* + * Known issue with this function + * https://github.com/cctbx/dxtbx/issues/497 + */ image_range_ = image_range; num_images_ = 1 + image_range_[1] - image_range_[0]; - epochs_.resize(num_images_); - exposure_times_.resize(num_images_); + properties_.resize(num_images_); + + // Edge case where num_images_ was 1 and has been increased + if (properties_.contains("oscillation_width") && num_images_ > 1) { + scitbx::af::shared osc_width = + properties_.get("oscillation_width"); + + vec2 osc = vec2(get_oscillation()[0], osc_width[0]); + set_oscillation(osc); + + // oscillation_width only needed when num_images_ == 1 + dxtbx::af::flex_table_suite::delitem_column(properties_, "oscillation_width"); + } DXTBX_ASSERT(num_images_ > 0); } - /** Set the batch_offset */ void set_batch_offset(int batch_offset) { batch_offset_ = batch_offset; } - /** Set the oscillation */ void set_oscillation(vec2 oscillation) { - DXTBX_ASSERT(oscillation[1] >= 0.0); - oscillation_ = oscillation; + scitbx::af::shared oscillation_arr(num_images_); + for (std::size_t i = 0; i < num_images_; ++i) { + oscillation_arr[i] = oscillation[0] + oscillation[1] * i; + } + // Edge case where the oscillation width needs to be stored + if (num_images_ == 1) { + scitbx::af::shared oscillation_width_arr(num_images_); + oscillation_width_arr[0] = oscillation[1]; + set_property("oscillation_width", oscillation_width_arr.const_ref()); + } + + set_property("oscillation", oscillation_arr.const_ref()); } - /** Set the exposure time */ void set_exposure_times(scitbx::af::shared exposure_times) { DXTBX_ASSERT(exposure_times.size() == num_images_); - exposure_times_ = exposure_times; + set_property("exposure_time", exposure_times.const_ref()); + DXTBX_ASSERT(properties_.is_consistent()); } - /** Set the image epochs */ void set_epochs(const scitbx::af::shared &epochs) { DXTBX_ASSERT(epochs.size() == num_images_); - epochs_ = epochs; + set_property("epochs", epochs.const_ref()); + DXTBX_ASSERT(properties_.is_consistent()); } /** Get the total oscillation range of the scan */ vec2 get_oscillation_range() const { - return vec2(oscillation_[0], - oscillation_[0] + num_images_ * oscillation_[1]); + vec2 osc = get_oscillation(); + return vec2(osc[0], osc[0] + num_images_ * osc[1]); } /** Get the image angle and oscillation width as a tuple */ vec2 get_image_oscillation(int index) const { - return vec2(oscillation_[0] + (index - image_range_[0]) * oscillation_[1], - oscillation_[1]); + DXTBX_ASSERT(image_range_[0] <= index && index <= image_range_[1]); + vec2 osc = get_oscillation(); + return vec2(osc[0] + (index - image_range_[0]) * osc[1], osc[1]); } - /** Get the image epoch */ double get_image_epoch(int index) const { + DXTBX_ASSERT(properties_.contains("epochs")); DXTBX_ASSERT(image_range_[0] <= index && index <= image_range_[1]); - return epochs_[index - image_range_[0]]; + return properties_.get("epochs")[index - image_range_[0]]; } double get_image_exposure_time(int index) const { + DXTBX_ASSERT(properties_.contains("exposure_time")); DXTBX_ASSERT(image_range_[0] <= index && index <= image_range_[1]); - return exposure_times_[index - image_range_[0]]; + return properties_.get("exposure_time")[index - image_range_[0]]; } - /** Check the scans are the same */ bool operator==(const Scan &rhs) const { - double eps = 1e-7; - return image_range_ == rhs.image_range_ && batch_offset_ == rhs.batch_offset_ - && std::abs(oscillation_[0] - rhs.oscillation_[0]) < eps - && std::abs(oscillation_[1] - rhs.oscillation_[1]) < eps - && exposure_times_.const_ref().all_approx_equal( - rhs.exposure_times_.const_ref(), eps) - && epochs_.const_ref().all_approx_equal(rhs.epochs_.const_ref(), eps); + if (image_range_ != image_range_ || batch_offset_ != rhs.batch_offset_) { + return false; + } + + if (properties_.size() != rhs.properties_.size()) { + return false; + } + for (const_iterator it = properties_.begin(); it != properties_.end(); ++it) { + if (!rhs.properties_.contains(it->first)) { + return false; + } + } + + // Need to create copies to get around const requirements + flex_table table1 = + flex_table(properties_); + flex_table table2 = + flex_table(rhs.properties_); + bool same_columns = dxtbx::af::flex_table_suite::compare_columns(table1, table2); + if (!same_columns) { + return false; + } + + return true; } - /** Check the scans are not the same */ bool operator!=(const Scan &scan) const { return !(*this == scan); } - /** Comparison operator */ bool operator<(const Scan &scan) const { return image_range_[0] < scan.image_range_[0]; } - /** Comparison operator */ bool operator<=(const Scan &scan) const { return image_range_[0] <= scan.image_range_[0]; } - /** Comparison operator */ bool operator>(const Scan &scan) const { return image_range_[0] > scan.image_range_[0]; } - /** Comparison operator */ bool operator>=(const Scan &scan) const { return image_range_[0] >= scan.image_range_[0]; } @@ -302,48 +489,64 @@ namespace dxtbx { namespace model { */ void append(const Scan &rhs, double scan_tolerance) { - DXTBX_ASSERT(is_still() == rhs.is_still()); - if (is_still()) { - append_still(rhs); - } else { - append_rotation(rhs, scan_tolerance); - } - } - - void append_still(const Scan &rhs) { DXTBX_ASSERT(image_range_[1] + 1 == rhs.image_range_[0]); DXTBX_ASSERT(batch_offset_ == rhs.batch_offset_); + + boost::python::dict d; + flex_table rhs_properties = + dxtbx::af::flex_table_suite::deepcopy(rhs.properties_, d); + + // Explicitly check oscillation + if (properties_.contains("oscillation") && !is_still()) { + DXTBX_ASSERT(!rhs.is_still()); + double osc_width = get_oscillation()[1]; + double eps = scan_tolerance * std::abs(osc_width); + double rhs_osc_width = rhs.get_oscillation()[1]; + DXTBX_ASSERT(eps > 0); + DXTBX_ASSERT(std::abs(osc_width) > min_oscillation_width_); + DXTBX_ASSERT(std::abs(osc_width - rhs_osc_width) < eps); + // sometimes ticking through 0 the first difference is not helpful + double diff_2pi = std::abs(mod_2pi(get_oscillation_range()[1]) + - mod_2pi(rhs.get_oscillation_range()[0])); + double diff_abs = + std::abs(get_oscillation_range()[1] - rhs.get_oscillation_range()[0]); + + DXTBX_ASSERT(std::min(diff_2pi, diff_abs) < eps * get_num_images()); + + /* + rhs oscillation arr updated before appending to reflect the larger + number of images of the appended Scan + */ + const double rhs_osc_start = get_oscillation()[0] + num_images_ * osc_width; + scitbx::af::shared rhs_osc_arr = + scitbx::af::shared(rhs.num_images_, 0.0); + for (std::size_t i = 0; i < rhs_osc_arr.size(); ++i) { + rhs_osc_arr[i] = rhs_osc_start + i * osc_width; + } + dxtbx::af::flex_table_suite::setitem_column( + rhs_properties, "oscillation", rhs_osc_arr.const_ref()); + + /* + If properties table contains oscillation_width, remove as this + is only needed for scans where num_images_ == 1 + */ + if (properties_.contains("oscillation_width")) { + dxtbx::af::flex_table_suite::delitem_column(properties_, "oscillation_width"); + } + if (rhs_properties.contains("oscillation_width")) { + dxtbx::af::flex_table_suite::delitem_column(rhs_properties, + "oscillation_width"); + } + } + image_range_[1] = rhs.image_range_[1]; num_images_ = 1 + image_range_[1] - image_range_[0]; - exposure_times_.reserve(exposure_times_.size() + exposure_times_.size()); - epochs_.reserve(epochs_.size() + epochs_.size()); - std::copy(rhs.exposure_times_.begin(), - rhs.exposure_times_.end(), - std::back_inserter(exposure_times_)); - std::copy(rhs.epochs_.begin(), rhs.epochs_.end(), std::back_inserter(epochs_)); - } - - void append_rotation(const Scan &rhs, double scan_tolerance) { - double eps = scan_tolerance * std::abs(oscillation_[1]); - DXTBX_ASSERT(eps > 0); - DXTBX_ASSERT(std::abs(oscillation_[1]) > min_oscillation_width_); - DXTBX_ASSERT(image_range_[1] + 1 == rhs.image_range_[0]); - DXTBX_ASSERT(std::abs(oscillation_[1] - rhs.oscillation_[1]) < eps); - DXTBX_ASSERT(batch_offset_ == rhs.batch_offset_); - // sometimes ticking through 0 the first difference is not helpful - double diff_2pi = std::abs(mod_2pi(get_oscillation_range()[1]) - - mod_2pi(rhs.get_oscillation_range()[0])); - double diff_abs = - std::abs(get_oscillation_range()[1] - rhs.get_oscillation_range()[0]); - DXTBX_ASSERT(std::min(diff_2pi, diff_abs) < eps * get_num_images()); - image_range_[1] = rhs.image_range_[1]; - num_images_ = 1 + image_range_[1] - image_range_[0]; - exposure_times_.reserve(exposure_times_.size() + exposure_times_.size()); - epochs_.reserve(epochs_.size() + epochs_.size()); - std::copy(rhs.exposure_times_.begin(), - rhs.exposure_times_.end(), - std::back_inserter(exposure_times_)); - std::copy(rhs.epochs_.begin(), rhs.epochs_.end(), std::back_inserter(epochs_)); + + for (const_iterator it = properties_.begin(); it != properties_.end(); ++it) { + DXTBX_ASSERT(rhs_properties.contains(it->first)); + } + dxtbx::af::flex_table_suite::extend(properties_, rhs_properties); + DXTBX_ASSERT(properties_.size() == num_images_); } /** @@ -373,18 +576,15 @@ namespace dxtbx { namespace model { return is_angle_in_range(get_oscillation_range(), angle); } - /** Check if the index is valid */ bool is_image_index_valid(double index) const { return (image_range_[0] <= index && index <= image_range_[1]); } - /** Check if a given batch is valid */ bool is_batch_valid(int batch) const { vec2 batch_range = get_batch_range(); return (batch_range[0] <= batch && batch <= batch_range[1]); } - /** Check if the array index is valid */ bool is_array_index_valid(double index) const { return is_image_index_valid(index + 1); } @@ -395,7 +595,8 @@ namespace dxtbx { namespace model { * @returns The angle at the given frame */ double get_angle_from_image_index(double index) const { - return oscillation_[0] + (index - image_range_[0]) * oscillation_[1]; + vec2 oscillation = get_oscillation(); + return oscillation[0] + (index - image_range_[0]) * oscillation[1]; } /** @@ -413,7 +614,8 @@ namespace dxtbx { namespace model { * @returns The frame at the given angle */ double get_image_index_from_angle(double angle) const { - return image_range_[0] + (angle - oscillation_[0]) / oscillation_[1]; + vec2 oscillation = get_oscillation(); + return image_range_[0] + (angle - oscillation[0]) / oscillation[1]; } /** @@ -473,22 +675,37 @@ namespace dxtbx { namespace model { } Scan operator[](int index) const { - // Check index DXTBX_ASSERT((index >= 0) && (index < get_num_images())); - int image_index = get_image_range()[0] + index; - // Create the new epoch array - scitbx::af::shared new_epochs(1); - new_epochs[0] = get_image_epoch(image_index); - scitbx::af::shared new_exposure_times(1); - new_exposure_times[0] = get_image_exposure_time(image_index); - - // Return scan - return Scan(vec2(image_index, image_index), - get_image_oscillation(image_index), - new_exposure_times, - new_epochs, - get_batch_offset()); + int image_index = image_range_[0] + index; + flex_table properties_slice = + get_properties_slice(boost::python::slice(index, index + 1)); + + return Scan( + vec2(image_index, image_index), properties_slice, get_batch_offset()); + } + + flex_table get_properties_slice( + boost::python::slice const &slice) const { + /* + Wrapper for slicing the properties table to account for oscillation + width being lost in the slice if a Scan with a single image + */ + + if (properties_.contains("oscillation")) { + double oscillation_width = get_oscillation()[1]; + flex_table sliced_properties = + dxtbx::af::flex_table_suite::getitem_slice(properties_, slice); + if (sliced_properties.size() == 1) { + scitbx::af::shared oscillation_width_arr(1); + oscillation_width_arr[0] = oscillation_width; + dxtbx::af::flex_table_suite::setitem_column( + sliced_properties, "oscillation_width", oscillation_width_arr.const_ref()); + return sliced_properties; + } + return sliced_properties; + } + return dxtbx::af::flex_table_suite::getitem_slice(properties_, slice); } friend std::ostream &operator<<(std::ostream &os, const Scan &s); @@ -496,26 +713,39 @@ namespace dxtbx { namespace model { private: vec2 image_range_; ExpImgRangeMap valid_image_ranges_; /** initialised as an empty map **/ - vec2 oscillation_; double min_oscillation_width_ = 1e-7; int num_images_; int batch_offset_; - scitbx::af::shared exposure_times_; - scitbx::af::shared epochs_; + flex_table properties_; }; /** Print Scan information */ inline std::ostream &operator<<(std::ostream &os, const Scan &s) { - // Print oscillation as degrees! - vec2 oscillation = s.get_oscillation(); - oscillation[0] = rad_as_deg(oscillation[0]); - oscillation[1] = rad_as_deg(oscillation[1]); os << "Scan:\n"; os << " number of images: " << s.get_num_images() << "\n"; os << " image range: " << s.get_image_range().const_ref() << "\n"; - os << " oscillation: " << oscillation.const_ref() << "\n"; - if (s.num_images_ > 0) { - os << " exposure time: " << s.exposure_times_.const_ref()[0] << "\n"; + flex_table properties = s.get_properties(); + dxtbx::af::flex_table_suite::column_range_to_string_visitor visitor; + for (flex_table::const_iterator it = properties.begin(); + it != properties.end(); + ++it) { + if (it->first == "oscillation") { + vec2 oscillation = s.get_oscillation_in_deg(); + os << " oscillation: " << oscillation.const_ref() << "\n"; + + } else if (it->first == "oscillation_width") { + /*Handled in oscillation case*/ + } else if (it->first == "exposure_time") { + os << " exposure time: " + << properties.get("exposure_time").const_ref()[0] << "\n"; + + } else if (it->first == "epochs") { + os << " epoch: " << properties.get("epochs").const_ref()[0] + << "\n"; + + } else { + os << " " << it->first << ": " << it->second.apply_visitor(visitor); + } } return os; } diff --git a/src/dxtbx/model/scan.py b/src/dxtbx/model/scan.py index 15dc2ca70..40cf591c5 100644 --- a/src/dxtbx/model/scan.py +++ b/src/dxtbx/model/scan.py @@ -112,17 +112,68 @@ def from_dict(d, t=None): Returns: The scan model """ + + def convert_oscillation_to_vec2(properties_dict): + + """ + If oscillation is in properties_dict, + shared is converted to vec2 and + oscillation_width is removed (if present) to ensure + it is replaced correctly if updating t dict from d dict + """ + + if "oscillation" not in properties_dict: + assert "oscillation_width" not in properties_dict + return properties_dict + if "oscillation_width" in properties_dict: + assert "oscillation" in properties_dict + properties_dict["oscillation"] = ( + properties_dict["oscillation"][0], + properties_dict["oscillation_width"][0], + ) + del properties_dict["oscillation_width"] + return properties_dict + properties_dict["oscillation"] = ( + properties_dict["oscillation"][0], + properties_dict["oscillation"][1] - properties_dict["oscillation"][0], + ) + + return properties_dict + if d is None and t is None: return None joint = t.copy() if t else {} - joint.update(d) - if not isinstance(joint["exposure_time"], list): + # Accounting for legacy cases where t or d does not + # contain properties dict + if "properties" in joint and "properties" in d: + properties = t["properties"].copy() + properties.update(d["properties"]) + joint.update(d) + joint["properties"] = properties + elif "properties" in d: + d_copy = d.copy() + d_copy["properties"] = convert_oscillation_to_vec2( + d_copy["properties"].copy() + ) + joint.update(**d_copy["properties"]) + del d_copy["properties"] + joint.update(d_copy) + elif "properties" in joint: + joint["properties"] = convert_oscillation_to_vec2( + joint["properties"].copy() + ) + joint.update(**joint["properties"]) + del joint["properties"] + joint.update(d) + else: + joint.update(d) + + if "properties" not in d and not isinstance(joint["exposure_time"], list): joint["exposure_time"] = [joint["exposure_time"]] joint.setdefault("batch_offset", 0) # backwards compatibility 20180205 joint.setdefault("valid_image_ranges", {}) # backwards compatibility 20181113 - # Create the model from the joint dictionary return Scan.from_dict(joint) @staticmethod @@ -154,6 +205,11 @@ def make_scan( deg, ) + @staticmethod + def make_scan_from_properties(image_range, properties, batch_offset=0, deg=True): + + return Scan(tuple(map(int, image_range)), properties, batch_offset, deg) + @staticmethod def single_file(filename, exposure_times, osc_start, osc_width, epoch): """Construct an scan instance for a single image.""" diff --git a/src/dxtbx/model/scan_helpers.h b/src/dxtbx/model/scan_helpers.h index 0f88a18ba..400bee252 100644 --- a/src/dxtbx/model/scan_helpers.h +++ b/src/dxtbx/model/scan_helpers.h @@ -26,6 +26,10 @@ namespace dxtbx { namespace model { /** Convert the angle mod 2PI */ inline double mod_2pi(double angle) { + // E.g. treat 359.9999999 as 360 + if (std::abs(angle - two_pi) <= 1e-7) { + angle = two_pi; + } return angle - two_pi * floor(angle / two_pi); } diff --git a/tests/model/test_scan.py b/tests/model/test_scan.py index 3d026c1d5..85d7628b3 100644 --- a/tests/model/test_scan.py +++ b/tests/model/test_scan.py @@ -1,10 +1,16 @@ from __future__ import annotations -import dxtbx.model +import pickle + +import pytest + +import cctbx.array_family.flex as flex + +from dxtbx.model import Scan, ScanFactory def test_scan_to_string_does_not_crash_on_empty_scan(): - print(dxtbx.model.Scan()) + print(Scan()) def test_scan_wrap_around_zero(): @@ -12,12 +18,438 @@ def test_scan_wrap_around_zero(): starts = [r % 360 for r in range(350, 370)] ends = [(r + 1) % 360 for r in range(350, 370)] scans = [ - dxtbx.model.scan.ScanFactory.single_file(f, [0.1], s, e - s, 0) + ScanFactory.single_file(f, [0.1], s, e - s, 0) for f, s, e in zip(filenames, starts, ends) ] s0 = scans[0] for s in scans[1:]: assert s.get_oscillation()[1] == 1 s0 += s - assert s0.get_oscillation() == (350, 1) + assert s0.get_oscillation() == pytest.approx((350, 1)) assert s0.get_image_range() == (350, 369) + + +def test_make_scan_from_properties(): + image_range = (1, 10) + + properties = {} + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + assert scan.get_properties() == {} + + properties = { + "test_int": tuple(range(10)), + "test_float": tuple([float(i * 2) for i in range(10)]), + "test_bool": tuple([True for i in range(10)]), + "test_string": tuple([f"test_{i}" for i in range(10)]), + "test_vec3_double": tuple([(1.0, 1.0, 1.0) for i in range(10)]), + "test_vec2_double": tuple([(2.0, 2.0) for i in range(10)]), + } + + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + + result = scan.get_properties() + for key, value in properties.items(): + assert key in result + assert len(value) == len(result[key]) + for i in range(len(value)): + assert result[key][i] == pytest.approx(value[i]) + + +def test_set_get_properties(): + + image_range = (1, 10) + scan = ScanFactory.make_scan_from_properties(image_range=image_range, properties={}) + + test_int = flex.int(10, 1) + test_float = flex.double(10, 1.0) + test_bool = flex.bool(10, True) + test_string = flex.std_string(10, "Test") + test_vec3_double = flex.vec3_double(10, (1.0, 1.0, 1.0)) + test_vec2_double = flex.vec2_double(10, (2.0, 2.0)) + + scan.set_property("test_int", test_int) + result = scan.get_property("test_int") + for idx, i in enumerate(test_int): + assert result[idx] == pytest.approx(i) + + scan.set_property("test_float", test_float) + result = scan.get_property("test_float") + for idx, i in enumerate(test_float): + assert result[idx] == pytest.approx(i) + + scan.set_property("test_bool", test_bool) + result = scan.get_property("test_bool") + for idx, i in enumerate(test_bool): + assert result[idx] == pytest.approx(i) + + scan.set_property("test_string", test_string) + result = scan.get_property("test_string") + for idx, i in enumerate(test_string): + assert result[idx] == pytest.approx(i) + + scan.set_property("test_vec3_double", test_vec3_double) + result = scan.get_property("test_vec3_double") + for idx, i in enumerate(test_vec3_double): + assert result[idx] == pytest.approx(i) + + scan.set_property("test_vec2_double", test_vec2_double) + result = scan.get_property("test_vec2_double") + for idx, i in enumerate(test_vec2_double): + assert result[idx] == pytest.approx(i) + + # Test incompatible size with property table + + test_int = flex.int(5, 1) + with pytest.raises(RuntimeError): + scan.set_property("test_int", test_int) + + test_float = flex.double(5, 1.0) + with pytest.raises(RuntimeError): + scan.set_property("test_float", test_float) + + test_bool = flex.bool(5, True) + with pytest.raises(RuntimeError): + scan.set_property("test_bool", test_bool) + + test_string = flex.std_string(5, "Test") + with pytest.raises(RuntimeError): + scan.set_property("test_string", test_string) + + test_vec3_double = flex.vec3_double(5, (1.0, 1.0, 1.0)) + with pytest.raises(RuntimeError): + scan.set_property("test_vec3_double", test_vec3_double) + + test_vec2_double = flex.vec2_double(5, (2.0, 2.0)) + with pytest.raises(RuntimeError): + scan.set_property("test_vec2_double", test_vec2_double) + + test_int = flex.int(15, 1) + with pytest.raises(RuntimeError): + scan.set_property("test_int", test_int) + + test_float = flex.double(15, 1.0) + with pytest.raises(RuntimeError): + scan.set_property("test_float", test_float) + + test_bool = flex.bool(15, True) + with pytest.raises(RuntimeError): + scan.set_property("test_bool", test_bool) + + test_string = flex.std_string(15, "Test") + with pytest.raises(RuntimeError): + scan.set_property("test_string", test_string) + + test_vec3_double = flex.vec3_double(15, (1.0, 1.0, 1.0)) + with pytest.raises(RuntimeError): + scan.set_property("test_vec3_double", test_vec3_double) + + test_vec2_double = flex.vec2_double(15, (2.0, 2.0)) + with pytest.raises(RuntimeError): + scan.set_property("test_vec2_double", test_vec2_double) + + +def test_scan_properties_pickle(): + image_range = (1, 10) + properties = { + "test_int": tuple(range(10)), + "test_float": tuple([float(i * 2) for i in range(10)]), + "test_bool": tuple([True for i in range(10)]), + "test_string": tuple([f"test_{i}" for i in range(10)]), + "test_vec3_double": tuple([(1.0, 1.0, 1.0) for i in range(10)]), + "test_vec2_double": tuple([(2.0, 2.0) for i in range(10)]), + } + + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + + obj = pickle.dumps(scan) + scan2 = pickle.loads(obj) + assert scan == scan2 + + result = scan.get_properties() + for key, value in properties.items(): + assert key in result + assert len(value) == len(result[key]) + for i in range(len(value)): + assert result[key][i] == pytest.approx(value[i]) + + +def test_scan_properties_to_dict(): + image_range = (1, 10) + properties = { + "test_int": tuple(range(10)), + "test_float": tuple([float(i * 2) for i in range(10)]), + "test_bool": tuple([True for i in range(10)]), + "test_string": tuple([f"test_{i}" for i in range(10)]), + "test_vec3_double": tuple([(1.0, 1.0, 1.0) for i in range(10)]), + "test_vec2_double": tuple([(2.0, 2.0) for i in range(10)]), + } + + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + + expected_result = { + "test_bool": (True, True, True, True, True, True, True, True, True, True), + "test_float": (0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0), + "test_int": (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), + "test_string": ( + "test_0", + "test_1", + "test_2", + "test_3", + "test_4", + "test_5", + "test_6", + "test_7", + "test_8", + "test_9", + ), + "test_vec2_double": ( + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + (2.0, 2.0), + ), + "test_vec3_double": ( + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + (1.0, 1.0, 1.0), + ), + } + + result = scan.to_dict()["properties"] + for key, value in expected_result.items(): + assert key in result + assert len(value) == len(result[key]) + for i in range(len(value)): + assert result[key][i] == pytest.approx(value[i]) + + +def test_scan_properties_equivalence(): + + int_diff = 1 + double_diff = 1e-6 + + image_range = (1, 10) + s1 = ScanFactory.make_scan_from_properties(image_range=image_range, properties={}) + s2 = ScanFactory.make_scan_from_properties(image_range=image_range, properties={}) + + # Empty table + + assert s1 == s2 + + # int columns + + test_int = flex.int(10, 1) + test_int2 = flex.int(10, 1) + test_int2[0] += int_diff + + s1.set_property("test_int", test_int) + + assert s1 != s2 + + s2.set_property("test_int", test_int2) + + assert s1 != s2 + + test_int2[0] -= int_diff + + s2.set_property("test_int", test_int2) + + assert s1 == s2 + + # double columns + + test_double = flex.double(10, 1.0) + test_double2 = flex.double(10, 1.0) + test_double2[0] += double_diff + + s1.set_property("test_double", test_double) + + assert s1 != s2 + + s2.set_property("test_double", test_double2) + + assert s1 != s2 + + test_double2[0] -= double_diff + + s2.set_property("test_double", test_double2) + + assert s1 == s2 + + # bool columns + + test_bool = flex.bool(10, True) + test_bool2 = flex.bool(10, True) + test_bool2[0] = False + + s1.set_property("test_bool", test_bool) + + assert s1 != s2 + + s2.set_property("test_bool", test_bool2) + + assert s1 != s2 + + test_bool2[0] = True + + s2.set_property("test_bool", test_bool2) + + assert s1 == s2 + + # string columns + + test_string = flex.std_string(10, "Test") + test_string2 = flex.std_string(10, "Test") + test_string2[0] = "Test2" + + s1.set_property("test_string", test_string) + + assert s1 != s2 + + s2.set_property("test_string", test_string2) + + assert s1 != s2 + + test_string2[0] = "Test" + + s2.set_property("test_string", test_string2) + + assert s1 == s2 + + # vec3 columns + + test_vec3_double = flex.vec3_double(10, (1.0, 1.0, 1.0)) + test_vec3_double2 = flex.vec3_double(10, (1.0, 1.0, 1.0)) + test_vec3_double2[0] = (1.0 + double_diff, 1.0 + double_diff, 1.0 + double_diff) + + s1.set_property("test_vec3_double", test_vec3_double) + + assert s1 != s2 + + s2.set_property("test_vec3_double", test_vec3_double2) + + assert s1 != s2 + vec = test_vec3_double2[0] + test_vec3_double2[0] = ( + vec[0] - double_diff, + vec[1] - double_diff, + vec[2] - double_diff, + ) + + s2.set_property("test_vec3_double", test_vec3_double2) + + assert s1 == s2 + + # vec2 columns + + test_vec2_double = flex.vec2_double(10, (2.0, 2.0)) + test_vec2_double2 = flex.vec2_double(10, (2.0, 2.0)) + test_vec2_double2[0] = (2.0 + double_diff, 2.0 + double_diff) + + s1.set_property("test_vec2_double", test_vec2_double) + + assert s1 != s2 + + s2.set_property("test_vec2_double", test_vec2_double2) + + assert s1 != s2 + + vec = test_vec2_double2[0] + test_vec2_double2[0] = (vec[0] - double_diff, vec[1] - double_diff) + + s2.set_property("test_vec2_double", test_vec2_double2) + + assert s1 == s2 + + +def test_scan_constant_oscillation_width(): + + image_range = (1, 10) + scan = ScanFactory.make_scan_from_properties(image_range=image_range, properties={}) + + oscillation = flex.double(10, 1.0) + scan.set_property("oscillation", oscillation) + oscillation[3] = 2.0 + with pytest.raises(RuntimeError): + scan.set_property("oscillation", oscillation) + + oscillation = flex.double(10, 1.0) + properties = {"oscillation": oscillation} + scan.set_properties(properties) + properties["oscillation"][3] = 2.0 + with pytest.raises(RuntimeError): + scan.set_properties(properties) + + oscillation = flex.double(10, 1.0) + properties = {"oscillation": oscillation} + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + properties["oscillation"][3] = 2.0 + with pytest.raises(RuntimeError): + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + + +def test_print_scan(): + + image_range = (1, 10) + scan = ScanFactory.make_scan_from_properties(image_range=image_range, properties={}) + + expected_scan_string = ( + "Scan:\n number of images: 10\n image range: {1,10}\n" + ) + assert scan.__str__() == expected_scan_string + + # Properties with unique output + + scan.set_oscillation((0.0, 0.5)) + expected_scan_string = "Scan:\n number of images: 10\n image range: {1,10}\n oscillation: {0,0.5}\n" + assert scan.__str__() == expected_scan_string + + scan.set_property("epochs", flex.double(10)) + expected_scan_string = "Scan:\n number of images: 10\n image range: {1,10}\n epoch: 0\n oscillation: {0,0.5}\n" + assert scan.__str__() == expected_scan_string + + scan.set_property("exposure_time", flex.double(10)) + expected_scan_string = "Scan:\n number of images: 10\n image range: {1,10}\n epoch: 0\n exposure time: 0\n oscillation: {0,0.5}\n" + assert scan.__str__() == expected_scan_string + + # Generic properties + + properties = { + "test_int": tuple(range(10)), + "test_float": tuple([float(i * 2) for i in range(10)]), + "test_bool": tuple([True for i in range(10)]), + "test_string": tuple([f"test_{i}" for i in range(10)]), + "test_vec3_double": tuple([(1.0, 1.0, 1.0) for i in range(10)]), + "test_vec2_double": tuple([(2.0, 2.0) for i in range(10)]), + } + + scan = ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + expected_scan_string = "Scan:\n number of images: 10\n image range: {1,10}\n test_bool: 1 - 1\n test_float: 0 - 18\n test_int: 0 - 9\n test_string: test_0 - test_9\n test_vec2_double: {2,2} - {2,2}\n test_vec3_double: {1,1,1} - {1,1,1}\n" + assert scan.__str__() == expected_scan_string diff --git a/tests/model/test_scan_data.py b/tests/model/test_scan_data.py index ace6c1844..10fd4e806 100644 --- a/tests/model/test_scan_data.py +++ b/tests/model/test_scan_data.py @@ -104,12 +104,14 @@ def test_scan_360_append(): def test_swap(): scan1 = Scan((1, 20), (0.0, 1.0)) + scan1_osc = scan1.get_oscillation() scan2 = Scan((40, 60), (10.0, 2.0)) + scan2_osc = scan2.get_oscillation() scan1.swap(scan2) assert scan2.get_image_range() == (1, 20) assert scan1.get_image_range() == (40, 60) - assert scan2.get_oscillation() == (0.0, 1.0) - assert scan1.get_oscillation() == (10.0, 2.0) + assert scan2.get_oscillation() == scan1_osc + assert scan1.get_oscillation() == scan2_osc def test_valid_image_ranges(): @@ -144,7 +146,7 @@ def test_from_phil(): assert s1.get_num_images() == 10 assert s1.get_image_range() == (1, 10) - assert s1.get_oscillation() == (-4, 0.1) + assert s1.get_oscillation() == pytest.approx((-4, 0.1)) assert s1.get_batch_offset() == 0 assert s1.get_batch_range() == s1.get_image_range() for i in range(s1.get_image_range()[0], s1.get_image_range()[1] + 1): @@ -167,7 +169,7 @@ def test_from_phil(): s2 = ScanFactory.from_phil(params, s1) assert s2.get_num_images() == 20 assert s2.get_image_range() == (1, 20) - assert s2.get_oscillation() == (20, 0.01) + assert s2.get_oscillation() == pytest.approx((20, 0.01)) assert s2.get_batch_offset() == 10 assert s2.get_batch_range() == (11, 30) ir1, ir2 = s2.get_image_range() diff --git a/tests/serialize/test_serialize.py b/tests/serialize/test_serialize.py index 849f9c955..22917baf5 100644 --- a/tests/serialize/test_serialize.py +++ b/tests/serialize/test_serialize.py @@ -124,9 +124,10 @@ def test_scan(): d = s1.to_dict() s2 = ScanFactory.from_dict(d) assert d["image_range"] == (1, 3) - assert d["oscillation"] == (1.0, 0.2) - assert d["exposure_time"] == [0.1, 0.1, 0.1] - assert d["epochs"] == [0.1, 0.2, 0.3] + osc = d["properties"]["oscillation"] + assert (osc[0], osc[1] - osc[0]) == pytest.approx((1.0, 0.2)) + assert d["properties"]["exposure_time"] == pytest.approx([0.1, 0.1, 0.1]) + assert d["properties"]["epochs"] == pytest.approx([0.1, 0.2, 0.3]) assert d["batch_offset"] == 0 assert s1 == s2 @@ -134,9 +135,9 @@ def test_scan(): d2 = {"exposure_time": [0.2, 0.2, 0.2]} s3 = ScanFactory.from_dict(d2, d) assert s3.get_image_range() == (1, 3) - assert s3.get_oscillation() == (1.0, 0.2) - assert list(s3.get_exposure_times()) == [0.2, 0.2, 0.2] - assert list(s3.get_epochs()) == [0.1, 0.2, 0.3] + assert s3.get_oscillation() == pytest.approx((1.0, 0.2)) + assert list(s3.get_exposure_times()) == pytest.approx([0.2, 0.2, 0.2]) + assert list(s3.get_epochs()) == pytest.approx([0.1, 0.2, 0.3]) assert s2 != s3 # Test with a partial epoch From 5851627cd002f1707b83faf8b70a482bf3760c2d Mon Sep 17 00:00:00 2001 From: David McDonagh <60879630+toastisme@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:57:57 +0000 Subject: [PATCH 08/10] Add oscillation fix for Scan.set_image_range. (#667) Add oscillation fix for Scan.set_image_range. --- newsfragments/667.misc | 1 + src/dxtbx/model/scan.h | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 newsfragments/667.misc diff --git a/newsfragments/667.misc b/newsfragments/667.misc new file mode 100644 index 000000000..c3baae2da --- /dev/null +++ b/newsfragments/667.misc @@ -0,0 +1 @@ +Fix oscillation not being set correctly in ``Scan.set_image_range``. diff --git a/src/dxtbx/model/scan.h b/src/dxtbx/model/scan.h index 3838aa51b..c4ef2f0d0 100644 --- a/src/dxtbx/model/scan.h +++ b/src/dxtbx/model/scan.h @@ -365,19 +365,23 @@ namespace dxtbx { namespace model { */ image_range_ = image_range; num_images_ = 1 + image_range_[1] - image_range_[0]; - properties_.resize(num_images_); - // Edge case where num_images_ was 1 and has been increased - if (properties_.contains("oscillation_width") && num_images_ > 1) { - scitbx::af::shared osc_width = - properties_.get("oscillation_width"); - - vec2 osc = vec2(get_oscillation()[0], osc_width[0]); + // Fix for dxtbx #497 for oscillation + if (properties_.contains("oscillation")) { + vec2 osc = get_oscillation(); + properties_.resize(num_images_); set_oscillation(osc); - // oscillation_width only needed when num_images_ == 1 - dxtbx::af::flex_table_suite::delitem_column(properties_, "oscillation_width"); + // Edge case where num_images_ was 1 and has been increased + if (properties_.contains("oscillation_width") && num_images_ > 1) { + // oscillation_width only needed when num_images_ == 1 + dxtbx::af::flex_table_suite::delitem_column(properties_, "oscillation_width"); + } + DXTBX_ASSERT(num_images_ > 0); + return; } + + properties_.resize(num_images_); DXTBX_ASSERT(num_images_ > 0); } From 31f73a5cb00fd1d7af4a308080d27de83db107c0 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 24 Nov 2023 11:16:53 +0000 Subject: [PATCH 09/10] Fix scan comparison when slicing imagesets (#669) Fixes #668 --- newsfragments/669.bugfix | 1 + src/dxtbx/model/scan.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/669.bugfix diff --git a/newsfragments/669.bugfix b/newsfragments/669.bugfix new file mode 100644 index 000000000..1e1d974b1 --- /dev/null +++ b/newsfragments/669.bugfix @@ -0,0 +1 @@ +Fix scan comparison for scan properties changes diff --git a/src/dxtbx/model/scan.h b/src/dxtbx/model/scan.h index c4ef2f0d0..e3cfc4ef2 100644 --- a/src/dxtbx/model/scan.h +++ b/src/dxtbx/model/scan.h @@ -442,7 +442,7 @@ namespace dxtbx { namespace model { } bool operator==(const Scan &rhs) const { - if (image_range_ != image_range_ || batch_offset_ != rhs.batch_offset_) { + if (image_range_ != rhs.image_range_ || batch_offset_ != rhs.batch_offset_) { return false; } From dbd7edea748f9e7db63a9bc3d990acfb0b9335b7 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 24 Nov 2023 16:15:45 +0000 Subject: [PATCH 10/10] Update FormatCBFFull test following scan changes (#670) Update test --- newsfragments/670.misc | 1 + tests/test_FormatCBFFull.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 newsfragments/670.misc diff --git a/newsfragments/670.misc b/newsfragments/670.misc new file mode 100644 index 000000000..d01880165 --- /dev/null +++ b/newsfragments/670.misc @@ -0,0 +1 @@ +Fix regression test after scan changes diff --git a/tests/test_FormatCBFFull.py b/tests/test_FormatCBFFull.py index 0b3d4a71c..c8ad43d12 100644 --- a/tests/test_FormatCBFFull.py +++ b/tests/test_FormatCBFFull.py @@ -40,7 +40,9 @@ def test_still(dials_regression): imgset = ImageSetFactory.new(os.path.join(data_dir, "grid_full_cbf_0005.cbf"))[0] if imgset.get_scan(0): scan = imgset.get_scan(0) - assert approx_equal(scan.get_image_oscillation(0), (30.0, 0.0), eps=1e-5) + assert approx_equal( + scan.get_image_oscillation(scan.get_image_range()[0]), (30.0, 0.0), eps=1e-5 + ) beam = imgset.get_beam(0) beam.get_s0() assert approx_equal(beam.get_s0(), (-0.0, -0.0, -1.0209290454313424))