From d17a30841a928b302abe42ffe01d67565d8332b0 Mon Sep 17 00:00:00 2001 From: Thomas Samuel Binns Date: Tue, 18 Jul 2023 14:41:37 +0200 Subject: [PATCH] [ENH, WIP] Add multivariate connectivity methods (#138) * add multivariate con base class * add multivariate imcoh classes * add multivar methods --------- Author: Thomas Samuel Binns Co-authored-by: Adam Li --- MANIFEST.in | 2 + doc/authors.inc | 3 + doc/references.bib | 376 +++--- doc/whats_new.rst | 2 + examples/granger_causality.py | 414 +++++++ examples/mic_mim.py | 432 +++++++ mne_connectivity/base.py | 11 +- mne_connectivity/spectral/epochs.py | 1068 +++++++++++++++-- .../spectral/tests/data/README.md | 30 + .../tests/data/example_multivariate_data.pkl | Bin 0 -> 96139 bytes .../example_multivariate_matlab_results.pkl | Bin 0 -> 3310 bytes .../spectral/tests/test_spectral.py | 515 +++++++- mne_connectivity/spectral/time.py | 398 +++++- 13 files changed, 2929 insertions(+), 322 deletions(-) create mode 100644 examples/granger_causality.py create mode 100644 examples/mic_mim.py create mode 100644 mne_connectivity/spectral/tests/data/README.md create mode 100644 mne_connectivity/spectral/tests/data/example_multivariate_data.pkl create mode 100644 mne_connectivity/spectral/tests/data/example_multivariate_matlab_results.pkl diff --git a/MANIFEST.in b/MANIFEST.in index 93b28376..5907df91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -40,4 +40,6 @@ exclude .github/PULL_REQUEST_TEMPLATE.md exclude mne_connectivity/tests/data recursive-exclude mne_connectivity/tests/data * +exclude mne_connectivity/spectral/tests/data +recursive-exclude mne_connectivity/spectral/tests/data * recursive-exclude benchmarks * diff --git a/doc/authors.inc b/doc/authors.inc index 7360a788..21a3fad4 100644 --- a/doc/authors.inc +++ b/doc/authors.inc @@ -11,3 +11,6 @@ .. _Daniel McCloy: https://dan.mccloy.info .. _Sam Steingold: https://github.com/sam-s .. _Qianliang Li: https://github.com/Avoide +.. _Thomas Binns: https://github.com/tsbinns +.. _Tien Nguyen: https://github.com/nguyen-td +.. _Richard Köhler: https://github.com/richardkoehler diff --git a/doc/references.bib b/doc/references.bib index b8a4059b..f3a20b3f 100644 --- a/doc/references.bib +++ b/doc/references.bib @@ -11,6 +11,53 @@ @article{AvantsEtAl2008 year = {2008} } +@article{BarnettSeth2015, + title={Granger causality for state-space models}, + author={Barnett, Lionel and Seth, Anil K.}, + journal={Physical Review E}, + volume={91}, + number={4}, + pages={040101}, + year={2015}, + publisher={APS}, + doi={10.1103/PhysRevE.91.040101} +} + +@article{BrunaEtAl2018, + doi = {10.1088/1741-2552/aacfe4}, + year = {2018}, + publisher = {{IOP} Publishing}, + volume = {15}, + number = {5}, + pages = {056011}, + author = {Ricardo Bru{\~{n}}a, Fernando Maest{\'{u}}, Ernesto Pereda}, + title = {Phase locking value revisited: teaching new tricks to an old dog}, + journal = {Journal of Neural Engineering}, +} + +@article{ColcloughEtAl2015, + title = {A symmetric multivariate leakage correction for {MEG} connectomes}, + volume = {117}, + issn = {1053-8119}, + doi = {10.1016/j.neuroimage.2015.03.071}, + language = {en}, + journal = {NeuroImage}, + author = {Colclough, G. L. and Brookes, M. J. and Smith, S. M. and Woolrich, M. W.}, + month = aug, + year = {2015}, + pages = {439--448} +} + +@book{CrochiereRabiner1983, + address = {Englewood Cliffs, NJ}, + edition = {1 edition}, + title = {Multirate {Digital} {Signal} {Processing}}, + isbn = {978-0-13-605162-6}, + publisher = {Pearson}, + author = {Crochiere, Ronald E. and Rabiner, Lawrence R.}, + year = {1983} +} + @article{Dawson_2016, author={Dawson, Scott T. M. and Hemati, Maziar S. and Williams, Matthew O. and Rowley, Clarence W.}, DOI={10.1007/s00348-016-2127-7}, @@ -25,6 +72,105 @@ @article{Dawson_2016 year={2016}, } +@article{EwaldEtAl2012, + title={Estimating true brain connectivity from {EEG/MEG} data invariant to linear and static transformations in sensor space}, + author={Ewald, Arne and Marzetti, Laura and Zappasodi, Filippo and Meinecke, Frank C. and Nolte, Guido}, + journal={NeuroImage}, + volume={60}, + number={1}, + pages={476--488}, + year={2012}, + publisher={Elsevier}, + doi={10.1016/j.neuroimage.2011.11.084} +} + +@article{HaufeEtAl2013, + title={A critical assessment of connectivity measures for EEG data: a simulation study}, + author={Haufe, Stefan and Nikulin, Vadim V and M{\"u}ller, Klaus-Robert and Nolte, Guido}, + journal={NeuroImage}, + volume={64}, + pages={120--133}, + year={2013}, + publisher={Elsevier}, + doi={10.1016/j.neuroimage.2012.09.036} +} + +@article{HippEtAl2012, + author = {Hipp, Joerg F and Hawellek, David J and Corbetta, Maurizio and Siegel, Markus and Engel, Andreas K}, + doi = {10.1038/nn.3101}, + journal = {Nature Neuroscience}, + number = {6}, + pages = {884-890}, + title = {Large-Scale Cortical Correlation Structure of Spontaneous Oscillatory Activity}, + volume = {15}, + year = {2012} +} + +@article{KhanEtAl2018, + author = {Khan, Sheraz and Hashmi, Javeria A. and Mamashli, Fahimeh and Michmizos, Konstantinos and Kitzbichler, Manfred G. and Bharadwaj, Hari and Bekhti, Yousra and Ganesan, Santosh and Garel, Keri-Lee A. and {Whitfield-Gabrieli}, Susan and Gollub, Randy L. and Kong, Jian and Vaina, Lucia M. and Rana, Kunjan D. and Stufflebeam, Steven M. and Hämäläinen, Matti S. and Kenet, Tal}, + doi = {10.1016/j.neuroimage.2018.02.018}, + journal = {NeuroImage}, + pages = {57-68}, + title = {Maturation Trajectories of Cortical Resting-State Networks Depend on the Mediating Frequency Band}, + volume = {174}, + year = {2018} +} + +@article{KlimeschEtAl2004, + title = {Phase-locked alpha and theta oscillations generate the P1–N1 complex and are related to memory performance}, + journal = {Cognitive Brain Research}, + volume = {19}, + number = {3}, + pages = {302-316}, + year = {2004}, + issn = {0926-6410}, + doi = {https://doi.org/10.1016/j.cogbrainres.2003.11.016}, + author = {Wolfgang Klimesch and Bärbel Schack and Manuel Schabus and Michael Doppelmayr and Walter Gruber and Paul Sauseng} +} + +@article{LachauxEtAl1999, + author = {Lachaux, Jean-Philippe and Rodriguez, Eugenio and Martinerie, Jacques and Varela, Francisco J.}, + doi = {10.1002/(SICI)1097-0193(1999)8:4<194::AID-HBM4>3.0.CO;2-C}, + journal = {Human Brain Mapping}, + number = {4}, + pages = {194-208}, + title = {Measuring Phase Synchrony in Brain Signals}, + volume = {8}, + year = {1999} +} + +@INPROCEEDINGS{li_linear_2017, + author = {Li, Adam and Gunnarsdottir, Kristin M. and Inati, Sara and Zaghloul, Kareem and Gale, John and Bulacio, Juan and Martinez-Gonzalez, Jorge and Sarma, Sridevi V.}, + booktitle = {2017 39th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)}, + title = {Linear time-varying model characterizes invasive EEG signals generated from complex epileptic networks}, + year = {2017}, + volume = {}, + number = {}, + pages = {2802-2805}, + doi = {10.1109/EMBC.2017.8037439} +} + +@article{NolteEtAl2004, + author = {Nolte, Guido and Bai, Ou and Wheaton, Lewis and Mari, Zoltan and Vorbach, Sherry and Hallett, Mark}, + doi = {10.1016/j.clinph.2004.04.029}, + journal = {Clinical Neurophysiology}, + number = {10}, + pages = {2292-2307}, + title = {Identifying True Brain Interaction from {{EEG}} Data Using the Imaginary Part of Coherency}, + volume = {115}, + year = {2004} +} + +@article{NolteEtAl2008, + author = {Nolte, Guido and Ziehe, Andreas and Nikulin, Vadim V. and Schlögl, Alois and Krämer, Nicole and Brismar, Tom and Müller, Klaus-Robert}, + doi = {10.1103/PhysRevLett.100.234101}, + journal = {Physical Review Letters}, + number = {23}, + title = {Robustly Estimating the Flow Direction of Information in Complex Physical Systems}, + volume = {100}, + year = {2008} +} + @book{OppenheimEtAl1999, address = {Upper Saddle River, NJ}, edition = {2 edition}, @@ -59,6 +205,31 @@ @article{SmithNichols2009 year = {2009} } +@article{StamEtAl2007, + author = {Stam, Cornelis J. and Nolte, Guido and Daffertshofer, Andreas}, + doi = {10.1002/hbm.20346}, + journal = {Human Brain Mapping}, + number = {11}, + pages = {1178-1193}, + shorttitle = {Phase Lag Index}, + title = {Phase Lag Index: Assessment of Functional Connectivity from Multi Channel {{EEG}} and {{MEG}} with Diminished Bias from Common Sources}, + volume = {28}, + year = {2007} +} + +@article{StamEtAl2012, + title={Go with the flow: Use of a directed phase lag index (dPLI) to characterize patterns of phase relations in a large-scale model of brain dynamics}, + volume={62}, + ISSN={1053-8119}, + DOI={10.1016/j.neuroimage.2012.05.050}, + number={3}, + journal={NeuroImage}, + author={Stam, C. J. and van Straaten, E. C. W.}, + year={2012}, + month={Sep}, + pages={1415–1428} +} + @article{VanVeenEtAl1997, author = {Van Veen, Barry D. and {van Drongelen}, Wim and Yuchtman, Moshe and Suzuki, Akifumi}, doi = {10.1109/10.623056}, @@ -79,6 +250,29 @@ @article{vanVlietEtAl2018 year = {2018} } +@article{VidaurreEtAl2019, + title={Canonical maximization of coherence: a novel tool for investigation of neuronal interactions between two datasets}, + author={Vidaurre, Carmen and Nolte, Guido and de Vries, Ingmar E.J. and G{\'o}mez, M. and Boonstra, Tjeerd W. and M{\"u}ller, K.-R. and Villringer, Arno and Nikulin, Vadim V.}, + journal={NeuroImage}, + volume={201}, + pages={116009}, + year={2019}, + publisher={Elsevier}, + doi={10.1016/j.neuroimage.2019.116009} +} + +@article{VinckEtAl2010, + author = {Vinck, Martin and {van Wingerden}, Marijn and Womelsdorf, Thilo and Fries, Pascal and Pennartz, Cyriel M.A.}, + doi = {10.1016/j.neuroimage.2010.01.073}, + journal = {NeuroImage}, + number = {1}, + pages = {112-122}, + shorttitle = {The Pairwise Phase Consistency}, + title = {The Pairwise Phase Consistency: A Bias-Free Measure of Rhythmic Neuronal Synchronization}, + volume = {51}, + year = {2010} +} + @article{VinckEtAl2011, author = {Vinck, Martin and Oostenveld, Robert and {van Wingerden}, Marijn and Battaglia, Franscesco and Pennartz, Cyriel M.A.}, doi = {10.1016/j.neuroimage.2011.01.055}, @@ -90,14 +284,39 @@ @article{VinckEtAl2011 year = {2011} } -@book{CrochiereRabiner1983, - address = {Englewood Cliffs, NJ}, - edition = {1 edition}, - title = {Multirate {Digital} {Signal} {Processing}}, - isbn = {978-0-13-605162-6}, - publisher = {Pearson}, - author = {Crochiere, Ronald E. and Rabiner, Lawrence R.}, - year = {1983} +@article{VinckEtAl2015, + title={How to detect the Granger-causal flow direction in the presence of additive noise?}, + author={Vinck, Martin and Huurdeman, Lisanne and Bosman, Conrado A and Fries, Pascal and Battaglia, Francesco P and Pennartz, Cyriel MA and Tiesinga, Paul H}, + journal={NeuroImage}, + volume={108}, + pages={301--318}, + year={2015}, + publisher={Elsevier}, + doi={10.1016/j.neuroimage.2014.12.017} +} + +@article{Whittle1963, + title={On the fitting of multivariate autoregressions, and the approximate canonical factorization of a spectral density matrix}, + author={Whittle, Peter}, + journal={Biometrika}, + volume={50}, + number={1-2}, + pages={129--134}, + year={1963}, + publisher={Oxford University Press}, + doi={10.1093/biomet/50.1-2.129} +} + +@article{WinklerEtAl2016, + title={Validity of time reversal for testing Granger causality}, + author={Winkler, Irene and Panknin, Danny and Bartz, Daniel and M{\"u}ller, Klaus-Robert and Haufe, Stefan}, + journal={IEEE Transactions on Signal Processing}, + volume={64}, + number={11}, + pages={2746--2760}, + year={2016}, + publisher={IEEE}, + doi={10.1109/TSP.2016.2531628} } @article{Yao2001, @@ -113,146 +332,7 @@ @article{Yao2001 pages = {693--711} } -@article{HippEtAl2012, - author = {Hipp, Joerg F and Hawellek, David J and Corbetta, Maurizio and Siegel, Markus and Engel, Andreas K}, - doi = {10.1038/nn.3101}, - journal = {Nature Neuroscience}, - number = {6}, - pages = {884-890}, - title = {Large-Scale Cortical Correlation Structure of Spontaneous Oscillatory Activity}, - volume = {15}, - year = {2012} -} - -@article{KhanEtAl2018, - author = {Khan, Sheraz and Hashmi, Javeria A. and Mamashli, Fahimeh and Michmizos, Konstantinos and Kitzbichler, Manfred G. and Bharadwaj, Hari and Bekhti, Yousra and Ganesan, Santosh and Garel, Keri-Lee A. and {Whitfield-Gabrieli}, Susan and Gollub, Randy L. and Kong, Jian and Vaina, Lucia M. and Rana, Kunjan D. and Stufflebeam, Steven M. and Hämäläinen, Matti S. and Kenet, Tal}, - doi = {10.1016/j.neuroimage.2018.02.018}, - journal = {NeuroImage}, - pages = {57-68}, - title = {Maturation Trajectories of Cortical Resting-State Networks Depend on the Mediating Frequency Band}, - volume = {174}, - year = {2018} -} - -@article{NolteEtAl2008, - author = {Nolte, Guido and Ziehe, Andreas and Nikulin, Vadim V. and Schlögl, Alois and Krämer, Nicole and Brismar, Tom and Müller, Klaus-Robert}, - doi = {10.1103/PhysRevLett.100.234101}, - journal = {Physical Review Letters}, - number = {23}, - title = {Robustly Estimating the Flow Direction of Information in Complex Physical Systems}, - volume = {100}, - year = {2008} -} - - -@article{LachauxEtAl1999, - author = {Lachaux, Jean-Philippe and Rodriguez, Eugenio and Martinerie, Jacques and Varela, Francisco J.}, - doi = {10.1002/(SICI)1097-0193(1999)8:4<194::AID-HBM4>3.0.CO;2-C}, - journal = {Human Brain Mapping}, - number = {4}, - pages = {194-208}, - title = {Measuring Phase Synchrony in Brain Signals}, - volume = {8}, - year = {1999} -} - -@article{StamEtAl2007, - author = {Stam, Cornelis J. and Nolte, Guido and Daffertshofer, Andreas}, - doi = {10.1002/hbm.20346}, - journal = {Human Brain Mapping}, - number = {11}, - pages = {1178-1193}, - shorttitle = {Phase Lag Index}, - title = {Phase Lag Index: Assessment of Functional Connectivity from Multi Channel {{EEG}} and {{MEG}} with Diminished Bias from Common Sources}, - volume = {28}, - year = {2007} -} - -@article{VinckEtAl2010, - author = {Vinck, Martin and {van Wingerden}, Marijn and Womelsdorf, Thilo and Fries, Pascal and Pennartz, Cyriel M.A.}, - doi = {10.1016/j.neuroimage.2010.01.073}, - journal = {NeuroImage}, - number = {1}, - pages = {112-122}, - shorttitle = {The Pairwise Phase Consistency}, - title = {The Pairwise Phase Consistency: A Bias-Free Measure of Rhythmic Neuronal Synchronization}, - volume = {51}, - year = {2010} -} - -@article{BrunaEtAl2018, - doi = {10.1088/1741-2552/aacfe4}, - year = {2018}, - publisher = {{IOP} Publishing}, - volume = {15}, - number = {5}, - pages = {056011}, - author = {Ricardo Bru{\~{n}}a, Fernando Maest{\'{u}}, Ernesto Pereda}, - title = {Phase locking value revisited: teaching new tricks to an old dog}, - journal = {Journal of Neural Engineering}, -} - -@article{NolteEtAl2004, - author = {Nolte, Guido and Bai, Ou and Wheaton, Lewis and Mari, Zoltan and Vorbach, Sherry and Hallett, Mark}, - doi = {10.1016/j.clinph.2004.04.029}, - journal = {Clinical Neurophysiology}, - number = {10}, - pages = {2292-2307}, - title = {Identifying True Brain Interaction from {{EEG}} Data Using the Imaginary Part of Coherency}, - volume = {115}, - year = {2004} -} - -@INPROCEEDINGS{li_linear_2017, - author = {Li, Adam and Gunnarsdottir, Kristin M. and Inati, Sara and Zaghloul, Kareem and Gale, John and Bulacio, Juan and Martinez-Gonzalez, Jorge and Sarma, Sridevi V.}, - booktitle = {2017 39th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)}, - title = {Linear time-varying model characterizes invasive EEG signals generated from complex epileptic networks}, - year = {2017}, - volume = {}, - number = {}, - pages = {2802-2805}, - doi = {10.1109/EMBC.2017.8037439} -} - -@article{ColcloughEtAl2015, - title = {A symmetric multivariate leakage correction for {MEG} connectomes}, - volume = {117}, - issn = {1053-8119}, - doi = {10.1016/j.neuroimage.2015.03.071}, - language = {en}, - journal = {NeuroImage}, - author = {Colclough, G. L. and Brookes, M. J. and Smith, S. M. and Woolrich, M. W.}, - month = aug, - year = {2015}, - pages = {439--448} -} - - @article{StamEtAl2012, - title={Go with the flow: Use of a directed phase lag index (dPLI) to characterize patterns of phase relations in a large-scale model of brain dynamics}, - volume={62}, - ISSN={1053-8119}, - DOI={10.1016/j.neuroimage.2012.05.050}, - number={3}, - journal={NeuroImage}, - author={Stam, C. J. and van Straaten, E. C. W.}, - year={2012}, - month={Sep}, - pages={1415–1428} -} - - @article{KlimeschEtAl2004, - title = {Phase-locked alpha and theta oscillations generate the P1–N1 complex and are related to memory performance}, - journal = {Cognitive Brain Research}, - volume = {19}, - number = {3}, - pages = {302-316}, - year = {2004}, - issn = {0926-6410}, - doi = {https://doi.org/10.1016/j.cogbrainres.2003.11.016}, - author = {Wolfgang Klimesch and Bärbel Schack and Manuel Schabus and Michael Doppelmayr and Walter Gruber and Paul Sauseng} -} - - @article{Zimmermann2022, +@article{Zimmermann2022, author = {Zimmermann, Marius and Lomoriello, Arianna Schiano and Konvalinka, Ivana}, doi = {10.1098/rsos.211352}, issn = {20545703}, diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 5016010d..a844c282 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -25,6 +25,8 @@ Enhancements - Add the option to set the number of connections plotted in :func:`mne_connectivity.viz.plot_sensors_connectivity` by `Qianliang Li`_ (:pr:`133`). - Allow setting colormap via new parameter ``cmap`` in :func:`mne_connectivity.viz.plot_sensors_connectivity` by `Daniel McCloy`_ (:pr:`141`). +- Add support for multivariate connectivity methods in :func:`mne_connectivity.spectral_connectivity_epochs` and :func:`mne_connectivity.spectral_connectivity_time` by `Thomas Binns`_ and `Tien Nguyen`_ and `Richard Köhler`_ (:pr:`138`). + Bug ~~~ diff --git a/examples/granger_causality.py b/examples/granger_causality.py new file mode 100644 index 00000000..f5d8316d --- /dev/null +++ b/examples/granger_causality.py @@ -0,0 +1,414 @@ +""" +========================================================================== +Compute directionality of connectivity with multivariate Granger causality +========================================================================== + +This example demonstrates how Granger causality based on state-space models +:footcite:`BarnettSeth2015` can be used to compute directed connectivity +between sensors in a multivariate manner. Furthermore, the use of time-reversal +for improving the robustness of directed connectivity estimates to noise in the +data is discussed :footcite:`WinklerEtAl2016`. +""" + +# Author: Thomas S. Binns +# License: BSD (3-clause) + +# %% + +import numpy as np +from matplotlib import pyplot as plt + +import mne +from mne.datasets.fieldtrip_cmc import data_path +from mne_connectivity import spectral_connectivity_epochs + +############################################################################### +# Background +# ---------- +# +# Multivariate forms of signal analysis allow you to simultaneously consider +# the activity of multiple signals. In the case of connectivity, the +# interaction between multiple sensors can be analysed at once, producing a +# single connectivity spectrum. This approach brings not only practical +# benefits (e.g. easier interpretability of results from the dimensionality +# reduction), but can also offer methodological improvements (e.g. enhanced +# signal-to-noise ratio and reduced bias). +# +# Additionally, it can be of interest to examine the directionality of +# connectivity between signals, providing additional clarity to how information +# flows in a system. One such directed measure of connectivity is Granger +# causality (GC). A signal, :math:`\boldsymbol{x}`, is said to Granger-cause +# another signal, :math:`\boldsymbol{y}`, if information from the past of +# :math:`\boldsymbol{x}` improves the prediction of the present of +# :math:`\boldsymbol{y}` over the case where only information from the past of +# :math:`\boldsymbol{y}` is used. Note: GC does not make any assertions about +# the true causality between signals. +# +# The degree to which :math:`\boldsymbol{x}` and :math:`\boldsymbol{y}` can be +# used to predict one another in a linear model can be quantified using vector +# autoregressive (VAR) models. Considering the simpler case of time domain +# connectivity, the VAR models are as follows: +# +# :math:`y_t = \sum_{k=1}^{K} a_k y_{t-k} + \xi_t^y` , +# :math:`Var(\xi_t^y) := \Sigma_y` , +# +# and :math:`\boldsymbol{z}_t = \sum_{k=1}^K \boldsymbol{A}_k +# \boldsymbol{z}_{t-k} + \boldsymbol{\epsilon}_t` , +# :math:`\boldsymbol{\Sigma} := \langle \boldsymbol{\epsilon}_t +# \boldsymbol{\epsilon}_t^T \rangle = \begin{bmatrix} \Sigma_{xx} & \Sigma_{xy} +# \\ \Sigma_{yx} & \Sigma_{yy} \end{bmatrix}` , +# +# representing the reduced and full VAR models, respectively, where: :math:`K` +# is the order of the VAR model, determining the number of lags, :math:`k`, +# used; :math:`\boldsymbol{Z} := \begin{bmatrix} \boldsymbol{x} \\ +# \boldsymbol{y} \end{bmatrix}`; :math:`\boldsymbol{A}` is a matrix of +# coefficients explaining the contribution of past entries of +# :math:`\boldsymbol{Z}` to its current value; and :math:`\xi` and +# :math:`\boldsymbol{\epsilon}` are the residuals of the VAR models. In this +# way, the information of the signals at time :math:`t` can be represented as a +# weighted form of the information from the previous timepoints, plus some +# residual information not encoded in the signals' past. In practice, VAR model +# parameters are computed from an autocovariance sequence generated from the +# time-series data using the Yule-Walker equations :footcite:`Whittle1963`. +# +# The residuals, or errors, represent how much information about the present +# state of the signals is not explained by their past. We can therefore +# estimate how much :math:`\boldsymbol{x}` Granger-causes +# :math:`\boldsymbol{y}` by comparing the variance of the residuals of the +# reduced VAR model (:math:`\Sigma_y`; i.e. how much the present of +# :math:`\boldsymbol{y}` is not explained by its own past) and of the full VAR +# model (:math:`\Sigma_{yy}`; i.e. how much the present of +# :math:`\boldsymbol{y}` is not explained by both its own past and that of +# :math:`\boldsymbol{x}`): +# +# :math:`F_{x \rightarrow y} = ln \Large{(\frac{\Sigma_y}{\Sigma_{yy}})}` , +# +# where :math:`F` is the Granger score. For example, if :math:`\boldsymbol{x}` +# contains no information about :math:`\boldsymbol{y}`, the residuals of the +# reduced and full VAR models will be identical, and +# :math:`F_{x \rightarrow y}` will naturally be 0, indicating that +# information from :math:`\boldsymbol{x}` does not flow to +# :math:`\boldsymbol{y}`. In contrast, if :math:`\boldsymbol{x}` does help to +# predict :math:`\boldsymbol{y}`, the residual of the full model will be +# smaller than that of the reduced model. :math:`\Large{\frac{\Sigma_y} +# {\Sigma_{yy}}}` will therefore be greater than 1, leading to a Granger score +# > 0. Granger scores are bound between :math:`[0, \infty)`. +# +# These same principles apply to spectral GC, which provides information about +# the directionality of connectivity for individual frequencies. For spectral +# GC, the autocovariance sequence is generated from an inverse Fourier +# transform applied to the cross-spectral density of the signals. Additionally, +# a spectral transfer function is used to translate information from the VAR +# models back into the frequency domain before computing the final Granger +# scores. +# +# Barnett and Seth (2015) :footcite:`BarnettSeth2015` have defined a +# multivariate form of spectral GC based on state-space models, enabling the +# estimation of information flow between whole sets of signals simultaneously: +# +# :math:`F_{A \rightarrow B}(f) = \Re ln \Large{(\frac{ +# det(\boldsymbol{S}_{BB}(f))}{det(\boldsymbol{S}_{BB}(f) - +# \boldsymbol{H}_{BA}(f) \boldsymbol{\Sigma}_{AA \lvert B} +# \boldsymbol{H}_{BA}^*(f))})}` , +# +# where: :math:`A` and :math:`B` are the seeds and targets, respectively; +# :math:`f` is a given frequency; :math:`\boldsymbol{H}` is the spectral +# transfer function; :math:`\boldsymbol{\Sigma}` is the innovations form +# residuals' covariance matrix of the state-space model; :math:`\boldsymbol{S}` +# is :math:`\boldsymbol{\Sigma}` transformed by :math:`\boldsymbol{H}`; and +# :math:`\boldsymbol{\Sigma}_{IJ \lvert K} := \boldsymbol{\Sigma}_{IJ} - +# \boldsymbol{\Sigma}_{IK} \boldsymbol{\Sigma}_{KK}^{-1} +# \boldsymbol{\Sigma}_{KJ}`, representing a partial covariance matrix. The same +# principles apply as before: a numerator greater than the denominator means +# that information from the seed signals aids the prediction of activity in the +# target signals, leading to a Granger score > 0. +# +# There are several benefits to a state-space approach for computing GC: +# compared to traditional autoregressive-based approaches, the use of +# state-space models offers reduced statistical bias and increased statistical +# power; furthermore, the dimensionality reduction offered by the multivariate +# nature of the approach can aid in the interpretability and subsequent +# analysis of the results. +# +# To demonstrate the use of GC for estimating directed connectivity, we start +# by loading some example MEG data and dividing it into two-second-long epochs. + +# %% + +raw = mne.io.read_raw_ctf(data_path() / 'SubjectCMC.ds') +raw.pick('mag') +raw.crop(50., 110.).load_data() +raw.notch_filter(50) +raw.resample(100) + +epochs = mne.make_fixed_length_epochs(raw, duration=2.0).load_data() + +############################################################################### +# We will focus on connectivity between sensors over the parietal and occipital +# cortices, with 20 parietal sensors designated as group A, and 20 occipital +# sensors designated as group B. + +# %% + +# parietal sensors +signals_a = [idx for idx, ch_info in enumerate(epochs.info['chs']) if + ch_info['ch_name'][2] == 'P'] +# occipital sensors +signals_b = [idx for idx, ch_info in enumerate(epochs.info['chs']) if + ch_info['ch_name'][2] == 'O'] + +# XXX: Currently ragged indices are not supported, so we only consider a single +# list of indices with an equal number of seeds and targets +min_n_chs = min(len(signals_a), len(signals_b)) +signals_a = signals_a[:min_n_chs] +signals_b = signals_b[:min_n_chs] + +indices_ab = (np.array(signals_a), np.array(signals_b)) # A => B +indices_ba = (np.array(signals_b), np.array(signals_a)) # B => A + +signals_a_names = [epochs.info['ch_names'][idx] for idx in signals_a] +signals_b_names = [epochs.info['ch_names'][idx] for idx in signals_b] + +# compute Granger causality +gc_ab = spectral_connectivity_epochs( + epochs, method=['gc'], indices=indices_ab, fmin=5, fmax=30, + rank=(np.array([5]), np.array([5])), gc_n_lags=20) # A => B +gc_ba = spectral_connectivity_epochs( + epochs, method=['gc'], indices=indices_ba, fmin=5, fmax=30, + rank=(np.array([5]), np.array([5])), gc_n_lags=20) # B => A +freqs = gc_ab.freqs + + +############################################################################### +# Plotting the results, we see that there is a flow of information from our +# parietal sensors (group A) to our occipital sensors (group B) with noticeable +# peaks at around 8, 18, and 26 Hz. + +# %% + +fig, axis = plt.subplots(1, 1) +axis.plot(freqs, gc_ab.get_data()[0], linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Connectivity (A.U.)') +fig.suptitle('GC: [A => B]') + + +############################################################################### +# Drivers and receivers: analysing the net direction of information flow +# ---------------------------------------------------------------------- +# +# Although analysing connectivity in a given direction can be of interest, +# there may exist a bidirectional relationship between signals. In such cases, +# identifying the signals that dominate information flow (the drivers) may be +# desired. For this, we can simply subtract the Granger scores in the opposite +# direction, giving us the net GC score: +# +# :math:`F_{A \rightarrow B}^{net} := F_{A \rightarrow B} - +# F_{B \rightarrow A}`. +# +# Doing so, we see that the flow of information across the spectrum remains +# dominant from parietal to occipital sensors (indicated by the positive-valued +# Granger scores). However, the pattern of connectivity is altered, such as +# around 10 and 12 Hz where peaks of net information flow are now present. + +# %% + +net_gc = gc_ab.get_data() - gc_ba.get_data() # [A => B] - [B => A] + +fig, axis = plt.subplots(1, 1) +axis.plot((freqs[0], freqs[-1]), (0, 0), linewidth=2, linestyle='--', + color='k') +axis.plot(freqs, net_gc[0], linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Connectivity (A.U.)') +fig.suptitle('Net GC: [A => B] - [B => A]') + + +############################################################################### +# Improving the robustness of connectivity estimates with time-reversal +# --------------------------------------------------------------------- +# +# One limitation of GC methods is the risk of connectivity estimates being +# contaminated with noise. For instance, consider the case where, due to +# volume conduction, multiple sensors detect activity from the same source. +# Naturally, information recorded at these sensors mutually help to predict +# the activity of one another, leading to spurious estimates of directed +# connectivity which one may incorrectly attribute to information flow between +# different brain regions. On the other hand, even if there is no source +# mixing, the presence of correlated noise between sensors can similarly bias +# directed connectivity estimates. +# +# To address this issue, Haufe *et al.* (2013) :footcite:`HaufeEtAl2013` +# propose contrasting causality scores obtained on the original time-series to +# those obtained on the reversed time-series. The idea behind this approach is +# as follows: if temporal order is crucial in distinguishing a driver from a +# recipient, then reversing the temporal order should reduce, if not flip, an +# estimate of directed connectivity. In practice, time-reversal is implemented +# as a transposition of the autocovariance sequence used to compute GC. Several +# studies have shown that that such an approach can reduce the degree of +# false-positive connectivity estimates (even performing favourably against +# other methods such as the phase slope index) :footcite:`VinckEtAl2015` and +# retain the ability to correctly identify the net direction of information +# flow akin to net GC :footcite:`WinklerEtAl2016,HaufeEtAl2013`. This approach +# is termed time-reversed GC (TRGC): +# +# :math:`\tilde{D}_{A \rightarrow B}^{net} := F_{A \rightarrow B}^{net} - +# F_{\tilde{A} \rightarrow \tilde{B}}^{net}` , +# +# where :math:`\sim` represents time-reversal, and: +# +# :math:`F_{\tilde{A} \rightarrow \tilde{B}}^{net} := F_{\tilde{A} \rightarrow +# \tilde{B}} - F_{\tilde{B} \rightarrow \tilde{A}}`. +# +# GC on time-reversed signals can be computed simply with ``method=['gc_tr']``, +# which will perform the time-reversal of the signals for the end-user. Note +# that **time-reversed results should only be interpreted in the context of net +# results**, i.e. with :math:`\tilde{D}_{A \rightarrow B}^{net}`. In the +# example below, notice how the outputs are not used directly, but rather used +# to produce net scores of the time-reversed signals. The net scores of the +# time-reversed signals can then be subtracted from the net scores of the +# original signals to produce the final TRGC scores. + +# %% + +# compute GC on time-reversed signals +gc_tr_ab = spectral_connectivity_epochs( + epochs, method=['gc_tr'], indices=indices_ab, fmin=5, fmax=30, + rank=(np.array([5]), np.array([5])), gc_n_lags=20) # TR[A => B] +gc_tr_ba = spectral_connectivity_epochs( + epochs, method=['gc_tr'], indices=indices_ba, fmin=5, fmax=30, + rank=(np.array([5]), np.array([5])), gc_n_lags=20) # TR[B => A] + +# compute net GC on time-reversed signals (TR[A => B] - TR[B => A]) +net_gc_tr = gc_tr_ab.get_data() - gc_tr_ba.get_data() + +# compute TRGC +trgc = net_gc - net_gc_tr + +############################################################################### +# Plotting the TRGC results, reveals a very different picture compared to net +# GC. For one, there is now a dominance of information flow ~6 Hz from +# occipital to parietal sensors (indicated by the negative-valued Granger +# scores). Additionally, the peaks ~10 Hz are less dominant in the spectrum, +# with parietal to occipital information flow between 13-20 Hz being much more +# prominent. The stark difference between net GC and TRGC results indicates +# that the net GC spectrum was contaminated by spurious connectivity resulting +# from source mixing or correlated noise in the recordings. Altogether, the use +# of TRGC instead of net GC is generally advised. + +# %% + +fig, axis = plt.subplots(1, 1) +axis.plot((freqs[0], freqs[-1]), (0, 0), linewidth=2, linestyle='--', + color='k') +axis.plot(freqs, trgc[0], linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Connectivity (A.U.)') +fig.suptitle('TRGC: net[A => B] - net time-reversed[A => B]') + + +############################################################################### +# Controlling spectral smoothing with the number of lags +# ------------------------------------------------------ +# +# One important parameter when computing GC is the number of lags used when +# computing the VAR model. A lower number of lags reduces the computational +# cost, but in the context of spectral GC, leads to a smoothing of Granger +# scores across frequencies. The number of lags can be specified using the +# ``gc_n_lags`` parameter. The default value is 40, however there is no correct +# number of lags to use when computing GC. Instead, you have to use your own +# best judgement of whether or not your Granger scores look overly smooth. +# +# Below is a comparison of Granger scores computed with a different number of +# lags. In the above examples we used 20 lags, which we will compare to Granger +# scores computed with 60 lags. As you can see, the spectra of Granger scores +# computed with 60 lags is noticeably less smooth, but it does share the same +# overall pattern. + +# %% + +gc_ab_60 = spectral_connectivity_epochs( + epochs, method=['gc'], indices=indices_ab, fmin=5, fmax=30, + rank=(np.array([5]), np.array([5])), gc_n_lags=60) # A => B + +fig, axis = plt.subplots(1, 1) +axis.plot(freqs, gc_ab.get_data()[0], linewidth=2, label='20 lags') +axis.plot(freqs, gc_ab_60.get_data()[0], linewidth=2, label='60 lags') +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Connectivity (A.U.)') +axis.legend() +fig.suptitle('GC: [A => B]') + + +############################################################################### +# Handling high-dimensional data +# ------------------------------ +# +# An important issue to consider when computing multivariate GC is that the +# data GC is computed on should not be rank deficient (i.e. must have full +# rank). More specifically, the autocovariance matrix must not be singular or +# close to singular. +# +# In the case that your data is not full rank and ``rank`` is left as ``None``, +# an automatic rank computation is performed and an appropriate degree of +# dimensionality reduction will be enforced. The rank of the data is determined +# by computing the singular values of the data and finding those within a +# factor of :math:`1e^{-10}` relative to the largest singular value. +# +# In some circumstances, this threshold may be too lenient, in which case you +# should inspect the singular values of your data to identify an appropriate +# degree of dimensionality reduction to perform, which you can then specify +# manually using the ``rank`` argument. The code below shows one possible +# approach for finding an appropriate rank of close-to-singular data with a +# more conservative threshold of :math:`1e^{-5}`. + +# %% + +# gets the singular values of the data +s = np.linalg.svd(raw.get_data(), compute_uv=False) +# finds how many singular values are "close" to the largest singular value +rank = np.count_nonzero(s >= s[0] * 1e-5) # 1e-5 is the "closeness" criteria + +############################################################################### +# Nonethless, even in situations where you specify an appropriate rank, it is +# not guaranteed that the subsequently-computed autocovariance sequence will +# retain this non-singularity (this can depend on, e.g. the number of lags). +# Hence, you may also encounter situations where you have to specify a rank +# less than that of your data to ensure that the autocovariance sequence is +# non-singular. +# +# In the above examples, notice how a rank of 5 was given, despite there being +# 20 channels in the seeds and targets. Attempting to compute GC on the +# original data would not succeed, given that the resulting autocovariance +# sequence is singular, as the example below shows. + +# %% + +try: + spectral_connectivity_epochs( + epochs, method=['gc'], indices=indices_ab, fmin=5, fmax=30, rank=None, + gc_n_lags=20) # A => B + print('Success!') +except RuntimeError as error: + print('\nCaught the following error:\n' + repr(error)) + +############################################################################### +# Rigorous checks are implemented to identify any such instances which would +# otherwise cause the GC computation to produce erroneous results. You can +# therefore be confident as an end-user that these cases will be caught. +# +# Finally, when comparing GC scores across recordings, **it is highly +# recommended to estimate connectivity from the same number of channels (or +# equally from the same degree of rank subspace projection)** to avoid biases +# in connectivity estimates. Bias can be avoided by specifying a consistent +# rank subspace to project to using the ``rank`` argument, standardising your +# connectivity estimates regardless of changes in e.g. the number of channels +# across recordings. Note that this does not refer to the number of seeds and +# targets *within* a connection being identical, rather to the number of seeds +# and targets *across* connections. + + +############################################################################### +# References +# ---------- +# .. footbibliography:: diff --git a/examples/mic_mim.py b/examples/mic_mim.py new file mode 100644 index 00000000..179ea620 --- /dev/null +++ b/examples/mic_mim.py @@ -0,0 +1,432 @@ +""" +================================================================ +Compute multivariate measures of the imaginary part of coherency +================================================================ + +This example demonstrates how multivariate methods based on the imaginary part +of coherency :footcite:`EwaldEtAl2012` can be used to compute connectivity +between whole sets of sensors, and how spatial patterns of this connectivity +can be interpreted. + +The methods in question are: the maximised imaginary part of coherency (MIC); +and the multivariate interaction measure (MIM; as well as its extension, the +global interaction measure, GIM). +""" + +# Author: Thomas S. Binns +# License: BSD (3-clause) + +# %% + +import numpy as np +from matplotlib import pyplot as plt +from matplotlib import patheffects as pe + +import mne +from mne import EvokedArray, make_fixed_length_epochs +from mne.datasets.fieldtrip_cmc import data_path +from mne_connectivity import seed_target_indices, spectral_connectivity_epochs + +############################################################################### +# Background +# ---------- +# +# Multivariate forms of signal analysis allow you to simultaneously consider +# the activity of multiple signals. In the case of connectivity, the +# interaction between multiple sensors can be analysed at once, producing a +# single connectivity spectrum. This approach brings not only practical +# benefits (e.g. easier interpretability of results from the dimensionality +# reduction), but can also offer methodological improvements (e.g. enhanced +# signal-to-noise ratio and reduced bias). +# +# A popular bivariate measure of connectivity is the imaginary part of +# coherency, which looks at the correlation between two signals in the +# frequency domain and is immune to spurious connectivity arising from volume +# conduction artefacts :footcite:`NolteEtAl2004`. However, depending on the +# degree of source mixing, this measure is susceptible to biased estimates of +# connectivity based on the spatial proximity of sensors +# :footcite:`EwaldEtAl2012`. +# +# To overcome this limitation, spatial filters can be used to estimate +# connectivity free from this source mixing-dependent bias, which additionally +# increases the signal-to-noise ratio and allows signals to be analysed in a +# multivariate manner :footcite:`EwaldEtAl2012`. This approach leads to the +# following methods: the maximised imaginary part of coherency (MIC); and the +# multivariate interaction measure (MIM). +# +# We start by loading some example MEG data and dividing it into +# two-second-long epochs. + +# %% + +raw = mne.io.read_raw_ctf(data_path() / 'SubjectCMC.ds') +raw.pick('mag') +raw.crop(50., 110.).load_data() +raw.notch_filter(50) +raw.resample(100) + +epochs = make_fixed_length_epochs(raw, duration=2.0).load_data() + +############################################################################### +# We will focus on connectivity between sensors over the left and right +# hemispheres, with 75 sensors in the left hemisphere designated as seeds, and +# 75 sensors in the right hemisphere designated as targets. + +# %% + +# left hemisphere sensors +seeds = [idx for idx, ch_info in enumerate(epochs.info['chs']) if + ch_info['loc'][0] < 0] +# right hemisphere sensors +targets = [idx for idx, ch_info in enumerate(epochs.info['chs']) if + ch_info['loc'][0] > 0] + +# XXX: Currently ragged indices are not supported, so we only consider a single +# list of indices with an equal number of seeds and targets +min_n_chs = min(len(seeds), len(targets)) +seeds = seeds[:min_n_chs] +targets = targets[:min_n_chs] + +multivar_indices = (np.array(seeds), np.array(targets)) + +seed_names = [epochs.info['ch_names'][idx] for idx in seeds] +target_names = [epochs.info['ch_names'][idx] for idx in targets] + +# multivariate imaginary part of coherency +(mic, mim) = spectral_connectivity_epochs( + epochs, method=['mic', 'mim'], indices=multivar_indices, fmin=5, fmax=30, + rank=None) + +# bivariate imaginary part of coherency (for comparison) +bivar_indices = seed_target_indices(seeds, targets) +imcoh = spectral_connectivity_epochs( + epochs, method='imcoh', indices=bivar_indices, fmin=5, fmax=30) + +############################################################################### +# By averaging across each connection between the seeds and targets, we can see +# that the bivariate measure of the imaginary part of coherency estimates a +# strong peak in connectivity between seeds and targets around 13-18 Hz, with a +# weaker peak around 27 Hz. + +# %% +fig, axis = plt.subplots(1, 1) +axis.plot(imcoh.freqs, np.mean(np.abs(imcoh.get_data()), axis=0), + linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Absolute connectivity (A.U.)') +fig.suptitle('Imaginary part of coherency') + + +############################################################################### +# Maximised imaginary part of coherency (MIC) +# ------------------------------------------- +# +# For MIC, a set of spatial filters are found that will maximise the estimated +# connectivity between the seed and target signals. These maximising filters +# correspond to the eigenvectors with the largest eigenvalue, derived from an +# eigendecomposition of information from the cross-spectral density (Eq. 7 of +# :footcite:`EwaldEtAl2012`): +# +# :math:`MIC=\frac{\boldsymbol{\alpha}^T \boldsymbol{E \beta}}{\parallel +# \boldsymbol{\alpha}\parallel \parallel\boldsymbol{\beta}\parallel}`, +# +# where :math:`\boldsymbol{\alpha}` and :math:`\boldsymbol{\beta}` are the +# spatial filters for the seeds and targets, respectively, and +# :math:`\boldsymbol{E}` is the imaginary part of the transformed +# cross-spectral density between the seeds and targets. All elements are +# frequency-dependent, however this is omitted for readability. MIC is bound +# between :math:`[-1, 1]` where the absolute value reflects connectivity +# strength and the sign reflects the phase angle difference between signals. +# +# MIC can also be computed between identical sets of seeds and targets, +# allowing connectivity within a single set of signals to be estimated. This is +# possible as a result of the exclusion of zero phase lag components from the +# connectivity estimates, which would otherwise return a perfect correlation. +# +# In this instance, we see MIC reveal that in addition to the 13-18 Hz peak, a +# previously unobserved peak in connectivity around 9 Hz is present. +# Furthermore, the previous peak around 27 Hz is much less pronounced. This may +# indicate that the connectivity was the result of some distal interaction +# exacerbated by strong source mixing, which biased the bivariate connectivity +# estimate. + +# %% + +fig, axis = plt.subplots(1, 1) +axis.plot(mic.freqs, np.abs(mic.get_data()[0]), linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Absolute connectivity (A.U.)') +fig.suptitle('Maximised imaginary part of coherency') + + +############################################################################### +# Furthermore, spatial patterns of connectivity can be constructed from the +# spatial filters to give a picture of the location of the sources involved in +# the connectivity. This information is stored under ``attrs['patterns']`` of +# the connectivity class, with one value per frequency for each channel in the +# seeds and targets. As with MIC, the absolute value of the patterns reflect +# the strength, however the sign differences can be used to visualise the +# orientation of the underlying dipole sources. The spatial patterns are +# **not** bound between :math:`[-1, 1]`. +# +# Here, we average across the patterns in the 13-18 Hz range. Plotting the +# patterns shows that the greatest connectivity between the left and right +# hemispheres occurs at the posteromedial regions, based on the regions with +# the largest absolute values. Using the signs of the values, we can infer the +# existence of a dipole source in the central regions of the left hemisphere +# which may account for the connectivity contributions seen for the left +# posteromedial and frontolateral areas (represented on the plot as a green +# line). + +# %% + +# compute average of patterns in desired frequency range +fband = [13, 18] +fband_idx = [mic.freqs.index(freq) for freq in fband] + +# patterns have shape [seeds/targets x cons x channels x freqs (x times)] +patterns = np.array(mic.attrs["patterns"]) +seed_pattern = patterns[0] +target_pattern = patterns[1] +# average across frequencies +seed_pattern = np.mean(seed_pattern[0, :, fband_idx[0]:fband_idx[1] + 1], + axis=1) +target_pattern = np.mean(target_pattern[0, :, fband_idx[0]:fband_idx[1] + 1], + axis=1) + +# store the patterns for plotting +seed_info = epochs.copy().pick(seed_names).info +target_info = epochs.copy().pick(target_names).info +seed_pattern = EvokedArray(seed_pattern[:, np.newaxis], seed_info) +target_pattern = EvokedArray(target_pattern[:, np.newaxis], target_info) + +# plot the patterns +fig, axes = plt.subplots(1, 4) +seed_pattern.plot_topomap( + times=0, sensors='m.', units=dict(mag='A.U.'), cbar_fmt='%.1E', + axes=axes[0:2], time_format='', show=False) +target_pattern.plot_topomap( + times=0, sensors='m.', units=dict(mag='A.U.'), cbar_fmt='%.1E', + axes=axes[2:], time_format='', show=False) +axes[0].set_position((0.1, 0.1, 0.35, 0.7)) +axes[1].set_position((0.4, 0.3, 0.02, 0.3)) +axes[2].set_position((0.5, 0.1, 0.35, 0.7)) +axes[3].set_position((0.9, 0.3, 0.02, 0.3)) +axes[0].set_title('Seed spatial pattern\n13-18 Hz') +axes[2].set_title('Target spatial pattern\n13-18 Hz') + +# plot the left hemisphere dipole example +axes[0].plot( + [-0.1, -0.05], [-0.075, -0.03], color='lime', linewidth=2, + path_effects=[pe.Stroke(linewidth=4, foreground='k'), pe.Normal()]) + +plt.show() + + +############################################################################### +# Multivariate interaction measure (MIM) +# -------------------------------------- +# +# Although it can be useful to analyse the single, largest connectivity +# component with MIC, multiple such components exist and can be examined with +# MIM. MIM can be thought of as an average of all connectivity components +# between the seeds and targets, and can be useful for an exploration of all +# available components. It is unnecessary to use the spatial filters of each +# component explicitly, and instead the desired result can be achieved from +# :math:`E` alone (Eq. 14 of :footcite:`EwaldEtAl2012`): +# +# :math:`MIM=tr(\boldsymbol{EE}^T)`, +# +# where again the frequency dependence is omitted. Unlike MIC, MIM is +# positive-valued and can be > 1. Without normalisation, MIM can be +# thought of as reflecting the total interaction between the seeds and targets. +# MIM can be normalised to lie in the range :math:`[0, 1]` by dividing the +# scores by the number of unique channels in the seeds and targets. Normalised +# MIM represents the interaction *per channel*, which can be biased by factors +# such as the presence of channels with little to no interaction. In line with +# the preferences of the method's authors :footcite:`EwaldEtAl2012`, since +# normalisation alters the interpretability of the results, **normalisation is +# not performed by default**. +# +# Here we see MIM reveal the strongest connectivity component to be around 10 +# Hz, with the higher frequency 13-18 Hz connectivity no longer being so +# prominent. This suggests that, across all components in the data, there may +# be more lower frequency connectivity sources than higher frequency sources. +# Thus, when combining these different components in MIM, the peak around 10 Hz +# remains, but the 13-18 Hz connectivity is diminished relative to the single, +# largest connectivity component of MIC. +# +# Looking at the values for normalised MIM, we see it has a maximum of ~0.1. +# The relatively small connectivity values thus indicate that many of the +# channels show little to no interaction. + +# %% + +fig, axis = plt.subplots(1, 1) +axis.plot(mim.freqs, mim.get_data()[0], linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Absolute connectivity (A.U.)') +fig.suptitle('Multivariate interaction measure') + +n_channels = len(np.unique([*multivar_indices[0], *multivar_indices[1]])) +normalised_mim = mim.get_data()[0] / n_channels +print(f'Normalised MIM has a maximum value of {normalised_mim.max():.2f}') + + +############################################################################### +# Additionally, the instance where the seeds and targets are identical can be +# considered as a special case of MIM: the global interaction measure (GIM; Eq. +# 15 of :footcite:`EwaldEtAl2012`). Again, this allows connectivity within a +# single set of signals to be estimated. Computing GIM follows from Eq. 14, +# however since each interaction is considered twice, correcting the +# connectivity by a factor of :math:`\frac{1}{2}` is necessary (**the +# correction is performed automatically in this implementation**). Like MIM, +# GIM can also be > 1, but it can again be normalised to lie in the range +# :math:`[0, 1]` by dividing by the number of unique channels in the seeds and +# targets. However, since normalisation alters the interpretability of the +# results (i.e. interaction per channel for normalised GIM vs. total +# interaction for standard GIM), **GIM is not normalised by default**. +# +# With GIM, we find a broad connectivity peak around 10 Hz, with an additional +# peak around 20 Hz. The differences observed with GIM highlight the presence +# of interactions within each hemisphere that are absent for MIC or MIM. +# Furthermore, the values for normalised GIM are higher than for MIM, with a +# maximum of ~0.2, again indicating the presence of interactions across +# channels within each hemisphere. + +# %% + +indices = (np.array([*seeds, *targets]), np.array([*seeds, *targets])) +gim = spectral_connectivity_epochs( + epochs, method='mim', indices=indices, fmin=5, fmax=30, rank=None, + verbose=False) + +fig, axis = plt.subplots(1, 1) +axis.plot(gim.freqs, gim.get_data()[0], linewidth=2) +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Connectivity (A.U.)') +fig.suptitle('Global interaction measure') + +n_channels = len(np.unique([*indices[0], *indices[1]])) +normalised_gim = gim.get_data()[0] / n_channels +print(f'Normalised GIM has a maximum value of {normalised_gim.max():.2f}') + + +############################################################################### +# Handling high-dimensional data +# ------------------------------ +# +# An important issue to consider when using these multivariate methods is +# overfitting, which risks biasing connectivity estimates to maximise noise in +# the data. This risk can be reduced by performing a preliminary dimensionality +# reduction prior to estimating the connectivity with a singular value +# decomposition (Eqs. 32 & 33 of :footcite:`EwaldEtAl2012`). The degree of this +# dimensionality reduction can be specified using the ``rank`` argument, which +# by default will not perform any dimensionality reduction (assuming your data +# is full rank; see below if not). Choosing an expected rank of the data +# requires *a priori* knowledge about the number of components you expect to +# observe in the data. +# +# When comparing MIC/MIM scores across recordings, **it is highly recommended +# to estimate connectivity from the same number of channels (or equally from +# the same degree of rank subspace projection)** to avoid biases in +# connectivity estimates. Bias can be avoided by specifying a consistent rank +# subspace to project to using the ``rank`` argument, standardising your +# connectivity estimates regardless of changes in e.g. the number of channels +# across recordings. Note that this does not refer to the number of seeds and +# targets *within* a connection being identical, rather to the number of seeds +# and targets *across* connections. +# +# Here, we will project our seed and target data to only the first 25 +# components of our rank subspace. Results for MIM show that the general +# spectral pattern of connectivity is retained in the rank subspace-projected +# data, suggesting that a fair degree of redundant connectivity information is +# contained in the remaining 50 components of the seed and target data. We also +# assert that the spatial patterns of MIC are returned in the original sensor +# space despite this rank subspace projection, being reconstructed using the +# products of the singular value decomposition (Eqs. 46 & 47 of +# :footcite:`EwaldEtAl2012`). + +# %% + +(mic_red, mim_red) = spectral_connectivity_epochs( + epochs, method=['mic', 'mim'], indices=multivar_indices, fmin=5, fmax=30, + rank=([25], [25])) + +# subtract mean of scores for comparison +mim_red_meansub = mim_red.get_data()[0] - mim_red.get_data()[0].mean() +mim_meansub = mim.get_data()[0] - mim.get_data()[0].mean() + +# compare standard and rank subspace-projected MIM +fig, axis = plt.subplots(1, 1) +axis.plot(mim_red.freqs, mim_red_meansub, linewidth=2, + label='rank subspace (25) MIM') +axis.plot(mim.freqs, mim_meansub, linewidth=2, label='standard MIM') +axis.set_xlabel('Frequency (Hz)') +axis.set_ylabel('Mean-corrected connectivity (A.U.)') +axis.legend() +fig.suptitle('Multivariate interaction measure (non-normalised)') + +# no. channels equal with and without projecting to rank subspace for patterns +assert (patterns[0, 0].shape[0] == + np.array(mic_red.attrs["patterns"])[0, 0].shape[0]) +assert (patterns[1, 0].shape[0] == + np.array(mic_red.attrs["patterns"])[1, 0].shape[0]) + + +############################################################################### +# In the case that your data is not full rank and ``rank`` is left as ``None``, +# an automatic rank computation is performed and an appropriate degree of +# dimensionality reduction will be enforced. The rank of the data is determined +# by computing the singular values of the data and finding those within a +# factor of :math:`1e^{-10}` relative to the largest singular value. +# +# In some circumstances, this threshold may be too lenient, in which case you +# should inspect the singular values of your data to identify an appropriate +# degree of dimensionality reduction to perform, which you can then specify +# manually using the ``rank`` argument. The code below shows one possible +# approach for finding an appropriate rank of close-to-singular data with a +# more conservative threshold of :math:`1e^{-5}`. + +# %% + +# gets the singular values of the data +s = np.linalg.svd(raw.get_data(), compute_uv=False) +# finds how many singular values are "close" to the largest singular value +rank = np.count_nonzero(s >= s[0] * 1e-5) # 1e-5 is the "closeness" criteria + + +############################################################################### +# Limitations +# ----------- +# +# These multivariate methods offer many benefits in the form of dimensionality +# reduction, signal-to-noise ratio improvements, and invariance to +# estimate-biasing source mixing; however, no method is perfect. The immunity +# of the imaginary part of coherency to volume conduction comes from the fact +# that these artefacts have zero phase lag, and hence a zero-valued imaginary +# component. By projecting the complex-valued coherency to the imaginary axis, +# signals of a given magnitude with phase lag differences close to 90° or 270° +# see their contributions to the connectivity estimate increased relative to +# comparable signals with phase lag differences close to 0° or 180°. Therefore, +# the imaginary part of coherency is biased towards connectivity involving 90° +# and 270° phase lag difference components. +# +# Whilst this is not a limitation specific to the multivariate extension of +# this measure, these multivariate methods can introduce further bias: when +# maximising the imaginary part of coherency, components with phase lag +# differences close to 90° and 270° will likely give higher connectivity +# estimates, and so may be prioritised by the spatial filters. +# +# Such a limitation should be kept in mind when estimating connectivity using +# these methods. Possible sanity checks can involve comparing the spectral +# profiles of MIC/MIM to coherence and the imaginary part of coherency +# computed on the same data, as well as comparing to other multivariate +# measures, such as canonical coherence :footcite:`VidaurreEtAl2019`. + +############################################################################### +# References +# ---------- +# .. footbibliography:: + +# %% diff --git a/mne_connectivity/base.py b/mne_connectivity/base.py index 76ee01b0..88951529 100644 --- a/mne_connectivity/base.py +++ b/mne_connectivity/base.py @@ -667,7 +667,8 @@ def get_data(self, output='compact'): ``(n_nodes_in * n_nodes_out,)`` list. If 'dense', then will return each connectivity matrix as a 2D array. If 'compact' (default) then will return 'raveled' if ``indices`` were defined as - a list of tuples, or ``dense`` if indices is 'all'. + a list of tuples, or ``dense`` if indices is 'all'. Multivariate + connectivity data cannot be returned in a dense form. Returns ------- @@ -685,6 +686,14 @@ def get_data(self, output='compact'): if output == 'raveled': data = self._data else: + if self.method in ['mic', 'mim', 'gc', 'gc_tr']: + # multivariate results cannot be returned in a dense form as a + # single set of results would correspond to multiple entries in + # the matrix, and there could also be cases where multiple + # results correspond to the same entries in the matrix. + raise ValueError('cannot return multivariate connectivity ' + 'data in a dense form') + # get the new shape of the data array if self.is_epoched: new_shape = [self.n_epochs] diff --git a/mne_connectivity/spectral/epochs.py b/mne_connectivity/spectral/epochs.py index 1ee2d1a2..eb766f06 100644 --- a/mne_connectivity/spectral/epochs.py +++ b/mne_connectivity/spectral/epochs.py @@ -1,6 +1,9 @@ # Authors: Martin Luessi # Denis A. Engemann # Adam Li +# Thomas S. Binns +# Tien D. Nguyen +# Richard M. Köhler # # License: BSD (3-clause) @@ -8,6 +11,7 @@ import inspect import numpy as np +import scipy as sp from mne.epochs import BaseEpochs from mne.parallel import parallel_func from mne.source_estimate import _BaseSourceEstimate @@ -16,8 +20,8 @@ _psd_from_mt_adaptive) from mne.time_frequency.tfr import cwt, morlet from mne.time_frequency.multitaper import _compute_mt_params -from mne.utils import (_arange_div, _check_option, logger, warn, _time_mask, - verbose) +from mne.utils import ( + ProgressBar, _arange_div, _check_option, _time_mask, logger, warn, verbose) from ..base import (SpectralConnectivity, SpectroTemporalConnectivity) from ..utils import fill_doc, check_indices @@ -61,7 +65,7 @@ def _compute_freq_mask(freqs_all, fmin, fmax, fskip): def _prepare_connectivity(epoch_block, times_in, tmin, tmax, fmin, fmax, sfreq, indices, - mode, fskip, n_bands, + method, mode, fskip, n_bands, cwt_freqs, faverage): """Check and precompute dimensions of results data.""" first_epoch = epoch_block[0] @@ -89,14 +93,39 @@ def _prepare_connectivity(epoch_block, times_in, tmin, tmax, n_times = len(times) if indices is None: - logger.info('only using indices for lower-triangular matrix') - # only compute r for lower-triangular region - indices_use = np.tril_indices(n_signals, -1) + if any(this_method in _multivariate_methods for this_method in method): + if any(this_method in _gc_methods for this_method in method): + raise ValueError( + 'indices must be specified when computing Granger ' + 'causality, as all-to-all connectivity is not supported') + else: + logger.info('using all indices for multivariate connectivity') + indices_use = (np.arange(n_signals, dtype=int), + np.arange(n_signals, dtype=int)) + else: + logger.info('only using indices for lower-triangular matrix') + # only compute r for lower-triangular region + indices_use = np.tril_indices(n_signals, -1) else: + if any(this_method in _gc_methods for this_method in method): + if set(indices[0]).intersection(indices[1]): + raise ValueError( + 'seed and target indices must not intersect when computing' + 'Granger causality') indices_use = check_indices(indices) # number of connectivities to compute - n_cons = len(indices_use[0]) + if any(this_method in _multivariate_methods for this_method in method): + if ( + len(np.unique(indices_use[0])) != len(indices_use[0]) or + len(np.unique(indices_use[1])) != len(indices_use[1]) + ): + raise ValueError( + 'seed and target indices cannot contain repeated channels for ' + 'multivariate connectivity') + n_cons = 1 # UNTIL RAGGED ARRAYS SUPPORTED + else: + n_cons = len(indices_use[0]) logger.info(' computing connectivity for %d connections' % n_cons) @@ -222,6 +251,8 @@ def compute_con(self, con_idx, n_epochs): class _EpochMeanConEstBase(_AbstractConEstBase): """Base class for methods that estimate connectivity as mean epoch-wise.""" + patterns = None + def __init__(self, n_cons, n_freqs, n_times): self.n_cons = n_cons self.n_freqs = n_freqs @@ -243,9 +274,76 @@ def combine(self, other): self._acc += other._acc +class _EpochMeanMultivariateConEstBase(_AbstractConEstBase): + """Base class for mean epoch-wise multivar. con. estimation methods.""" + + n_steps = None + patterns = None + + def __init__(self, n_signals, n_cons, n_freqs, n_times, n_jobs=1): + self.n_signals = n_signals + self.n_cons = n_cons + self.n_freqs = n_freqs + self.n_times = n_times + self.n_jobs = n_jobs + + # include time dimension, even when unused for indexing flexibility + if n_times == 0: + self.csd_shape = (n_signals**2, n_freqs) + self.con_scores = np.zeros((n_cons, n_freqs, 1)) + else: + self.csd_shape = (n_signals**2, n_freqs, n_times) + self.con_scores = np.zeros((n_cons, n_freqs, n_times)) + + # allocate space for accumulation of CSD + self._acc = np.zeros(self.csd_shape, dtype=np.complex128) + + self._compute_n_progress_bar_steps() + + def start_epoch(self): # noqa: D401 + """Called at the start of each epoch.""" + pass # for this type of con. method we don't do anything + + def combine(self, other): + """Include con. accumulated for some epochs in this estimate.""" + self._acc += other._acc + + def accumulate(self, con_idx, csd_xy): + """Accumulate CSD for some connections.""" + self._acc[con_idx] += csd_xy + + def _compute_n_progress_bar_steps(self): + """Calculate the number of steps to include in the progress bar.""" + self.n_steps = int(np.ceil(self.n_freqs / self.n_jobs)) + + def _log_connection_number(self, con_i): + """Log the number of the connection being computed.""" + logger.info('Computing %s for connection %i of %i' + % (self.name, con_i + 1, self.n_cons, )) + + def _get_block_indices(self, block_i, limit): + """Get indices for a computation block capped by a limit.""" + indices = np.arange(block_i * self.n_jobs, (block_i + 1) * self.n_jobs) + + return indices[np.nonzero(indices < limit)] + + def reshape_csd(self): + """Reshape CSD into a matrix of times x freqs x signals x signals.""" + if self.n_times == 0: + return (np.reshape(self._acc, ( + self.n_signals, self.n_signals, self.n_freqs, 1) + ).transpose(3, 2, 0, 1)) + + return (np.reshape(self._acc, ( + self.n_signals, self.n_signals, self.n_freqs, self.n_times) + ).transpose(3, 2, 0, 1)) + + class _CohEstBase(_EpochMeanConEstBase): """Base Estimator for Coherence, Coherency, Imag. Coherence.""" + accumulate_psd = True + def __init__(self, n_cons, n_freqs, n_times): super(_CohEstBase, self).__init__(n_cons, n_freqs, n_times) @@ -297,10 +395,236 @@ def compute_con(self, con_idx, n_epochs, psd_xx, psd_yy): # lgtm self.con_scores[con_idx] = np.imag(csd_mean) / np.sqrt(psd_xx * psd_yy) +class _MultivariateCohEstBase(_EpochMeanMultivariateConEstBase): + """Base estimator for multivariate imag. part of coherency methods. + + See Ewald et al. (2012). NeuroImage. DOI: 10.1016/j.neuroimage.2011.11.084 + for equation references. + """ + + name = None + accumulate_psd = False + + def __init__(self, n_signals, n_cons, n_freqs, n_times, n_jobs=1): + super(_MultivariateCohEstBase, self).__init__( + n_signals, n_cons, n_freqs, n_times, n_jobs) + + def compute_con(self, indices, ranks, n_epochs=1): + """Compute multivariate imag. part of coherency between signals.""" + assert self.name in ['MIC', 'MIM'], ( + 'the class name is not recognised, please contact the ' + 'mne-connectivity developers') + + csd = self.reshape_csd() / n_epochs + n_times = csd.shape[0] + times = np.arange(n_times) + freqs = np.arange(self.n_freqs) + + if self.name == 'MIC': + self.patterns = np.full( + (2, self.n_cons, len(indices[0]), self.n_freqs, n_times), + np.nan) + + con_i = 0 + for seed_idcs, target_idcs, seed_rank, target_rank in zip( + [indices[0]], [indices[1]], ranks[0], ranks[1]): + self._log_connection_number(con_i) + + n_seeds = len(seed_idcs) + con_idcs = [*seed_idcs, *target_idcs] + + C = csd[np.ix_(times, freqs, con_idcs, con_idcs)] + + # Eqs. 32 & 33 + C_bar, U_bar_aa, U_bar_bb = self._csd_svd( + C, n_seeds, seed_rank, target_rank) + + # Eqs. 3 & 4 + E = self._compute_e(C_bar, n_seeds=U_bar_aa.shape[3]) + + if self.name == 'MIC': + self._compute_mic(E, C, seed_idcs, target_idcs, n_times, + U_bar_aa, U_bar_bb, con_i) + else: + self._compute_mim(E, seed_idcs, target_idcs, con_i) + + con_i += 1 + + self.reshape_results() + + def _csd_svd(self, csd, n_seeds, seed_rank, target_rank): + """Dimensionality reduction of CSD with SVD.""" + n_times = csd.shape[0] + n_targets = csd.shape[2] - n_seeds + + C_aa = csd[..., :n_seeds, :n_seeds] + C_ab = csd[..., :n_seeds, n_seeds:] + C_bb = csd[..., n_seeds:, n_seeds:] + C_ba = csd[..., n_seeds:, :n_seeds] + + # Eq. 32 + if seed_rank != n_seeds: + U_aa = np.linalg.svd(np.real(C_aa), full_matrices=False)[0] + U_bar_aa = U_aa[..., :seed_rank] + else: + U_bar_aa = np.broadcast_to( + np.identity(n_seeds), + (n_times, self.n_freqs) + (n_seeds, n_seeds)) + + if target_rank != n_targets: + U_bb = np.linalg.svd(np.real(C_bb), full_matrices=False)[0] + U_bar_bb = U_bb[..., :target_rank] + else: + U_bar_bb = np.broadcast_to( + np.identity(n_targets), + (n_times, self.n_freqs) + (n_targets, n_targets)) + + # Eq. 33 + C_bar_aa = np.matmul( + U_bar_aa.transpose(0, 1, 3, 2), np.matmul(C_aa, U_bar_aa)) + C_bar_ab = np.matmul( + U_bar_aa.transpose(0, 1, 3, 2), np.matmul(C_ab, U_bar_bb)) + C_bar_bb = np.matmul( + U_bar_bb.transpose(0, 1, 3, 2), np.matmul(C_bb, U_bar_bb)) + C_bar_ba = np.matmul( + U_bar_bb.transpose(0, 1, 3, 2), np.matmul(C_ba, U_bar_aa)) + C_bar = np.append(np.append(C_bar_aa, C_bar_ab, axis=3), + np.append(C_bar_ba, C_bar_bb, axis=3), axis=2) + + return C_bar, U_bar_aa, U_bar_bb + + def _compute_e(self, csd, n_seeds): + """Compute E from the CSD.""" + C_r = np.real(csd) + + parallel, parallel_compute_t, _ = parallel_func( + _mic_mim_compute_t, self.n_jobs, verbose=False) + + # imag. part of T filled when data is rank-deficient + T = np.zeros(csd.shape, dtype=np.complex128) + for block_i in ProgressBar( + range(self.n_steps), mesg="frequency blocks"): + freqs = self._get_block_indices(block_i, self.n_freqs) + parallel(parallel_compute_t( + C_r[:, f], T[:, f], n_seeds) for f in freqs) + + if not np.isreal(T).all() or not np.isfinite(T).all(): + raise RuntimeError( + 'the transformation matrix of the data must be real-valued ' + 'and contain no NaN or infinity values; check that you are ' + 'using full rank data or specify an appropriate rank for the ' + 'seeds and targets that is less than or equal to their ranks') + T = np.real(T) # make T real if check passes + + # Eq. 4 + D = np.matmul(T, np.matmul(csd, T)) + + # E as imag. part of D between seeds and targets + return np.imag(D[..., :n_seeds, n_seeds:]) + + def _compute_mic(self, E, C, seed_idcs, target_idcs, n_times, U_bar_aa, + U_bar_bb, con_i): + """Compute MIC and the associated spatial patterns.""" + n_seeds = len(seed_idcs) + times = np.arange(n_times) + freqs = np.arange(self.n_freqs) + + # Eigendecomp. to find spatial filters for seeds and targets + w_seeds, V_seeds = np.linalg.eigh( + np.matmul(E, E.transpose(0, 1, 3, 2))) + w_targets, V_targets = np.linalg.eigh( + np.matmul(E.transpose(0, 1, 3, 2), E)) + if np.all(seed_idcs == target_idcs): + # strange edge-case where the eigenvectors returned should be a set + # of identity matrices with one rotated by 90 degrees, but are + # instead identical (i.e. are not rotated versions of one another). + # This leads to the case where the spatial filters are incorrectly + # applied, resulting in connectivity estimates of e.g. ~0 when they + # should be perfectly correlated ~1. Accordingly, we manually + # create a set of rotated identity matrices to use as the filters. + create_filter = False + stop = False + while not create_filter and not stop: + for time_i in range(n_times): + for freq_i in range(self.n_freqs): + if np.all(V_seeds[time_i, freq_i] == + V_targets[time_i, freq_i]): + create_filter = True + break + stop = True + if create_filter: + n_chans = E.shape[2] + eye_4d = np.zeros_like(V_seeds) + eye_4d[:, :, np.arange(n_chans), np.arange(n_chans)] = 1 + V_seeds = eye_4d + V_targets = np.rot90(eye_4d, axes=(2, 3)) + + # Spatial filters with largest eigval. for seeds and targets + alpha = V_seeds[times[:, None], freqs, :, w_seeds.argmax(axis=2)] + beta = V_targets[times[:, None], freqs, :, w_targets.argmax(axis=2)] + + # Eq. 46 (seed spatial patterns) + self.patterns[0, con_i] = (np.matmul( + np.real(C[..., :n_seeds, :n_seeds]), + np.matmul(U_bar_aa, np.expand_dims(alpha, axis=3))))[..., 0].T + + # Eq. 47 (target spatial patterns) + self.patterns[1, con_i] = (np.matmul( + np.real(C[..., n_seeds:, n_seeds:]), + np.matmul(U_bar_bb, np.expand_dims(beta, axis=3))))[..., 0].T + + # Eq. 7 + self.con_scores[con_i] = (np.einsum( + 'ijk,ijk->ij', alpha, np.matmul(E, np.expand_dims( + beta, axis=3))[..., 0] + ) / np.linalg.norm(alpha, axis=2) * np.linalg.norm(beta, axis=2)).T + + def _compute_mim(self, E, seed_idcs, target_idcs, con_i): + """Compute MIM (a.k.a. GIM if seeds == targets).""" + # Eq. 14 + self.con_scores[con_i] = np.matmul( + E, E.transpose(0, 1, 3, 2)).trace(axis1=2, axis2=3).T + + # Eq. 15 + if all(np.unique(seed_idcs) == np.unique(target_idcs)): + self.con_scores[con_i] *= 0.5 + + def reshape_results(self): + """Remove time dimension from results, if necessary.""" + if self.n_times == 0: + self.con_scores = self.con_scores[..., 0] + if self.patterns is not None: + self.patterns = self.patterns[..., 0] + + +def _mic_mim_compute_t(C, T, n_seeds): + """Compute T in place for a single frequency (used for MIC and MIM).""" + for time_i in range(C.shape[0]): + T[time_i, :n_seeds, :n_seeds] = sp.linalg.fractional_matrix_power( + C[time_i, :n_seeds, :n_seeds], -0.5 + ) + T[time_i, n_seeds:, n_seeds:] = sp.linalg.fractional_matrix_power( + C[time_i, n_seeds:, n_seeds:], -0.5 + ) + + +class _MICEst(_MultivariateCohEstBase): + """Multivariate imaginary part of coherency (MIC) estimator.""" + + name = "MIC" + + +class _MIMEst(_MultivariateCohEstBase): + """Multivariate interaction measure (MIM) estimator.""" + + name = "MIM" + + class _PLVEst(_EpochMeanConEstBase): """PLV Estimator.""" name = 'PLV' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_PLVEst, self).__init__(n_cons, n_freqs, n_times) @@ -324,6 +648,7 @@ class _ciPLVEst(_EpochMeanConEstBase): """corrected imaginary PLV Estimator.""" name = 'ciPLV' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_ciPLVEst, self).__init__(n_cons, n_freqs, n_times) @@ -352,6 +677,7 @@ class _PLIEst(_EpochMeanConEstBase): """PLI Estimator.""" name = 'PLI' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_PLIEst, self).__init__(n_cons, n_freqs, n_times) @@ -375,6 +701,7 @@ class _PLIUnbiasedEst(_PLIEst): """Unbiased PLI Square Estimator.""" name = 'Unbiased PLI Square' + accumulate_psd = False def compute_con(self, con_idx, n_epochs): """Compute final con. score for some connections.""" @@ -392,6 +719,7 @@ class _DPLIEst(_EpochMeanConEstBase): """DPLI Estimator.""" name = 'DPLI' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_DPLIEst, self).__init__(n_cons, n_freqs, n_times) @@ -417,6 +745,7 @@ class _WPLIEst(_EpochMeanConEstBase): """WPLI Estimator.""" name = 'WPLI' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_WPLIEst, self).__init__(n_cons, n_freqs, n_times) @@ -455,6 +784,7 @@ class _WPLIDebiasedEst(_EpochMeanConEstBase): """Debiased WPLI Square Estimator.""" name = 'Debiased WPLI Square' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_WPLIDebiasedEst, self).__init__(n_cons, n_freqs, n_times) @@ -498,6 +828,7 @@ class _PPCEst(_EpochMeanConEstBase): """Pairwise Phase Consistency (PPC) Estimator.""" name = 'PPC' + accumulate_psd = False def __init__(self, n_cons, n_freqs, n_times): super(_PPCEst, self).__init__(n_cons, n_freqs, n_times) @@ -528,15 +859,389 @@ def compute_con(self, con_idx, n_epochs): self.con_scores[con_idx] = np.real(con) +class _GCEstBase(_EpochMeanMultivariateConEstBase): + """Base multivariate state-space Granger causality estimator.""" + + accumulate_psd = False + + def __init__(self, n_signals, n_cons, n_freqs, n_times, n_lags, n_jobs=1): + super(_GCEstBase, self).__init__( + n_signals, n_cons, n_freqs, n_times, n_jobs) + + self.freq_res = (self.n_freqs - 1) * 2 + if n_lags >= self.freq_res: + raise ValueError( + 'the number of lags (%i) must be less than double the ' + 'frequency resolution (%i)' % (n_lags, self.freq_res, )) + self.n_lags = n_lags + + def compute_con(self, indices, ranks, n_epochs=1): + """Compute multivariate state-space Granger causality.""" + assert self.name in ['GC', 'GC time-reversed'], ( + 'the class name is not recognised, please contact the ' + 'mne-connectivity developers') + + csd = self.reshape_csd() / n_epochs + + n_times = csd.shape[0] + times = np.arange(n_times) + freqs = np.arange(self.n_freqs) + + con_i = 0 + for seed_idcs, target_idcs, seed_rank, target_rank in zip( + [indices[0]], [indices[1]], ranks[0], ranks[1]): + self._log_connection_number(con_i) + + con_idcs = [*seed_idcs, *target_idcs] + C = csd[np.ix_(times, freqs, con_idcs, con_idcs)] + + con_seeds = np.arange(len(seed_idcs)) + con_targets = np.arange(len(target_idcs)) + len(seed_idcs) + + C_bar = self._csd_svd( + C, con_seeds, con_targets, seed_rank, target_rank) + n_signals = seed_rank + target_rank + con_seeds = np.arange(seed_rank) + con_targets = np.arange(target_rank) + seed_rank + + autocov = self._compute_autocov(C_bar) + if self.name == "GC time-reversed": + autocov = autocov.transpose(0, 1, 3, 2) + + A_f, V = self._autocov_to_full_var(autocov) + A_f_3d = np.reshape( + A_f, (n_times, n_signals, n_signals * self.n_lags), + order="F") + A, K = self._full_var_to_iss(A_f_3d) + + self.con_scores[con_i] = self._iss_to_ugc( + A, A_f_3d, K, V, con_seeds, con_targets) + + con_i += 1 + + self.reshape_results() + + def _csd_svd(self, csd, seeds, targets, seed_rank, target_rank): + """Dimensionality reduction of CSD with SVD on the covariance.""" + # sum over times and epochs to get cov. from CSD + cov = csd.sum(axis=(0, 1)) + + n_seeds = len(seeds) + n_targets = len(targets) + + cov_aa = cov[:n_seeds, :n_seeds] + cov_bb = cov[n_seeds:, n_seeds:] + + if seed_rank != n_seeds: + U_aa = np.linalg.svd(np.real(cov_aa), full_matrices=False)[0] + U_bar_aa = U_aa[:, :seed_rank] + else: + U_bar_aa = np.identity(n_seeds) + + if target_rank != n_targets: + U_bb = np.linalg.svd(np.real(cov_bb), full_matrices=False)[0] + U_bar_bb = U_bb[:, :target_rank] + else: + U_bar_bb = np.identity(n_targets) + + C_aa = csd[..., :n_seeds, :n_seeds] + C_ab = csd[..., :n_seeds, n_seeds:] + C_bb = csd[..., n_seeds:, n_seeds:] + C_ba = csd[..., n_seeds:, :n_seeds] + + C_bar_aa = np.matmul( + U_bar_aa.transpose(1, 0), np.matmul(C_aa, U_bar_aa)) + C_bar_ab = np.matmul( + U_bar_aa.transpose(1, 0), np.matmul(C_ab, U_bar_bb)) + C_bar_bb = np.matmul( + U_bar_bb.transpose(1, 0), np.matmul(C_bb, U_bar_bb)) + C_bar_ba = np.matmul( + U_bar_bb.transpose(1, 0), np.matmul(C_ba, U_bar_aa)) + C_bar = np.append(np.append(C_bar_aa, C_bar_ab, axis=3), + np.append(C_bar_ba, C_bar_bb, axis=3), axis=2) + + return C_bar + + def _compute_autocov(self, csd): + """Compute autocovariance from the CSD.""" + n_times = csd.shape[0] + n_signals = csd.shape[2] + + circular_shifted_csd = np.concatenate( + [np.flip(np.conj(csd[:, 1:]), axis=1), csd[:, :-1]], axis=1) + ifft_shifted_csd = self._block_ifft( + circular_shifted_csd, self.freq_res) + lags_ifft_shifted_csd = np.reshape( + ifft_shifted_csd[:, :self.n_lags + 1], + (n_times, self.n_lags + 1, n_signals ** 2), order="F") + + signs = np.repeat([1], self.n_lags + 1).tolist() + signs[1::2] = [x * -1 for x in signs[1::2]] + sign_matrix = np.repeat( + np.tile(np.array(signs), (n_signals ** 2, 1))[np.newaxis], + n_times, axis=0).transpose(0, 2, 1) + + return np.real(np.reshape( + sign_matrix * lags_ifft_shifted_csd, + (n_times, self.n_lags + 1, n_signals, n_signals), order="F")) + + def _block_ifft(self, csd, n_points): + """Compute block iFFT with n points.""" + shape = csd.shape + csd_3d = np.reshape( + csd, (shape[0], shape[1], shape[2] * shape[3]), order="F") + + csd_ifft = np.fft.ifft(csd_3d, n=n_points, axis=1) + + return np.reshape(csd_ifft, shape, order="F") + + def _autocov_to_full_var(self, autocov): + """Compute full VAR model using Whittle's LWR recursion.""" + if np.any(np.linalg.det(autocov) == 0): + raise RuntimeError( + 'the autocovariance matrix is singular; check if your data is ' + 'rank deficient and specify an appropriate rank argument <= ' + 'the rank of the seeds and targets') + + A_f, V = self._whittle_lwr_recursion(autocov) + + if not np.isfinite(A_f).all(): + raise RuntimeError('at least one VAR model coefficient is ' + 'infinite or NaN; check the data you are using') + + try: + np.linalg.cholesky(V) + except np.linalg.LinAlgError as np_error: + raise RuntimeError( + 'the covariance matrix of the residuals is not ' + 'positive-definite; check the singular values of your data ' + 'and specify an appropriate rank argument <= the rank of the ' + 'seeds and targets') from np_error + + return A_f, V + + def _whittle_lwr_recursion(self, G): + """Solve Yule-Walker eqs. for full VAR params. with LWR recursion. + + See: Whittle P., 1963. Biometrika, DOI: 10.1093/biomet/50.1-2.129 + """ + # Initialise recursion + n = G.shape[2] # number of signals + q = G.shape[1] - 1 # number of lags + t = G.shape[0] # number of times + qn = n * q + + cov = G[:, 0, :, :] # covariance + G_f = np.reshape( + G[:, 1:, :, :].transpose(0, 3, 1, 2), (t, qn, n), + order="F") # forward autocov + G_b = np.reshape( + np.flip(G[:, 1:, :, :], 1).transpose(0, 3, 2, 1), (t, n, qn), + order="F").transpose(0, 2, 1) # backward autocov + + A_f = np.zeros((t, n, qn)) # forward coefficients + A_b = np.zeros((t, n, qn)) # backward coefficients + + k = 1 # model order + r = q - k + k_f = np.arange(k * n) # forward indices + k_b = np.arange(r * n, qn) # backward indices + + try: + A_f[:, :, k_f] = np.linalg.solve( + cov, G_b[:, k_b, :].transpose(0, 2, 1)).transpose(0, 2, 1) + A_b[:, :, k_b] = np.linalg.solve( + cov, G_f[:, k_f, :].transpose(0, 2, 1)).transpose(0, 2, 1) + + # Perform recursion + for k in np.arange(2, q + 1): + var_A = (G_b[:, (r - 1) * n: r * n, :] - + np.matmul(A_f[:, :, k_f], G_b[:, k_b, :])) + var_B = cov - np.matmul(A_b[:, :, k_b], G_b[:, k_b, :]) + AA_f = np.linalg.solve( + var_B, var_A.transpose(0, 2, 1)).transpose(0, 2, 1) + + var_A = (G_f[:, (k - 1) * n: k * n, :] - + np.matmul(A_b[:, :, k_b], G_f[:, k_f, :])) + var_B = cov - np.matmul(A_f[:, :, k_f], G_f[:, k_f, :]) + AA_b = np.linalg.solve( + var_B, var_A.transpose(0, 2, 1)).transpose(0, 2, 1) + + A_f_previous = A_f[:, :, k_f] + A_b_previous = A_b[:, :, k_b] + + r = q - k + k_f = np.arange(k * n) + k_b = np.arange(r * n, qn) + + A_f[:, :, k_f] = np.dstack( + (A_f_previous - np.matmul(AA_f, A_b_previous), AA_f)) + A_b[:, :, k_b] = np.dstack( + (AA_b, A_b_previous - np.matmul(AA_b, A_f_previous))) + except np.linalg.LinAlgError as np_error: + raise RuntimeError( + 'the autocovariance matrix is singular; check if your data is ' + 'rank deficient and specify an appropriate rank argument <= ' + 'the rank of the seeds and targets') from np_error + + V = cov - np.matmul(A_f, G_f) + A_f = np.reshape(A_f, (t, n, n, q), order="F") + + return A_f, V + + def _full_var_to_iss(self, A_f): + """Compute innovations-form parameters for a state-space model. + + Parameters computed from a full VAR model using Aoki's method. For a + non-moving-average full VAR model, the state-space parameter C + (observation matrix) is identical to AF of the VAR model. + + See: Barnett, L. & Seth, A.K., 2015, Physical Review, DOI: + 10.1103/PhysRevE.91.040101. + """ + t = A_f.shape[0] + m = A_f.shape[1] # number of signals + p = A_f.shape[2] // m # number of autoregressive lags + + I_p = np.dstack(t * [np.eye(m * p)]).transpose(2, 0, 1) + A = np.hstack((A_f, I_p[:, : (m * p - m), :])) # state transition + # matrix + K = np.hstack(( + np.dstack(t * [np.eye(m)]).transpose(2, 0, 1), + np.zeros((t, (m * (p - 1)), m)))) # Kalman gain matrix + + return A, K + + def _iss_to_ugc(self, A, C, K, V, seeds, targets): + """Compute unconditional GC from innovations-form state-space params. + + See: Barnett, L. & Seth, A.K., 2015, Physical Review, DOI: + 10.1103/PhysRevE.91.040101. + """ + times = np.arange(A.shape[0]) + freqs = np.arange(self.n_freqs) + z = np.exp(-1j * np.pi * np.linspace(0, 1, self.n_freqs)) # points + # on a unit circle in the complex plane, one for each frequency + + H = self._iss_to_tf(A, C, K, z) # spectral transfer function + V_22_1 = np.linalg.cholesky(self._partial_covar(V, seeds, targets)) + HV = np.matmul(H, np.linalg.cholesky(V)) + S = np.matmul(HV, HV.conj().transpose(0, 1, 3, 2)) # Eq. 6 + S_11 = S[np.ix_(freqs, times, targets, targets)] + HV_12 = np.matmul(H[np.ix_(freqs, times, targets, seeds)], V_22_1) + HVH = np.matmul(HV_12, HV_12.conj().transpose(0, 1, 3, 2)) + + # Eq. 11 + return np.real( + np.log(np.linalg.det(S_11)) - np.log(np.linalg.det(S_11 - HVH))) + + def _iss_to_tf(self, A, C, K, z): + """Compute transfer function for innovations-form state-space params. + + In the frequency domain, the back-shift operator, z, is a vector of + points on a unit circle in the complex plane. z = e^-iw, where -pi < w + <= pi. + + A note on efficiency: solving over the 4D time-freq. tensor is slower + than looping over times and freqs when n_times and n_freqs high, and + when n_times and n_freqs low, looping over times and freqs very fast + anyway (plus tensor solving doesn't allow for parallelisation). + + See: Barnett, L. & Seth, A.K., 2015, Physical Review, DOI: + 10.1103/PhysRevE.91.040101. + """ + t = A.shape[0] + h = self.n_freqs + n = C.shape[1] + m = A.shape[1] + I_n = np.eye(n) + I_m = np.eye(m) + H = np.zeros((h, t, n, n), dtype=np.complex128) + + parallel, parallel_compute_H, _ = parallel_func( + _gc_compute_H, self.n_jobs, verbose=False + ) + H = np.zeros((h, t, n, n), dtype=np.complex128) + for block_i in ProgressBar( + range(self.n_steps), mesg="frequency blocks" + ): + freqs = self._get_block_indices(block_i, self.n_freqs) + H[freqs] = parallel( + parallel_compute_H(A, C, K, z[k], I_n, I_m) for k in freqs) + + return H + + def _partial_covar(self, V, seeds, targets): + """Compute partial covariance of a matrix. + + Given a covariance matrix V, the partial covariance matrix of V between + indices i and j, given k (V_ij|k), is equivalent to V_ij - V_ik * + V_kk^-1 * V_kj. In this case, i and j are seeds, and k are targets. + + See: Barnett, L. & Seth, A.K., 2015, Physical Review, DOI: + 10.1103/PhysRevE.91.040101. + """ + times = np.arange(V.shape[0]) + W = np.linalg.solve( + np.linalg.cholesky(V[np.ix_(times, targets, targets)]), + V[np.ix_(times, targets, seeds)], + ) + W = np.matmul(W.transpose(0, 2, 1), W) + + return V[np.ix_(times, seeds, seeds)] - W + + def reshape_results(self): + """Remove time dimension from con. scores, if necessary.""" + if self.n_times == 0: + self.con_scores = self.con_scores[:, :, 0] + + +def _gc_compute_H(A, C, K, z_k, I_n, I_m): + """Compute transfer function for innovations-form state-space params. + + See: Barnett, L. & Seth, A.K., 2015, Physical Review, DOI: + 10.1103/PhysRevE.91.040101, Eq. 4. + """ + from scipy import linalg # is this necessary??? + H = np.zeros((A.shape[0], C.shape[1], C.shape[1]), dtype=np.complex128) + for t in range(A.shape[0]): + H[t] = I_n + np.matmul( + C[t], linalg.lu_solve(linalg.lu_factor(z_k * I_m - A[t]), K[t])) + + return H + + +class _GCEst(_GCEstBase): + """[seeds -> targets] state-space GC estimator.""" + + name = "GC" + + +class _GCTREst(_GCEstBase): + """time-reversed[seeds -> targets] state-space GC estimator.""" + + name = "GC time-reversed" + ############################################################################### + + +_multivariate_methods = ['mic', 'mim', 'gc', 'gc_tr'] +_gc_methods = ['gc', 'gc_tr'] + + def _epoch_spectral_connectivity(data, sig_idx, tmin_idx, tmax_idx, sfreq, - mode, window_fun, eigvals, wavelets, + method, mode, window_fun, eigvals, wavelets, freq_mask, mt_adaptive, idx_map, block_size, psd, accumulate_psd, con_method_types, - con_methods, n_signals, n_times, - accumulate_inplace=True): + con_methods, n_signals, n_signals_use, + n_times, gc_n_lags, accumulate_inplace=True): """Estimate connectivity for one epoch (see spectral_connectivity).""" - n_cons = len(idx_map[0]) + if any(this_method in _multivariate_methods for this_method in method): + n_cons = 1 # UNTIL RAGGED ARRAYS SUPPORTED + n_con_signals = n_signals_use ** 2 + else: + n_cons = len(idx_map[0]) + n_con_signals = n_cons if wavelets is not None: n_times_spectrum = n_times @@ -547,8 +1252,24 @@ def _epoch_spectral_connectivity(data, sig_idx, tmin_idx, tmax_idx, sfreq, if not accumulate_inplace: # instantiate methods only for this epoch (used in parallel mode) - con_methods = [mtype(n_cons, n_freqs, n_times_spectrum) - for mtype in con_method_types] + con_methods = [] + for mtype in con_method_types: + method_params = list(inspect.signature(mtype).parameters) + if "n_signals" in method_params: + # if it's a multivariate connectivity method + if "n_lags" in method_params: + # if it's a Granger causality method + con_methods.append( + mtype(n_signals_use, n_cons, n_freqs, n_times_spectrum, + gc_n_lags) + ) + else: + # if it's a coherence method + con_methods.append( + mtype(n_signals_use, n_cons, n_freqs, n_times_spectrum) + ) + else: + con_methods.append(mtype(n_cons, n_freqs, n_times_spectrum)) _check_option('mode', mode, ('cwt_morlet', 'multitaper', 'fourier')) if len(sig_idx) == n_signals: @@ -624,8 +1345,9 @@ def _epoch_spectral_connectivity(data, sig_idx, tmin_idx, tmax_idx, sfreq, # accumulate connectivity scores if mode in ['multitaper', 'fourier']: - for i in range(0, n_cons, block_size): - con_idx = slice(i, i + block_size) + for i in range(0, n_con_signals, block_size): + n_extra = max(0, i + block_size - n_con_signals) + con_idx = slice(i, i + block_size - n_extra) if mt_adaptive: csd = _csd_from_mt(x_t[idx_map[0][con_idx]], x_t[idx_map[1][con_idx]], @@ -639,8 +1361,9 @@ def _epoch_spectral_connectivity(data, sig_idx, tmin_idx, tmax_idx, sfreq, for method in con_methods: method.accumulate(con_idx, csd) else: # mode == 'cwt_morlet' # reminder to add alternative TFR methods - for i_block, i in enumerate(range(0, n_cons, block_size)): - con_idx = slice(i, i + block_size) + for i in range(0, n_con_signals, block_size): + n_extra = max(0, i + block_size - n_con_signals) + con_idx = slice(i, i + block_size - n_extra) # this codes can be very slow csd = (x_t[idx_map[0][con_idx]] * x_t[idx_map[1][con_idx]].conjugate()) @@ -727,10 +1450,11 @@ def _get_and_verify_data_sizes(data, sfreq, n_signals=None, n_times=None, 'plv': _PLVEst, 'ciplv': _ciPLVEst, 'ppc': _PPCEst, 'pli': _PLIEst, 'pli2_unbiased': _PLIUnbiasedEst, 'dpli': _DPLIEst, 'wpli': _WPLIEst, - 'wpli2_debiased': _WPLIDebiasedEst} + 'wpli2_debiased': _WPLIDebiasedEst, 'mic': _MICEst, + 'mim': _MIMEst, 'gc': _GCEst, 'gc_tr': _GCTREst} -def _check_estimators(method, mode): +def _check_estimators(method): """Check construction of connectivity estimators.""" n_methods = len(method) con_method_types = list() @@ -748,30 +1472,24 @@ def _check_estimators(method, mode): 'not have the method %s' % msg) con_method_types.append(this_method) - # determine how many arguments the compute_con_function needs - n_comp_args = [len(inspect.signature(mtype.compute_con).parameters) - for mtype in con_method_types] - - # we currently only support 3 arguments - if any(n not in (3, 5) for n in n_comp_args): - raise ValueError('The .compute_con method needs to have either ' - '3 or 5 arguments') # if none of the comp_con functions needs the PSD, we don't estimate it - accumulate_psd = any(n == 5 for n in n_comp_args) - return con_method_types, n_methods, accumulate_psd, n_comp_args + accumulate_psd = any( + this_method.accumulate_psd for this_method in con_method_types) + return con_method_types, n_methods, accumulate_psd -@verbose -@fill_doc + +@ verbose +@ fill_doc def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, sfreq=None, mode='multitaper', fmin=None, fmax=np.inf, fskip=0, faverage=False, tmin=None, tmax=None, mt_bandwidth=None, mt_adaptive=False, mt_low_bias=True, cwt_freqs=None, - cwt_n_cycles=7, block_size=1000, n_jobs=1, - verbose=None): - """Compute frequency- and time-frequency-domain connectivity measures. + cwt_n_cycles=7, gc_n_lags=40, rank=None, + block_size=1000, n_jobs=1, verbose=None): + r"""Compute frequency- and time-frequency-domain connectivity measures. The connectivity method(s) are specified using the "method" parameter. All methods are based on estimates of the cross- and power spectral @@ -790,11 +1508,15 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, %(names)s method : str | list of str Connectivity measure(s) to compute. These can be ``['coh', 'cohy', - 'imcoh', 'plv', 'ciplv', 'ppc', 'pli', 'dpli', 'wpli', - 'wpli2_debiased']``. + 'imcoh', 'mic', 'mim', 'plv', 'ciplv', 'ppc', 'pli', 'dpli', 'wpli', + 'wpli2_debiased', 'gc', 'gc_tr']``. Multivariate methods (``['mic', + 'mim', 'gc', 'gc_tr]``) cannot be called with the other methods. indices : tuple of array | None Two arrays with indices of connections for which to compute - connectivity. If None, all connections are computed. + connectivity. If a multivariate method is called, the indices are for a + single connection between all seeds and all targets. If None, all + connections are computed, unless a Granger causality method is called, + in which case an error is raised. sfreq : float The sampling frequency. Required if data is not :class:`Epochs `. @@ -804,8 +1526,6 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, fmin : float | tuple of float The lower frequency of interest. Multiple bands are defined using a tuple, e.g., (8., 20.) for two bands with 8Hz and 20Hz lower freq. - If None the frequency corresponding to an epoch length of 5 cycles - is used. fmax : float | tuple of float The upper frequency of interest. Multiple bands are dedined using a tuple, e.g. (13., 30.) for two band with 13Hz and 30Hz upper freq. @@ -840,11 +1560,21 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, cwt_n_cycles : float | array of float Number of cycles. Fixed number or one per frequency. Only used in 'cwt_morlet' mode. + gc_n_lags : int + Number of lags to use for the vector autoregressive model when + computing Granger causality. Higher values increase computational cost, + but reduce the degree of spectral smoothing in the results. Only used + if ``method`` contains any of ``['gc', 'gc_tr']``. + rank : tuple of array | None + Two arrays with the rank to project the seed and target data to, + respectively, using singular value decomposition. If None, the rank of + the data is computed and projected to. Only used if ``method`` contains + any of ``['mic', 'mim', 'gc', 'gc_tr']``. block_size : int How many connections to compute at once (higher numbers are faster but require more memory). n_jobs : int - How many epochs to process in parallel. + How many samples to process in parallel. %(verbose)s Returns @@ -858,7 +1588,8 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, when "indices" is None, or (n_con, n_freqs) mode: 'multitaper' or 'fourier' (n_con, n_freqs, n_times) mode: 'cwt_morlet' - when "indices" is specified and "n_con = len(indices[0])". + when "indices" is specified and "n_con = len(indices[0])". If a + multivariate method is called "n_con = 1" even if "indices" is None. See Also -------- @@ -888,11 +1619,11 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, "mode" parameter. By default, the connectivity between all signals is computed (only - connections corresponding to the lower-triangular part of the - connectivity matrix). If one is only interested in the connectivity - between some signals, the "indices" parameter can be used. For example, - to compute the connectivity between the signal with index 0 and signals - "2, 3, 4" (a total of 3 connections) one can use the following:: + connections corresponding to the lower-triangular part of the connectivity + matrix). If one is only interested in the connectivity between some + signals, the "indices" parameter can be used. For example, to compute the + connectivity between the signal with index 0 and signals "2, 3, 4" (a total + of 3 connections) one can use the following:: indices = (np.array([0, 0, 0]), # row indices np.array([2, 3, 4])) # col indices @@ -903,6 +1634,15 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, In this case con.get_data().shape = (3, n_freqs). The connectivity scores are in the same order as defined indices. + For multivariate methods, this is handled differently. If "indices" is + None, connectivity between all signals will attempt to be computed (this is + not possible if a Granger causality method is called). If "indices" is + specified, the seeds and targets are treated as a single connection. For + example, to compute the connectivity between signals 0, 1, 2 and 3, 4, 5, + one would use the same approach as above, however the signals would all be + considered for a single connection and the connectivity scores would have + the shape (1, n_freqs). + **Supported Connectivity Measures** The connectivity method(s) is specified using the "method" parameter. The @@ -928,12 +1668,31 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, C = ---------------------- sqrt(E[Sxx] * E[Syy]) + 'mic' : Maximised Imaginary part of Coherency (MIC) + :footcite:`EwaldEtAl2012` given by: + + :math:`MIC=\Large{\frac{\boldsymbol{\alpha}^T \boldsymbol{E \beta}} + {\parallel\boldsymbol{\alpha}\parallel \parallel\boldsymbol{\beta} + \parallel}}` + + where: :math:`\boldsymbol{E}` is the imaginary part of the + transformed cross-spectral density between seeds and targets; and + :math:`\boldsymbol{\alpha}` and :math:`\boldsymbol{\beta}` are + eigenvectors for the seeds and targets, such that + :math:`\boldsymbol{\alpha}^T \boldsymbol{E \beta}` maximises + connectivity between the seeds and targets. + + 'mim' : Multivariate Interaction Measure (MIM) + :footcite:`EwaldEtAl2012` given by: + + :math:`MIM=tr(\boldsymbol{EE}^T)` + 'plv' : Phase-Locking Value (PLV) :footcite:`LachauxEtAl1999` given by:: PLV = |E[Sxy/|Sxy|]| - 'ciplv' : corrected imaginary PLV (icPLV) + 'ciplv' : corrected imaginary PLV (ciPLV) :footcite:`BrunaEtAl2018` given by:: |E[Im(Sxy/|Sxy|)]| @@ -965,14 +1724,32 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, 'wpli2_debiased' : Debiased estimator of squared WPLI :footcite:`VinckEtAl2011`. + 'gc' : State-space Granger Causality (GC) :footcite:`BarnettSeth2015` + given by: + + :math:`GC = ln\Large{(\frac{\lvert\boldsymbol{S}_{tt}\rvert}{\lvert + \boldsymbol{S}_{tt}-\boldsymbol{H}_{ts}\boldsymbol{\Sigma}_{ss + \lvert t}\boldsymbol{H}_{ts}^*\rvert}})`, + + where: :math:`s` and :math:`t` represent the seeds and targets, + respectively; :math:`\boldsymbol{H}` is the spectral transfer + function; :math:`\boldsymbol{\Sigma}` is the residuals matrix of + the autoregressive model; and :math:`\boldsymbol{S}` is + :math:`\boldsymbol{\Sigma}` transformed by :math:`\boldsymbol{H}`. + + 'gc_tr' : State-space GC on time-reversed signals + :footcite:`BarnettSeth2015,WinklerEtAl2016` given by the same equation + as for 'gc', but where the autocovariance sequence from which the + autoregressive model is produced is transposed to mimic the reversal of + the original signal in time. + References ---------- .. footbibliography:: """ if n_jobs != 1: - parallel, my_epoch_spectral_connectivity, _ = \ - parallel_func(_epoch_spectral_connectivity, n_jobs, - verbose=verbose) + parallel, my_epoch_spectral_connectivity, _ = parallel_func( + _epoch_spectral_connectivity, n_jobs, verbose=verbose) # format fmin and fmax and check inputs if fmin is None: @@ -991,9 +1768,24 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, if not isinstance(method, (list, tuple)): method = [method] # make it a list so we can iterate over it + if n_bands != 1 and any( + this_method in _gc_methods for this_method in method + ): + raise ValueError('computing Granger causality on multiple frequency ' + 'bands is not yet supported') + + if any(this_method in _multivariate_methods for this_method in method): + if not all(this_method in _multivariate_methods for + this_method in method): + raise ValueError( + 'bivariate and multivariate connectivity methods cannot be ' + 'used in the same function call') + multivariate_con = True + else: + multivariate_con = False + # handle connectivity estimators - (con_method_types, n_methods, accumulate_psd, - n_comp_args) = _check_estimators(method=method, mode=mode) + (con_method_types, n_methods, accumulate_psd) = _check_estimators(method) events = None event_id = None @@ -1037,8 +1829,15 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, n_signals, indices_use, warn_times) = _prepare_connectivity( epoch_block=epoch_block, times_in=times_in, tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax, sfreq=sfreq, - indices=indices, mode=mode, fskip=fskip, n_bands=n_bands, - cwt_freqs=cwt_freqs, faverage=faverage) + indices=indices, method=method, mode=mode, fskip=fskip, + n_bands=n_bands, cwt_freqs=cwt_freqs, faverage=faverage) + + # check rank input and compute data ranks if necessary + if multivariate_con: + rank = _check_rank_input(rank, data, sfreq, indices_use) + else: + rank = None + gc_n_lags = None # get the window function, wavelets, etc for different modes (spectral_params, mt_adaptive, n_times_spectrum, @@ -1050,23 +1849,36 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, # unique signals for which we actually need to compute PSD etc. sig_idx = np.unique(np.r_[indices_use[0], indices_use[1]]) + n_signals_use = len(sig_idx) # map indices to unique indices idx_map = [np.searchsorted(sig_idx, ind) for ind in indices_use] + if multivariate_con: + indices_use = idx_map + idx_map = np.unique([*idx_map[0], *idx_map[1]]) + idx_map = [np.sort(np.repeat(idx_map, len(sig_idx))), + np.tile(idx_map, len(sig_idx))] # allocate space to accumulate PSD if accumulate_psd: if n_times_spectrum == 0: - psd_shape = (len(sig_idx), n_freqs) + psd_shape = (n_signals_use, n_freqs) else: - psd_shape = (len(sig_idx), n_freqs, n_times_spectrum) + psd_shape = (n_signals_use, n_freqs, n_times_spectrum) psd = np.zeros(psd_shape) else: psd = None # create instances of the connectivity estimators - con_methods = [mtype(n_cons, n_freqs, n_times_spectrum) - for mtype in con_method_types] + con_methods = [] + for mtype_i, mtype in enumerate(con_method_types): + method_params = dict(n_cons=n_cons, n_freqs=n_freqs, + n_times=n_times_spectrum) + if method[mtype_i] in _multivariate_methods: + method_params.update(dict(n_signals=n_signals_use)) + if method[mtype_i] in _gc_methods: + method_params.update(dict(n_lags=gc_n_lags)) + con_methods.append(mtype(**method_params)) sep = ', ' metrics_str = sep.join([meth.name for meth in con_methods]) @@ -1080,33 +1892,35 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, warn_times=warn_times) call_params = dict( - sig_idx=sig_idx, tmin_idx=tmin_idx, - tmax_idx=tmax_idx, sfreq=sfreq, mode=mode, - freq_mask=freq_mask, idx_map=idx_map, block_size=block_size, + sig_idx=sig_idx, tmin_idx=tmin_idx, tmax_idx=tmax_idx, sfreq=sfreq, + method=method, mode=mode, freq_mask=freq_mask, idx_map=idx_map, + block_size=block_size, psd=psd, accumulate_psd=accumulate_psd, mt_adaptive=mt_adaptive, con_method_types=con_method_types, con_methods=con_methods if n_jobs == 1 else None, - n_signals=n_signals, n_times=n_times, + n_signals=n_signals, n_signals_use=n_signals_use, n_times=n_times, + gc_n_lags=gc_n_lags, accumulate_inplace=True if n_jobs == 1 else False) call_params.update(**spectral_params) if n_jobs == 1: # no parallel processing for this_epoch in epoch_block: - logger.info(' computing connectivity for epoch %d' + logger.info(' computing cross-spectral density for epoch %d' % (epoch_idx + 1)) # con methods and psd are updated inplace _epoch_spectral_connectivity(data=this_epoch, **call_params) epoch_idx += 1 else: # process epochs in parallel - logger.info(' computing connectivity for epochs %d..%d' - % (epoch_idx + 1, epoch_idx + len(epoch_block))) + logger.info( + ' computing cross-spectral density for epochs %d..%d' + % (epoch_idx + 1, epoch_idx + len(epoch_block))) out = parallel(my_epoch_spectral_connectivity( - data=this_epoch, **call_params) - for this_epoch in epoch_block) + data=this_epoch, **call_params) + for this_epoch in epoch_block) # do the accumulation for this_out in out: for _method, parallel_method in zip(con_methods, this_out[0]): @@ -1123,12 +1937,11 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, # compute final connectivity scores con = list() - for conn_method, n_args in zip(con_methods, n_comp_args): + patterns = list() + for method_i, conn_method in enumerate(con_methods): + # future estimators will need to be handled here - if n_args == 3: - # compute all scores at once - conn_method.compute_con(slice(0, n_cons), n_epochs) - elif n_args == 5: + if conn_method.accumulate_psd: # compute scores block-wise to save memory for i in range(0, n_cons, block_size): con_idx = slice(i, i + block_size) @@ -1136,26 +1949,47 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, psd_yy = psd[idx_map[1][con_idx]] conn_method.compute_con(con_idx, n_epochs, psd_xx, psd_yy) else: - raise RuntimeError('This should never happen.') + # compute all scores at once + if method[method_i] in _multivariate_methods: + conn_method.compute_con(indices_use, rank, n_epochs) + else: + conn_method.compute_con(slice(0, n_cons), n_epochs) # get the connectivity scores this_con = conn_method.con_scores + this_patterns = conn_method.patterns if this_con.shape[0] != n_cons: - raise ValueError('First dimension of connectivity scores must be ' - 'the same as the number of connections') + raise RuntimeError( + 'first dimension of connectivity scores does not match the ' + 'number of connections; please contact the mne-connectivity ' + 'developers') if faverage: if this_con.shape[1] != n_freqs: - raise ValueError('2nd dimension of connectivity scores must ' - 'be the same as the number of frequencies') + raise RuntimeError( + 'second dimension of connectivity scores does not match ' + 'the number of frequencies; please contact the ' + 'mne-connectivity developers') con_shape = (n_cons, n_bands) + this_con.shape[2:] this_con_bands = np.empty(con_shape, dtype=this_con.dtype) for band_idx in range(n_bands): - this_con_bands[:, band_idx] =\ - np.mean(this_con[:, freq_idx_bands[band_idx]], axis=1) + this_con_bands[:, band_idx] = np.mean( + this_con[:, freq_idx_bands[band_idx]], axis=1) this_con = this_con_bands + if this_patterns is not None: + patterns_shape = ((2, n_cons, len(indices[0]), n_bands) + + this_patterns.shape[4:]) + this_patterns_bands = np.empty(patterns_shape, + dtype=this_patterns.dtype) + for band_idx in range(n_bands): + this_patterns_bands[:, :, :, band_idx] = np.mean( + this_patterns[:, :, :, freq_idx_bands[band_idx]], + axis=3) + this_patterns = this_patterns_bands + con.append(this_con) + patterns.append(this_patterns) freqs_used = freqs if faverage: @@ -1169,7 +2003,7 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, freqs_used = freqs_bands freqs_used = [[np.min(band), np.max(band)] for band in freqs_used] - if indices is None: + if indices is None and not multivariate_con: # return all-to-all connectivity matrices # raveled into a 1D array logger.info(' assembling connectivity matrix') @@ -1186,27 +2020,24 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, this_con = this_con.reshape((n_signals ** 2,) + this_con_flat.shape[1:]) con.append(this_con) - # number of nodes in the original data, + # number of nodes in the original data n_nodes = n_signals + if multivariate_con: + # UNTIL RAGGED ARRAYS SUPPORTED + indices = tuple( + [[np.array(indices_use[0])], [np.array(indices_use[1])]]) + # create a list of connectivity containers conn_list = [] - for _con in con: - kwargs = dict(data=_con, - names=names, - freqs=freqs, - method=method, - n_nodes=n_nodes, - spec_method=mode, - indices=indices, - n_epochs_used=n_epochs, - freqs_used=freqs_used, - times_used=times, - n_tapers=n_tapers, - metadata=metadata, - events=events, - event_id=event_id - ) + for _con, _patterns, _method in zip(con, patterns, method): + kwargs = dict( + data=_con, patterns=_patterns, names=names, freqs=freqs, + method=_method, n_nodes=n_nodes, spec_method=mode, indices=indices, + n_epochs_used=n_epochs, freqs_used=freqs_used, times_used=times, + n_tapers=n_tapers, metadata=metadata, events=events, + event_id=event_id, rank=rank, + n_lags=gc_n_lags if _method in _gc_methods else None) # create the connectivity container if mode in ['multitaper', 'fourier']: klass = SpectralConnectivity @@ -1223,3 +2054,46 @@ def spectral_connectivity_epochs(data, names=None, method='coh', indices=None, conn_list = conn_list[0] return conn_list + + +def _check_rank_input(rank, data, sfreq, indices): + """Check the rank argument is appropriate and compute rank if missing.""" + # UNTIL RAGGED ARRAYS SUPPORTED + indices = np.array([[indices[0]], [indices[1]]]) + + if rank is None: + + rank = np.zeros((2, len(indices[0])), dtype=int) + + if isinstance(data, BaseEpochs): + data_arr = data.get_data() + else: + data_arr = data + + for group_i in range(2): + for con_i, con_idcs in enumerate(indices[group_i]): + s = np.linalg.svd(data_arr[:, con_idcs], compute_uv=False) + rank[group_i][con_i] = np.min( + [np.count_nonzero(epoch >= epoch[0] * 1e-10) + for epoch in s]) + + logger.info('Estimated data ranks:') + con_i = 1 + for seed_rank, target_rank in zip(rank[0], rank[1]): + logger.info(' connection %i - seeds (%i); targets (%i)' + % (con_i, seed_rank, target_rank, )) + con_i += 1 + + rank = tuple((np.array(rank[0]), np.array(rank[1]))) + + else: + for seed_idcs, target_idcs, seed_rank, target_rank in zip( + indices[0], indices[1], rank[0], rank[1]): + if not (0 < seed_rank <= len(seed_idcs) and + 0 < target_rank <= len(target_idcs)): + raise ValueError( + 'ranks for seeds and targets must be > 0 and <= the ' + 'number of channels in the seeds and targets, ' + 'respectively, for each connection') + + return rank diff --git a/mne_connectivity/spectral/tests/data/README.md b/mne_connectivity/spectral/tests/data/README.md new file mode 100644 index 00000000..ea9da2bd --- /dev/null +++ b/mne_connectivity/spectral/tests/data/README.md @@ -0,0 +1,30 @@ +Author: Thomas S. Binns + +The files found here are used for the regression test of the multivariate +connectivity methods for MIC, MIM, GC, and TRGC +(`test_multivariate_spectral_connectivity_epochs_regression()` of +`test_spectral.py`). + +`example_multivariate_data.pkl` consists of four channels of randomly-generated +data with 15 epochs and 200 timepoints per epoch. Connectivity was computed in +MATLAB using the original implementations of these methods and saved as a +dictionary in `example_multivariate_matlab_results.pkl`. A publicly-available +implementation of the methods in MATLAB can be found here: +https://github.com/sccn/roiconnect. + +As the MNE code for computing the cross-spectral density matrix is not +available in MATLAB, the CSD matrix was computed using MNE and then loaded into +MATLAB to compute the connectivity from the original implementations using the +same processing settings in MATLAB and Python. That is: a sampling frequency of +100 Hz; method='multitaper'; fskip=0; faverage=False; tmin=0; tmax=None; +mt_bandwidth=4; mt_low_bias=True; mt_adaptive=False; gc_n_lags=20; +rank=([2], [2]) - i.e. no rank subspace projection; indices=([0, 1], [2, 3]) - +i.e. connection from first two channels to last two channels. It is +important that no changes are made to the settings for computing the CSD or the +final connectivity scores, otherwise this test will be invalid! + +One key difference is that the MATLAB implementation for computing MIC returns +the absolute value of the results, so we must take the absolute value of the +results returned from the MNE function to make the comparison. We do not return +the absolute values of the results, as relevant information such as phase angle +differences are lost. \ No newline at end of file diff --git a/mne_connectivity/spectral/tests/data/example_multivariate_data.pkl b/mne_connectivity/spectral/tests/data/example_multivariate_data.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ccd9f2a3732eec869041b9bd07bf44f5360085ca GIT binary patch literal 96139 zcmX7vcRbbq`~M|Lktlg<3t1s$OFa~oN|Gcki9%YEL?LMrGLuRoDx~a@z4zYRag1{~ zi%N;g_xbt#{N;voym&od*Y&vWkH_V?=(#l4|Nrn?aqZSs2UjKQ>y8(d$fp+_FIk8B z@t?DCynfZ{mW|Ct$Iy^a*)Xo#3;y5Ft21+kc(W!DXvi}((VBx@46l7+Z8Uf#yRLRz z(FwCz3+lIj9k?&C*nP-j7~XCUo0C+!!Tz#`x=X1Cxm^jp8;(*SxxU@N`|B8Pe17Z^ zoyUgBbL)crTK&*%-{Tf*(F;%di$&A?`3N2C*uHM07ProFoFih`n7;n9o3)jVMJENG z(KMKlnUPbym^6ayBVkV>ENZb>gx{NGIf9WZ&1ZZYi?L^i_))vl{iuDR$V&Rjz=q0p z>6=fRaD1Ce4Iah{=({ZJjH&vj@ z(JcwlIWXSuJ<^DDB<%IMB{5@7Q_18TVI(4BpOQ!Z0m$dF-Nca5YACub(Qy zW5et3A0BCg=;>vj8WUR)MPF9(ox2yB$1l3^Z*7P417F)jMJl+qYMczYkc|iO0wR1q z;}F=a@1C%&8TBn*uaC&HkoLVuS+ti4N?hnK-kpJXcXj&e*`o~Xy&lmK7&(eSw+9Vv z2V+2$_t{im-inMp=f0Z?QgLL%SJ5QyVFV9Lsc2P?A|NRtVMu2Ld|Gk!-AmY*yI^ujNoPGPQSh^8UjZW z$IMK=BH7F2L%mQ3Ryc0BZ6P=YsUrEzRYxnqWw5~TT=NCkYaaG|y|Ed;{%o7nJvRo= zX>E;`#vY6@Kb`%0xea!&ZPK>M)nZM-rf;wK+R%0}{{DfCPVhX;XkJAx!sbQar>`fD z;!(tft!Bm)>pPDTXH*5lfv zUIeCX@N`n_fZL|o2`76-a%R^x3aAAU?{ic-p_4t!PC3s()c!xC#Xq}ow#rf8He(!H-`;e65H|{4IS~OD zxn_)-Jrpc(F@1i6hE{({9B}%5MFj_kLAX z|HgtpZ#}a}l>v|6b65Y!w84q{&|xXzL9l<7aWh7`!Lvc^PFN5Jf=fTR7cu+c6`VDx z_^us-T7ngsX?;jP(IDEZTmkRA$HV1jR8$G?)7m*Z24PidY)}9bv9?MXMZ3mPA);+s zv~Um&83$a2XIsG^;~W>HI|gr_$1_>JRD@6NF5_ ztM3)zq|_x_o_;D!7x>KFO&G%L*{!j&3WLyhr1STM4nc$BU^je>154_P&|hQapzPEw zyEjC^ZtXJ$%U+Ga>9=3=gB?S7_FR{*|3^3MD{K_l2k9`^oGgv2A47KTd0P8aHE0_e zTMjvLAZZx5U0QJ*Tw>lYJH=dZc1y8z#qbbJ`VJ>=v7tlA=%d2{^+p(aY0EbJM@P{+ zwVh4cZy-B)>S3EkI?DHrT`YM)$E#feYfsyA(4&*^r`C`Ho6L3U{K+k#K8)<6oM^?F z^1k{LX*HPq?0B0;>=P1K4HZA*3e%ll5ZOAMkfd|`J|bP@V!ZA!X4E#S-2p#2W2z?a>UvN7a7 z>lm6GUq&y(v&g%9e)W&zMq1O>Ri(qQ*;li9b8$Bgq#K-C9z6Gh&lx?z z+lf%g$)L-JIxy2V#mGu-!n%X578>6wkz=y>^LtWHrW`n9Q|ps%kT>q zOGq7<$}G$68-PNVjOjb+9%QlZR^BP4LEWmwN6w=IDc=iU=;{wa(8Bk~TT)-FsQMlz zp&Xd+o0dHB z=FXNO?$o1|Wq~y0O{#>77+2$YPtC9A1rx|S(fRYx@m^fMx^u0Z_ArDfj0<~TcHxu$ z`#(Wk;}g zT-)^VY@`0NCX`Oaw7yk^6Z zbw*tY%Lk}>G(u}p2NO5vb<^mKAF9aqlGnerSRM50bH&+C=s z{@(Mk(0oaS_qu|#n}uyK2)y-Pa& z`@lH#-@0Yo`8I)7oleZ+n@m{pNDlqzPeZ|XM`3ds6>SDj2fqC-Lev@;!7Q;Dct2Es zAn|7ay~b-_Y}m#`XW?Yn>7^48u}QdfWFrR@-kPYY05)zu`ut_wj)nXHIhT3T_srj& z)=FrYfS>j8t%qI|ArR*nYfDNH-7|kF{uB!~Mm(&ejzid!W@h$Ds0%G^7VgKs4`9u@ zQ-)UVJt*3H^YZTB;fN~c<}O;A0*mmf?=HWGaQ%MX5>K;XSSl9`kHpnM>_egS%+*%# zaXpk9DyHI1*EyxCP$vFm$A`ohkn{aZJLwizK5C}Dwp~&s&*Plxl@Ce<&^{=}$LXOX zp*Ll0dSDouYl1!B#J3~xX|9^-?+>UMd)XBDy9r#Q`xmIyc;GO9>oFrICjQWzZVUL7 zV^K>)M&aBz5<5y~zKgd)J6iiNtG*3C<-Yqo&uE1N^Pt*?qXV#!-4lE+zXxl%g3ZMD zj=^$L^{0I98yGQ*_UQ=sf=lMs3(rA6WZhT4xmlJ2r$hENwlA4z$o!Y@*h0mfAfLat zBPdWv6!e)n+KY^nxX{qZz?V*Gk0AMev^CA;*FA2;k}ZOJ~zxm9U(vkg>ahwWIm`d9{>nEIhL`&p3J>bY>1 z)`ETA#>)?Cbi;W2n3q=fJ8&h$Eq*vjL9_g;lok&ve&`>pi5RBAd(C_yKQkSUOS;z&l}ygVJUJP8@k?o>cIkm@%mn}$V;=HGmN&4ekp$cd+yNnL*=@9$S1f&Ck+ zwiuWQaK2D4G&X8hP7#Q8gOY73*_$&&dN8ti}-QVW-bxUuo9 z@jI9L%rMq`v>!LW)&_w)#~HF}vkoC5uc8<9-;0y>Si*qM_66z~6oU#-ROAB*=BX1r1U+9&eNyhMmp^=}BoO z%7ue!Y_5&re7QS=LHg-=6|dtvs)w+TdwEBv;wbuRSUExO=%{nOQRNgg0;cJiH9v^H z$f#Nuw`0u+7BLceX7)~CNtY$u!~=rl`!zb=d#Zn`M!g=-m63AObFJ9^UC`( z5&uiX)|&MBGb>(xVBP43Y@6Mt&Sn;Lcbt47#hna+in&b3I5HniJlVNAjDxWa{?YOg#NVtxgalLIuHU-xsQjk&3K<%fj=R35uzF$X?c8-E zG#e8}A=)R%JTm*~fY;caL3l)d9^Z0-if6*{LPbRH3(7y$xkB#uB4M^v|G#n6h4ufC z-cNMJ)5_<~(PhYJG+!kwNC#ibE$+q@8E7h>HuXA1>W~~w_C#uJNdGSABQM3Sj00Q5LHX++HUccpANAgy{t*lSfL$Jyn@pUBtsDXlwFR}4E&w;uRr{r zh3FXH?@DnaWd8T*Kb}eInDJIQIqx#?i3e>Jv+l?B0NG+AQg;O0mCjwZWTSlV+V2jr zG`yer;is~Y0r^zdtUucak?E0dc*B{E*&UAWPmwxct4IBC=+O`&+hjFeED2Y5Di$27 z-hhadze+c#HPC$iZ~UcGA3U#o;oNY|fcTAf`5B*rK|8||3=ki}vEG;eMQ>mr-qP6e zUEDaBH#eQRd#41HgWsHNX40V>5p`2H_!GpGz1EixwL($Oc;fve6-)A~y!X}*leub> zbZ8|LWrK2E`>1sEiA*i!$)Mw#=;pXvyT?#u6)#}q-i`wMS9@hROsq=%He1)k#CJ^- z?X4mcSihdF)Y`y?M)ZbS+U7C5k5RFGxM>6*mmjI(7?9V8^Q<{!TSex~LXrBfY0$0R z=W>2^4Giz9SS&r!fMl~!>5Ar7D2%JJcP{LL|7`I5q$Q~*i3xw_in_3|W5Iq?Pd1)% zWGuMpESP=qajkaez({7YLZQ3^ePw>#2RAZcq}0D}YAXXa+^e&tsiVl#_4+DUJ&0Y$ zl@=`T>_zUcvvHl?BS=sGVQI0v8(c?j`K)j9#A1QqzlPo9{OwDO+qgOlHG%{CvcGX4 zvrwSFrgsP`mE3X* z`JLZ)S;SEgoO^w1@*D%Z_xnle+c97j>b*>v=o6|{(bc^}18}Gr`5L4Yhp5Dd90`*m zY`-U=5R=~p_l==jtv%R8f9}ywyj+7&h1We<5_DK74qE=%&BjyGiidI$Az@_*z%lP*IB949cj%%%iri}E>?>6l?6H0O1Mb5K>1>dREH^nG_usMA7 zBQ{RoP}=2FN5jvu=Dbv?Arz@cus?5`0L*ekhNn4Dk4kp?JgXY>p=cI(^OkL* z$9G}U!mUB&axt02o`$d|7mB#G2w(7upfAwx#Fsd| znDObnGj z?OCYD#NhcS#m&+qX!7=Ro?ldl!+yrAOJ0pYWzU*Rw+$Jv+`8O`lbi|nSSS1P`@?W4 zIzITbsS)Oz3$qVuwZTQXE^4PuCye-g)jQw!V`ATHt3Bj-E&s4;K~W+D!dvBQHyk4U zVz<=p7rGqC7uUR-+|h~<#jp3)wvzdIgx`M6UM9F+2KFEDbp@B;nPFAQR>a;AXglpx zg3EToo01LaP;PxB`Q%Y5)VW8`Id+a<&9bwn-@O>Ze_^_Nv<+Ff@?ZJ>6(`x?ldz*$ zv}NO%tIVUMtsID(HB?#H4B^Rtda6qa=W;sC*4h1f49QXp)Dwq3;_naTwTt(%QPiE+ z>3=N*)J{2O$emWaL#%TQqa8{+6Rz&@W&1MZaH|n*49cPU>m!#|!YQvT zYJFTM^7gYDH&NM0Z&}O3;;BK|a3-tbZwaE>jK501YKBmSW#HOl?POlcyB?l32&ZM{ zo$p7QAu765cIrb13d7EwxEMPOhTriTlM6Jg8f!QtCqcMF-|nxjQo$sBkaa5gozC8P)2HqDyp$t+RMvlG6gs zS+!c-J+-i4>B>3=4WoYPs;cLeL!_>>?vQ*rf$Ue0Q-1C5#yQH_#1ES(@Gce=>fc-g zkt~UBW49&<-B>OC$fywluYNAQIx%}{?fIC* zehkp#L>&*4xliGOZFFM;9FF-2+7z~eVtiAM_U1EmkHt%e4JP96n86vl5-KFFW#kW} z4B_p~Rp!ZrCr&(&e^R`#4aRy7T6ZVA5jDG6ezYSN$|jfN$88I-YfVn={jv^dDQ|Y+ z+s**rlCS8y(`#3XzgXt1)wkZz5YIu;r6Hn%N^E2DSF4kh5`K{fd8pQ~tICByRkJg)t zaNZ-vz>U(+hfUs2fXDM=}2LDKkTjfMM15MFk>eaSN_ggzzC_IHlKu6E5emrpqe z_WH*-=sE;?=vs+8In|ii$LHS?Gzyi4AGt##+4!-1NXs~D0F)Q5w3hxFJieZ09NOQ8 zRb43!3PN=7SglxbrL7ERzN#N(&#|CgS?S^-*o%i=i(>?i^g{99c`8o|nX8w75-+$j zjPkE5-gLV&Aa%>svFCmnOy8C&(|(Zu-^K2>;ekPjP-5j2R~Er+H9kyz>qSF_fu#d~ zKXe|^rC$ASF39s+b)Wky=;4rJqwT z1&I?fA#b<{mw9`}z^JSYYer1WzO1D{!no{?&535j%NPs@2~B_^yqo{)k$%K{SM?6| zBK3Fune#M_Gr|%Rdp9s3t>L8u7-3AwYbt;n^g; zm|>JHG0Q(7S&Oqg={`o;hq13@Dq(%kC_=1se4>aB-MPA{ zcKQwlLCNVnLgcxYI8?0GAjm@h_(j&YXf}R+IhOU-jOd#1&E|1o!??T5xFB?T2+S*I ztWy%(VEIz){EOrQe-p(nFM2Je@G#gjyZ-p~!5_qo5bLc0+oD&w>0!5I7& zEz^rl8OMkFr&qfYohfhsnNODZYo1b(O~JmO;e2GTH}^&wVn1%>h<6Uclj|+-eZqH5 zR3-NPnP|aCh=!u>0flK#SHw_W=e)EBhP<=%lQ%WBFrf6A5Hm2 zcw@YqzTj)Z2~S?K`K&OCfs^BBj1N*##Ju(DJDmx)mAe{)2`5R5S@GTQFAIIAE5}wd zYM@)d_t^VGA9RlWa-Sh{&z?n(vX*V7;a$K+ZZqd5h`%{f-}pNTFC&L4tm}z>Imj^; zBR*JN$*0KCg~OO}Z8!a4k_*py_HU;P32+Mc(tPo67%|%U3Tx}=2(3QRsK2BN_N#_> zIK~gaV~g{rE&q)pd1`Oa-+f8ga`}jV*qw1uZ1)blUd_Up$c8nof_-p#xz?(py8)sv zU80}-?E*V*|4K!|r<&NYN;~RmATHQsFzVk1iTQ(Pm6s0S@7KG-k9?WP+Su9f;O#iv zU4`Eq%%;FVx6?MaIUh>eWt&fH3<6H?s$QE@5p_^lc|xHJ`kR~&*Y>tzoxsNkO@U^p zOPcsCm2W~of|O~h6$@`<{*y?3HGy$Wm)W6nY+UBN3Y3XrLRTbUexZBW4CBCo0 z9&^}BlKSqjwA|;UQYKX1uQeVHXAy33E!OGGJ6O)BHYk-QBkED)-}4pKSbX*|O@?r> z$uht1j~zLxaJrg|F;<4cTbK=k?Y)7Eg~H%+=VC1YXwYR3f{H7m^;ip zhNvGGjkm9?fTm)@62aKvlX+70`0!g{iCB$N&l!L)!A z*6sM`Ucj91V_}`i=~wHgYvCs+ljVy}P)9B#mzDKF_R|mH==XUjqMu%PwwrLNh49;! zpN#dUb=CodvnMwRZxS0yhvj9@Fni_zT)qh&mW(2PNu{E}Fn1%qeO7N1@}%L5h0e7l z4ej7^zV)WAz!VKen_v9?$$@Rg;^WjCELeHlUAsja!6~(Zb(O@26iQ+mED%UXSD|Ud z>&GMTd**yw&#N30?!V>vmiHoRgMyZVPcDAF&)PH}K}Xx=mL0>{EEIA3rq|I&5GA#& zP$D}Y&w6`g_h0LR_JOgX6VVJLwq|*qcwdUiu{z131jP!g_w$ZkfFvx~2WXc~O@-bVUblRkKU=DCA>6~-Povu|j# zVHXx3IuqM~K!rayH&5lG$X#Id@z0s?e(tHV^L{o8PD`JV)15%qwm!*1(g$^Sdv_^v zh2o+8QhGO;A9xLDpZnhs{jA6yvG!+Rh4nU{H>nn%mY{^fd%NZdDywz`-}Q5cSe;&0wxBd;M}V1GIm# zgM0K?;8NS&J9IP}PbSB6Q{K^Wf9*-tI%_6|4{nS+MgCqUd+*QANK%h`MwV{3%7J*o zz3BDi?+Wk_3$&A`f?lI|)r-}aK-pX2*Au-D@GLvIx2bpn4gO-bRdaMO z7LA-3Iy8ca@KGTi!pRc#zOI@g{`id(XB@Pn$6+`2`rz$JHiGgk8HY)oiMMZA@bv8n zf|eZe(GjA+&BE)qhfWc&<4CP}9vyxQ*pHXY)M0YNy;q8vY1pvIXZM}iaR^(sX6q=k zA!bJ%y-6#Bx$HB}Zok1ELhjHBV~Z^4!Mr^s9-b5FKq5~*J*+H`yJY= zZByXt^UPjkJ<$bAD&EXIE5|-7ewRnWY&2Y0^WUFX4w^jX-%AGd0x@p|Zh2FY`@H*( z!&4^sxh0Ojy32+qX9ZqKH^QdePwCF4W^9K4LJvFQ!^&@4D`b(5YspG{bki}EPSK9T_ZfJaaW?E=2op%P08g@NqMjA}FTq#E1>N*v{4>TlP zNiFsa97o3w$*7}MB#*_jx25)eEuuQFYU?HzBTu^YW)z18=Zh^5_*z*|w*PBlrjUzC zuad)_{pFww_g-y(nTjs26=*Nbht5jlEUEZjY_SoNp882aXhYPYicKXr#M|lot%imc zffnKALD}>UgD(bp5v#=DPTXk=*WH(BZ06& z@TD5b*pWW=Ec#@E&VAw||9sk}@Qv_-v;vLNoot-?vs5Q$Ll4y7_g_^FYDQIFG4~&S z8aA1|;T6xNAn^Hu?LKiFMB00f>1^ynU4_KErDl}~yf5W#-=2@~-q5|$U)c~B^YcAh zl835#DcK7o-(Yh}|B8!gH+HV)u59CD;w<}C$0Ct=Of;eXtBZDt%8a3p`ho1a0aXAlvMs zd11B>BO-U@f^2ALnAHEaE2Yo<#G~UO$ql?n-g)JsD+`+6cgH)+G-BdK zr}cDtH;O)aI;tFdfZWJ><8+dj^1QGqSXDR=hr50^EWJR7w&~R4%HlqJY2%x5*6c)+ z;1Pb2C8dxrb^jcAkNA~4I^7QvUz9Stq;d0NDhi@j9#tlK|30Vp!2W@8I5Xr%zB!Xz zQ9@Wbol;~Z$pl#9a{=uZ#4 zHrj$m(Iw^o9bzM=uYQFVvlWw?J|Vx3*Ffm8{JPA5JOsBNw_9|x3!gO|1$PTjK^3*q z|GTdfbt02+EwG2ac>dI4K7CLL)D>|FuCg@&bL|wL%zL zYb<@h=|xlipWF4%-^1Y8jUJIhX)tu$s+K`~ziKyQ!PATAC_B9M)~UxOVA5wYxBGP> z`KE$g3!RD!RG#X)yQ<*#ezqiuZyz!jh<4h}WD`F?rm>zuIIxI-mUML$ zyw@AN>-~@Tyy6WP7g;r;t1n)mwX4obdyF92zlvG=sR%qfs;O^v z+K_n3toyU;I4lQNTQn0sR$~}`T*-uvA(cCwkN#3IvtHxZo*bg1+?l5+P5Jm~{n2YV z;qy&*dt=AL3BT3uGB*zFfk;od+Xj*s5?jGAvPr6k+hXxu=XuA`mwS~SW#0(1RVTdo zZ*}9!v1*ecxp4$qR>jsi#UPZfQ9exaTG^jf-tVvPM3L6Hb+7Ata8NO!E&k~sOsIcL z6xPx(zUjqX#c4KJHSfLph~HLnB`NV1PaEMI`B!Md9Vp71&XW$w#Fh_Qt{Op1q#M+( z?ETe-1!_i%72N3%x%Nazjm${`!Cn*LE9hvRJZ};Z%*Jl#om=Lp#IK*N+wMa86_+~K zBKHOFAf6{{@X_Z3sImWXH|n>-Q-4-Sp^k!&dyX-K&kUg@hfglvstc5u#huvy3Cfib zv!yK3H@Xj>%+#i$!mxgW#*S{>_^s#hv|?%f_eXIY`&gQtt$9ZfM}qBm4yi$?|pygu8)m+fI8)4c41Ojs|j9QD{U{IDK> zi<8WwB?$K^Nzy|z$ro~)iMA|hf$n8VlT%@z;jJ_hR!{2v^#O&%n0-vJ<3l=3g2o}s zdp19|mWmat7DqW2HpH{jWq-HvbjR;YZ{nn1shV2sjsl5_6Vqkax2oZiLb$*M|{ zUkthIV{J>rj;a;cxC7}>5{o?0DL(=impHW=4juMqX6(br9QdN5Mbajpr z6TQ@I>wc~eh3iChBP&@rRifUyPNNBRo8p7tUK&Tlz`-Hu+%A;p1@B}kG$ZF))kA@_ zCKRkyQ_DNn3+BIH3)V?ZV7tVs_pL<(kbJAj3`-!~UBlb#B*`T_S;sMnAbAP%sH6p3 zvO1w2@>g9*QzZVBlm&PExrG$@w^3>L}oCK+7vFv=g7;qej^y zJu?=vk00j^;3-0^rp(5vq(+#YIA^xrwhMjHiyTuU2B1{jSdwy?aL_6K1)_(kXn5!} zm)Dl%A5V4(HLrMCuFL3RmhMOz#+4|0W zzNe^M4=T^9=j;E`VQTHX$KwtMEtl)uZ>9EPyDt~>Yzw*1R*KV+Q%v~!Ithf4`7SrI zNBVgM@w+~X{}9!nK+Mp2{xPX5kvGh$eNq^3+;5l=s@R2H$(MQ6yNJ&g!FxV4r56tQ zs-r7|Qqi<13$22wkWHAVK67apk{W{ps_G-4gmh`q}her}si8~n;d z(aE#~XYFjb9cP$p8B-CL%@F>oRYCIlj41by^=O)ikCrg(Li)(wx53ZIyy$8eG9*g; zu-#1O4PrDHpM9fuS-T!l9qM;33=O04!Rw#mVNBR8o;V}UW})(K+oi9j9B{pQD5+4%3Ucoe0gc?_mnl%}|@4|<%Mb6~-1(HY3iYR(BP4qmMl|BDwlWh3SM9Hmw z(~OS8KSjg>SWxfO`jK^g46ppoQ*Y=p(4=^3&~l;x4~5(B%vp@WVAU2k&!k?oi#_^o zLv(6N&t8ww%cQQKG+f=kx&sfTi<7p<*I;yqO^XD{YlObZ8hDXki!I^i@7|I8O{V(e zpP~Qr=^A6pGE6dIk|Jd=`IrNjYx6M{N$Gf8BOH8^)ZcGgI%_Rk-0BY;e%VXKh6$)JRDgx0mHqX={t5bqHJ-H!C7kteD5U6-LRj){lQuFVA3yX zMKO%;BvHBW0ho<-t9W z3OuzsDZd!J6D_Obt(T?5Z;KQ1QnXF%1=KjXAn zEj%gQ?kb8g(0ma7$j`J3V#D866R8a1CqH6Ee`Djj<^D)kCWG+H;X0k=ZP0Bp`B@m1 zh&I=`ZG8WFz!xPx!Bv)x5{azWmUB&T>VEkY%Or+i!b14HHVZm|D>WhxM1gb~sgl?vf$or_q8X6}+am zNiM5sZjVsOn%7uC-}U8)aR+oV#~S@#wLv!O-GOVQ4xe>OTd~W03>|_u>t6(tyj705 zB)?ESO7&QBB3x{&@G)B1ak?3vYxr)}X+1+>ff=(rWf+MCSIWn1>Y+u~xn7#YhQy5> zjyaiQaO~TDOtYmG?W$7u_3jdmzK!y-ZoUf-tm~f_e4xN?dywH>yENoWG&Q(3Q_v=A zop8s1gNirW1G(uF_{*By;xEDm&;E|cMfOC$@<%U8{MiI=p}i~hM=R0Ks0;n*#>8~p zz}p^48n)H2nrr*0Pz^QN&TTpb1KFJu24{($`&%q_>0%GeLsn$inp1FMaonFY`UFmk zZ5iWZkR06Z?$Wm_TVb)BtDvO32a%&;32x#H1ioDODcWKX9UYV?U+y9N{PlK6=!Gs! z2+o|owx$>U%gkC-yobp?NVdnL(0G!UP_|x}GXQs?36nF%WIy0tb?;nZK0G(?piPx` z;u*J!p!mcnEdQReT6>?Izr2`(5$6e?k&7}dS9}NFVU=s2PW8hwM2}$_5)bVQZy4es zESz`s>z&{k!`d6HW-^}sKdi_-SG0&bu6;-<*l$djL|)n7J_T^z=Vn;69!hlyX0pAo-}ekt%qa0IgX zqz2n#ix42@n0dH&nE2rbbkul8F#1D{B5KZppr-mTUIQf<lAoTlh?0$&I(TL#MKuA3G$U}IH+pQYF>#!KN(m4{S?gWnp<4Xq~g2Zrd5LhB+n}G$lxQX%evKzF2rxF!iM<3?^oA% zVD^iql4d{gv%2k%roU!kzBQtze2#{C#o60e3TY6V%}vue&49}Jrb3UwF<9-`pTDHI z3oMEX*X^$)kFHGlFY*N8Oe6bRxBUzQQ+jcYN^c2r93t(Xy0k)kB7IYuK^JE4YY1?= z4}s$zAb3)Z4a1oB*m>7}C??JxyR@4;2e;Ud{uO7yLUy0XlBPZw`90ieeXs| ztDfn|ll_<9JHmhcPmeo>Tuivxhq79w)n~0caQK+z!x=FqIy_Zw`{~kgwEUO!H==Kg zmZi{Y504{ed_!r01L0=cyA3Y(KZAG%ce`R@6~uQ+dHYn7++@dOjw^o&)Zf-;Yfdsq zjzO$2|H=@IQbhjSDo5^r!t)cnl07I^`*>MrAsrKZ6(J&B#OK>iIX>V^_Nk;VywXTX z2K!3cKf{3*%&>fa9KF_#&@Zgwj~-n(IZZM0*u;XH7bWajh~Gy3&SZL3P(}#D6TxzY~9Ut_z{p z_?~O+u0p0pm*|E*;tPvEHvOa9hb97L-X!J2>|*$})eVGOEc1=xBKm7>!?lY+S2&O` zUn7xDa!#v;e!pA(a~QftUS}NNKSLKS`g@#uE_98x%P+ndffD~nyP-%2%)HJgc#wRB z;`^u_cjtz2%Y1c&R?{$62>;!W*EFDl9Sx}D;Oo58lOCwmg?0c?k4Fw%#-IplqsSxiLdr>-72CW7a?k#=85Fh?0 z72o|Da{;@j?D<*5zu@n;aht%*8qtE4`||Pan}mmQ3KKaK5!%+&HdyJm7y5l6dz2fR zA4C+E;nJXw()2>&?|o2W-`)5edY2oA_iB+nHq}SAl|M**jpwi2BiW6tH?%=rlIIHl zy>~F(J)h))m9!OuNuFL+^24V4G}JsfGox{j39DX5PM+l;-ldEtFdX_o|G*nwPxb{3 zMGN2gZY29Qt-sVIn?HbyM)}q@cLgm*A$%$I46GAX-jaE45PUrkKkn!%KwXY}8n*`> zqB5&{T8Xc^1ib;*a~SYGe|f+%rvM$bwsL6_WKU8hwzy+cD!iweHCpK{@F+ZRFPY>l z_$lxI^bsHLt585YeI4{50J?GB*gZjE!wvN~PE}dgN-)8#dXic%!l? zyp`-%Ihu77-SErdah0S!*>hc^wmxWIDrT-eTBQ240`@tI{@FStPr0Ps^1qo%DE;0u z_q>tl-$&2W9Ii3&SW)fzKO3_DWXQE$&X4dXn*Zl*wp7e*U2|nRnSrS7UKviWtD)d( z-&ucQ9N|-3vdh_>$mVb?X0Fzv>(-lqKU_`N^=i@ca%TosD)CAw&$AKSU|*3|OXlWn z3+cwE=rBI=pVP))@lg5q?Vh$Z2brZJnS8Be53|mVPyF`~{te!EyN=zD#~z!$ehX>A z1smmT$wb17pDw=Kv89-Bqm&>Z#+!L(}E>pE?FIj_8hU7NE*Ui*G`@j+#~Q!d;QLHKgk!^Rd0N6 z^%UY)cR2Gr&%==$I~~K%li&M;Y7qIh5}VZ2f)sEN{KDzbP@-bW}GM6rtp+Ml_d|$|D8Mx+`%Ima8!|wNgQA$_4 zAX^r7f%^{?p2tU}Pq%!6gx=q4l6^h+E%GcgT9@QU)W(>imR;at`G1nEi-zG%;bRXJ z+we)#?9!KUI^uTvelKXq#l%9I*%9)5RmeC30cB$@A*s z+%M2g2=doi5Cp4V8)IZ2k^Iyx;X?l+8vYDq%o-JqLNNP@ z9YIIa``zc9+mL_RNIQ2NmQXG7l>7TMo;@~CGm*^|1WlD?-Uya}ptZd^H&MPRQuX_?~CgbgnD zthTleV8y)y$<8N7k$G|;b>{%N?;@&GeC5RdkrW@RlrKbV%Ef}Z&Q=)a8CYL+CUsN$ z>VfJ@jR?K_Tit2J2s~}f&u%gL2;E_gD-MVIQLnw_1Q&Uq2B8P+yBE75Ei;YD+766) zmUC8|X+x95g%734rBMCgF~3is8{b0?M*Y)gAar?&Ltmx9bpAzB;+FgPc2VovWvE>Ja(Fetf512LipXw>AGGUk}d}wpYJ{ zGHcaS>yucp?CIi{5q<;b*cUf9+LGL(qK=(Cl|1)UVQrVKR&bW_{}e)U?5S!a2ewoZ zZn<4mtyO@AJ^H)-^{;ZE-&*}ay`c&xeNVfz5x;_!9UAX?n1k-l`+2JmGVx+n_~}uF zaZI~Jnm#4Ds5w2Mx79gRtb2X^>HUT>P(A$vJw`ggpR9LEn&gWQcD`P%DZ+u3e< zxH1|j$Sl6@5i!|>gkM`$^O5|v!JP%p-ls@@)cAhQQlh_wUW#&FF_~CTsk==dqQlVX z%#r=e>u|tNAxQc=`T2;V>OfPHEBeI$);p>OTC*uFg(57nFVK5-%8bj8|D8fI`4R_-!F_yibzC+WK}}8ik5@ONJuH9k`-koWUs6cB41>$LXuhb-h1yo zA3R1vh4eeWf9sWad_MPm&bhAZeSlpeI%gOCm;Kx4U(goex$Rc;o2sc9G6n>!Owb3( zJ@YT*sv({eI!-t2f26~S^gA}!aIS*i7d@{B5g2PJ6@OkB2cuG-4l%2KaG`pxQ;hS5 z#hrP*ILc0N?f%(PjlR(Y!>4i=&Md&Yzb2LJ$k!MNQ^+}drxhyH$sa-u{{93~OA8+n zpy9M{vK;y&;$v#q_L=m6m(cTho};tyXFL64aMc9pd>Od-hH?mWE9pr7dg9#YBE!yZ ziVx86U)qS8UV(;+0N`I$ z4xK#~JKF)%u&`}9w1~N9vAy@a(N41<5k$$}Xg>hv6%WoXrgcLLQ*FpopB}Isy3xk} z8s9HC^H#2G68oP^9-=Qg;1?^s!*V40)`#vD+#bRAK3(+OQKnhoxG|~Lj{c%}YEJTg z^e>nfycgdkO|&5FU3#MVgESt!9+jx z=^UgDe_dz(F$J1pUvr44XN~Z;{>ysM0DO*p$4~v81(y|@7yY`6@NYx2ME=_<{GPjg zzWPZqR50rnhCG{qi*p>m9mi^OgpQ5`eVH*|$msM|;r9NUDazFgV4;3=jU4qB@{|_?{{=#HPOGLW$E){^V97BV4(j0iM2_!2! z)c_%+#)#zg5QMpU9tg$UMy%P~E3LadK(bvDorC$wNCJQK%NF!g=hc2{J41lc6!pQ= zICob_^0*i8)dhh>TOPULQFu$RA~N2=di~T5-mzdJtP6K8DcxU&{e#P0M~QPF6tT7D za%>8?zvSvOV*h3=^~7&tq#ceeM1QK}BZA=f<2(PFra*EftF!`h^b?a(^9r|T;GX*) zQyFbP+&TVh*0t zE|t=wPd(XxvbsfL8O}Sjho8i{RLaaupLhHt#E6qTmM+F~U57ow>rF1q#P+)y8qEL) z?bTiy#Sh^2#iqEQA`#riZwE6cCqUMgM}}i95mJvOxb_xwgK$;mS)V5(Q0MY8GW5m> zOrLxrbg>0>f_pNxkWMqf6aPb5X` z0&MK#P%ylOzRb-@u|GJ+>`Um1U%T4@M>uLuXx!+AEg`mZbybU)>$$Pd3iFD0fB8>V z#S`%PT~}Af=jMm|dXg;Waxa8@aNoo^an}7ML$S}f5bwGva&u=8eoq!@*&oKbKwZ}J z>-i*LGU;s)l?UT+sdOL|1yb4OU%w<^IpjA(}J7 z5a!T6u})D3ORPY_@q+XFH?qLVtI(Bs8|R>AE52p>g5lisEA7zhSZ|bBUQ|UR^h6H?;clRuNz>@N;WE`65`Rl2#42WJ1{!9lHyw*k=}B zlhnOagY({(AATLH2a*)?lan86;U(|6!N$o|*id^hAv0GAWBzlatHbDL`#n=vdO08B zdTb}cN^x#Y<+a0#zL*cO#VVEcv(S*+7tn#6r@ObLGZI7Upd&}aUuh%-q+SX*9M32N z3K~bl&wqqBq;@Lt*sI)L!r*S@{ORDr7&-*Ytv$WMfJ?ix)&5uYZv5 zH31dlhjtk&-9gkXS*oXG5+3DbkzUarLj8VCyg;ARhba6!9Re)PjvTjv*6cA>p+0nQKH@$TkqmWn9l=(sLryv392(QSuEr!9YMU!|b9{six ze+nNE`k}DLiS#=9Gp}aeGHS-UL78IlqJC6AeC3ELV4Rx(TQgGDvBT)|5Nj`#s%--2 z@C&~ecG3T9JLF}wh`G-LQbzm?ov=~*VC!sg2Sj{c*yhIhHR(9thS;51;C?P&#$dk$ zpX;7m8hu%YT522jGSqd4>6ikH1u@?_T6=ouLIvhzsC($~2ND)Laiuzi1BZ9wWfUo;$@9 zo{Plb{jZ}rMqxcfs8>wYG(opvO)J?v%(XmVLV8_qC@x&GW{o?e2|)Z!L zgtYE~lxYb3#sAH*U=lRFRGbV$2VlnGRgvqBQ8-m;{g}TF{jKLdAJ$5&f`Z(uwsQY9 zf!y{<@*OYq?9^)P&q#Pw>M9y5mrhQ= z+q9OYYxwzP1@vF;>%jh`r=8oWv<^yKh7ANQt3WrWkWLMAGDivpW12CasyIp>cH4Ry zxVn#%X5SiyUDLnGGS{cT!n5|-6xIQp)$1>kk#oi2>E&OKI#96RXIh!jRj64RIoOFl z#FOC{^sFRS;rz2VdtDfk5clnphhVys#8Yi3k z=kQ)WaLQ?bG7N+%W9P=^vcQ?|#K9Cg)Fxn>LMsaOYwdD;Cc z7Mg_JFsJmiE9gsoDpE0=hJAMn{eL_7`*)(*U%<4s2>eme$(vSv;2_;jyJ0pBN&{!~ zE-Q_|LSPxkqrXd#$o#vK(`FQ0UsRNASWLiZM1Aw@<}!#*CK6A?55WqrW{U;e7zFQe zi=+gWLyaa)^Q>YY*j+faNRp2JH}aU3orC{DgRzzI_~bnFlys(6UaJ8Eq7*+5)d)yL z67`>@R04Sm`{(2MeismykGr|7!gnFRJ1lI-(~-HA?EkkLeK{{(+yqxZ#Q1jCf4wW< zLvv7xhI$q#?yZF#U(W`&w=4`_FppUg{nPcgc_Qq_K72yLw*vLd3%+azXQ7EVVpXBP z118d%2)1m|psQ+B#vc;}3V**I5Bt}Q_h!ps>Hp6SNUo(`Zb6;I>t=X*v;*{+=iA#JvU9P=RIv6!mjh&jf6?13~lYC*r@|;6X?L|5f=^a_kWE5A zd*gABP+mO0kDL+ZWt@aJ9pS7If%u#nyuFK_7T}*z9o15;1#Tby5}IY?GO=ikn|~Pu zN&l8fTBd1usBCq^e-VA1)z&Q?r6tf(L!YWDTMKfPOC1%;b5P7=-SPwTDv$T6RE!Z9 z!6LR-`Pu)^RT_%vx_8*u*hj5OXRQD)%jw+)&Ph1P7j^KEa3>@tT?ygcnS@}4IGw9* ziBOjxk&!ds4cvcx1t$`4o_yu~cwlP=7#DA(^1sXmiPodvOAl2+M!SgCb=2wO_aZ15 zUlc=@u6>TDZ91$U7LDsc|477r-Eu>&BH-H$8N6|P3ASC7>M4{_S9)3P_wfnlDmBkg z&!eB8ghhs8)wuyS#?O4`u)uzxM)?NS;|Vx_Q;3S15&h~v>rxLS3<8t$3)QPn%3$U7 zqp36a9@bwPX|AdwLTkxepMK;O*{OHyixA78-{?>OYSavn@udHl9%}-{=>3zyewb&m z{&K15PzSKAw?ABD!8(72N#T4;9+Zhbq*$Qsgxrwsup0C~c0?q;HVhmFH6$afH~5FuLO;lEt&=znRfGJ)lWRXBRv?`&+lre5 z@AK-k&LftUaCI2x#5AK|=c`J;gXiw$GY{`8CiTLl93#IG^tT#PeLLYFJOg0@`5jw- z@g6uB`jkey2;!SQxwv7SEogc+h4(fQ?oR|boxr?qbt9=qGdcEA=FhobVEre1GnRDU zjcJ(4Hhi_*+zNbG?&|VLb;3Z=vm`b0IxxP-l12y^fzRqSF?_{Sa8|J@Wwves@*D&I z&<71*KA7TYWOyHp)>rE!A@8&1r;BS8#~658P-wS3m;r8Aok%rzJnx5u67rv=Be!NC z#SrhyZC#bt-}ab?Dp)h5J>3PO^iN&J(U&Dvu&xXPVL&D=`|Dsf=4wUSuG^ASK`*sJ ztQE~Hn5@ZY*{35fv2pJm_3$jjc05n7jh#c^RQJ=yJ56-f3Y$8XP!ls-x9=mp4N-7 ziXhPyyhms*zd~+-Zc*opv))ml&Cr)cPK6wn&jv{fnA5#yZ!AY&J_v$uJgKv&Q1>|f zhki^I^N(F|92r!75PqiK-SG$N2l`~2ruR`F9vqw@Pb&qh`e)(v=<9Kd{O0cB(hpKw z!AS>WhCnRYGt&v@x56gTw=P9suA4`1QyZUmIU8;M$BoFV-4v&`(VYN~Ip2blMHJaVz#W z139GLsM}w@CJ>NI+XElMuYXI%xkZ&1b&Gv-KRlH&bxXmzLD@3n!krIoAn}9GyO}Es zeVMc64(z=UEp#Nk?@v2K4=rTxBA2WF^ojihzZsBz)@j;ybOv0yuj!;AXR=3%-)zUb z8t&2w%@0fsK!?`NDdO!screvcu$jC9-j{!-@1p-FgjSWz_5tRe1uDKi?Lb|lXCy61 zJ_!O_8A@4q`XGZi?nDkJn?Pwb%qQ9$ud|Rxj-}E*rzOsLP^i9X%Yo;^K(6_&F6J=jE~ht|vthr%`uoze zhm#=OYwxEnodaEUk*6zSCg5t&VWt%?%nMw5|KJ`&6ZDZWt{?Km{KBgP`e~O@kH^8a zdQBmmFbX;!#E1ul$_YA`J4=^#~C0xlU&6H5yp~+nT~=eoORYDsvir^c(@BhPtqI^owdg z5U9V6d}ZGQU15GWFR%2nSjwDP0X>bbw_TSzVQpIIEZxN6Jb>zv&%`kcS~)=i&8-LUSb0~MjV4{o|U)SHV=0b9W3+fv9E z6#Nmswq}LCWrE^A^QWI7y`1^s7WnTJW?%xKW;$7#Vldi+ydNFV$DHwIU6TY<6 z3fUmOI(hO8>Vp5^O)i)?0-oEKIJLeLz$(wx)_}Sn{EV)+zIr$gU14u` zW>Whg;N$*tqS&wVU0o{>5L<$|((kV(r+XoGub-WnWD!WHxv7@*>fo>~%WjimE0q6H zk?@+IL=N2Ehx)JIK@s{iD3J#<5nps9UbzU|Dz%wQ3qQlj3lcNAdn=$Tk>B4=*9QjH zzjfqNiQqTpOnJcybZs3oAdxy5)Z%+G19ptST_cg6yemj8p=T0MXGD@^7&s5F~AV1)j>9zI+s2`u(xDCuIVW z>!m4pK1ZzrcGIcs*P_s0Q`f9Y_p}QR1`YUFUBP^5^>^Ce7RVFiU#g{yUqJ2{U2K%Y z0_tGwD`}jMyCzRu}5Ek)IxdDIAJU>pT2y50sLEOC8CeFX&XEz7N9?ZdC-@8+3 z0u#_ErM{r)v5dMllZ)11Kit=7Vo6cM{Q1bgO9z&kL1?y%;{wGD5Y7jFLTOgp@qKDhuX-+HIQ!WSUf&29D#>Pj+_1qbe--rgFj@rw@Ut+8BUj{Ka%@VkPS zq5<>POD4vD&*2=uf;K+(E#S*2U`h%eTy%@^N)8yJCIu3~|&VqNHe zc|gC2IkAGX4zZe%$TK4=|C5IAS!gqr=nY2#n9Ygals&Two%!Oty~zEu(`wn5l#O}K z80DH54dXz3dofeE54m2?LS!5zzi2c=c_$t7B^k&ggkes}>?J>6B-I zTOj&uE>n;3A8CQ$dC6LG^znEf%r^G;ZyE@H#TXF{4dhH}b(fHjcya3EehcJCD=_TP zJKf8Lk0PDwpQ(mne%vBwmSP2_!%m(^L(Y(LCV%{VS~cuGU!XTd-HSHd!!qPZ9$azz zS2yz==frmN5ds-?Kyk~f&_B8h1PKc1T|b6_ZP|5}(iwde8%yW%2$y2aw^sa}p$qy_0eR$B$J!?dP}G}iaaD8%Hj*^X`H|K` zwcd4Z;@u@+UJd?XorUkqJLQlCn`-#2I!~6Dl8yP+$TfDBPMG*qTkwBgM1l|Bj}p`; zmI^+q-`>QWb|KM{W^xp?E=MsSb!t{;`Zs?nuYeWJ2IvY%g{%;^lraY7Z9e_d^LVFAseipmTozQDecc=Y=jarBuSXFF!MgL$7( zO0hbxWaI@3eLtJgf&O?`nmVli56g(tF8pePG5tijR`+GtYYb9kahiZ5gU_ui*pZ7b z<0}?NL~fj}{mVFme(;G6{kQQH{f4ha+5dA!|Lv1Kl1;0xFz@mvcdl?2MiU6ndpIz^ z_#i)^{tfa{%tBJqZ!AN##+-Bx)epaJqr=GxVWtPx-0;dyD7G$iJvjky}U>QgJ|xAmN~{6PN#`BQgv$6Q!2Z)e!A7At|Apz%*mr@aP1Cb8ov46MMn+7gm| z+ta|NdbhFQAl5UC0R(Yg)WsOjwn+xff`>?PqU4<(_!qOO6^I<%y1B}qZV#8ircruZ ztpo3!BRhI=30a`q^;^08Hu~EoIc2GuW}r9LK2jKcr(`Pb1D<5#Fj+WKcVc}8q~yN7 zX}1r@Jk(H7lRy(h>}gbMY>tD!YLLV0P^_abEiii>$OV!ogh~F28nEK9uh>u82Wn%_ zDr_s$A<41gXFbfoK+v&$gf~^lmo;cvRBna>#wKS?-fU3z9QgKv1n=?v;+Ex$eL(5( z&_N4*L&4v+mJCYq`xX8u^Y6nr+)?lQt384F*9xkioX6(i-oUT$GcBl>^HHn@{vHS0 zdMiPC^gl$>|FfwSMqlgb;@jdUgP=t`AItG`1#U(7<#VBK`b+On593+$fp%cD!owd{7kEqWeIu|zf68*oQDDnhx4{3 zMUdX`^uICIB?!n1^wUJXPCQN6!<*l#q4eP?hi;x(kS%z1kL}7FydnQIeh25prCaY$ zkA=*EO`iUvxC!)+nUJUleMA1-Qm10`*&ztwemz@ZhaCQcn>xP_O(PHZo^3MrBUxTu zfB)%sf!ed#kwLLf5VtXtuhNG3$kT1ZqK{i)SCaI}NB?Q~xsSUl8}q{#*sdL1=$HcD zwdMd7>q)q4;Go~)*a@@3dgdX>kLtTv%{!ordCejhv8Vdzlcq4LKD<;04MEnOKSN6( zd?Z5E{PP5qSX|=g4H|*TJ1^2s8qPu?*~KfI|K{Mg?r1@ST_<#l$dTq@4zO;!=gIwz zVVH{vk&xf+g}|R<(gH8XkZ^C=AWL-5Y6P9Dqd2khG4km@88Ozd6*E z>E5|K8WKTY*rT9ZG1!miGQYiZF$uX!1!+{5Yf+C7BTsE(oC38Qvdx$7AzwL#%Ag)U z*Fk;Ctg=%?a8YYrd{Kb?##;9g-7h#lxytlCy=)w`^=QbC>32d0h2iajhaGV7siDCR z<`OTO-KU8{-l|%Mo6@d&KSXO&&r$SMqrMdf(D#=Rm3ZG#&XJiv+n_FFH93P^e(C>E`~2p>DDayck)&%vju zE@k9IQ600Xzruri75dKaxKdg+nzVM1hef81NLxmF_pi7-BofSI+FC4jE zJI3K20lnMkENTYfty%6!{Cb8}6?DI`|5gtCF<4aFi|6?hn)vxHc)Xa%)KG={I~o|J zeR1yxW3wQ8e8K?e=u&*4LH$fvf`3~g3i*K!zn1hcht5SlbL;}w7(5~qXtaD8+oN<|W?qCn!1~fRpjR6IOLVVtFGlJ--Jso9E}=*kFEVf67o0ZyVf} z3F*Ctel6p03K`2_oEQGtChPrB34HTxXN>cE!OrsR#a7G@?+_Zc_}M3bgOx8UWe?;(OVE`tpkA_faw@ z%%`}TnK}-|l#YDF{(N@t z5#SxTUm0#Y4?4f5zqg2@Z}R=J)ro(TVC?R;yckph@mUo&)MLNFVGr7?QAPyF2#IJG zyYPRm)yo(0{K&Idrl}o8-k;j;K$Ls?GV)I^r9|>|f~;g!Mq+OxFiyTJ7|tGqV|izi zeQWXObXUy?97Ns2qAYS0b5lnwW36M*H_~&rWi}e;?jv#R%>MBR4Hlf2IE>?o|g z;gB2sSq>yJ)-r9!$}wNds>qIhccMw(b18~;F!y3gvSCEtjN_YKIl?T&Q1v<5R!l&G z{uD>h4Xi(WITv|?kXP+(z)00T2o!gV_f@+LgTIgXnFs3&z^aoroQZzd+bTICLU6zE)r+FI&kjjMwa&;^(#9OYI})&qaZ$#!1k{O{lgDhW)-pj zuds>Ld#3aj8sdEtBFQ>o#oz&@HwpT<^8-niuC&RC=fb2SDt5L`*P<-H+J;+)GWo?Bgb4^x2!HD|vjlden%Dr{Y zT_zKT?p!@eL?740^Hl{J{5cE^-!8t5MBN}JK=|wT8Av+B)X9i*Lz0xJ*%L;K&=lkz zn1lO10^VuN&LfZ1=t8JhXZ0jJG0Y3rQJ4aqw;7vSshB^~pR(7~cRPj{{$W#i3?F8YTu4d6fA?FU@c_eTB0sY2r zt9XIJxKmRZ!HhB(ch4B$H#X*hxND2a^oZDHZdpmB=i+cT`6 zE=xHwz0O&IAUkXKofsnA6G}U4posmM=*3T;(bq_lDafm$-wP9Lo=NYRa84RZZC{B# z@@F6ueRuB?5$9RIBT8=KezKR!-bo}kaDKp4^1W&fIbCN@hU(CM0@~cI zOMMQ#V6a09SLac8xxMy!+G`P_m`|KI#5V`rl2sxD*xzo~zOb=Bk6g&0?r~A7HaHim z`6TNo?l*AWl=e(o0GqIC@462}*h!2e^
+9ZrPBU;)-1exv=?V;rm_$&YL!OzD*2h)k%{ATNfB9=0 zbHO$b%pUT@Bd=b^_(d}j4qm;-lK7?*{9|V5hER_k$PzhTi+;TLk#uSH%>>+gVs_`u z(K=Wa45WQsIt|>Bn}r&+s5h^PWtHGQl&s*3+NOo6&?=ZfNi&9AtM2?hf^|RCyNt71 z+Dt(CC(5g#HF%yErN|iaBQIO|dEW9{H z`dzsS5=Dc7e41ULs4V{Tqs$x>y~}j@i~LZ(PnX^Lw{d=Cn&&Z$eMv`5)y3WXG&oSq ztRz8+_vsEp(XB_QPo~P3+|x(}4LWK4Rjl*km*cM=t8D}F9fLbpF5+`%HI8d1DqxX$ zzJKrZDD3ysIhU`4{DX%RU!t;5Pd4j6t%`p0E8~%-b85K1Pln=~2I?3UVpayRy#?S% zd9c{|BXYUl+3_V`L|wqy_>>zWN<=NR#o}WLK(co$K_|T#{@XCSq3YHL$<_v+p6p<5 zjlDkEdT9mL4mpkp_vC`Lru|y_6YTfGqsARDAEL{V!C$P?49Rx!bRy5E;lI1%j#S9W zsae-zcRAb*PsRMd8SO_dxW0sPIL<$(NJWYzaIU0G2{IwbVI3Gfc~uSjn25WH<=3Vf z;AxUZQ6tXN%IL*mrwG`y=m7H(u-R?{xI1`IKfl27<8BmAyKR zIk-llFTHSP27&}!LSheKE;^gO?cpOgDCFU8D#3hEhr+yOoD$A8Rt`Fv$xnmr{>o&h zvqVsGSmDm2!o6P){1|ji2jJN6A6-WQ>spo5(wvPIFvY1o9fv%sy=O|f0=_HIT+X4i z`DhV%#zGZljhf)&*HOFY6_|^XE;{f}8-2mDDL~Z{41AnqDSuYF;Mi9h`j_YzWm68F zJ-*Tn5BQG;d>TcrAX5SxBRBd*N#E-I#N0y4KUEnovK25Dnpk{MQ3V|);}_WcU}`N-nqHDTn3So`KbuP|MJpYPN|gfM@QCgiX5q-O~P7-4`ua|Is7 zR5Y6-$6oka@masVa?tE|%Fz+32H%>vF0(nzSChmF1|$D^yKji}gzyY73$n1%^bmkF zca;8}B>@Kh$(=lRJP!^U$X+ckSpp{OcQ5LZ$3cF}FQ`3a5mq=BG9@k#!S09qY9W|2 ziMs4^+6ezW({$ZNt#}*ADwk_saBl+=;mJg@|N7xr6OxXPd6 z&l6#GIy1kf3<|tk=0^uH7in_jef1yw`hk*-;|oMs(ywdL=)pQi*7`_51MbV?_(`{a z8adwol3SVzD$yt3G&G)s`7R?fqCqBd0yK{ssk7ySvB6(5we&(re&W1jP=h)w@y!Lx zVce_7VOO7o`S5j#Vk5q&aY&_5A0A0rfhMYT5f3HQvoEa`J|L+FACsM9JC;jOMf2I2 zYNZGDRi)&~i*-OpbKiT=W*jPnZXb}(Z-s?9=1h6P8Qd>GZCqzL1t%Eto0lKYK(N)s zXRFh#koWxRp($zHfAceA{rmhV@JNmx{&fiTXJ;;M?o#vxU5O#6>>q@t33_Yo#3}gj zu1`NubRHVqt;~}!*EbS>s$xs29f&u1on!s#Vf$0fF^6*_&?}ZT9NsYpq%ntL3R=;3 zL1$Yg$(;xjI?|h0aEF@wz9%1c+$UgWqm42txfdMIcxyW1o`@Tr%%|s$4nf^$O%w~( z{n9V#tKJZipHeh%J+J)}tcm%0Pn3>=@N=DH1s5W4DDaJPz3+yPhinxcR#zbV);iNA z?8AxIu5yqt%|cGtm;Z*)PwDcSH8#0;4oq7)4ZNS#z{gdM43bL|pee8tb4f58b{h!{ zzWClhK6lwSKXL^|d0TW{NYKZ1Bc<0?tqu6IUmw+CZiRtwq5Beds^M?GxPyKQ<~A=J z7|Jc42k~d}uX}lrd#{$9U_X`(6)VxhT=!ZrKX)s+UcLwv+AfM_ArDKbXz9)!i)FC# zjVlV!>ja*zvi38g4ba0k@Pt*S5t4?le?2k21O*@e{a}*9{eWLrv(tr|zJ^ z@f;dZVqmMp=fNZ59fdnH&H;q?zx^+u1tOzfebqu8n3Cs=xHEE?TSf&ubgnc(L07DG z4Fm4$$;}i|J2(u-_G=Aka}%MCe`A&zeU>eo69uUu887b6Fl9%-k)B@VJ}E zdIfW$jQ)Bi423v<@Um+P7iocx2VIszYVoi;Wq9MFAMTIYZobk$UJn(|t4&wZKY+!f zs|k-emmtF5qK#9j4CYT??QiLsfKrOBF0OlcZ(nfBJ8&C)mk#4|_41YI0}^>dl8Ahu zCc&M~*gD|orijkR^W5{o`3bKH^gV0N@=T&{z2TEUa+(VEVMn}U_JnZ1LZPqLk)uT9 zVXwNCT$}|?iQ0_gIG?!nOlwu&a~cjymApFts2LgzF5Tc^_zwF6i0)!|PcA%=uV6v{ z2f?JC_g*K~Elb>%f^LygH5;lvRLH@NxZU87`wciQidy(!&Z#4>&47XC z8+;wJ?A0qmE{)4odZm#D7>IRXQ)I{d4d-AxdK)4B)~~_3YV6xyu5pQz&qC&#nCq$q zV?Y(ka;@m%5(p1F|6*E8fEu%b+fR{WC`?{yaI%ggxRLJfGnkECS5kIv(`fRcW-`>hE{{o(u3BKO<&|hmt zOn3e}ihFVQrC2?vg!nG$VdI$@Fg>WWD#fz^WEwu}+8&*tea!FM34=}uU*%NDLVeM5 zs`9@+!tzh@8*3)P4rKkey{2~dc7KoHv8WHDHsN&=Vo4{K$BC&1pv% z2O-@cL%e7UeS|lhoA*pyf$(yg7-W+N4z*{A@$Q}Q?cUP-TkPjnNK%*#PowTy8e;xR zWEyhBL=yO=P`4qkh!etkm!G3BB8RdGw%K{wEMB4SrReg{Pqi(e&6}t=wlD!tke@qp9qZ?`Pf36rgVc_H`d5TXVCKMJlHh@U7_mLZsf+XDdjYA}RWaAv!K)b_ z?B4@rui0g%@jlfax>U5w)dAbia&La%_u*z=bT(iv7PxvZyiK1Zf_7r$%4w}JIPa|< zyNdUomsjxDn`T6)W<8;H#CjFZMQL18{Efc$I6^{fZ4Vf�e~KjKOKumVM#|E0Fc{ zHEp72FMJReyyuOeng!ttZ2#1d>v)9Yo!ufma^rptulS%pOL#M$lKFTB zX!dNeyJ{nsz~!;Z$k{n)P}>Om___&PKhE2ybgaO9z?e!CCCH-VXB49$nc>=eC=Mw}FvY=Q3Dkon zf93bzvq^9;sq#Q5&hJjyvUaVcOrX!^2IUeX=22A4^Hv%bp-hG2Zw%Wia0}HqO?qQq zJ9SU%O$+V^B9LEaCScwzrkGX``-UfX?ub@K&Oye>Q?tL&U-q%+{&nq}xMy=iZtRps z4a}&~Sf^Ar!cz&obB~apB|y$SNr{zYe> zwgHenX=jXuKDBQW!a|VS}EL_4+s;oG{!ws*(Y$ zJd!?7ZsESWeW?Rt=uZxz7z&yv;NI!7^NF2X*cVqsXQ!ghryo{oF`9&YnfD|Lopi`? zFP~$+iMhL!Fu4Z_!0Tv&`RNTUx9RCY5)y@AJfwi=9MFHoT`(Eh^&o$Jn@|ZjqM2T?vId+IX&zC=DDcdu#(AI;EuINOAs?{Hgc`a;z1yma<=7C<2!U z4WGv`kHWF2aQ_o>RJb@iMI*C1LEx9oVY&b7jt_eBq}JLXMXA$O2)QSV@fqd+q3$%a z_~(sm$`G_)r3yZR^G{{D%tJ>TYN6;;Zlh7oFpS30===$sf%n^N?pN1l;kW$mIg!*3 z*yzdn#In{8#(Y8=gRUiD5XPYR&2B|%u%nJm&&T~M1GcA+uxAv&gZq7B;>d5({bN!s zhkjo3Q&Z`RI1ijz6Rq{NMcu+=-h__31gNg^p0!5~mdbJ_N&aLXctj1p6jUF8AIq@Eob<8C!n3x>H^{E}8gUT04s2xtelsq>E==lPiMd#gf5UE3{mrlu zYMqdJX#h5+Kd-$(zld1+(`+7EB4pn1-EBi3sDzDs<2>%~w0+@oumm|Chn3$Rmkpl; z>vFQHD@ygyY5s1C$c}ppi{|A;8&^mRP5zN{_%(CsR))9RF0)KPGTOqF?@0GW8b}rO-9k$*- z)dbxV6cpxICslDiwlCVkJ$8A!tfAk~Cy*%IwO_gu28E4p9$H79&9t0=#@lY7{l{!Y zpmj&w!I8C22E1`fPnR@87Y)J>{yEi6-c`Z+t%A+~hh5 z6B1hnG#}G|V{+2b^iQ&<_<#eozwgG<5~z8ymLyi^L4~m4tLq}G@Un~|nZ0WQ3a*Fef581i?)%^5 zld2*IqsF{+sBZ|evaI5Zk=Hu1m$De?-iZ5SC5?UKxUJyZZy~iQm@jfCr-5^@18`aI;ahP9_#GBb#@VC$`4xxydQ%t3sw=xUV^j7 zxRYZq-9B-kbQ$*<|Q0>3!Zv#Uwt zu#(_(Mso^%CqyOoYkXqYe+N>!;Jo3pStH~;#&b>bt*Dg2wy~;%wfYi%o*7rpa-n}{@hiRT%`7M>EB@|_ zye$LIP1esa2!tb<{(>*-z}+dbcAxAFRM41yn?g=A@fn|W^8)fOuVv>{=GFnfAL&M5 z2yy`_FY~se-|wJ}=$xt(@)i>cDQ%F0J7i-^aoVICj91@IEuh|OZK1Hhpk5F0!nF62 z=Q9B$U%z_&6n)HxyXns#LI3VfZA!avJ9IFdy!SOF1DLhAKdXz(fH9L=>RIGVx)n~& zoTkNn%{xtzAlwJEGI!sqGPeUMb&qsE&TUgRFF(lgn1JN)5YGYoE{IjrA|o@vT&9v` zzNs8?h2)eNnzt*!g`GpMen0w=8&*zyJzNYVvP~+LmzqI2as4NyByudx!niMn<32Z% z15YmCd!b9Wes}6qH0;k&QOy;{y2aXc&;awTB)Xj++VQtcNNIO^OTVjt*Q)(K$gsV93I zb157b{)LQL$3X2FldA#Pmy+`zx)e-41&{0>`CA2bz@mXxs)_Fas5U%cNtNcI3}ys&HZJ_6i3agu2>qzf3g&Sv>@<9j=%_D7-R3v$zpzHW)MLR8lE z`j1}-;45*uhS-ez>HgevXuFAgo5*R=Q+wl3aMFOBiKYbV8E(+GC87`aq4!IV!}vMu z>)U$VRRG2Xe=aP`=0NKBk8@T7$Um!QqHRzgg?zQK)+?_kz;k@g^6jTt*tkC5t&xd) zJO!U`?fseopVCx?C)njIW^_<=U><~etFi48KF>Vzq!0J+UqpT7*Ckh%KRI5XRQwo1@ex3z&uAaD{E69+d4F%-S_Dp=Y`wzM`2UwY388Ype5L^Z*OvziAu@lfY4yztun8u+ zHqqnWnwFL?#5?uy)haS!Wr+xf4xiBxK|fYQe`m6%bsogTv+GHqpHa11x12O<9^yI< zp2_{Z49trK#% zJjGAU=jVY>vZ#4TVF$tXZJQ5q59D|4s+zxCI;C%AIhA;{fYda z&GB5(Uy}}f?2VthUDioCoUb$Z-qsjPlDxchs?EgAPb{WNfJWS)V;F;XY*7d1l7+7G6$$d%Y& z5J?J)?uKPErT|CcGN{ITP!+LE00k8#*o~D#zDcN?hYFsL`-~R{Ybt>D??A0D&N;fT zeKmX-io6IdJ6HS6dEi%W{>&!R1UmjE@6IJw0%g2@U>lyxd?C4x5-(@r{j!PtH15%x zIUsY0kEIrJdX%0X9-M~Lx37P2a3;d@qmOHzq#?IDz}&cbv`x_(N;6bQBzOa>wk?b#h?k}k3_xY_d8*ESsIAN&t& zXHDur_V$s`gEA{%pCrv;f_Wp)$hcqe9Q{C6^Sg{esSzaq$}w&rCz2vB`RTF==E5J9 zg!MZNK|yBRO*>cQah|33;)F3MKQkntXhVRsSoe8(NzBFAu{!9<$Abf7Le>wF1<<1$p3)BLy7l5);?4?oEWwZ7L zX@mp~fWfuvE~l7LXWpR9?v#RgDcY7l7CZ~V#bW9*w+{BZ)}reMbTv?+7VsqGcqDH;1Cf4!0llmqE(Do5h?5ao+3vVJ7qlycQsFY5-}Bn`@Zg}d944<6W0~@ zP#-jT;o(Hk`+R824I>I~M4qt(P53BH5v-nT^LTM_0cc~_gV~TCc7~ckrkJDi%G@P2r2c;%jUDUj7~l;$H#yGP3N}g=4h`F_@HFow8#$vJ4x!_UephMEJ|f7h~d^PoO5AnfWdjll)z3N8`K zr)h#nVVxTp-B{N?SygpJoyFw!ELy90JWtoW2d}#}LM#pEh>$<#qH}Gf=hq_-D_(R? zAg~HZwA1yYA2sLVJrZ+)u%-=UVdk`9QC(2XtU!^`j(U)MSI;XG=$pN4d{YbGSF=v) z0sDH?3Am@aGEQZJW4)>BY$yS!oIhLMHJ<{fEIMf|Z`?bY`zkt3qTXToimM9xMK@Bb zhv{%H7BBzW?<;8*9BBMYCvAhdrEUgHN_rCzt)F-{EC=@!9KKfW=KXM4wn=@tn?;-HA zyZ9^07W=VMR@L7Wqp-BgZ}=E>vaVNHG7WX6;MScxtl>*z5R_%ePP$PDih)mQoTNs9 zlq2*#`SLK>i7xJ5$9$G_+oWepSXUCy-wvThPQ>#J9-f=%GZI{3cH|oDgV3;2x*uJY zAo)}JqoG?8NIy0!xc?S&0-onB)}pS^S)GR@_aEkZ`p#F!SCj%pVoC2F`m{;1MQKv_ zCSZx^ip1POVi=9vl~z!x6{c{#fcG7#t38is54^e) zzQ1FCGqf0sKeZ_7hVGQKsA>n?{|$D~UEnRiJ$y)Hy*qLV^Wg!R^mkZcf)~2?ia>)% zr$*!OJR~2RCcafu=|6&NbWo3^O}tPWiqBP}CAI>(>fy`xH~)4IHNvi=@nfSn0^IyTc<97j z0#~~m#4U42f!4g3y`iuJD(H?`a6IdW86{#_qgvEYkB`kCmHl6zF(i2f^O89?MVqfX z&jMlB=-VCHB9N228pr&p3wWg{yhh6^f$Fz*8eqRm$rNUD4*RlLl3`gNN6dGOy2MiE zGy`6lL(xv-Bk=cY=aYTCO+ez`(RmH~#hSi2?=im#c*0M9>2hTicpX2SNn`PU4&keg zYi77lxSz#JlwS^y)kH-W#tCquM)s85Y7>lwCP~a=E?Jqe2?lPKAkQ)8{VmJ`C0aYR zX-lX?eR0l*!tzQ`zCQeF26dB(U1zVxB0u)Ek&dFJYX^u1rfNP#T|zpMFV9D9^kLSE zGL3dEK&UYnGXs7Ofw@(iPuPc!xIHRr>X`zIp-@drQ=A(w+X{U}j;q>jLE3M~+xBtk zJ+xfc2HJz4_*s!Rcx+SS=D|tSmrl(L*hgdk_UCg_fzAggV{Cnt9W##HN;hLO&2mUf z6CaiCABFB$+aw#PTZmr}{&Zb1J&t%j~{st0n??aogWetuo$Q(v0nh^ z^_kfv4k{|KanmNIp&ix*5SBJo_m8e=g4EbcoBCOvaXRzwMozD5- zIhYbRhznIi{_3f!>G30|^F1+NX+xI^k{+cMhcHJ@Pkbgcr*{fScEts5)Q`e^!^yh2 z7hPb}`=FDBw*&Uz^Vr$yVd!f97CBOr31UgQViR?ppyE4`WxR zvzN2NsS|47vSg+vqi=jIFsl8_7~Bxj9g)Lx^Q^}WFK6VF@h___msO$Ojh&sG0`;;I zR&R>g(9dXbZFN~r409;jP}fyZr^*_GM6Jk*o8&4)=^lGmp2b z%|Lm6=;{3(m><&{m>#{+3Jc*fi${<@-T0_OjTH4nav82SqA{l`sQ4#q!_O+HV2wQ; zes&g=20TkuH*vpS_ThsLo@4LCpA3z2k3#DhDtR-pu5q8W|A;w}R@ubF_YP-3AW>o0 zKC1<&d8cf5F97q?e?140aW5iwx!*Mjbr+Fi4=XSSvdZVdK0%HeARYQpIFEXw^46PL|Ijz=II3r1Z_x%; zXDtU08KY0gP<3)xWd;Jrbea8oQD3x>F#b+@3K;)P^g14>gQ{<>CHqd7!WR81KyXjeaJxIsuSM@zm3`_2y}BvYc>e<6TA(H3>_+ZyWwK%L zBF@KG_|9Fbc?%&X{yq_um=9GEQ)`ZOYJiT$+1j-QP)$+yAt^xqOYzD@gUNCzB6`{; z%ask2-HxwH7JtL0ouJh&@(PY{4L*Hegzur&AV+W+>PA%?Db)xmAojOc^pkuwc zOx7BM`}+xRG?S2*Ly|*$a5EkH4Np@%%0Yfvxv5!$7S8i>e!b1@_eXyFp3P<4e;I66 z{LK5@4at|z)x`34LT17;Z#!QJ=-Rw^f8)z2e6t_z-$On}CRI#ETnhG4ONoPZUb!IF z%6IiAp$xiu8R9ytPeae~j<7Y;Ls)3E~ErAO4_$JhE$oPI00g>pG+a_{I zB)45D^OxqpHS*8X#~amf`Bc^AIJ!A7dRF!_fCc?(fq8Ea3>HJ#2q(=hexDw`WSlfC zC7>!66Px2*j&tQ*ExN%HkofzGzlOI2`{AJWx5$y4z4+qAEAk;|YRTHRC5;B6FvEh1 zt9X8qjOyO$jD{jJs=}9FQFpUtBp9&28?3|5p8js!0^|C%ea_pHuzfdVKwb}hKNJE? zE|OjFsp-5bD5Hbxu;d4bq^tk`%W0*U*+5sQo@OpFWBouT;@@ExeUf1Ul z`oEO}&_p(8c;jRa5T)rawoTc?5!e1BDo3!t&AiflDX0SOMy3@ydkunx?vaevodM7r z4_NxY4r8%>knB>7_oWkOA;mm$N{(a>4R6b;^4)r7(x@+?nTajPQDkQA?wGro{ z%PkU$4KQ>vlEKcQ8BTVJ1gBXBLMYv~%QwynczsSuGADz8+{tZozlwQK-e13HP(1*? z^~NrLOmXj~_qcs)J`u!+Oc&BmO@ZVsqJ#_Htx z$U-l1md_+;Mhsg0bDM;g{`Wc7!N@5PoX=*#91o*Q<7|AAs6*f|Rkvn_PH5PdEkj0erp zPcArh#NNYy8WJvSD|X3DfXQvypRr3gSK>U-|5vjZ98LMv7o*T8%k@1qiVAsA+qMgT z<%U3ty(6cksT)FWsk@!X#T%Uq zMUe->#`di3$Kz2DyspOHIPnLn z-q}rJ2;yJlvZooP!Y{p+JB5GHXRg(-PWq(*7NbOo&f=U$`|H1#ag@kaz89b-%Z7V` z&B|8ZF!bH)QXIL`IS&!aM)=PR+;wS+Ix9F<2aSq^jzkpd+9_Ktauls1qw85D5sL^%g zG7$Z^_@G)K6#d7mf{W+}rJZEnY}B6ys7>v#w?HoQKRx=MDAd`=c70wRNe3cR3D)Ct zdSKV+mqUhJ{}Q9>OJ9xpz;e6O&IRWNR}0pciBQk#>DjZYkNZ^D^OO&G`_kY;?Px5= zQUy3Lc&`wprvlU7MfObj1+Z{TZ<)sLY2{y4;QbKHg-Ey{nKp%eG^vg=AP*t8THE}C zDFG;mKYN+a;{G9$>3m{oBRqQ%`Ba853_{VAAqE z{R?mO$x}-uEHf0tV8K|(%yq2aN``nF-{D>=v?49CuMpxNGn+W5;5}Mh!ej`j+bB4< zIO30an&~?fNt(#Jyr??=1pW3;J%8vMA#ZD^+-p1LbuFCn+MYMUdUeV8wPy?Za(uFe zL({#IfQ)FIjvkt z%9fJDtxy)#_c?H;3Fbuh-o1M=1T<$N4|%Dg{??Nu`YA7Rn5Vus-w7xKhfJ$6j`Btz zJG!1iqEZj0y7Y6eGkPGoZ!jr%qX_gO2iM3`(TC-^wD{H(=N}T^EGFCh;J(T4&WGrK zam`Y16YK* zC#Z}C$Ew6(uD6-K{%wIExY~X$`LHnN@?O_azTkrOpK*Q~VRr_GY-*EdP;bxK!tnDL z`l8Q~&QYcRX#=Od_SeohZ|!aoU&=@Qpk9eh{6R9zAtYBxB|g@Q^8=MfQ^=uk`2BF6 zTiXgUC4>9j@xC8=D_`%Dn-8IjeEXDCQKzjxm;B(|EF59kV6uwE{jH*;EFlf=;lG!4 zBRu9%N1Ht^R5S@qrQWqMQ8iG+S#n)GzY+8#)x+y9W?jf0u@TR#!OHhApe z^soxwU#qjn%Sn?5!I(}+)9wTEN)mFio}mw3J#_0e*|-0B=nS#v0P_^CvQZGQPjXmp z9+5n14>xG1sEs&r&gXqlPq`!qOwtLi<#BbO-T9E)_2C>GJ(c6$F+L5K$(LP?lO|#H za%g7QHtNu<9(D#@sRV{E#8k=@$enJ#Vn6Z&a}2t!={@zU08hR-LKf{D_RV!=$7+y+ zD%>vnaR~L(A5RttTteQOqLPq0B>p9k6wH} z0-afsZxYatA3!2~iEZzD|SOL=xsKeVUS~3wYsxeu)p@!@Z*q2Bkg@J8(zNM)241Pk_n)#}*6+%n|OnqAm^ z_8mEOPRAM#ovH*Tvc~nVxL4Rd^;Y&ba-N(1DSkH$E`p~tn$!=-hN16ol#WA+ssl>SUnol}Xa&0$ zisUvP{qXPN7UL1oVQ?7|b7c^o1^RQwvV88-V5~@S&Kq@fuWxFWxHyeKuR_)5{k)S< z^6-mh#VyqB9=@TLP8a~yt>vqc=VPJe`EuK_y%w05p%K?I9s)h;9}|md{ji}V{X&Kx z^On7ty;ZSZvM8#ck1Z;K78^YwNe!Gg?+UD1Mo+-mR%%L5%vCUxVSe!29DQ#J50A^V zg~0i*TuSe(Fz>4W5{K#2I7yn5Ccd0K76$9UAcL2l~tqf+GKOHTiFd?sED+SLM! zmMP8uzXF#=4FDz?pEPT|9LFq*kj8N;@!*{bD^m7xtgeC4&pD?MEJ? z0ApVg@<$!Aj_>-sd<|NCPtQ`KzqIW|tB}bK`Y7`2^evQ8f6ywhRBMqA&gb=Q7uPUv zv`xx5AHU~))w#(I8g(G}@ZraRM^V74`9dK1eoq?e8`CU z<7ckIt+%j$ze2SymPHZg8$U-~52;r}Jk{@qLef>Ji@Z}_gnLsy*}>9d708FZ&+2gi zeS2=AlKwe51mrPik>7cPbB^WXAtMR62aJEoJ5MwXd5m&rq+RDgRVZm+e?kZJ_{4N< z-DpAmle2@g%p0J;b^4-ZbOdOPz55-V-3LR=8e-|T$OXwJw-&=WtT9R6U(1kQ%n2FH zH}k~%=(>To;#@6g`M$2Wgd8BXhLW3Y*cZjQuQGFn4#WA(>dEZrVJLh*ZOJLE6&7~S3VJW;Q(786{zf&P@ekmKgRQfENF)N}f1-XJ(w zHrzGf@dK}_mIz;b?uYCyF3wy>u5os;_aW4oFJ9}DFmC$^$%JgX@-LWQO7_cp%Nlhs zgU#Nax|ly1QiJLC=--TeFrxIK1qcaY=Nmd3z~%7nna@GUO{Lb~cflOzA=(2oq&m2_ z&Q|o$T}(i|0MY4RX^lW1M8`T<+zJZ!gBzr7O#qkKKkh7tA$S#G5@*2C0ea0hr@u_5 zL*2*SnB4h3XlZuvynARIK1z&SqnJeB_CN7bnkzU@J=0%mU6}(q({%^yICDWYn5H|v zs1#Bo*)ru6Mqz9uSK_+>>RX6E5rV{#%N|0UtDJ~_7D8D3Il4?xc;hphFM!-E&5Y-Q zn8SGHE$Y{??mfueLOFbK8t(86>^pL;8&=;q5d9M)Kr$;^MOWtl+&!@}?#5V!_jAH~ zVwDo0C*0e-{R-<%&0Rv2)=bz&D$wEY2 zsp66fpHIkSGcpZ8#Ru(>u{qSgUZt70mnsIPtK0LT2L_;Cdk9|~+#C0<(B^KUjyEl- zMNFLlddnl69aZf>ZzlPGTomW*qO%XzQJ0pnc7VYc{T4U3+Y6SjS3pNkjuo=l-0=OVZ4!bQ;_w9t_`aGc+-|yjy$?@nqa8M{4A`KgbvYnZ1^%Ufb4^q7$TF38^ z-+IAC7r6$e$ElLI#$n*&y3h^e2zjPm*!K|ka?iVKCYmVBd@_L3XscPbQ45+~w&@!{z~e}^h?SP#1yqhJAUGY49FPE0|LrgC2Ke(XP=z9s%& zr&SY5AAe@G9rLPQxGqFjL5q{Se})F?L-T_8lW(IxqjHw5pK%nJ?TybJL(X2%$s2bX zAC==h`1#AfX-^=bCl_@?4!D!i6uSn#FJrt8Yb(g1mH2b#0?V~BD7xesG$tDa8VeIP ziCa~0$1_~Op9gcih)K+n3bH}D)c5W5*+FRH?ELJH`DA`!v9)hoFwcf2@{l2W478w| zTzWt9YEDZDi2u(icu8#~--7<38#z5+F|-i zUdIpgyWLgyyIZwe_WvBrrqe$k%&pv3Rq;M+zAdY17(52wpB9w=$-%ra{&kiC3*=K~ zoqhfKYcr5*J zfE9%iSz+cL%wx$)R6~8UgH*B6YtaJmK|GC{b2bd^tJhdC9t2eb-@WhVsJBiJ@l9|K zhH$FCSGxxAzC5hHMVg3tv3WlZF?!%0yL^DPZUDK0hR2I&>x zHh}XA`b<&YQ4kqgeMEu&P_QWzCMW*ipZ{dF?{N<7vv!$$MuNJJha`ge-N^mRn|LhS zihKi}YGR7?Ul1m9h5aqo#}c#kMAAA#5PZe$Z{TU{4{z+O%BPuPe%fp(zr-Y@wS-J~ zY2utKKxUXZrVNs@D^AWM&m!vRwte3D0np7}I)7SV4lX^3Ps>Vehrm~hSA*1$e`o$- z?f2i8N8<&gKJWc!GyO`~Y;d zZQd%sGX*kk!Rx_Gz=i`RHARZ&Q9>o=6bx|CJ%?aRn5jAp*|xqlcO z1{?6hfw#oRUq{n_e>IboSH4ZMWDJNq%^`@ z3irGucPVp`qZifDsP5hjMAjnPGKL*c&Fkhpgt<>QH*Ajl3#tXH9&7rOX*fqd@g(I> zTnd=&_i6kdH2~R48J9ocem6NLTB*IM2dI(`1oIJh!N%J2KX=6XAb&MXyF|4Dh8TJT zfBPrF2#@cJA=y4)F%uH)JcD{RngJ>PtIaU`{IGvK_LX1Qq8E?Ym%@OGQ_8AP7r1cp zej7P40dhI2%N8{i@}G+$@|5KG zt6`R3)<&?q6bSL0E^O~x!I{G@Txzcgy1vy!NzLTIXy*vsonpL?4^Rl}x0C>Xe}neE z1E?>_KFODb`Y_ker`Ehw^TC4NgXrm-T8OT;B^Z#*!JCUFzLgoMPYVpC7G5-c4P+O%e8u|cbLPG^-oDaHw=Bd@7(CGGGP5G zVkx?XI&I0T($w;DnZRorm=lWpkx9oF4UhZKuh=e9ldoO_^v#{Yq}cCnZc0$hcoTq$ z&LY%^W)k{dYHBqKpbwE-xWMZv>NL$Zvg`RK!RWW8&pYJrRwyufIU{GzKW?Ou?(-<> zXp~1)X(u3wx_Dlz2f0hmYJWbVFS_uVXTt4Cn3llIA@8qhaG5+8&0^p4;`3KI1Sv#nzkNI|jz{<}ODvF@M{@Gu_8< z7W&0fssb^$cWCIHtyDz{urz$Au9(Mp8(&eud8=PAxEXk&7U!N(EONxz37s&Mdx}Hv z?Qh%@eF`yhpM@)KL$giKk#AA9;T4u#2{YaP7fg{icKV#ceS6GVRCd==Kh=f$Q&L~Y zmJ2_ifa0D>+39g0bA10+Xc%?v;!M`6FNT33EGpvBY4n*6n=gLC{DM3Q(-FEf+!G{6 z_dYs^oXPI=?1-anu=9e;Fd6xGQesM{1jspb8xOlgj2%!bG3v!wXBApjOn=$G$Vy+*Im>+g!aGbMHpL z+Xg;~q^3&Kib6c}*w!_+7X&5W&3)!o8XS~|Sfro$Yx%Jj891B?Cn`o(nsLC1# zXWYLxys~0mel!Rb4RnOogn0-zYkT?HZXA3(o>RFCeuppQ#cI$0VgI!2Lbt3s4wRG? z!LhXjAbK2BsOp{qrc*ZqX=+;_@X(rFBG#n|*<_DtuTKNDW(Zjb2iCuuJJ%DmJK%nv zgTP-e)LTh5)cFra0)e6I1+PR63{0J-)3QNb0;=Akqa8ue#HG6d=a+I@TwzZR{RXjH zid_eSTA;ZkBEDb|=ShP3+;OPe*VqZjb7n7tl1p8Nhft3pm>rkcI9~(D7#{U8yBEV~ z+L88*xnW?Gc2{%Dn1HB;?3%;u{jhU+pi%r}2~cSNe$hYG3+Fy1g9-A`=*Kk2UA#Jx z&l~C9hR?%47|*Dj;*Lq=QOx z7M@4Roo{uyR1HS<911WvKxPKQ7kr zY4mTq_zv9Tn+8$}cZw9ugDVp?Xr9(Zu2<#k5w$$juTB567Lv|}&s<-{N$Igpxc-zq z^&9~Ptm^!-VBgnPTr7y(%4cD`Sm$tMPK+Bf1v>1I-GHSFX26)dsXEKRn_QgH) ziLR0U%eY@;pp{VPu7b>Z@)HS9@wufjt$5aAuJ6&T%VD#arza>*y&_==U(i4iaikCG z9)}bLj$+Qz&G{l8^fRd+9&)J95%MHR7+F?--!PBf<%t zS@3VXdbiaTb=dSp*)2G~l^i!&uG)x)j1%_~EAjIU_eiafqyPMc=y)7o5Y9m+HHl?~ zCcrEB{9D6*{JpenSW!|p0;}5DPkp?-pz(3RBe)?uxfDCX8I+~{1cO?-|y)D&^@uE>2^Xd5Ct&%|1-UmlI7gt;=$W-aza+DlOp;|U*f4&0@V_2t z(ws7>!D=45iV8Yk>ZU=U%Wa~pLztsI6R^a)-VATNWo}&GKu(1CGpZ}qlMvMzs&0w? zY%Lk_$a}~YjB2k;(_)(iI`#MaUw<5dusO#)r)S8|U+Jb(NKJx!)VjRP7v^Eo`Q`q0 zKFrS?lNgu8d25~@X+&EU>hs-3YJOh9IWo;{nZH5(a3i3GnEOQ&T*#Y@=jq{K3bME8HyBom?H|V+!s%f*{mVlj z8FDuLtav`Wr|cn#ET0EE`yTn4tVwu%ieAqt0`=@2-eQ)*$UCiA zV)VT|h*iX#rieu4(f5D+p_7SK=x{3Li2ND07nh2MOvCSyf1n5TWa;nAm(edTUF~af zt{vjWgdgSwPC>G~LH8@u0eG_K;Sgk32(PGAbfYo{VXKK|ZKV}Csb@6p<$Sxqf3#s` zhX?svdSph#%N?*&Qle}75WfeTf%Vyg$j4HzdU*?f$JP%Pnz?X3ClReYWFLea-T>b5 zbWiM0D7kB59qJ*(!fnBUYX(}v?n>~=4Z}z3my|@PQ+j?Y^pc_I4A{n62k76ze)i)p zsVeM~i4^*te27&8RhekR?S^Wg%ap$0-ub`(tXbxAV+u%^d#HawpPusJc}=kn>|cpr z)RhGE!ld%yuU59WUwy|TRu)CCVwF`BYhu;!?r&`Cn z-sz4x?yFKqbLk^1*L&z-hO6YCaPXtP3!co_+rOw!o zl3NNB9mRX|8B2EgQQbMv@|&~1ls*H7TUuDR`6-2EGaD1YVtQU!WB~Ia*Z0>_2mOKM&X1H+ZOC19`YlNi zm7I8+ZdXC#+lq)#7}7LU%Sx(-50QU4SHF~l-><3L2Of@ryM;r|cf1!Y0;aRy zShs@5l-kX!Jr%%8^82SW_DMXJOisn9zqkGKks}H9Feb0c3-~wsK=gurN~9s`lFxq2 z6d6N~@RdgkQCAya+SD#~H>(w>R9iXoFpoI*@Hx9;?{vu0lp?#DgY%~2*FoVWi`N(UMjmBxvF9yN3|96V;%a{ zJdj+z6IfI<9}GY22d&|+yCqAV(6&cyH%eOpEqW`2F66*CP+JH|yN*MM`f6-h7y7Nu zKXDw!d{}?lfj3#<$PLd(c(aK7y`ZnnRGJr1*VXAPb`d#KBo4bRvFKC%UcC8c@C9-oO*71^AbGX5J8e*#?wX_BEpKukv3?ML5?e z)I1*lt&)g)5&ENV<9Cr)^Mks-8F?CA1w#Zo*F4Bfdae9T4S7gPthdd?8h}NsWjLj< z5E_zPX6*Ek7h$@3_Jm*^Zr-kW{-%BSev?u69K!hlM*-n8a_~IL&C>_# z>Vc17H$p=)1;H=B-<8lBhIh$D)~Dr=clm5YozlG&K0ILS@^@;6%<&rx=g=o5S=1TR zanKQldMelU9BLrzpnl&9?k)9lb_P7Jd1Oo!McW{#vx}jkwhXzN zTgpet2@v?>+>t+A=r1@+CJ~Qw;PCFfGf9Kkk6mWh=Q)DsAus77*0|u@c;W18&KYY18putd9KL4B6@d%?`Y3BMD_i)J8zE9Gx+3{%eN{) zecNS`o?!+y9kYiHV*g(Jy<5}`_s)8E)^xrpV?NQr(aO!b7KmjEm41W1_ZLSu@3WNZ^t*L z^^X9blD0+K%Q0YVY7R)j{3%|>?2d!J<;YjCP!nc}hXgNqLvBN?gW?!_*!MX>eD25F zo2Dhe`jg{W>7xZ;9Nc9-qlS9uK6%A55v=<&Wj!h{)PjelqH>5@6P$bf*vOG#81C*x zJ5ZkLgkrzGH6vb}qiH``lEZ%ZX3yXEK!$oIo`6UmGSpqSaGqECP>6Z-t49K8%V3=C z73(kLQc_&{v3VZnI$qf$>4KOeJm0r|@($e~tf8B;p{oT%C_f#NszH5Nrim}ZsSb#r zdB4gvo&nJeIvdJ&@%_5}N5a|xbH#{Es*RE{=Pj0U#_k86uP?=Z>TNWDv&F z^Uq)JXJC3nMz;ua;!JZX4@aUO#z^=Z>6-Z{M0R>)@v);m{nc~QZ>Ur15?VR&umN>c z^4w&f6Z~r7P3rjc2rM4r>w1`)1;oVE86v63scQTbnlPOP zUcV%lhWHmyx4U*EytxeY?*DWO${B^l*jmc^BGefr-x_@Gm;#d8EYeeYy&xFT?_2AP z{QtnZpDZ+8P?Ntfoa2ji`8}0K{r0G*@BT@Di4X&s`g3=NQCHhEe)w6*Y&;MGVpfe) z(N}$q!F;tN98O=mK-vF)9YFlXiRJq|+&IY8u77nFk~hpwQf!O@@3E>cBCT~G$Hvc_ zU59;1qR^OsaS7a#ms#*h!1u5$W-D{O7G5cPJeoqE+i@q4B%jZd@Ss*QUjludVLoPs z?lLpzli;#DVUIlREjp^bb!1xnr=%r%tgvQcY5E{?(W zPMAdaEG$ryu+ih`a-BonIO5bS&|rW-|K&gV{}eAsY0 zT;&%IuiI&XoWotO*uyXv*Pds&xoiU3rwo}mai3Nvy_Wi%X#px!bt~^?cEaUjPCoBG zPQd#DJ)@9P^fSwphbBB51FCepw?mGdP;k`58`!@%``c0d_@DyL&k zB-9~?$8^0>oa1so=4~y%TsuAr6|Yj>>axv&*|bjYnNK(u3%qzs6!jW2L`beJ|>dc6xaDhKb7{kXoC9ornMd z+H8-fk-s*1N6sMoKspTl)g3YzOotBkcOt9E7dmG5O_3XYNGsOIM_S!5hemNoT;atm zNPXl>l-TNp>Wtd0DT5jKwDgYi1J;*|gjvZK`mJC+L8iEFhMZOcyZV-LfG`rnd zFDP+;$bX-nsGt&FXBb4NVXo%VYSFNKIP#6NKHpz!L4Hx)7Y0$iB8cCf^?klo2vY9! zy?5|Fmz!r6Kj(+scnvn@pMt&6#cflkPtya7zfGbmc~e02YR*mz)?YzXRcp8UQ-SE9 z8so<8L;xKUQ*!xC~bD2AK>VnBE$SczbHqov}eF@nG5~ai6fpMUbaNJ-DAVT?i;rUN^qCW*9>{T%SfFzFh+I9LU2YPWK6r3mFRR036f)I%Sm#kU z6TaG3ZKsFa7pAIdlkN`S)o0T%!TrV>U77gYzC*qFkZL)PKl)@MmV26?7bgQz(K*_?DOGTT-sFTK z=Eii<2@!sMErTXewbZp&Y2a5=w?2XUNTP3T6)yfkV3)H`upe_Lt<+{%zJ+66P_&-# zHPR?x|9(cPr~-9D`;!ilh#^=0bL#ORvv}}e@~fHEFNa)>xP?y&MbP-7C8guZEO^iN z9=fW5+@=uhimQ-gLS8YHgnmDFwy2Zq#4SMWx1WxWtRGkt-Uo6R_2Qm#{Y-Vw4@eae z&}ghIh4U;X7LH8q;Hn`s_y+w|%n65?^KVbX6_sr_0?swWM3~lh6$-$tCN*{Y$#?WE zGu+x!!TR?}w`Lu29_kD5>6(Cf0)=1kH^`$Q~({j#XZWqTZ zJVz%4Ra725{<0Qm6RdB#Y@iSB3X?Joo(GP9e9MJ~nxOD>wy`blJ5R?ih^B}k_hXcx z?B@TY@)LEWH#CId(NY!FgKdqZ_33<>PSc7eP#b8TG_e1D2bID&fTv z*(-&T2{6A%b}(IP0(OX;gX~e?Y<%hPesk=1`9E%Y@!O(byVY>R@WdY&r}jK?f(`c! zzBBv$dr`mtPE{>w4|S^EOZSDQQCB>9l1n`|7ABJ^kMQH(NzmH2^2Bl@@H#x*Yg_4n zfWB^98rpg29Fp7r_i^~nj}X1avC`A9dUk-K6M_-E*TxvFKe*;J$v&b z_SN17{+}yrL5^|{?#Jc9-47f^k$FSld@Yi+#&a4RQm>i)Jl6&MK0+&&Oqfp`X#D#C z*7vWjST6|VBftCCv(i(T*LWtRb6kuGeMqsQ?*c35!RxJc*eKSwW)a4nK++6BZW=_2 zVZ9)bB$9OUB-W2?S$i_53lP+R3es?7m?Q@XHNaU{@FT&L$Yo`#Ni>C#Ja_`D+0r0E(aA=dQg^%Sg!8QHs! zy-;n1xSe&ozQ;VT<1yw_n+!5sK>ve7&EqrnB^I#M^{6ro^Am=4XJhEhqoA|O z{?(akoG<=fmXSnG15t;}xb?jd&~sLQb@o;Q#EVO|z0*RUgzpLQS=5J~l%~^7N=^rd zi;jf{GRz=fHtlucB-W3VVdgz6$Wc)+F)=nLfO9qp%Ms*-rNty-1lO4b1iVD%8y$+yruUk&~GjT_CqV_lkyK1}r8d@3%qUti+6xJ`Lta-nuhqCDYmu zns$-{y}w3bFo!uT;`=mkJu^A`6zgKrlxF2Qu~CSbU2hien}C6a-pfSU6Cf^jG;{yP z96qbCyA2gXU>G)7rsp&ai$WDzLb&e>vIsJ3*qjE1_2&H@$j#BaaJjYU&H!}KoX=h3 zYk@AU4b2Z&Kg9EW@s1EiKd-etX=)?t#jwG9c*`m&g>wU|*&*fHp?-IDJfVIG8hGBxL!e@#Gj zM{Q}D8$aJ918Mwf22h@?RhjxV2Kfv*DJ8#KaG$oL@>~SxSxx%SBJw9-P}-~|#$pU0 zd%wbl(*j&|891YXzC4fI1xj_RN;noH{9We_=3IWReKmg_c?0s_c*O%I;i(AGn?cNh zQrc0KPr5b*^QV6ubG?oHL%GFYW_dWDT3S8Od@2Lz|7g0=*cCv;Pxv=9SP!cg%Ohm2 zk3vZmkA>}32viJ1@>oR zLZ}y@J>fg=Hu7=ODLOsY#c>bV$r%0W$|RVdzvW5#zrNs2gPIlUuh0BVsr))q4&0lA zK_)o&4&YEN+D=-4$xe3tcfT@$s_+RfXUiB=>Oc5;H+ckJA6xbAeAfjh=vHHI;{Dg9 zxWN2s7WH1g9e?bQxxNKj* zAe8>8nA&B`Wq7$iU<`RP^csKXo}bEwGZ&rzxnMnBVSR9?9DOB4^S8ND$B@g?ojIzH z_nlN;Aytz(<{cd<`Dl*5m;9%W(mCfwa1QM0`VzUPHNp1zQpoxE66fvFyN0~CN-gUs z#YqSv@hQ|8D*#oBu*=jqCm#}XyW2}q2`fb;O-^>`b2>qEXAnTK;k9 zt)#&)(9&sF{YmNoYO+8=Jwpk+y|u1w4w&2OuUyHl(+$k~V`smNp}y-tc;s+;6C6m5 z7i1b4g?qZrQFdoA=WguPS)mx5vvfWtxuWD*v(ufbsPR7knaAqt#kcoYh zMv+=C@uwMh6m%nB?i=z}*I3k{F&p>ETa0wNxL2Wm)=GP121wU^!>*@}gOyv%r=U-j zAdydNB!cfFm+Gje6(#aE9`5)(6GuKF$>-89lr^w(-fF`O_4m5{Q{EOns7J4qdyzbv z2P>4_TQfH350=il zq5mUI`%(OU zTew&5ITBqXEsDDS3dxa^Vwfj$(^1TT41J?n60YWqdZ040Fu?j?4C3w=Xpe6a;Ajr( z=#hUjuz2 zM&-fd3aHZj%O;zLdrb?;yOD)0Q2Ldy{Gb#4YhHZk!V**AL(1liyb0zB9)ZZz?^EC` ze!tQ8C+cz)YzMC`BJbw2@2^SpL5q!#`dPO$;O||^`^R+Hd6)Yg9u~^mp2I>!n zT9zXUyXxTn;8j}{tShV)-bh?1Mc&G@<{ZKgJWp%e)U@`GfC1n1y2oB6T#Ry!WN5F0 zTSXF)@hk(7(gF>CKcTO@ZTy#bOgW6-9)CTGoc#z9+0!&qEzrhv((UN6VQ5+QbTW{h zgJdU>{m-#ZluZ-!@sLBkW%roo`X!tXG-o^&=q?5`mA%?^@;F$sygE{je9c#?YoFgO z_ku|1`hOIicRZGF8^(>2QIdq@hg3pHB?@&aL_!G34hacGW`rcGYzf&F*<1GBdvA}u zwm|5u`{P** z`o~Tm%Jo*p^Ns%EqZCK*IUCuNvhA-)}(eXk2(} zd=dG{UQ40v6X@rFwlee$_}Guxb`G>d(~&w$Nq5Z4nBTD3KU4{+(j1&)@P znW;pWdl(K}s-?N7n`%SE%Q$d_C9Dw~nO9`p0>qTGJk;~x9++|d53OvttN zB=XFdf@joY55n6~r$70NJ?(iLB;*AIS>qn;&M7M8Re$V7h;QOAMW2wH(`{W2=MMPc zo_KW==jv&uGq;7Y2PE4`iY6R$E!>X-_l({R0ab>I)OkC+Z?nH^zcGY;GJV5H{ji3o zyNL`rs7K#^pzWjKg1q?u&mV|BU2K9=b`kO; z=tE9@FZT7V5ppQDzVALqAEfK~mWNmBF$crqI>eBHdl&Y3saf2Y|KRy_hy_$ps*L-D6jxwgRsc;+Q3VqmMt} zh`;(|BLrLymVANy&1CMMN`KE|o}MSg!0|{o9AGT^`VjqAb%(;0|57)@VIseR39=UO zpkFcuh4exI&DA-&P4}~sjFlS zgH%?Kf4D04N9Y;~bl*ZB^3b}Ut!f1DYyW3**nuzZJ~Y0A{;zwSBThrs6VNktWdBOcEX2p$ zBFLQU1pd1jtrY*~ZWW|Vv(YsJiA~mvK=uw$Gn2oV>{E>T>cJfcscy(lJ9kmAVg+=V z56WZh#Ju`7G>dII+^bF^Zgn!$f$zxEx zNj>8lhkaMg_GMyk24UyRyXe1~ZJ?d@slA!K2f3rIkGW2M0|l>39N(QL;m+~M@&MmH znDeBPS^=DERkh7*4kSWXTcJSt!5R?UbKG*;?ZE!wH)^|*s0$7>aJD85L5Pd0gHUKU z+<2G7!J)eX5@Go#Z{5M3wJ$OgmB|yJEaw!XnNTui1tOiUw_i; zOjvIN;VP!}XIvdnNF^N3lZfx@y?oKhV4Q1(-b4pY6oZaO`1gBBbNG8)f2es5^qa4pp!vL{Tt_O&SV~)9qw8Pz_;_@VXG&495cgoalfkhJi^pnji8^+J-zmSBCw2zlkWl1RP73UQ zmi9eyKWP+BM^)#BBCkzBHPbJOsR+tuZ%S~;*`jC;Jz#h#oZrAqAkT_HJRIsmN7z3n}wli*)|k32^Yxy#$_H}m5OK)9rF zOqD4d`PneR_y>6bS^^Dx#ED?o(4Bih9l0z!oPR8DkHfDML>A`gr}_*U@`}FmP{nKD zaLj2L+GJ=Nj$wW=dgS|N-kn7_|1ILIOU)Ae(Yf8@RE+*HCfEMsl<3#GPj79|hCbKi zba4Wn*W;5l?{?9@*Hf;jQZ=&zN;FF(Sw6^TIdMv>&Isq=-5$>}^p&m;|2wCR-+P{X zVR^nQ^_W}R{zkJn2I=gi7bz_9-#tWXbFjVw#)~SLAEgb!v{B^7?rI+h%gx;Ah@Jwb znZ{kxgdym1)EzAk!#uf4iNoId99$ct;jnUTg%R41Up|kcPtJ^@+Y0Zkn$Rk8&7=)d zWls+8YvOYzc=UOGa56Zhy*c2G{>6-o6<=vsR)FSQ(F@9i446uBE(l1%K9+hx$9Cxv zU=z8N|6fu!lnLBpd2(S9BHFu;=A#aFn5B&` zhu4?9ys8lWHwya%Y=hh*u>THu{95Gf=s&s4cR#}%{qDm&x0G4X5AonO)pfHp zIMSq+EsQ!QRV%4OKLz&1{tdK_!MS7A%2YOI?;FVFDX+L4#_xBd=kvIpI!ImdamhfR z!;jBm89)3-K+mk2b=iCl_Us1gj&S0=y3?uiRLBSzhN;gR8+1d><;RYR{>bSPG=2bS zcu#Y@cqit?GEjH)>38jSf}hvfJf<-0P5Ja*cS#EOSeF+(R+6hAV%wg~ZKwh&`BQ^*%2baeuJBefsG$e~3MLBwH+I5PtW(ajFWPhd)xy z0$)jxi}!e8m;O9*ifT`lmCj88;c`onSz{i=>rWC~H1S?IIIa5nN*fHZ+9WOxV@~Y} zvBL+{`Q{0~;(hU)yzDjQAC`}PIk~bSOM?**D-+YJNb!eQd~14}^MdgI;3sag1u z9HP!0f?R-}-nMBHJkMRipRy;60sZ3TgNxYz@$8)v8zwa&RNz$Ao_!f~{c%vLV5$S# zi}&IT`le_6gyW^w;D-_^2F0g>a)?n)otEn}_e<@nQ(psD3H0JG|`)2pv#&eFGl#!J&vH(0-1?EpL zPeD1!D9x#iWuQ!n3=aIg2xihHl`LP7zr01fW=a1Q2%wW(`wGrC#I;=V-GQj@&!?~#*Nu?I1+=hX*s0#v!Fzxy53ih1#5 zMFsS|6ixKx9c`IK067)eexM}_s7A8Z}*(s(IrUT zHq!X`rVYMjKe`2*qhK&me6BsP2U-J;40X4m&oM>4UCy8l90s4%H6Uj(fzH$UiEJhI z={3a)#pZxHKab2Ro(vq?=Z)+1_l4qm`0?lS+gdFG zye>{# ze&7o!{1+^M`}1_og!WMMnY>(@84xT39`;NB+>2)6OQ69;X=UUqG3|Xxxq&@TY_vz4 zJh4x@Gxylp{Q=N1(YfG;y-D=P=r1*}%)#&mwVxOr`m9xI+Bi#!f%(9~JD=4NuvuSc zZ!TK`q4s0xQ|NzBC81$`a%utI55D}*Eev~zh~-L%F_(PG=BD(KfqrFS-% z9uE{h=d=XkCr@~(C=uYlPicLbBY3VFi52#k4Z?LbIuj}Mdz3wiJ2&o{1Vk+oDbkVO zU<-ObR3zhm@(c09lEY3AvENuv_pKHPHx82w##919r!c*#paFf1PqUuehWKk@KZV0_%~&WAiF9>vZRm6(S$nt6UwU|Jt||I0q&n8!tWvcR|zK&HMhgn9GwtB)Dg_0%q)=B6{)O%2@1RWq!2-0!%s+ z+TDABpADKW+Eu`2bI5%^?6;Wz^>6qDL6wd|t~(d*N~o`u}mQ2J2cx+GW}{hN5(z7~9+St?9Cm;+7u z>ts%N&yp2jabev8T);aUIy1&NcR+|EOt_VJ%c~5=2@aT>Mx#3m0P&HVV)Od8=`%Nka&8 zcaeqpM#$fvWXL2bob5pVWfu(7R6(dT#qX@~Tqt}aZInEWy>c_hV&1U!!s$^Cs(0vX zX}f&wDqO3A{Y2d>snOUYzL4KE>OYS@u($(TI3ENWw(FbX+_{~$kfeiLo#dn8%w)p@ zU{L82DIa+T&Itq&$Z%^%|F$dqnaZ0?Uc<39?N;kU*F3{c-GaL@7; zF9PQHYF+d3dN{Kh(eu-B9`ds8R-_}h*86UE9}nhdbb}q`SM}4tDy97pKLzgJIZjgQ zjWk2Qa~+S@3-z#0d8U0wX2L$Ry z&fM{+1hKUoTFI+N|r6?1%E$5!rldVGa=@ynYm$WPX{ zl}qotF$v7%7Lih!bGT38E7oE`K25I8c}MigxcJ;jrftB!n8{lw4&k0E?A=Ky3P27J zahcfl^+wobs|X4Zn9lQCIoezF!mc zozxY-#*F82>BDEiFUukImM4WKM<*zp`89hP{mQ0Knwip(Gj#hIOUZNFLC`oMu={y= z0qiO44IflL$9u)Mdd^F{ z?p;9pxu!qvJo40>WggidiGl3gCC9%Pv7e0I?BdVwcrWx8e8*au4t>{cq9SK-evk;? z>p=dAa3;sgI=6Nhr!M<6j(rkEMrG$k^vmF8DyvdTLI+eOhn}X47y{3J;{l^G*f;(<-szTA9mI2`sl7o?kMawzoYdu@e%<3P-Bs|WUv@ZB=Dp$x}KG$-E@{4Uao$%5l42 zZm$GM7wwM6egx1~;tWxY9e`(lz4xEsUf+&qgJ8FU{KdXejk=P$#~%ZTMqm^WC@R(Z2J3X`_<3K?qSz^!w} zCx2rM3hIuA2483cg?_JTp;t3NPCn5o@~i@M#>0H z++=OU=rV`E*2;&0tg#t1MSnk`#6D4_TWSH^6{vspsGna&pDD3-dE`}Q%mM#CVE28u z8T=MGHe`MvuUkF*cHV(eus2++)m=mX%=h>^@q*d^*Ym?^o=kv5J8`?y`4t!vyb^0w zI}X8fEslfznC}Iy{z}xB(ic|9pfwo?J_c784t`9YbgP(a z2RgUg6qFYlK*e-+DLiHYa``l!DtT~Uz*Ea>SB*NuSmt1BJEP~O zunavtEA4}&V?ZUy{D=no_ncYE+=)K4!{M#Mz~A`&&2KbS>I*LcXBE#Ag*SaLFCZc) z?}R-xCHm#}y%!;@<_$<;4p;Np$?6TZItZ;EI&=@`cpS!0w}Zb9_5+r-FvDZcIG6W(xCs@e3EBDejWX1 zC-$};?f(GzNVoA(yr=jH8ghBYqAuX^*c=j&=THAs*6n#0(ANs?tOhPXnR))Le$->R zoSV1l(KlqOF5#4~J_fOJy!CmyqadX#aa}}y9$vqoI?w;I8+e)orL`xeh_Xtcs=Cd90Bq#XKAU7@bC3_ z6s`3bd$et!E(`f@B6b_+T#?)Mi%_+y9E{v@fAL~Qn|5IFN_&13=RDQb-UDY#J3#Wk zfVrdCN3J+1^32*0=gWeVh7Ve?Uwv6Sr?I;nW}^mvvKlwQfljG{{klmg{`)dC4fplj z^#Y&MuvfNiKx%;F=pr;_k}Z0tV*i#o`%dHWZs?pUdAaoYQZyujpow;=t?OOVuJ@=zN0 z9EQB7_cL6vH?1t$vi;jAocU$Qc_elg^Uchvj?uXwnzPSWE;cD!=``2xi zhQ7fgY%kryTuF51=C|33NKmNHB{_t7&&<0wGAz%IgOI*- zVleA6u%|ctJ8!)LCF5seGaq+=lI7@C0nCXJ#7%RaU5$cA#}&liBk#d$`@NLcP9O48 z;^ah)f?>5x|CO@xEG*ibceUhe0m0o1tTM^itIHIkK!3Xt2pso~9to#IsH|J+8^>nk zcN+At2QGnm-OtncE|Vbq-J;{!A>=MACjL5&ex!OSrSvIt)bGeywS&LULh97#RXfaw z@T}YV(c;hBwEN-uzY64q>E^dFp&!BHYdZ}EFLIxo@kSP10(bAK<$L1XBj5ATwD}Bj ztNl+;B%luJDxFO61iAGFEEFz_LL(sAKzB|b`D8u;d^?K$1c+Fva?r&-pQW59Z(cvf zz4~JPd)A6susfZ>nx~4s-|>^|4$cFx{3PS&vMD~-rB=SZrsXg~)3VbhGYl6yW*IuW zFc(O#b4frJ`Mx%V7=l!=-<`} zVPEva+AUVOIqbE+bUlQOYy^s<=nTEJ7h&XYt^Xo&1$c6TU$mMuft#w{M^Z`bFFPQ3 zYEj4;h!n>%#{Rh=ulYY0?*r2?^rma#&%;5yZ+dIivG>F8G_RP9krm+6f8633fcK54 zOR3k82f>&x*_iPw8wga1?)Rw)VDK*OY z&+r~QKY<)R=D9(MV~wyL#=2&kg8batcRpIU$1U3pr{hNdG%4}bL|X~WYj*h)llFGQ z%QNBKnm9*De|uR^iF{8Q)~uuF1&ZN))S0Io8Y>Vc_UgD1`s`JB8Arks#~@>7^Tyc` zyif2=gl9X9!mPnnqIK46uzlrA!H1kWg7YCB;NuIo!6P9plSps54D&B$(Ywj<52bxo(0bKi++=q$QykE zrZY3xTUVK#6uv*UcMs(?qfSdh`scH4LuPB=%YeY}C6&zb`FkM8VgJeThG=xNJ(&x0!8rT8X)>|fUJeWNDX17@jl^0W7Q z;PveXG^FU$5ThZI9X*`{bz!CtvQamn`OWm>m-;QJHl@eX{OB-y@cU0`NK^TJ*uL4>tD2 z{wurne}4C4Q)2vFna$n3qV8c%s^*kTi8TSJehY~u;vOJ+*E64As2TIQ*PkEVtbnrN z&YfeHu3+`q@MB$3BT)J%^#3`IeOy}83x8kEgR6x?#wX?luzKSgeW<1z*zWb4Y2bOD zc+hCf6Z1d;KNriBt2&@ACoowVIiNQ(sx_Y0O@OwX($5s+;(YNtJ^5l8`(aNu#&_PT zfidp07n8Fvr&q!Coh+gNC?~ZJ3v(f_^~uIpojmLhpo@EM&N2=k`ttbRJ{^NRrfYH6 zk(YG9n)9tP?I`GtYFrG|8UbsU!P-x#9|n$pXeawL3;!MjKRK?p0A3vSx~bF)AkSqn zt29c0gGnLnh7-um)Yu-3!+q)DD-=x9*aJ~d(?FUYk30^p*ok9!ZiuT*Q@lNbe$%uG zQbvI^D12cp9(8>ds9$R6EOQS)S^u%{b_v{H^i>;AtY(7$^molAoYS3mF0pVtddVc&xS1=LgQ)E)^{wmwmx-qd&wYi~!Hv#m^PueZ1(6F+DT( zUod<*@rDQYS%NHQ`pU3>o@m}6lOfC!e7be}h+5l$T&dGB;7bvtpPYG5b1oHZ9Pb`# z$DTEd)QjKvQl{XM?)uixUCb9a2*Wr5r?N#S#2Gm!h$LrkH#4^$jO%}Kbh$Ff4I z_=GS1{fRecE+l8eCqZr(=T$scMHODrtxrOWgo&yk@>FDGlhsIT+u(1;1dI6Rj zWd(g!1|UHt&bJ@)K3p!TzGuYd(SNP{#Cxw3m{XtI`pqJb(v^!NvJ`U@b;ImUpX*@o zuEor24b*4UReoLH#$47FPdQ=i6aV+3vFj81fH;m=W;D%oz)Eq~w|9Y)kRDkP*2?i6 zE;ErgwPFr1{kOOst5YXj3+#(Z?Vp661(}04(ceT&IQIK)0G5jwi1UQQubRjnf7Z9`=X<`-K>{(dgUJ{rneLrx#~G{Y@B zg!53v^*?(lLdc7uTR&Ct-y|>;sy00-$LE(V?UgcHF)*xd7V(W^k9mLmk@q?1GaovZym%LcJE`_J%JjW@8LfJeak9ieZ zz8%`_5!j2Qwdw8a0fp`*jx<-~nGhNKOLtAdU)@`Cx}Hm*5*2nj8-1D4KXt-*kyjbp z`*o8ndjLT0tD4%aWgw5bFn;4M_U)8t2!ye;!L&^+)dc1PKV?N(9}mX8e|srSczqj) zwVHOBru#v3vUk5Foj-_;auwY^`4dd-^R=ij?@JUE&F)h!534p8Br5Dvpsr4|u=r0A zoLK8N8^(T)UZdfmpNhy$?5|BSmYxGX@x?9g*im?{dzr8K^CDQB=GMOEgZi87=)EZ9 zn;Kj?L3eT!c}W(HiT}_an*91^M>_i25;0nK2K!(_StDa@%0|G8^hWeb-7q|DVXz6x zo(6$FGX+WPahIoQX<-l_M!vHEr8gbs&_$0MU#h49fpumZ)|DQZJy}{_pI8g_@ygp} z*zdWznJD+crUYm`Lr3;Dh9FVm1jT!k8JJFNSCwZ(9VF*dw}{dZ$dc}wO73+5Sj&47 z?cj5KCuyBRyadwU4LST(m$~1w5`z?tJlE`&_ zdVSVcx7RZ13sPQ)TGdPqf#+h+q{!#I_)HnhAjOrL{4@F`> zSkKv_oS0=;>UO7Ob3#41u%5aCb0NlodH4MXanCrGeJ7C|?=3mnM~x|Jv6uIrp~R;} z$e3up(1-h;)O(h0K^B<9{kLgA9_R%udlWza!~A<__fUwk5b8Qb>-PL6n2UI#u-D%- z3fyNB^OC-I0o(0I*G3;lfS=ouo^{MauuwP)QHr!d`fwMcU}ZiWa0xNZ!al*MJS(XW zas&{c5mzA>B7mBc)a=FPX85bEcq}%(0eerE-WscP0#&EN#Fq=W2c6fxW3N^XHQ#s% zZU2xDaA1V}WA`L@iTd8Pe6)xh*=#d9oU;$S%=w$KfcmNG+&fm$Wr%l8ym;s#-n-bp zU1Wd^V0)aV#BsbBY%5JZvhr189uZ7yow~uIQuOvnStp1TFpa3OnzUNT8~f2DU2B|L!*CCNij()6U<-8d zy%f{UK>bGA^|>|bhHBhyXNEj6$C^U=nj85W%#E+p$i!E`)Fn_(4(~tYy_^1lwS{1= z!s4-4G7ssKF|@LtRrvGDSKs_R37g}m4>#%b!f3`DLm`1>c%3*(Vm5*NOGC{>=Z10! zh_(!0L%pUd?=@wjZ41zmaAf^O9q7}TQp`J`j$30Z`!(`2`i*xtnrcQMOlQ}q zRRMp`xcYe}0pxwApE+Vfn+Lq!SKdc52x^Ph?t|y08)x70EAW6X+gM5|D-o1Qo>@^F%Byh&G5L_>Prt<&R1tv^`a&L1x!McuHBUY;e zzJIv;<)r~~w?$4UX11X}-0459c-CR)GN$MC%?>sTf zo`R&JgY4{Gv*0&3b+h7l9`4n1G<9$H!f!5}T6)Q8kXciZtuOA!KUW#ZR)YQ-qrT64 zPv)UUnR-9_1^Q2ID;fH)Mgl$aS>|05%#(~(yS|*rfm63DNEZXAVIE_j!JP|mc$v6y ze7Xge!CNK;`=FAuRxAcw+d;G2_19YA5*#31I^en00Kbm?IkkHS{kdX0Z&WK?VQ)25 zUFQq>=e9YQ+7!#7$HYV`S-Km7Nc3;@ZA`-2Z@oliu)tXm|^_eB8#niNm_nPPvOTa@i`A8$SEJr_19ZyN%BnNg$D)7YPN^`na4T*7|QCxqzp z=+A1I9;RkRU8`GSXfZt$jx>h}W#sk-0Nn2)9}Z{*ES9JHFSE8KNfM%$9VXL zukpKLAToV>R}b^d4cE3PqJQ;+3$wBG4BpevUp+oUi}MXF{b1%n>0UTyQTkKz0{SES ztWtcwVgBaE=BJZl=o{K0_l?ah0`iVy|3r6?yVBv&WpH&8cpP4uk_9w?7~jR|o0uCU z&bdTtkG{wr(MQpfV0L*bNgVwgfkU@v*cDLsY`kP5^|1r)m@{`Q9LMi%%;Th& z`cvR>$lHXiE(5yGX^C?e^nu!i#5ZL5$or|19=Rto2WOPxnAznPVaTc6ntHz)WRAaS z@@qr>YD6))=Wopa2@Kji!Q8Od0dkRwr;`v+pT(YudA@NYE0Qz06OfzeA$4?a0GftK zJe>G3&nQ$)C__$tc50q~tXnxW79TxLhCV^*#PUIJlM!GcCV9Si1@}&kjoe1Goj^xw z#&b(x2@0;P$^udg{aoY>-M}13;}ZB#3b|bTS_hq5 z&5!SGE`s^_Y(aAD4ez_~pxpaeE6CU_39?=4M6M{++`+@$P((qV&Nf>Mx8F}cYuKLy zwkUzcHor*FcDZXcd~Fm?>5TqJTp5Ew#s{8nQ0EXWeeq-GS2dWA&bc1LzNO931M3@< z*jus{uHZY8375L)>Q7!7gUqQ$$x-BauANKGNX=YPoy~pU zM(Ux={%OIbAJHIvF+pvu0(nB6e!3O~L*P6%7uJ{40n+9xS_%~1VAw)kr-^%T<>M)B zCa!pH3(Xz)e?QZo%q>JS7=xKD z(aUWa1lZ`Vq%g$YI{Yy;gV9e-cQ( z+;WO#K%aMyBt_Ii>?hE#wqC_vk;&-g%rAOx5sE@M`%~dao_+Z+}V%ze~n> zM`wRaN;eIp4Z|;6kj;+td0yE2}-9yJJEZ&e8gCb0iwjXW#;%{*}4Gmm|X zx=Q?kZL+kDN&t;pi?Nt@;Q9Qe$c1kNh99vyT*aK$?5j(MUn7_2Fu&mzIquK@()~T2 zXo>H~BiynliqI!bAvk{naDRMkalZ!l8!h?n`dz%^5ViE;?cKxZe?EKZq}JICP%@GC zIQ$0tPA}0E?c?6t)PdcMHmD2Y>_i#uP!Cvr&9}b)1G&OORO07xUV3N3MdssxdB`&p zv;X1TK`+MrFj^ROo45ptutMY??GW7&=9_~`OChVRj75m6yjT7K|DQT_N9?^Nac)UV z`Z`@w2a!Q#xi@NhfV0HFC@5nTZWD4Iwj&>haEGgDB6S5y>)(8M@og3gB%VAX{e^x2 z15)X0Q}}(+8cJ5WR|`HfL%te>AsBcWX(NntafQ^SErrxB@VLtNEinUmg`fVF6dhfL zksrE+waDS~aNzB+XDvbhwerKok{O`W7h_{YA5NLG$%P3#r>>WL@EguSAFg!0gQ`tG z{D^$GZiYTfBB%b#&s8kpb@sWd7Ef?5%Hy|aNHzc_lIaQe?fXF}L@})X*CIsrWzEt$ z3_*bRB&nJlzNeqsXVs4PL*4Y@0Fj?%5LxB8OwWz_;O+F@7UWb6J-KT9m3IgV$_sAI zvm!V0&P%R5*-`N1G?!r z&CD;MA9yEb{P_e3(`eZW;62BTu%^K4ItW&qvv(G|P!I6B#^oFR7RoeRYWdNRV30RE zag2cZ5uve|Jc(uK7|E{Z#rN^Iz3b2V|8|4)heErzQhDI|Q%>}vQ5W*9g&Q;!ryw@= zrA22C0s7RX-%?DVZ{ky6-5>rc&@XIhI&lK~SiX!8y&CJpzCG1aA>3nGr$yy(+qHpJ z-nDGTW4I5#+a^xmzIFac0H$WD7)BFKP`pVgF1$E9>^oC>YyI{}W+E zj;i}$Rm=3Zwoo_>UVhC^S86`c4;C}&}#)^j))Z;6GeaNdmkB^ z@qDu+n%k_3RKx*HUNgZDoLlF)yvpW8^@c>#xkheYvyC0oH7q*YMQ+ z2=%|+NvL~#amJss8 z8G}3RPGj!kt^OG@OjZ%CPxG!nU4iIVDa=L4CExq{R6rvf?=`FQoqOna5pQ-}Eg_o$ z>GsR9nZmh1&!Z*#Sr>EVe?qH0(2o3wh!4ii*r#YGHFD_R6r8S&Dw1elgf5Z1>|asDV+~Vz#bewm2RPb_?!lt zv~kKU!QC+B6ES%2RJ30Yc(jW7BCnVB9+-CwaJL?{L0|S>9;23|>L{rFcX!N1WEobQ zh1v|myTFa9SCY7t0PEFVy6L+;5LVqK($0!G6i&KMOU#k7b&D!x=G7wC=B;o_>>@}S z-S~1DIWNU7npQ5&@EY0l)YpLg_jH#Z-2;%FmFH}F9{I0*r;ON9 zw6=nE zx`eS=NS3wycwejme%z#Vn+w5vHe+Vq@?Z}*H~X2>568iU6CxcBcuxpWIj48+<1h#a zlVy7*3~4s&g$5&b0bNp+on6z~|2@kFBtGu+K7g-uY$+IL)OE zG1Q}fenD+OfqxpxyT~0sTF*kHE{E16>Re zyw1}uNO}~|kU}v7>PaIJSFM&osFT(;R1NtJpTAHH^iBio!CQK%$Wv;`@78IHn};*y zmPNB$n1AC{`Q=-TdT~2xzXbAM*hoe$FkEN`bGK8?F`U>d5qzF6pZYUozhN+UecA_B zTm}u4JGfV_I>}UZ2y^|XvpBR<7J%VZfTRk=GH`ZW|5$|m^rH^m^;s8|LE8VRVK<=? z_$_YTZ9tv+*(!@&zU34oUp*?>c?o9lZE_`|w&oK@=Kjuk1FW!9HNW9dKeYr!7-BZ>%5Z7fa zNKTl7f8HVNVD z@4vqU5Ev9|`$CNz)7@n+4cvE$sdHrP%jHAXaF5}?9MsKh*N*mnj)xD&?9OCLp^isH zzPNt58Mz3Gd2)AqAitXG(<*Hhm=?z5*$!dvr()3Ct@${pP31t_JjIZt40-wt-5d;*sV<$lXbuDd4k@g>%7&C2k;ZH|D08K+uD6U>u)cxUi4? zI_diq>6k++4f^oK1aq{UE7O{nQBQKF7W0s13xb^&%>n|`op6$JEUN7?aw$rbBh-)A zW3S$qpuTH;$fZBH?dCfO3R%)Z1MHY*`sJE&u^%~Lq~UrL)|ihAD+tm^Er4B}hmpgW zvmm-zUO@HWAsqhEVx?!@0o{CDB)wFNVEaXmu?V^MBvV5Fbrz1|{@}Gu(NZi-o#H>c zJ~j!TZ%<~Jx})y;Du!|HU?J2AQ^V;y^933TH z28A&8eSPGfw+srdZ=@|j@P}%OE!^i#I`>(aO?3d>Gx7Hx+N~h;;mfxxJhO1#%<1FN z^(Ek0IN`O6x{Spv=ho+2-{I-is9^ce`0p-VgPZy9;h)ktT2<%4>c|0ACY3gjt$9Qo z(qD->1IIz1&`G$ys!!Tsf!rjgo7_6cYp{6iSLKSF<^v@ppA=C?v&bobx+<0p42rr5 zf0i(x@m%YmwVxzkRP_wvvIQvT${Pv$tlJm;>Vi$dCUbHJ*>~5%|Py4s>-$8o?(z? z6db*&)&P2?IyJBRJAu42>+4*?3@9=A>%2!kh{cG23yW?&@}o@aVi>$JI361 z&_~3>{kA2d0c3)llthAufU>3Vfgf^FBr@vXJ#b$Eu24s!c+7iqIZVUb6av(HRA2F# z8;1bZ1Zqaiy``M-6%amv{Ox-R>pGW_4<-=2@zSIR419CW5bbn8bgGDlKuHL!{a4&7qpL_?_hE!gtH{MYuCn!D^P!1kds>DZ?!5I7I!F%o@n zYEt}$5&A@8kCGW{AiuB8AU&+$Qa`Bo*v7pvTLMuA?FL?QJcmRZvY7h{aWA?b{~{bY z=DUw>#WyrS+S;vciZ}usqi>_O8$y2nvvTUIB}34&va-jNi(DmHaiRq&^lkFqGrIOR z6J)JM7`Ga_vA@SGQV97bPk)xadew=2BOIyw-!mvCb_DMJ)hkn1}`P2fO@~ka;pRP@3MiH|HD1aycI|6 zb>uAt$^Jgo@SqYhxK&#Xye43~Ug`J0s7z>hIC$`RQ9sz(6=-)pMV`!`gf7{$*o&|p zpt6nnJW*r(PuC;f&_F4oow7ay)+afv^>Huseb>`piFXAoGWhfce<3$3?mqL?+g0%R zF~LPJe-7RjEA!i&N8kFPlZo>ulA!y0U*8)$0$j;npt9aVKahmu-}Kr!aQ#XXWU+?6 zWJ#(W)8G+k9MSFN7+Zn(`tvVGEEj-9IpGyCbr<-MTqv=?+>1n%h4-V4CP;MeJX4K5 zr-$0q{z(O{fFJw((NA3E@R~ATXnY6zA_kk!u^?~2)}~`+Nv93E)H-IWa31H67AC3@ z8G<`iU)!mehrw&&a<;<~?$y_iv<`(1fVzmGP6(kNbXC1~UT|X0l|e32&=qyb8BUsy z2IJtO5Zk{rF%3GuhnHIBW?)m$e1>PV3>Mgqo<50r1U;X;Jm=Tge?Gw0cm#PSRq=uc zEhUB^^iMu@iBvW~)k&fZ^!@%xrVD$D+`XB`zymtzi?CEa9d&dA|G(wr|GP?F3IFj& zR@Cdx0H4CML&G=bA>P(&<2c@bxSVI03ej)6yrn-e8H9SvrH^^Nykl_hv1nr}ey&3o z#&g%!&51mB}HHp&r{na zj|k-Sy>gD?zmL5$t2CzxZ?3kY^8DVZP#yDKOKW6|zsuo}l341ywh4$%Z}d&GEdtGo zQ)a#W&G2xkiFc270sgk?Wpt-{}6&Y#!cuHm-Rs<9?Uv^d|AI4iK5GW6dbT z=j-a)meegjpidXi?`p;K&Z;M$34P98TxAE&7fynFY`(D2l@VZk^ZWJfsTH`LDl7d; z1p6m;zgm#{_QKHctNqjX+}yf!N2dkzR<>;>sp+YB&$y*KR1tySizxE|mcl`Z`7rUR zQVBoDA8bD*pW<9eDCLu>8iBGON9hl`euwnKu{WuM>fqB?ar+hQKTTibPIbw}-qus| zjD>}*K;iA=%epPox7u>G6&g% z-i7n%hl)1l+0jc#0wZaWlPRaB;DGa=jBl9lpZcnjaW8xtF5TCT2uA-%iC&8G8RH?) zbBhor$6iSOq_O%I%)`wl-8w$bmjjy@rRyu~yI|_!3-;&zb1)`T7m`|wI@MhslQoK7 zXxs0k(nh`@V-GdQ_`3?w^f-L#^#tZQ#HY)@y`6#YE+777p&r39aWF>?_p1e!2K7z& zo?fN@&05Vo1o2i!;`t*almLZYA;-z)SbYRXC6S;INVHj!a+ z2zgR_|NgVx*&TxM=4ei#nQl;%`a1aES{-C{ihkZe-A*+^+;NU>1o$4_QVUWafQd0J zkk3QEaD<>?75;wYejF+k&P~V{nlkyb_#nn~Oj_T%ov(zB#Wg#_!*Qm*5w_1OFC2pE1WSJI&+1 zR6;}I2V^3rZej6eE#Bv*8zUNLkSD|!XUqGqb_sN977p2AZ=K#5{@4HUBM(lA#)$1~ z4eSJe?Gvr(1mQ=fB>@vXkX9D_ryg@ijcw&gPc+NGc{6A!+8*_0tD&1t$TMPCy0lU} zHU*^#X0M*;r{hx#3$V0ou zE#7E505VcZ#q@aousWxAn8khr2m8)s*{Di*Vn#bJnzH~eEl)}gL}6}r&_(7c=I=|- zdy`zL!~Hy+*yL1A2;647oT5iI3jA-Q4#|q(Ua9&!$rGGI!Wl$QHmLOj*?W_*2D>Tf z+^yv9nw^7T+ixLtm=|sQa(eCjL>DNCixvhHt{}(#wdhOii@1^-@yPTx`oxt~V|HI7 zf7701QmU#Hme=})k8bt>>vR?C_HHb23jcZMav!efE?~LR0F6Qk6V{fb!RRF}gT!&z^g9Wq||d9X?rZKk`fv=#}N@jhcg&GR7iP zktI+m&XOoLNB&}qfbVO(SCx8xJ||k82p0!$H7zbKLp7(wF|p}R`2Onf`sgyAJO`W2 zc7M|$yXeUMgq$kiItYE@D%1h64x(RXq^^hY$r!}OeDwP#JPd!ErO8QeV?WKU zHPvfx=dc%gQFb1APpqT!mPN?*W7Xm@c<{Fh3hBleJqlXEcQHiyoJJ>{kB}6ecs>O` z4)Z;Z55s=Vi3{g$eVGL!gUVN=MlHb4dq%JzDjm!UID2A{W8%W0)a@^`492_bd8w?Z zXD2f~h@Hp&yvfgL)QQOX(9s?_utvY++3zi)xurPoT_0FzONNDFa-$r~+oi`A`6u$= z-XWB*IN1|((Za|0!?xyuiH-V$0%I9izy3;e{@oxLILu#Z<*tVCnh(k4$>=}y2@7j= zLm$y*cSxyDDMS-E{z@B~1HR9}?-&Lq!C|oKRs4}9C<}L`h{qmEHY!Tp!=|X4WUX?f z&`-lnCa$)_Y1rpI-J?i%;TP1uz80gLi#&(3=5RTC0_3?W^X@k;fL)X+BOB@+2JI=I zo~pNl9g(|b7~3dtnTJsZ>tlZi`-eaI4ak@JqG(;^*$E6hb8*_PMKE(Y^_{~T%#VB# zNnD`B{c1;N!11?jP_!_sJ(61nb{)3~oZe!7ptMA%_y_g^iW0w!e+9UQ-G%;PK0wXd-RZH@C^!Iht6(Mi)d~wJYZXv0xNdQ}uJ%9B9_lvx?pFrM zX4DxEWJ;lkO@!#9Bl^dXz;*y3eI-T3$TD*!f&7TFR7)s5PHpO$|1I_Dnwh^dpQmxJN{socp6r>c2 zoj`hmIX*uQbEkwS_W4dUf~HU0{l2;p(5p-1m_CcY=WCu#f^X&_!$!_M2+t|G&HmYn zQY@P{;cF+*84XQ57 zZel;Bg@c8KJo3!ko}Kx~i+zHwo;^oU^6=xaVS2i?2#_wi1W0g^$Gg|n%=Z1Zw z9ZyAWi=&>?U?+QiYz|b+e8t=HrZH~;8UF0O5L-|%DzY#H-6j?f-y-kNX6V<5>SjDB zcz3=~dxIP!(-1;Qar7B7_nf6QE&% zW4a4-6L+vjo+?M?rSLQeFf%0R@}qCk^tVdr#0Xs8ZngC7UBP^Ts0mf#Jo3`+oelY~ z6niYn(;fcH2GUgXBU;M~&~kB0trPpb2-v!}_&r-eBFq&WEjYnPFO(Kji=9!AQS_pPNGFCfjch;Hv6VY%%NvQ`t}dE+HqTqO#@f zw&(;LDn0wpHVAW*Ms$%1-#TFH1_Sj3_5tWUp|eWSZ`hKKNZ9@BeZ#=f-7}G50QTVyI#`1g_tH+4iUfb3VbGgN<*m`1mn*Ya_zII4S4^dM(+S-0`CjNoCHxhPr?c zt8qzjZVgQGFcOE}tb-%Lk(B>lA^&*ySPc8wIr#lE>ZT&|9h44eX2fd$VRsV>3;=0aCI{K!s9?)QM#|j`_Hk)g^2O^>*qV&Fx zfvCng9UkN)DQ)qUIpCf+{=(Aqk=0V{DR&QwQpFzH`o=s%p>FW2-+oTwx&Y38aW|*X z-z%7`kxfa1{C!0V#vrR1h*7dxU}V95_PjTlQv9R9YAd31PSmR_V44Jo&vLPQS*LH;c&2%61?$?}S z=IsX_Ma99r2J9o0>09ad&w*twKYerb(Pzd)%l`e(`r=3p!m&vT%^`*b0N zS@!}+OcC|0k_;nnBp_k*ZztGH%|^wN+Q9254W`pn%fKxkP_T(Q;+tgB?F#I55em03 zN?(fs8`Cl-o8N7~@yzL!g8D41`t^jp{1J_N&bQ+vrgOl#XBU+x-47lS(?|3hyTGlb z?yfQJTYD>atRlRs;qJZpldW{P$2{OSizFNdk1sumRr#pj>8=!16;{Ej2g~nQrl2s@)G&u`2sU>UUi^PP^!ifiF)W`A@k{QD=^@h)9Fp4WrG|cc zQGWlYdhOu#!QAwMV9XId;KTz(85EeQmQ+hX5V`0&aRBJ2;n_H)9X68ETHHz^bGJxj1Cx-@-9 z6k2c3({xS`!xl#fkLL*Py{U8xa-H$HGc(`(%$Ez%;dUEW2xnoAdgg&K3F^AOLK%1P zoT8HRD>|cqJcG+DAJ?Chfz+KW{?0Pw1!doh-DXOMEw7=hL4k2NGkIZ(iUT<#V%(Ho z%J}_HJ!KAF`DzhC

z1V9jr>?9(J6r-*lG-ztx5vQe{Ri$v+c|h-_d<0~6!#BjM2r?*^n$!% z;k;{MCxkzg@rha+hOVvJgp;^8aPHv#bogc_hzg636>d%e#eGul0ql_s9-53AuSQ?F zv{1gw3G{DTcLZuujRK`1dD&A*?47AjEN1M=2XWh@XD(cr0I_1v8YO2u=U%s0Y0FoF zNLH)Kx=LIyq88taF46{vXfc;ok>gXP(*o~*8cVG1 zq28{+_wKhU_6EH5_;sfXei;uPW>y-7<^Q%CqLJ4(I6`#( z#SzRYgtnR6KS~EFd&koX6czCFF!4|Nv$Me8II-x5eLl4wmI{SpLlDiLrSjAqxv=cA z4S`1&;OAPH#I8N|{rYF055~EkJFw11j}d(tf)+|ma_DnyYwwjq|CFijHtnWK3rHu2 z2Fc^Rz!}T(zIC?%eM$?*teo26rSF4Ox^Cpz&`ec+ql*H$d(<(zGgYw4^E^s2U>c^V z=()mQ&H@?z^udJ&yvMEF6hDzv3h6Hzi*xknAm247>gZGC*SS|4g>mOY=wg>r%*cJfHvGjiItQ+pSn zuVs$nIjtA`=Ds$qd94jv?mcM^5*|Z;4OCE-4#Ce9aS&)3gRcJI4o}v3pt%OqkrU%! zJp1Fnv8Tw7K6~n}GWOt(Dw|5Gh(y6EQ;|kKVGSHxXS~~rd0RPi|5tHO^Wb@d!0F~H z)J?V82`O(NuT1D#t6@M4xn&&LA?KLE&z_!H)sw_rJ6dINn1RB|gtf!0-EbbKq^c_9weeI{=b zLUBs=%^aQ-tEey5EoO_H+yV} z@5Z!1VEE0{=_}X+^(xwN6#3MYCt~)3P&beF+E2+H%K-*=jfcVRlhC^u-a9x@0PC`^ z&kADyYR19@xyT6Snm8Vd-8wT1;zIw-3Ox&;Nm*|EIo{`FE<9$fZ@|8Q0ivWrxjO7Q ziIjYPs0Uc@sVYAWEQjkZkaS8y|D${1Fu+_ z@nbGIyepq7EfCnwEEM0F$Mfw_6@B05MqsICiSSTefJ5uN6Eb2mz%FgV_JO`YFT&45i!Gum%g|rbRus+F4_t~L zDK31*@9&L+W5dA?nCrMaGO;@f`Wn0v3ERlo<3IB&CHyCpK5C@0Qtt#IqusE486RkD zq2}GjIj6E@Q^8#t=d8t~tG_O1fl7z3YLxs6^via5Jo3bR`r!r9qPXc>*`*HdD8ej-En`LS2ftI`$iGRDZKx3*b{B5-jgz6|6F4YZ! z`noUa_IZ4+b84a&@!stAY@Ss>xeFM{qqr~L_`i=y;2`T+BvgLeq-eR0T$i(KH@lI0 zFTAMT{ZDTMXb;ipp1RlzDUq6o0y8mpN=(eYtkVJ3-7B;5J=j~?bf7GZJpimC(kH^| zkk1`*RaSL+5%@bQD)c`tL8(b9=WOc`9ACCL?Uaf6*(VZDd(zt=p`!C%ApIDOj|v|M z%XL98&)wpUKUioRMnAEh?~MSBs4;k()G7aE205j&#rnb2RiI=TB7EMc3l5W$Jd(m(933kY z-*w!Bp83mpLKgdUi>|Ke2L2rd_B=80K;3@ttRd?x=8PyZL*=gF^B7+j`DHx21rBnA zt=addAdgStCdVl1CwxPpJ*Mb$vrBc1`8ojp`=4K5yio;{xh(m&y-+WIOgkZoemK%0 zrK~}+T41GqyC7*l1QMbl>0~y&&_R0DE0J^&F5Rupen2t+{E5d%>hOMXV~?fh2==#5 zzg9oHhUb3gtLY?{(HUSHn=mtVLymGR*qr}23z_WccUjv9ab8Z)D{Sn;9;dV6)X%YZ zf@bnEFe4YZGQR4|^ARZhaFC`T9|8-yEU~nUi=ZE_Y-3|T01Q{F1+9jV`zLwf+C)(` z^6!oPA7h?kb*?x;c`p(;QjMbh@& zIJyMk2$%`}nF@xbOQA~B14 zugn*~c3(owVH)3q{GumfkNZHBw#1^rRpjtpkMjM2{`!&5angidJg*y$TxiDgS%^O~ zlGPv=KL4}(Y{M}Nw&TAgT->q$dP{*)|4k7TS+IHgucHsk^Pl6&7v#uoOt*=)VBWOK zaId{^5u&{xScxq)!L?uVG!MlwcOXJWorD|=-(EwW8}BEO<4I0geHZ6>Ekz>_$3!?z zNB?Jr4)>T^J#8QL`(ca4DE8aC4mc+h?xJc~>IBZp`CH3H@Aj7{*-4onGF?dANC+K06(K2}1l_Es;A> zFv{r~sK1{KdZt&N9!S-IJaMk@0`6 z<>9-qeAC)E7c~EkdvS9Bm;@aC#@!|%knC-_g9DzY!UuBfTG*q0>fyy0(gGNb z8oBDdlL@g;&n6K% zt}s)@BA*8?j{k*!Nk)!(Q1Yub<}uK|lD&2s_qElPQdtV-Ly-5Hq$XMwdq`wc^PhAW^IgE zK|XW!^`!-pI=H1)>B1X@e&j)t`_kys>$;flc3Kv>*_A$@%I8brsO!gOafeCB+~~7s zNbLX@dXILZt});b)^c2s#@_WFyCs#51-M1*y40kDz2!<|9{q&lKyvTut+m&9FG>qZ zvhzmY#9^sMSqt<*(46g}jh_PX5(4u&%xAs0a_n%r!6Xcd@tJ=xMs8FGi8-#yRMRk-hy_^N zyu%a!fBvgay6OM@5Y@J>r*kf1e*b|uaq3z-l(d~%k-dh`5wYomU;RbCI>=9=AMAgV539bX#TClqponE)=TL0} zC_Jd*U&cH(Rk7}UG2Eky|52M~z#P|!6Awv4F(*3v!I6zvB?nR!G=HzJ#bAKf7N8sFQs7Bd6i}{>%?g8^TAWaNVGT+Js&H7UsL4ZlO9%ve{({jGI1X0B1a8AGengq)zr2n$z%w<4+AM2xBCaE~M6e841Pt*yVem*iaL)}1pprPewhkLo< z@!2|?70AEWs4FjuIj8^P>9={y!0q~n+4zY*%qP|_60;WT4cOAxnRFk^A>}3&zzk0Q9#pDGWy_*xJEOjth6vApYg84sEwS-RG zzch8YhH)QV0n+)d&4?(xN1C=%9G7bbU(@F#UdV|Mtfj5}SBriu)st^ikr%Q5{+#3; z+7aj#Yo4Pk>W50pvh6JG?K~52{MB{%47MKu%IS=W1CKkluRxuH;w`#IP2= z8PXbqC(MtDv(Df=XmsV-XIkuW$RYW5Ql^>Cld{ZEd9Is~AKgQd@U7Q|kS z9da?mo}~eX3o_l!U`=?m!0~b?%(3wJrHYRM*`Jdqjwen*@27+iqWiNDB*Q@AgndcQ zsZO(`o`9yyxW=`HRm}_N8}M$rm77{z<`RR|}MWJ-p^R8Uh5mxY&dB>t+{Rz*ukIT$xL!~Nq{Z@QvJ&SY_{ILUM10N!K-X2A z7InSH+)Bl$Up&9svZ{soE7p;sksa(wAaeLrwSBn(NME|iYW6Mx=?ynV9E8HYZ^jzW#8=nhdnnLw==#s;eNI!Rj(5Fk&}`e=WYv(0kuR+ zXFAShUJVwqMwp+;W_GcW%0s>fr^$1b2z)O!KFQion*;G~6O$LH_*CCxRm|G33dwi)G`RfsV9k0=^MIS|c;@Ma1lc^K$Tld3x(4YCm+TVGo zvmTP9u7zDm{8P*CmI)zE`JaNfVEH&(Gnoaoc< z9sFK?u5w`d{2KM?j=fyZts3Au^_}NG;!fBMjivTUM?L;Fk$q!7_C`oN8Q#gky?816 zl(IG6BfeQt29zzrmdoQ>H-afJ95*?ekhKEeV`!gDbIgHT`3;#K?J@Wz^TA`l9{WjW zc$v>UO@l_W*|BaQC4mgK;5W6nE#T@(_o}I{b zO&;nEJI;qD}@%)cJx%uiU?x~I(A1%W@p5+G7djl%;7uIU@&L7VMx{447 zOWeEhyzD2TYMX=|rN*CEb^4(6?Duw_q*h?B`~IaZ7e6PG!NUwzbx^aNEOHu_AXu%R z_`6v>lr`OuvsE64Uygn2FCzwF$4&N|SraUEra0b3&H|!_{VuF*qIY*f7}&eYHEEcWb<=KrGGi`eDgIaO~_MzaTIN^VyLRM{37W zS7P>0e1iSf)K%ql)I;$1$E?&C`c}O;GPQ15V4l4Fxa1D%zDsLnlYTFkVM0hEjrrIl zTx*DpcE62$cu77UZt(^17UR397CQs?uJdd1l!uQ+ZBj}6WS>j2~ zL0zVHGtio~0Y(#R-M+JA0#_De>zwQa{BvfvqaW}>3dvwtCwK@)tZ-aefT`@${V&p%k^gt3 zKwe=a8u3T9GGDu;pd4f$44gZT@G^p@X+uW)`O zUO)dF|K8eueoF1;G`vpsZDGP3+drAf<0I%Nkykr3de~wB#0$S>8s0-rhqUkGI`nsM zsh4H>dETcob=ZE*buR@t#q&F*?iXSHz%`pX27NA^hO)}7zu?(7cKK$Z zR#-9P{aax*1*GHEzgKQzucp7Ubs^?3`?hLh7br2Gc)pj+MsFHaIPS~SV$SM~p)<>Y z2kwQI-NbB!hT#q|yVYg%Rev`R8X?-oo`edSSpwY^n2#A9pg%ne%lw6I)h}nDlAp#n zf36f})oy@(@GEZu{GStvR2AayEue5D$pLe|oF(#HAM3%Xug;U! zY#OHN*LW4~j)N&@czYZAx4Lh*p8e|438T+T!Y8fT&E2O?{$Eq{lOe-BIfcIdI!A2`{3Bv>sM=->y*k!G{2fW2zg2c zE7SeBhcBSv^J>K0rPdef*TKtR_DRL)3GSC3J7_c(p+DC^vJ9YCgk;358oVt%Hw&(#OQZeyh!I{V>bsi zZ!II2LvgQB*{?2MhwsynjnR^VCiukF@p&J6>IHx7z2SUa0leXg3wx-WWoXX@-5MDM zDK9R?SL(gs(E8;Y4zhl3<(U<3# zd&`cgOfBj{e*;tEe}l*mKmWn{z0t)bh`TW;aZ_pqxHN;PD$XDmmGa!%D(P5Yjt-!W z#y$|KiJCUYiE60v;NX3S|8Ds+35zpQeK7kq_kFG<-gj~hm=nCapxc+_Pd;-O5EBe6 z+P-ZEfn}+7BfM`{GR;^m7o%Pim1CpChdm-pA`-@HMKEq|%Gymn542Cpiy36=q43}2 z3>9`ll)=Zh39$BlQq@e=+Xn&=e?hCFroRA&-jV zc-ATO8K`h-C*&iaL_cQcUCZA^5NO*t(!J6RA1Lc2_52s0TUpj;1-TlkJO0y(RH)ZR z9B{-2=73q}g}yaT)X8MnwNKCgKp!J#Y`aVtu#)7{X$ZBza^&>;XUMgXxk_lkkNv&> z8WZJiDYRnVT664IR|E7X`5MIQPs06*19qbKlhDY}{LEIeIYsRsPeOD>k-Hx~?pSWZm^8~-2J)cc~6kDNBXr7om_P>{)cZj*N9(C)U z&Z|L>&Uj(#56r1vHA8m3>N{IYDzV-lp;wkiB&x)ch zvMD4baytb1xUARu*1Ry!#nJU?YONae@hDgAC+K7CyB$M$x*z05V{Hg`=3%QahTOh+ z5_IV%4hdXFzk%{=Q6=&MaBdp7c^P$bg1`Xf7UZvG=hd2;#dkpzlnH*5!adP{-^j`( zh9N#>zZ9mB7k?p5U~jnrdhI+(-t=N0pyP?6enIS=j5nru@MHvjAG>oNT*u(Xn1N>y z?hl@^S*8;DA#Y@w!~AH-C`6Zh5GbP@f_Lmuc^~oTDKy{N#b4&`uLoiD4-fS$?<_#-EBd1`IM?6S*Dc`c1;dKpXL9c2eNXWA+7&#P(rX-c zYjDnu-UwBAha8pp*7hBD&sn%~TtHAlZ4nCT6SWdZyWvk;sMi;k8BqR*?Ktg?P`?qZ z_73-+)Kwe%PadIfrsP7DCh{h@s;q_^RI#5YYMYH}4|}cKhG-dHII7Md1-aQi-XwbCmO%exibVpc7nuP=i(qLN5%M5%Q*Iwve1*@pUV}I zY>$qQL%g2i;g=sVmz6$sVF)=O#uN*Jd)CrlG7x=%3Dl6msNuDX|KZIPMS+NwV`K%_mAEJ@e;fehXHQY052`j)8 zp*CO>mH|w{4YtPjIzU;ROe#pB1>|fUERJLTua;lKu9y(HNoO8Nn^Haj?X+G~UF00Z z=*vW%^g?dnURy-9(HNY6vp#)41G&igMlp8h(?D77FRNNrB^0rn{X3nz05=cF-vZ9* z`-?;S(ReRoPq`J|Pu+&+dN#8ja{QWoqIL!=CZI%)baNQLU#%rCWvi zU??s!1N%@2#?lF0h0-D2F@yb}A^}YGW@x7{PsybqzoSGykG^Z%0n}9jr_q-f?6dQz-Y;K( z&e8G3-;||L8yb1>g~ufPHl}HM)HDoXE(2$q1n{1%BgS8Y&$q9s@R3hs-7vxV36|&9@!>o zV?Pw#+^2YGh;!a8wW_{>IVi8*vecl%UTH$FpL;aO=Q3g=HgReL# zK=C*{IQC5F8Sc#k>uu(;{=+(q@vqd~XI)1QXo4vN8YLBu$s_1brR$I}A&!c#oMkj^RrygLM5W zqJ#EU>?I;R{Ra1%{UXhhA{xli6;I*3dlK(;W4G*PbTF@Z=_Ttq87*MsePMMQxqwCf zpC|Ofo57!B(cXlx5yWn|DSSg8``6Fq3B>qZ?kgaBp7Ikxz64f=}J3KSb9A zrApEy^H!lCe^x>MAHfJzxP=K5q94Sv(CIEM_NKPSo$<-4#(kRq@^u%fVdyMgEwDN~ z1bU3wN0^KYVVRjD{8kV0zc`I9^Oj{N0&stqb_W7REm2NoVk|fIu7N-%WI3mB37Ni)oKsa zUvV!f5m3{Ox}n#tYKw}UDVV+UR$2H~Ijl?1=OxymZ@0(%!}0(=H-(?7ZcOyUFOndJ z7Ubv-{)nxu2p$4RP>B=^>LG;IDito>RRY2iLtJ1p2Bp4FSicH@a0QDf*6(S9PVEH`v#zFx1 zd4+O^?!;m5a}nWCD)vI@h8wK7eH@0OhH}oyx5!-*s1bXTfcM&c`)yl0koEZXxuI>7Sh#@LhrR8G~=H4Cmo)PL`VOSInm`Z;v&6 zz<%+n>jtmG7olFpk6RDjJAFcYaoaqOVtcHEH}uS zgm03!0MUu8PTOzeaMywMNzWGMa}*wbo4Gy-Qamq)mKFM-&Gc`O+ECZ!AC=&eed^VsKXTS9Gl9199#`T`7Fh*Twj6cD>=*iia|K((f?x@`^fY*QhsJh z)I%UcNA*W6K z&mi{|`+V^56yg2Hj2uqR;?bnbRWN&>G9d$hPyh163dJURa6cxs!_I{Mz)NetT=6~l z?8IHtDzjb)f8RyK`63vabbKm5{u+aGTVoHOeVK;-2NHfStLNa>oNfFz?ju7F)jp3y z?vnO^>QvCrdU*FnNo?#4>JsdffeFVKp|5baDg4STG$~!X^IRVL%r1IS9oxpdX}~?% zgTEP|ijC>Em_KKAd3R15^Zg@g-|`u5pnh!F)qK&b7KAkRZNI*a1ydb8;&O7#^||}@ z55K5^cxBB@r^hS6ZhMDD_kAz0X0@uv-Sr1{%V-%7?_PMO9b=N0-47c{?JqsJmcU2N zPW*yiGstKpimS-?0|f<(|2fRt{$<+uTlf(bxu~mY-mNCAoh|rk-3r&%=0YZ1$UCTBtyuBp{k042Jje?iFcc!cx_WJ(R?KVyT$455xQAdq*WWto9&!MG>wWsN zk3JG3Mz@b;woM?g_@M3!KXTiDdfq2XKpwz!DxvsF?f?4~bKJG!Kp~dM`}K!OfGD3+ zwwQn4RJ{1R_;VL9uxFfTvTT7XTNg^aaPO7Q#QlSla0IqIZr{FOUj_s={k|*Qh43S` zkm8K)0?1LPd+AtKfVbZXzRUQ0s9)RtekCRzl(YA4w62$f-jdKas*+)_JQCZcliG~^ zy&@gse!Vbl-t3zlIRGA;%KiKJeYwHK?5;<$0DL2pa|-&HJ4{o5p=ve`xodrWFIcBw zywkYKbE6f;`7^|9r+eX4oVJOo8P4TYJ~tOLd%^V~+ed2TrT^vGo)tj<@};`uV!ix+ z$P>NXN4>KMJF!)P&Nx?d5%78CC04;qxl?W<-djhFHaohsqoD6>ef}o;m&j6%957&R-7~`N=yV<^8@yYc zGa3drnr;mZv2HLvk-AEI2{}})7ZUX`|0on5#&;X}vb>fDl8&FopfrN!qXDlUkbz`q z%={P#TeQ@3KE{3F@jFt1IEQTw(xp>Ib-~b;|FRD(r$L&M`iP-A>W7POTv-*XA+VGn zE*AIJs=@3&r?xslM*Ripoe1RDguGBP+%1H0;(a1Y=SI>$b5k7CSuI%-`^Du8rjm!qNYX3)ppQZaY21%&Qt^;a^~LO|1f(u&l=JARjjKY!4d68xQ6=wc#rQGE=~9K-&wj{9Pw zc{QLO@;l98E(PQazZ(BUUd78q`ahLFI)QxQ^)6C1sXkD zwzmL%ZRw5=Lg+Bhq{m2G(uSNBxB6k$W2l4snC%rpKk`VVKMtVJn*1rd)phiB))ren zcW}Wyo)l4VrWW#w`d^wFpnhOZ9$ZmTF%D|#x35ixBF{H(r>S%jpUd8zm{pS$NQ#To z$m?B(vY4`KpD}-W?vYV+>5pmXbt=PHBl?QfldGQFqF&# zmR(GGtP3V9MkHL@Q71aAy%UvC3&9jXWf+0?10k|4N17aHXB`g7Kt0PxUCaJYXE%uV z2_Ny#%m4zWYC~nrJ>|U846Of%d9nc>Np5NEFE)^Me2+evlcPlmyt?@PuqesUD(ZmE zT&@z+OY`vWOozvt1?;6TI6Sgsh&q-6w;~m92RQhqZC7}eLTRcyeH%0S;8`8N&?zDZ zFFsRtM1K@H2#Gv@(ATr?GtXFo{-yW#4pj#R&jY(7$#)L)KaTGNEEJ2+fwksWssr?` z6bEiFN*+RAbHC);ee7YD=-dsW^v1s2o?sQilvId3+$q8QY#vPgOFA20HUn4C%bmQ# ztU0tSozcU+1N~jp2qW!vG%1PM{E^&OM4W){^C1L zcYGdw@vmJEV-A!}jMzrmv>Q|-w{8CNw!@Z@-|X|RxED|Q^>zDvH8`^*)ZKB#{wTfJox+w(DR2);`=4Ui5Y&w#_n@)qVDIiU9fEeZZT zn={5ulV0FoE7e$KnhXE^TT$U{L|+9X-=5c6AJm)_3yj70ca&y14JG>mm}dTv)Y;63 zkH_B?m9?RMRwOw+9I68^CI`IxPAtJKe#@WK(rwTVOe5F?hxj&$LLP$oIA1rUOvCFeXqB`A=i~Tx_{`a4g1z9 zMOhD9$NlndxK-kQIkwrWupIWzh24*J$G*;cg2FvG=jBJaaR^kP-(mgQ=o8$Z}Y8m*ibo$X+Aa|qj1lh)K?3qvb zlR8SzwerYkW=NYH4o>2E9Pa?_Y~Z} zl79_3Ls zfjlJcMwQ5e&T68nIcowuy{zpV=-bzJ;GvYrT?9qDVoUMmNnoRm&Eq}301H&cgU&)@ zAR{wi;!=wK%)VsicdmnA__NbtsRnx#%n7IVPT?Ny?VES>84K{s?t`GA`y}>qYu`0l zi2?B<`wsJ?6R=8fYsOiz07_R_iHX;+SBp@uy&%6A-1m=tJHdxNA9>e{#k|Xbz-vsL zU`7EhJ1GZC^P&Fu=u2CrW)+|raBmLp`Q(+WZ%kE?``Sz^bH8L3w0{RFooL3rNWMn~ zhf4=k(q(^iV!-E`E`RqE=@{tI{RpEns)T&nnsLiN`Is-g(vbWr2Kq*6l#0`4AkwGx z4C|pma1QOL3zmJ=0$wU)?}zL9A+tKZ^c&7;1e%v= zBiVC+_$kkHo?17E%{XbMNzKBwKz?quhqzxnk^I33^%=o}wtp0%wJ`LAm9Ir&20n(8 z&iS=ZL6n*$OB(7T=6VX5&G?+|%tYSM(^!JAxNvgj2J~CfDfK!QjDlfnd63IC_I^dK zWj*!9pMyahA>8OH05T?dYBS6LVkx zC4pAY-RoCOk3=#*cz-e;la+)@8LwqW)rN?V89* zX~FmJUq_3V^IqVt!`9^Oh(51;hTJHEez?`*WZsHgw<|yO7+G-tn_X`7hLWcX93xGQ z&q;SfX6pIg6+EXo4-D>~{5S}#fix%AUn5_fi}Ts@`y;?PprPwHJPzkvwU%z8J{aD* z)q2Ei5PNApT)T|^WM!$_&4f~TPKXkXHl<^~`!ye*D9v2hUUL)QrJaL8qQeg*`IbTA zLChOQ+=B}F_ImMr2>_w;fBaY9N5blv0`hwsU7$p;O0B=!4i9X$j>sTaNWUaN+Oww{ zWb-c`^1{9mFa8npF?`?B?<7dt;JlMkH)6l=5IKLG?Y&!#$Q3KJN|uRV0-j&t!ugW9 zu%KsYsU?U0OB(n6iko9Vgqvu(Cgg6H4~Snd@CN!Y3tt8FXWNmsdimy{-%6%n)&F`j zD9`^kE4^C={0@mr(v!s?n@ZlKR6Y#~3{{-m=${+5moj6=&t*~4+~?{AJb#lfQTw{o z0MD%}$JnQnVE(#X+nWLG)eK_`e!)2f!fz^DzF_X>TF8M{k!%+{mS0|`NW=NlNp|vy z&Mdlv^FVruLHW9F0`?B*oDDjPxrfWv*PLp{U{iv! zh5?~&Wf*e5MbMuPg@*7s7{{*nqHaJ;u!P$~~+X$+J|o+5`s zr2XLx&ZF^DHTL(IFsH_hcNonKIr#43e*acI`cIBfBoSBy zf%LvWv2J;QBlin-3x@)(*1lV}(Y|KW6m z9D7Q4C~g&CpM@E{cg&yw@}q8B&CQRDL*@fyGayevNWI)M2RwkC`J?U@?*GnKD#g;C znShY*IXf@kVb7v}eJN*qJDk|B;fVB_fyHx^(vLW+K&Wy$B4rbMWnXHgXrQjo;F4Q+ zX)PZzimi>`XdnmEyfnY>{vaq8mehn4<2^Sc+p*WGPjPW=H%UVDW`vCnUmS&k*-bc3<4(&Q!SWgDa@x{!>7UX=+B*S7UmV8~dW54*?mm za#Ymq*yT%)wGm5W9Y-?_!LF~=429Gr=Xs;gA2-U~m?V;!Fe7+b|M7s7T z$>oFpg8aEV8Q!2oW8?XT^6vom=ZNbGb=?>PAGSTVnx<4>gYrM#tu)w`P%2Hwc{`Kzab_V{(INEQuaZe)*Iuiq`r=+=#dd!w z`gokeoww-zp@Xp0e{_BHSwHHzo~CN;;=ss$3Pz5eJ{ zFxmq{!4lhk*$}{WMP-~3b?y>}Kgd;cQXswU%QSOaH!xk4z88S|Sl;}{_3Sp&5E<2` zW^RvlKKk{d@f_*}JK3iND^Zu>KFZ^O{M5QW_LE0*kUyzvv_-w85gu~h;?Q;=!qy`5 zo>;77V#%S8vR5a-$ZVK*EQtb7_XNjA@9qW%Ujv`?Cq0nUJXSQok`4~hHv&~TyP%TI z>&omEV8w!TUm8aFz@S`0P$x4GJCu?UQxgLiA{8!H|Cjw$A9FUT>8h>nqY&*bx3?Yh9ZTk|LfM&c{`#Qw$JtYDAn6#R2)2FT zeb=|Cq@)d;MAs~qv@4N!{avs6`Vg3^9+AF^`lYJpV;dN4Fu$5_>~1mYm%WDsK6K*V zn1w_a-4xl3KE;*;nOvA>!~d7EB^%Svk$Ta<$G~1;o72(b9`U&slvV%++Q+`D$e0~<=wq?_$lh)2z$IL zBvb1#SKQ;kBTFJw=) z{nfG+U?hE8Mpk1O_g}xCosl6zLTF3LY1}8>aD1T3+&K+hlBVYT5yAyp`aZlg&bU-hQQg#X6R|HR((E9h} zfo9fDiB~?;aL)1!kqgC5MvCX1El3C%YglCoTQt8<7TI>%9UA`%dr+Ojw`gh?< z)Ll#;x$HHrnq!uc>F1EsW=f16 zdWyM3!V(G&y2yFw{q)1-bt?#dF}!K^0{5*t^TuB>XVm6Sjz};8^Of&EF-qP=0zq4G zDJroFb1>bKwPVTf%rwMki4i$e@!avmzqK$D?c<~M8|Nxp{O#veM?up%Yrh=&yJU>H z+xYSPNm5DO7u)j@{L~CMJkE}T-OgB_Y|OLOAfJt3R3C+9W$%v)gXkAXYB~_KGzK0* zJC5B#US4#5j+nYKzAvK}QWnr3G#Q&%!hwC?i!LF;(^}-qZgTIOY^jG=T6*@4tyJj0 zIn*it5xKT~lf!;FsN-~~avT!D{3JPM&e4Cq^nHhm3Gub?-DTnN>{1{6(cj`5DbfRW z77s-K3QjDFbwnjo42Vkp?~B|C?=1ieiwXN#sz1>F*3UuG?&NbL}V2 z4YYNMu>@cydh3d248Vt!lk0DL@__ZYai}5cgv%;#ALeK11S#_;rWfSVC;Dlo<2TD0 z7%=s{l6nPwp?Bm;MTXu%J2^6AbqsS+i!RvQJ&l|q6BRas9r6k}*xMBxzQN;2{>wY? zo|z`S@$^H~Ez|^)Vng0l;5@~)K+(n-LQwyhgEzI-tJ<=%3 zg>y}=xPbGCWXPFYqA1EVK$z^@YVB*`(amM5a%$zVNQP)5&pO&X`fleQIPM)13 zBG=x0!u9o=xApLFq3CX^7701h`wx}QBj4?)NMfb{>Y)0n%b#Pe)Zk&&x^sHS!K}-e zHM&0yp&GK4uQ1PM{Hm`dS-BnLfA^+(=OB+!XehDnDRSx!_*X0WNbvWFMhY`>1qoj9 zJ6jjZKm|jVQ|P!KGBPSV!-(~FZR%s!=Xa1M>L_f_g?W50X*O)F<3LLc%wc(iydvwN zZkN9q==*mcI@^!jgC}Md>LGYP4(5}t3MvA{hZl{UOr~I6xyW9(WC$`gixE7WFqcly zB}(km2;{CF9sFT32*2J1)>db=gSZ^~LUdy^XdAT-%}L=raPo-s9DfK1*6F%BU|+y& za49j?8vWQ|@zm|8mrpWlDRn4n0A&dw$#C?2*oae4t&Jt2Zd_Bispcn$9sTUG8}+U_ zqd#`;^euq-$H9!!c;4F=cycZa4#SFCoX3NmT`({-(-fha3GyEF?Fuz&6_W^?Psb4CBj;Js1F&)jA?efg_E#j`K_c0p$j_sa(VAPYXrluiT$kJ zoxni40&$ux7WO6m3pAe{O=O$ zN(V^W{Lzfxf%@?*LnRLGLAambVRk|qa|!N|{Nn4M0RM!d+kSk00Yo;^299wElxLA< z5U&T5RK{IVVef#|91d1ne+L^SCU)MY1i0P2JD)eW1LQ|49@gSKpJ6?jETxKm%XOjp z74HTxdOlYE5c}0)3$OFucdc>)HP{NS{_sGe&2`c!W7BIcP{5}eT3fiY@{BZt2 zpYP6{!oA*z;D8AFYE9?=^H<44U$W=5*jm(qsTOSDlYdnWcTxnv6VI2@37!QS9}UW{ z1}$4JA^%JC*S&OnzXDU8h}mNVAW;{RPwG#=e2&nAOPBh=d}Fam0`?<oJ?>XB>O z;gHTXH3D0CD@K&8hhZtUDkw3e8sZM*UN|$4`E-4w#s}q5pSd(#ShOz*IqPj(D11+%LD~byfGz0b z3sJIYzw`YY?rpV0Kdm)Fv;^^iX?MpOxDDP`?s|9s^!T~N%u>Sb_NS{tM^al2eFVbyT znnBgtX#F<&tA^a{+TY+@*+ebuW>7Kunk}rAR*(~$l1^IP#!P~NX3E=n5*d;t?*4VE zX@|Oo{tq?8KJa4t{B1Al`@Ya;pMOF>QuJ|#|NlgVZMfm)e9H^}Uv~AnWgit86~=Vs yVAP|K;3(5583X1pM$IT=W8)Kl@$-Lw3>aLa>W_#MrUX9$^yM22oXi0NBeagm@MzeQg$FkSQQ#^m4%k_A;`R#)yh=~i@WPjfX zXSe4!gJs+CM!qsC@<;w1>o~Hdh=4S z26OJ7m2;+b2n(Zc`cAD0eQ%Qnc0FmtwZj@?+VfSoH+{Y^v#AF6%=X@JI$8(sDUUlz zBXv-o3S76@x&m*+WWK-tvI&FiF%Qk^7Rc}E%&^{54=q8ZnMKhm==GGne>GPO`-ziV zgx+4qjWYuyEp1J(t+DNu_31$7u-(W|XA|rcw$w;EHbL&Nrl(3}3sNfFzxo+Bq3*R- zu~1$cBJGBS0^J($zIa?$Y)>_EHl!?6lfLxe$*IKBd{nxJui;eYK|;pW`)EcL>Wv5F zzl2mn@agw4(_PnbM_F{u;D#!wh)9?|=WT)c^9S|6ExU%^Hl?u3A$72Fer9Mp(}F=6 zqlLkw4y@Bnbm+F~2CMg&;U~uq1gI9BOg-3wz~mh*<7#c-GfiPR-FSA+Zx=)K23#yhFAoHEWB%SMD5SO%=cHA=7mwVCNgQ+}KgB;<0 z7&cl8=c(Sn%Fm2Tma5$taDJlUlh%dU)z;&&-Z$`IrQ)_e>pslZ=X#i(?ZjxL4EpEm z!3=YJoVuk7yPJKI&70fNTgKsD_o@SDgKZZambb#j``VZCu38AY(hiq}RUzMl*?XwA z8t$_#$Sr9^&~~fiXV44{rlg_u-Bt){w0wX5Aep25(3hFD4KSHU1Fuyl0vUpG6*{e0 z79GJqq1BAf$8x%Tg*vcCu-7R|upRj^Z+`L{ZUO)1ZBh;+jo{MgN|E^E8lL{Xe$*hn z7Sn;PT+(7SIAFEE;Tw#`1SeO4FPOfEgP2PymJyi;sP-hXUW6_6fvTs|5z zuLcgW+?nWmym($IAqgB&jl%JR5nu!#_;r(S2!eSW3lxhSus&sSW5(o3tUPG?+1&Uf zN>dJUh6IAZ&CE4=S{w`dnpXRZB`m~jJvE#blLXK zp;?v3^`J=@WZxWZ*~I4uy5iIx0Vx*RGE2lCu3{sg{3j3X87AmoM;>bT@e{=Hb(^=mJV zW=G-Xn$^1(z9zzbJa2{SiwIo#P?r76BY)T}-Pv_T(+LyftE@7LY{AjXeqWSojnvTc zi>vO1z~ISxW8NQv;lDPyE1~6$a&) zO7RPtamZuMo^v-(!&Q3;`-XySeB3axG*TxWqYi-*D$S|5Wv+Z8IX(y8*`FVB94=%3 z*zb>2B=g}~kPt4t`!ep+6omHh=V7}Di>1nw1HV;yUR+LT2v%r(l$@IiK0D)G22ZlU zc`D+|;7`W8;#?>8TsA5y_6BptO5ozJw3CaO1yS3PF7xRU%-RcG2sX>bf}?_0;iqD# z50%Xte^-vNDNFOxkqY$W9oA{=ErExpujht4#i(u>V!5p;0rRzs-q9TeusJnd5!9c7 zVL#D)m3`q@roUM!YTG$%ZQS*Z|Ifk3a7zse?Ef}!cKN*tT$WanKYxn0S4>u#c3BA*Cm#gPJTiwK0xAC{MT zmW6zl-NKx8G758}AC!DyK}WP;Uw&T-#ESD;h?R{IM3r%9ZH!pOY9Z=YQLv6Z9X-xhHc`O6^O4z~aD2Q_iB zW2=%KDADufQ9%x%)e8n?q+BYr zoaD={{WeT=M)4iD`D2jOPZ(qm1(Wlpi;;y#NWc9&8b^dlfA3ORrTrx5%RH_xL*ARr zuG`H%1B!i?e_N3+DB&~lhSLF{EbZ?zc}f4B=Q?QXLqHkdF+K8x#O@fkt(_M@wchfh z*@uC8`dOE2G#pgFrpVRRQD8U=h&}9Kfr@v2V__Nt%2v3r)g%^_lWwHEMJyO2G?k~j zV!*h6ePaJ=QYUNlI`<|y|32=_R0HYXK)>2b3k0>nSM}H{f1Vcm<<%2$pe~y&=r2zIC6{FrKbu%+91kXSvQqwQ3fcPb3sJ8svcC-_^fn=VdU$GzH8U95lZ> zaZ9b81I7=(%fw5PT&;8GWKS-1Twj#eF3W*VZSJnNm=T9x5|W0 zxIvOYS|%7FEIGN!3^4d?c-4)`Tn4@aG1Fu&mAUtSO2mVz&zOqeK<0`r-0Q!T{GY%^ z^=z$BP&v6WgSUwur@pG2d6U=`J!`^E^l~%p_J|qbPp_gueVX9X*_7fYN9Lj1m^`YF z2Kzt$SmI!7>EZToi?pt49iY2_dZnMDKI92X?7PsNsYEL42Ck?^M-s~GG`7&$V60}Y z7%w9d5P$Hi`b|RpBZC6tIug64L=KfFg4(EltyF#BH(~s^%Bgdm?4ig}pp3 zL~?VxWG3>6g?8{65B=y0s=0w#FnJmjCm=1N%pDYW_E5x%bD+}CP2B!M>b-qtnm3XW zn|vaDLoR}H{PTj)bOfkr;mj~PSqYJmiqrEtP5O;V-k-lnR(M!@lEcd&_~=C+rO$#o5U|$X!UI(MkBQ@@ z1don_TjL#qcTlrhyNP@Pc$Fj_l8I#sw8XWv2p-p&`zhaWab534-6)ZUt5x}Dmjv=${n-jBVyT(=Osy{R-dw~rsD?;Lu9khx zi%4>}sK$dKGMA5uQltu*<8HM^-dQ36*XcNI%Dk7p3O6u;{(Y~uAII|j;?p-97SYoKn-$yL%=bhBvNO`O2+Dq^R+S zxs(~PTIy(Rs9!c1AGE@!7f9SYGMU(&O{BNQkNM!6l^Zg2RWm_-5vpc>O#vg$&CAY; z%%LW*Eb<3}e^XYm#&v@G=%|F)GlG|U(y8P(!sCqQ*p-cBk5>NMwzbiTwU$VFT3dnI z+GVAE;yB?Y&&M~|7SwQ6oi#tv;bT23hM_n41lIfQiy(7OPe3dzg zhEJLB`hk1Tk^r)wDA)C4#|ZEGXN=a#5I#DKHdgTy9O5xU%72oWdcyf=_a!ilv<_9R YBRsuaoG|%`M2|Ad>cg!6TEtWB|Aef;q5uE@ literal 0 HcmV?d00001 diff --git a/mne_connectivity/spectral/tests/test_spectral.py b/mne_connectivity/spectral/tests/test_spectral.py index 3286bba8..fa8cf44d 100644 --- a/mne_connectivity/spectral/tests/test_spectral.py +++ b/mne_connectivity/spectral/tests/test_spectral.py @@ -1,6 +1,8 @@ +import os import numpy as np from numpy.testing import (assert_allclose, assert_array_almost_equal, assert_array_less) +import pandas as pd import pytest from mne import (EpochsArray, SourceEstimate, create_info) from mne.filter import filter_data @@ -408,6 +410,301 @@ def test_spectral_connectivity(method, mode): assert (out_lens[0] == 10) +@pytest.mark.parametrize('method', ['mic', 'mim', 'gc']) +def test_spectral_connectivity_epochs_multivariate(method): + """Test over-epoch multivariate connectivity methods.""" + mode = 'multitaper' # stick with single mode in interest of time + + sfreq = 100.0 # Hz + n_signals = 4 # should be even! + n_seeds = n_signals // 2 + n_epochs = 10 + n_times = 200 # samples + trans_bandwidth = 2.0 # Hz + delay = 10 # samples (non-zero delay needed for ImCoh and GC to be >> 0) + + indices = tuple([np.arange(n_seeds), np.arange(n_seeds) + n_seeds]) + + # 15-25 Hz connectivity + fstart, fend = 15.0, 25.0 + rng = np.random.RandomState(0) + data = rng.randn(n_signals, n_epochs * n_times + delay) + # simulate connectivity from fstart to fend + data[n_seeds:, :] = filter_data( + data[:n_seeds, :], sfreq, fstart, fend, filter_length='auto', + fir_design='firwin2', l_trans_bandwidth=trans_bandwidth, + h_trans_bandwidth=trans_bandwidth) + # add some noise, so the spectrum is not exactly zero + data[n_seeds:, :] += 1e-2 * rng.randn(n_seeds, n_times * n_epochs + delay) + # shift the seeds to that the targets are a delayed version of them + data[:n_seeds, :n_epochs * n_times] = data[:n_seeds, delay:] + data = data[:, :n_times * n_epochs] + data = data.reshape(n_signals, n_epochs, n_times) + data = np.transpose(data, [1, 0, 2]) + + con = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, sfreq=sfreq, + gc_n_lags=20) + freqs = con.freqs + gidx = (freqs.index(fstart), freqs.index(fend) + 1) + bidx = (freqs.index(fstart - trans_bandwidth * 2), + freqs.index(fend + trans_bandwidth * 2) + 1) + + if method in ['mic', 'mim']: + lower_t = 0.2 + upper_t = 0.5 + + assert np.abs(con.get_data())[0, gidx[0]:gidx[1]].mean() > upper_t + assert np.abs(con.get_data())[0, :bidx[0]].mean() < lower_t + assert np.abs(con.get_data())[0, bidx[1]:].mean() < lower_t + + elif method == 'gc': + lower_t = 0.2 + upper_t = 0.8 + + assert con.get_data()[0, gidx[0]:gidx[1]].mean() > upper_t + assert con.get_data()[0, :bidx[0]].mean() < lower_t + assert con.get_data()[0, bidx[1]:].mean() < lower_t + + # check that target -> seed connectivity is low + indices_ts = (indices[1], indices[0]) + con_ts = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices_ts, sfreq=sfreq, + gc_n_lags=20) + assert con_ts.get_data()[0, gidx[0]:gidx[1]].mean() < lower_t + + # check that TRGC is positive (i.e. net seed -> target connectivity not + # due to noise) + con_tr = spectral_connectivity_epochs( + data, method='gc_tr', mode=mode, indices=indices, sfreq=sfreq, + gc_n_lags=20) + con_ts_tr = spectral_connectivity_epochs( + data, method='gc_tr', mode=mode, indices=indices_ts, sfreq=sfreq, + gc_n_lags=20) + trgc = ((con.get_data() - con_ts.get_data()) - + (con_tr.get_data() - con_ts_tr.get_data())) + # checks that TRGC is positive and >> 0 (for 15-25 Hz) + assert np.all(trgc[0, gidx[0]:gidx[1]] > 0) + assert np.all(trgc[0, gidx[0]:gidx[1]] > upper_t) + # checks that TRGC is ~ 0 for other frequencies + assert np.allclose(trgc[0, :bidx[0]].mean(), 0, atol=lower_t) + assert np.allclose(trgc[0, bidx[1]:].mean(), 0, atol=lower_t) + + # check all-to-all conn. computed for MIC/MIM when no indices given + if method in ['mic', 'mim']: + con = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=None, sfreq=sfreq) + assert (np.array(con.indices).tolist() == + [[[0, 1, 2, 3]], [[0, 1, 2, 3]]]) + + # check shape of MIC patterns + if method == 'mic': + for mode in ['multitaper', 'cwt_morlet']: + con = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, sfreq=sfreq, + fmin=10, fmax=25, cwt_freqs=np.arange(10, 25), + faverage=True) + + if mode == 'cwt_morlet': + patterns_shape = ( + (len(indices[0]), len(con.freqs), len(con.times)), + (len(indices[1]), len(con.freqs), len(con.times))) + else: + patterns_shape = ( + (len(indices[0]), len(con.freqs)), + (len(indices[1]), len(con.freqs))) + assert np.shape(con.attrs["patterns"][0][0]) == patterns_shape[0] + assert np.shape(con.attrs["patterns"][1][0]) == patterns_shape[1] + + # only check these once for speed + if mode == 'multitaper': + # check patterns averaged over freqs + fmin = (5., 15.) + fmax = (15., 30.) + con = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, + sfreq=sfreq, fmin=fmin, fmax=fmax, faverage=True) + assert np.shape(con.attrs["patterns"][0][0])[1] == len(fmin) + assert np.shape(con.attrs["patterns"][1][0])[1] == len(fmin) + + # check patterns shape matches input data, not rank + rank = (np.array([1]), np.array([1])) + con = spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, + sfreq=sfreq, rank=rank) + assert (np.shape(con.attrs["patterns"][0][0])[0] == + len(indices[0])) + assert (np.shape(con.attrs["patterns"][1][0])[0] == + len(indices[1])) + + +def test_multivariate_spectral_connectivity_epochs_regression(): + """Test multivar. spectral connectivity over epochs for regression. + + The multivariate methods were originally implemented in MATLAB by their + respective authors. To show that this Python implementation is identical + and to avoid any future regressions, we compare the results of the Python + and MATLAB implementations on some example data (randomly generated). + + As the MNE code for computing the cross-spectral density matrix is not + available in MATLAB, the CSD matrix was computed using MNE and then loaded + into MATLAB to compute the connectivity from the original implementations + using the same processing settings in MATLAB and Python. + + It is therefore important that no changes are made to the settings for + computing the CSD or the final connectivity scores! + """ + fpath = os.path.dirname(os.path.realpath(__file__)) + data = pd.read_pickle( + os.path.join(fpath, 'data', 'example_multivariate_data.pkl')) + sfreq = 100 + indices = tuple([[0, 1], [2, 3]]) + methods = ['mic', 'mim', 'gc', 'gc_tr'] + con = spectral_connectivity_epochs( + data, method=methods, indices=indices, mode='multitaper', sfreq=sfreq, + fskip=0, faverage=False, tmin=0, tmax=None, mt_bandwidth=4, + mt_low_bias=True, mt_adaptive=False, gc_n_lags=20, + rank=tuple([[2], [2]]), n_jobs=1) + + # should take the absolute of the MIC scores, as the MATLAB implementation + # returns the absolute values. + mne_results = {this_con.method: np.abs(this_con.get_data()) + for this_con in con} + matlab_results = pd.read_pickle( + os.path.join(fpath, 'data', 'example_multivariate_matlab_results.pkl')) + for method in methods: + assert_allclose(matlab_results[method], mne_results[method], 1e-5) + + +@pytest.mark.parametrize( + 'method', ['mic', 'mim', 'gc', 'gc_tr', ['mic', 'mim', 'gc', 'gc_tr']]) +@pytest.mark.parametrize('mode', ['multitaper', 'fourier', 'cwt_morlet']) +def test_multivar_spectral_connectivity_epochs_error_catch(method, mode): + """Test error catching for multivar. freq.-domain connectivity methods.""" + sfreq = 50. + n_signals = 4 # Do not change! + n_epochs = 8 + n_times = 256 + rng = np.random.RandomState(0) + data = rng.randn(n_epochs, n_signals, n_times) + indices = (np.arange(0, 2), np.arange(2, 4)) + cwt_freqs = np.arange(10, 25 + 1) + + # check bad indices with repeated channels + with pytest.raises(ValueError, + match='seed and target indices cannot contain'): + repeated_indices = tuple([[0, 1, 1], [2, 2, 3]]) + spectral_connectivity_epochs( + data, method=method, mode=mode, indices=repeated_indices, + sfreq=sfreq, gc_n_lags=10) + + # check mixed methods caught + with pytest.raises(ValueError, + match='bivariate and multivariate connectivity'): + if isinstance(method, str): + mixed_methods = [method, 'coh'] + elif isinstance(method, list): + mixed_methods = [*method, 'coh'] + spectral_connectivity_epochs(data, method=mixed_methods, mode=mode, + indices=indices, sfreq=sfreq, + cwt_freqs=cwt_freqs) + + # check bad rank args caught + too_low_rank = (np.array([0]), np.array([0])) + with pytest.raises(ValueError, + match='ranks for seeds and targets must be'): + spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, + sfreq=sfreq, rank=too_low_rank, cwt_freqs=cwt_freqs) + too_high_rank = (np.array([3]), np.array([3])) + with pytest.raises(ValueError, + match='ranks for seeds and targets must be'): + spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, + sfreq=sfreq, rank=too_high_rank, cwt_freqs=cwt_freqs) + + # check rank-deficient data caught + bad_data = data.copy() + bad_data[:, 1] = bad_data[:, 0] + bad_data[:, 3] = bad_data[:, 2] + assert np.all(np.linalg.matrix_rank(bad_data[:, (0, 1), :]) == 1) + assert np.all(np.linalg.matrix_rank(bad_data[:, (2, 3), :]) == 1) + if isinstance(method, str): + rank_con = spectral_connectivity_epochs( + bad_data, method=method, mode=mode, indices=indices, sfreq=sfreq, + gc_n_lags=10, cwt_freqs=cwt_freqs) + assert rank_con.attrs["rank"] == (np.array([1]), np.array([1])) + + if method in ['mic', 'mim']: + # check rank-deficient transformation matrix caught + with pytest.raises(RuntimeError, + match='the transformation matrix'): + spectral_connectivity_epochs( + bad_data, method=method, mode=mode, indices=indices, + sfreq=sfreq, rank=(np.array([2]), np.array([2])), + cwt_freqs=cwt_freqs) + + # only check these once for speed + if method == 'gc' and mode == 'multitaper': + # check bad n_lags caught + frange = (5, 10) + n_lags = 200 # will be far too high + with pytest.raises(ValueError, match='the number of lags'): + spectral_connectivity_epochs( + data, method=method, mode=mode, indices=indices, sfreq=sfreq, + fmin=frange[0], fmax=frange[1], gc_n_lags=n_lags, + cwt_freqs=cwt_freqs) + + # check no indices caught + with pytest.raises(ValueError, match='indices must be specified'): + spectral_connectivity_epochs(data, method=method, mode=mode, + indices=None, sfreq=sfreq, + cwt_freqs=cwt_freqs) + + # check intersecting indices caught + bad_indices = (np.array([0, 1]), np.array([0, 2])) + with pytest.raises(ValueError, + match='seed and target indices must not intersect'): + spectral_connectivity_epochs(data, method=method, mode=mode, + indices=bad_indices, sfreq=sfreq, + cwt_freqs=cwt_freqs) + + # check bad fmin/fmax caught + with pytest.raises(ValueError, + match='computing Granger causality on multiple'): + spectral_connectivity_epochs(data, method=method, mode=mode, + indices=indices, sfreq=sfreq, + fmin=(10., 15.), fmax=(15., 20.), + cwt_freqs=cwt_freqs) + + # check rank-deficient autocovariance caught + with pytest.raises(RuntimeError, + match='the autocovariance matrix is singular'): + spectral_connectivity_epochs( + bad_data, method=method, mode=mode, indices=indices, + sfreq=sfreq, rank=(np.array([2]), np.array([2])), + cwt_freqs=cwt_freqs) + + +@pytest.mark.parametrize('method', ['mic', 'mim', 'gc', 'gc_tr']) +def test_multivar_spectral_connectivity_parallel(method): + """Test multivar. freq.-domain connectivity methods run in parallel.""" + sfreq = 50. + n_signals = 4 # Do not change! + n_epochs = 8 + n_times = 256 + rng = np.random.RandomState(0) + data = rng.randn(n_epochs, n_signals, n_times) + indices = (np.arange(0, 2), np.arange(2, 4)) + + spectral_connectivity_epochs( + data, method=method, mode="multitaper", indices=indices, sfreq=sfreq, + gc_n_lags=10, n_jobs=2) + spectral_connectivity_time( + data, freqs=np.arange(10, 25), method=method, mode="multitaper", + indices=indices, sfreq=sfreq, gc_n_lags=10, n_jobs=2) + + @ pytest.mark.parametrize('kind', ('epochs', 'ndarray', 'stc', 'combo')) def test_epochs_tmin_tmax(kind): """Test spectral.spectral_connectivity_epochs with epochs and arrays.""" @@ -472,9 +769,9 @@ def test_epochs_tmin_tmax(kind): assert len(w) == 1 # just one even though there were multiple epochs -@pytest.mark.parametrize('method', ['coh', 'plv', 'pli', 'wpli', 'ciplv']) @pytest.mark.parametrize( - 'mode', ['cwt_morlet', 'multitaper']) + 'method', ['coh', 'mic', 'mim', 'plv', 'pli', 'wpli', 'ciplv']) +@pytest.mark.parametrize('mode', ['cwt_morlet', 'multitaper']) @pytest.mark.parametrize('data_option', ['sync', 'random']) def test_spectral_connectivity_time_phaselocked(method, mode, data_option): """Test time-resolved spectral connectivity with simulated phase-locked @@ -500,30 +797,109 @@ def test_spectral_connectivity_time_phaselocked(method, mode, data_option): wave_freq * epoch_length * np.pi + phase, n_times) data[i, c] = np.squeeze(np.sin(x)) + + multivar_methods = ['mic', 'mim'] + # the frequency band should contain the frequency at which there is a # hypothesized "connection" freq_band_low_limit = (8.) freq_band_high_limit = (13.) freqs = np.arange(freq_band_low_limit, freq_band_high_limit + 1) - con = spectral_connectivity_time(data, freqs, method=method, mode=mode, - sfreq=sfreq, fmin=freq_band_low_limit, - fmax=freq_band_high_limit, - n_jobs=1, - faverage=True, average=True, sm_times=0) - assert con.shape == (n_channels ** 2, len(con.freqs)) - con_matrix = con.get_data('dense')[..., 0] + con = spectral_connectivity_time( + data, freqs, method=method, mode=mode, sfreq=sfreq, + fmin=freq_band_low_limit, fmax=freq_band_high_limit, n_jobs=1, + faverage=True if method != 'mic' else False, + average=True if method != 'mic' else False, sm_times=0) + con_matrix = con.get_data() + + # MIC values can be pos. and neg., so must be averaged after taking the + # absolute values for the test to work + if method in multivar_methods: + if method == 'mic': + con_matrix = np.mean(np.abs(con_matrix), axis=(0, 2)) + assert con.shape == (n_epochs, 1, len(con.freqs)) + else: + assert con.shape == (1, len(con.freqs)) + else: + assert con.shape == (n_channels ** 2, len(con.freqs)) + con_matrix = np.reshape(con_matrix, (n_channels, n_channels))[ + np.tril_indices(n_channels, -1)] + if data_option == 'sync': # signals are perfectly phase-locked, connectivity matrix should be - # a lower triangular matrix of ones - assert np.allclose(con_matrix, - np.tril(np.ones(con_matrix.shape), - k=-1), - atol=0.01) + # a matrix of ones + assert np.allclose(con_matrix, np.ones(con_matrix.shape), atol=0.01) if data_option == 'random': # signals are random, all connectivity values should be small # 0.5 is picked rather arbitrarily such that the obsolete wrong # implementation fails - assert np.all(con_matrix) <= 0.5 + assert np.all(con_matrix <= 0.5) + + +def test_spectral_connectivity_time_delayed(): + """Test per-epoch Granger causality with time-delayed data. + + N.B.: the spectral_connectivity_time method seems to be more unstable than + spectral_connectivity_epochs for GC estimation. Accordingly, we assess + Granger scores only in the context of the noise-corrected TRGC metric, + where the true directionality of the connections seems to identified. + """ + mode = 'multitaper' # stick with single mode in interest of time + + sfreq = 100.0 # Hz + n_signals = 4 # should be even! + n_seeds = n_signals // 2 + n_epochs = 10 + n_times = 200 # samples + trans_bandwidth = 2.0 # Hz + delay = 5 # samples (non-zero delay needed for GC to be >> 0) + + indices = tuple([np.arange(n_seeds), np.arange(n_seeds) + n_seeds]) + + # 20-30 Hz connectivity + fstart, fend = 20.0, 30.0 + rng = np.random.RandomState(0) + data = rng.randn(n_signals, n_epochs * n_times + delay) + # simulate connectivity from fstart to fend + data[n_seeds:, :] = filter_data( + data[:n_seeds, :], sfreq, fstart, fend, filter_length='auto', + fir_design='firwin2', l_trans_bandwidth=trans_bandwidth, + h_trans_bandwidth=trans_bandwidth) + # add some noise, so the spectrum is not exactly zero + data[n_seeds:, :] += 1e-2 * rng.randn(n_seeds, n_times * n_epochs + delay) + # shift the seeds to that the targets are a delayed version of them + data[:n_seeds, :n_epochs * n_times] = data[:n_seeds, delay:] + data = data[:, :n_times * n_epochs] + data = data.reshape(n_signals, n_epochs, n_times) + data = np.transpose(data, [1, 0, 2]) + + freqs = np.arange(2.5, 50, 0.5) + con_st = spectral_connectivity_time( + data, freqs, method=['gc', 'gc_tr'], indices=indices, mode=mode, + sfreq=sfreq, n_jobs=1, gc_n_lags=20, n_cycles=5, average=True) + con_ts = spectral_connectivity_time( + data, freqs, method=['gc', 'gc_tr'], indices=(indices[1], indices[0]), + mode=mode, sfreq=sfreq, n_jobs=1, gc_n_lags=20, n_cycles=5, + average=True) + st = con_st[0].get_data() + st_tr = con_st[1].get_data() + ts = con_ts[0].get_data() + ts_tr = con_ts[1].get_data() + trgc = (st - ts) - (st_tr - ts_tr) + + freqs = con_st[0].freqs + gidx = (freqs.index(fstart), freqs.index(fend) + 1) + bidx = (freqs.index(fstart - trans_bandwidth * 2), + freqs.index(fend + trans_bandwidth * 2) + 1) + + # assert that TRGC (i.e. net, noise-corrected connectivity) is positive and + # >> 0 (i.e. that there is indeed a flow of info. from seeds to targets, + # as simulated) + assert np.all(trgc[:, gidx[0]:gidx[1]] > 0) + assert trgc[:, gidx[0]:gidx[1]].mean() > 0.4 + # check that non-interacting freqs. have close to zero connectivity + assert np.allclose(trgc[0, :bidx[0]].mean(), 0, atol=0.1) + assert np.allclose(trgc[0, bidx[1]:].mean(), 0, atol=0.1) @pytest.mark.parametrize('method', ['coh', 'plv', 'pli', 'wpli', 'ciplv']) @@ -671,6 +1047,115 @@ def test_spectral_connectivity_time_padding(method, mode, padding): for idx, jdx in triu_inds) +@pytest.mark.parametrize('method', ['mic', 'mim', 'gc', 'gc_tr']) +@pytest.mark.parametrize('average', [True, False]) +@pytest.mark.parametrize('faverage', [True, False]) +def test_multivar_spectral_connectivity_time_shapes(method, average, faverage): + """Test result shapes of time-resolved multivar. connectivity methods.""" + sfreq = 50. + n_signals = 4 # Do not change! + n_epochs = 8 + n_times = 500 + rng = np.random.RandomState(0) + data = rng.randn(n_epochs, n_signals, n_times) + indices = (np.arange(0, 2), np.arange(2, 4)) + freqs = np.arange(10, 25 + 1) + + con_shape = [1] + if faverage: + con_shape.append(1) + else: + con_shape.append(len(freqs)) + if not average: + con_shape = [n_epochs, *con_shape] + + # check shape of results when averaging across epochs + con = spectral_connectivity_time( + data, freqs, indices=indices, method=method, sfreq=sfreq, + faverage=faverage, average=average, gc_n_lags=10) + assert con.shape == tuple(con_shape) + + # check shape of MIC patterns are correct + if method == 'mic': + patterns_shape = [len(indices[0])] + if faverage: + patterns_shape.append(1) + else: + patterns_shape.append(len(freqs)) + if not average: + patterns_shape = [n_epochs, *patterns_shape] + patterns_shape = [2, *patterns_shape] + assert np.array(con.attrs['patterns']).shape == tuple(patterns_shape) + + +@pytest.mark.parametrize( + 'method', ['mic', 'mim', 'gc', 'gc_tr']) +def test_multivar_spectral_connectivity_time_error_catch(method): + """Test error catching for time-resolved multivar. connectivity methods.""" + sfreq = 50. + n_signals = 4 # Do not change! + n_epochs = 8 + n_times = 256 + data = np.random.rand(n_epochs, n_signals, n_times) + indices = (np.arange(0, 2), np.arange(2, 4)) + freqs = np.arange(10, 25 + 1) + + # check bad indices with repeated channels + with pytest.raises(ValueError, + match='seed and target indices cannot contain'): + repeated_indices = tuple([[0, 1, 1], [2, 2, 3]]) + spectral_connectivity_time(data, freqs, method=method, + indices=repeated_indices, sfreq=sfreq) + + # check mixed methods caught + with pytest.raises(ValueError, + match='bivariate and multivariate connectivity'): + mixed_methods = [method, 'coh'] + spectral_connectivity_time(data, freqs, method=mixed_methods, + indices=indices, sfreq=sfreq) + + # check bad rank args caught + too_low_rank = (np.array([0]), np.array([0])) + with pytest.raises(ValueError, + match='ranks for seeds and targets must be'): + spectral_connectivity_time( + data, freqs, method=method, indices=indices, sfreq=sfreq, + rank=too_low_rank) + too_high_rank = (np.array([3]), np.array([3])) + with pytest.raises(ValueError, + match='ranks for seeds and targets must be'): + spectral_connectivity_time( + data, freqs, method=method, indices=indices, sfreq=sfreq, + rank=too_high_rank) + + # check all-to-all conn. computed for MIC/MIM when no indices given + if method in ['mic', 'mim']: + con = spectral_connectivity_epochs( + data, freqs, method=method, indices=None, sfreq=sfreq) + assert (np.array(con.indices).tolist() == + [[[0, 1, 2, 3]], [[0, 1, 2, 3]]]) + + if method in ['gc', 'gc_tr']: + # check no indices caught + with pytest.raises(ValueError, match='indices must be specified'): + spectral_connectivity_time(data, freqs, method=method, + indices=None, sfreq=sfreq) + + # check intersecting indices caught + bad_indices = (np.array([0, 1]), np.array([0, 2])) + with pytest.raises(ValueError, + match='seed and target indices must not intersect'): + spectral_connectivity_time(data, freqs, method=method, + indices=bad_indices, sfreq=sfreq) + + # check bad fmin/fmax caught + with pytest.raises(ValueError, + match='computing Granger causality on multiple'): + spectral_connectivity_time(data, freqs, method=method, + indices=indices, sfreq=sfreq, + fmin=(5., 15.), fmax=(15., 30.)) + + def test_save(tmp_path): """Test saving results of spectral connectivity.""" rng = np.random.RandomState(0) diff --git a/mne_connectivity/spectral/time.py b/mne_connectivity/spectral/time.py index b11a9640..7c1aabe6 100644 --- a/mne_connectivity/spectral/time.py +++ b/mne_connectivity/spectral/time.py @@ -1,5 +1,6 @@ # Authors: Adam Li # Santeri Ruuskanen +# Thomas S. Binns # # License: BSD (3-clause) @@ -12,11 +13,16 @@ from mne.utils import (logger, verbose) from ..base import (SpectralConnectivity, EpochSpectralConnectivity) -from .epochs import _compute_freq_mask +from .epochs import (_MICEst, _MIMEst, _GCEst, _GCTREst, _compute_freq_mask, + _check_rank_input) from .smooth import _create_kernel, _smooth_spectra from ..utils import check_indices, fill_doc +_multivariate_methods = ['mic', 'mim', 'gc', 'gc_tr'] +_gc_methods = ['gc', 'gc_tr'] + + @verbose @fill_doc def spectral_connectivity_time(data, freqs, method='coh', average=False, @@ -24,8 +30,9 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, fmax=None, fskip=0, faverage=False, sm_times=0, sm_freqs=1, sm_kernel='hanning', padding=0, mode='cwt_morlet', mt_bandwidth=None, - n_cycles=7, decim=1, n_jobs=1, verbose=None): - """Compute time-frequency-domain connectivity measures. + n_cycles=7, gc_n_lags=40, rank=None, decim=1, + n_jobs=1, verbose=None): + r"""Compute time-frequency-domain connectivity measures. This function computes spectral connectivity over time from epoched data. The data may consist of a single epoch. @@ -44,20 +51,29 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, ``fmax`` are used. method : str | list of str Connectivity measure(s) to compute. These can be - ``['coh', 'plv', 'ciplv', 'pli', 'wpli']``. These are: + ``['coh', 'mic', 'mim', 'plv', 'ciplv', 'pli', 'wpli', 'gc', + 'gc_tr']``. These are: * 'coh' : Coherence + * 'mic' : Maximised Imaginary part of Coherency (MIC) + * 'mim' : Multivariate Interaction Measure (MIM) * 'plv' : Phase-Locking Value (PLV) * 'ciplv' : Corrected imaginary Phase-Locking Value * 'pli' : Phase-Lag Index * 'wpli' : Weighted Phase-Lag Index + * 'gc' : State-space Granger Causality (GC) + * 'gc_tr' : State-space GC on time-reversed signals + Multivariate methods (``['mic', 'mim', 'gc', 'gc_tr]``) cannot be + called with the other methods. average : bool Average connectivity scores over epochs. If ``True``, output will be an instance of :class:`SpectralConnectivity`, otherwise :class:`EpochSpectralConnectivity`. indices : tuple of array_like | None Two arrays with indices of connections for which to compute - connectivity. I.e. it is a ``(n_pairs, 2)`` array essentially. - If `None`, all connections are computed. + connectivity. If a multivariate method is called, the indices are for a + single connection between all seeds and all targets. If None, all + connections are computed, unless a Granger causality method is called, + in which case an error is raised. sfreq : float The sampling frequency. Required if data is not :class:`Epochs `. @@ -103,6 +119,16 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, frequency. The number of cycles ``n_cycles`` and the frequencies of interest ``cwt_freqs`` define the temporal window length. For details, see :func:`mne.time_frequency.tfr_array_morlet` documentation. + gc_n_lags : int + Number of lags to use for the vector autoregressive model when + computing Granger causality. Higher values increase computational cost, + but reduce the degree of spectral smoothing in the results. Only used + if ``method`` contains any of ``['gc', 'gc_tr']``. + rank : tuple of array | None + Two arrays with the rank to project the seed and target data to, + respectively, using singular value decomposition. If `None`, the rank + of the data is computed and projected to. Only used if ``method`` + contains any of ``['mic', 'mim', 'gc', 'gc_tr']``. decim : int To reduce memory usage, decimation factor after time-frequency decomposition. Returns ``tfr[…, ::decim]``. @@ -119,9 +145,10 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, or a list of instances corresponding to connectivity measures if several connectivity measures are specified. The shape of each connectivity dataset is - (n_epochs, n_signals, n_signals, n_freqs) when ``indices`` is `None` - and (n_epochs, n_nodes, n_nodes, n_freqs) when ``indices`` is specified - and ``n_nodes = len(indices[0])``. + (n_epochs, n_signals, n_signals, n_freqs) when ``indices`` is `None`, + (n_epochs, n_nodes, n_nodes, n_freqs) when ``indices`` is specified + and ``n_nodes = len(indices[0])``, or (n_epochs, 1, 1, n_freqs) when a + multi-variate method is called regardless of "indices". See Also -------- @@ -159,11 +186,11 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, (i.e., time window length). By default, the connectivity between all signals is computed (only - connections corresponding to the lower-triangular part of the - connectivity matrix). If one is only interested in the connectivity - between some signals, the ``indices`` parameter can be used. For example, - to compute the connectivity between the signal with index 0 and signals - 2, 3, 4 (a total of 3 connections), one can use the following:: + connections corresponding to the lower-triangular part of the connectivity + matrix). If one is only interested in the connectivity between some + signals, the "indices" parameter can be used. For example, to compute the + connectivity between the signal with index 0 and signals "2, 3, 4" (a total + of 3 connections) one can use the following:: indices = (np.array([0, 0, 0]), # row indices np.array([2, 3, 4])) # col indices @@ -174,6 +201,15 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, In this case ``con.get_data().shape = (3, n_freqs)``. The connectivity scores are in the same order as defined indices. + For multivariate methods, this is handled differently. If "indices" is + None, connectivity between all signals will attempt to be computed (this is + not possible if a Granger causality method is called). If "indices" is + specified, the seeds and targets are treated as a single connection. For + example, to compute the connectivity between signals 0, 1, 2 and 3, 4, 5, + one would use the same approach as above, however the signals would all be + considered for a single connection and the connectivity scores would have + the shape (1, n_freqs). + **Supported Connectivity Measures** The connectivity method(s) is specified using the ``method`` parameter. The @@ -187,12 +223,31 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, C = --------------------- sqrt(E[Sxx] * E[Syy]) + 'mic' : Maximised Imaginary part of Coherency (MIC) + :footcite:`EwaldEtAl2012` given by: + + :math:`MIC=\Large{\frac{\boldsymbol{\alpha}^T \boldsymbol{E \beta}} + {\parallel\boldsymbol{\alpha}\parallel \parallel\boldsymbol{\beta} + \parallel}}` + + where: :math:`\boldsymbol{E}` is the imaginary part of the + transformed cross-spectral density between seeds and targets; and + :math:`\boldsymbol{\alpha}` and :math:`\boldsymbol{\beta}` are + eigenvectors for the seeds and targets, such that + :math:`\boldsymbol{\alpha}^T \boldsymbol{E \beta}` maximises + connectivity between the seeds and targets. + + 'mim' : Multivariate Interaction Measure (MIM) + :footcite:`EwaldEtAl2012` given by: + + :math:`MIM=tr(\boldsymbol{EE}^T)` + 'plv' : Phase-Locking Value (PLV) :footcite:`LachauxEtAl1999` given by:: PLV = |E[Sxy/|Sxy|]| - 'ciplv' : Corrected imaginary PLV (icPLV) :footcite:`BrunaEtAl2018` + 'ciplv' : Corrected imaginary PLV (ciPLV) :footcite:`BrunaEtAl2018` given by:: |E[Im(Sxy/|Sxy|)]| @@ -210,6 +265,25 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, WPLI = ------------------ E[|Im(Sxy)|] + 'gc' : State-space Granger Causality (GC) :footcite:`BarnettSeth2015` + given by: + + :math:`GC = ln\Large{(\frac{\lvert\boldsymbol{S}_{tt}\rvert}{\lvert + \boldsymbol{S}_{tt}-\boldsymbol{H}_{ts}\boldsymbol{\Sigma}_{ss + \lvert t}\boldsymbol{H}_{ts}^*\rvert}})`, + + where: :math:`s` and :math:`t` represent the seeds and targets, + respectively; :math:`\boldsymbol{H}` is the spectral transfer + function; :math:`\boldsymbol{\Sigma}` is the residuals matrix of + the autoregressive model; and :math:`\boldsymbol{S}` is + :math:`\boldsymbol{\Sigma}` transformed by :math:`\boldsymbol{H}`. + + 'gc_tr' : State-space GC on time-reversed signals + :footcite:`BarnettSeth2015,WinklerEtAl2016` given by the same equation + as for 'gc', but where the autocovariance sequence from which the + autoregressive model is produced is transposed to mimic the reversal of + the original signal in time. + Parallel computation can be activated by setting the ``n_jobs`` parameter. Under the hood, this utilizes the ``joblib`` library. For effective parallelization, you should activate memory mapping in MNE-Python by @@ -284,6 +358,22 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, if np.any(fmin > fmax): raise ValueError('fmax must be larger than fmin') + if len(fmin) != 1 and any( + this_method in _gc_methods for this_method in method + ): + raise ValueError('computing Granger causality on multiple frequency ' + 'bands is not yet supported') + + if any(this_method in _multivariate_methods for this_method in method): + if not all(this_method in _multivariate_methods for + this_method in method): + raise ValueError( + 'bivariate and multivariate connectivity methods cannot be ' + 'used in the same function call') + multivariate_con = True + else: + multivariate_con = False + # convert kernel width in time to samples if isinstance(sm_times, (int, float)): sm_times = int(np.round(sm_times * sfreq)) @@ -302,12 +392,45 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, # get indices of pairs of (group) regions if indices is None: - indices_use = np.tril_indices(n_signals, k=-1) + if multivariate_con: + if any(this_method in _gc_methods for this_method in method): + raise ValueError( + 'indices must be specified when computing Granger ' + 'causality, as all-to-all connectivity is not supported') + logger.info('using all indices for multivariate connectivity') + indices_use = (np.arange(n_signals, dtype=int), + np.arange(n_signals, dtype=int)) + else: + logger.info('only using indices for lower-triangular matrix') + indices_use = np.tril_indices(n_signals, k=-1) else: + if multivariate_con: + if ( + len(np.unique(indices[0])) != len(indices[0]) or + len(np.unique(indices[1])) != len(indices[1]) + ): + raise ValueError( + 'seed and target indices cannot contain repeated ' + 'channels for multivariate connectivity') + if any(this_method in _gc_methods for this_method in method): + if set(indices[0]).intersection(indices[1]): + raise ValueError( + 'seed and target indices must not intersect when ' + 'computing Granger causality') indices_use = check_indices(indices) source_idx = indices_use[0] target_idx = indices_use[1] - n_pairs = len(source_idx) + n_pairs = len(source_idx) if not multivariate_con else 1 + + # unique signals for which we actually need to compute the CSD of + signals_use = np.unique(np.r_[indices_use[0], indices_use[1]]) + + # check rank input and compute data ranks if necessary + if multivariate_con: + rank = _check_rank_input(rank, data, sfreq, indices_use) + else: + rank = None + gc_n_lags = None # check freqs if isinstance(freqs, (int, float)): @@ -354,26 +477,38 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, out_freqs = freqs conn = dict() + conn_patterns = dict() for m in method: conn[m] = np.zeros((n_epochs, n_pairs, n_freqs)) + conn_patterns[m] = np.full((n_epochs, 2, len(source_idx), n_freqs), + np.nan) logger.info('Connectivity computation...') # parameters to pass to the connectivity function call_params = dict( method=method, kernel=kernel, foi_idx=foi_idx, - source_idx=source_idx, target_idx=target_idx, + source_idx=source_idx, target_idx=target_idx, signals_use=signals_use, mode=mode, sfreq=sfreq, freqs=freqs, faverage=faverage, - n_cycles=n_cycles, mt_bandwidth=mt_bandwidth, - decim=decim, padding=padding, kw_cwt={}, kw_mt={}, n_jobs=n_jobs, - verbose=verbose) + n_cycles=n_cycles, mt_bandwidth=mt_bandwidth, gc_n_lags=gc_n_lags, + rank=rank, decim=decim, padding=padding, kw_cwt={}, kw_mt={}, + n_jobs=n_jobs, verbose=verbose, multivariate_con=multivariate_con) for epoch_idx in np.arange(n_epochs): logger.info(f' Processing epoch {epoch_idx+1} / {n_epochs} ...') - conn_tr = _spectral_connectivity(data[epoch_idx], **call_params) + scores, patterns = _spectral_connectivity(data[epoch_idx], + **call_params) for m in method: - conn[m][epoch_idx] = np.stack(conn_tr[m], axis=0) + conn[m][epoch_idx] = np.stack(scores[m], axis=0) + if multivariate_con and patterns[m] is not None: + conn_patterns[m][epoch_idx] = np.stack(patterns[m], axis=0) + for m in method: + if np.isnan(conn_patterns[m]).all(): + conn_patterns[m] = None + else: + # epochs x 2 x n_channels x n_freqs + conn_patterns[m] = conn_patterns[m].transpose((1, 0, 2, 3)) - if indices is None: + if indices is None and not multivariate_con: conn_flat = conn conn = dict() for m in method: @@ -385,18 +520,28 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, conn_flat[m].shape[2:]) conn[m] = this_conn - # create a Connectivity container - if average: - out = [SpectralConnectivity( - conn[m].mean(axis=0), freqs=out_freqs, n_nodes=n_signals, - names=names, indices=indices, method=method, spec_method=mode, - events=events, event_id=event_id, metadata=metadata) - for m in method] - else: - out = [EpochSpectralConnectivity( - conn[m], freqs=out_freqs, n_nodes=n_signals, names=names, - indices=indices, method=method, spec_method=mode, events=events, - event_id=event_id, metadata=metadata) for m in method] + if multivariate_con: + # UNTIL RAGGED ARRAYS SUPPORTED + indices = tuple( + [[np.array(indices_use[0])], [np.array(indices_use[1])]]) + + # create the connectivity containers + out = [] + for m in method: + store_params = { + 'data': conn[m], 'patterns': conn_patterns[m], 'freqs': out_freqs, + 'n_nodes': n_signals, 'names': names, 'indices': indices, + 'method': method, 'spec_method': mode, 'events': events, + 'event_id': event_id, 'metadata': metadata, 'rank': rank, + 'n_lags': gc_n_lags if m in _gc_methods else None} + if average: + store_params['data'] = np.mean(store_params['data'], axis=0) + if conn_patterns[m] is not None: + store_params['patterns'] = np.mean(store_params['patterns'], + axis=1) + out.append(SpectralConnectivity(**store_params)) + else: + out.append(EpochSpectralConnectivity(**store_params)) logger.info('[Connectivity computation done]') @@ -408,10 +553,10 @@ def spectral_connectivity_time(data, freqs, method='coh', average=False, def _spectral_connectivity(data, method, kernel, foi_idx, - source_idx, target_idx, + source_idx, target_idx, signals_use, mode, sfreq, freqs, faverage, n_cycles, - mt_bandwidth, decim, padding, kw_cwt, kw_mt, - n_jobs, verbose): + mt_bandwidth, gc_n_lags, rank, decim, padding, + kw_cwt, kw_mt, n_jobs, verbose, multivariate_con): """Estimate time-resolved connectivity for one epoch. Parameters @@ -428,6 +573,8 @@ def _spectral_connectivity(data, method, kernel, foi_idx, Defines the signal pairs of interest together with ``target_idx``. target_idx : array_like, shape (n_pairs,) Defines the signal pairs of interest together with ``source_idx``. + signals_use : list of int + The unique signals on which connectivity is to be computed. mode : str Time-frequency transformation method. sfreq : float @@ -443,19 +590,34 @@ def _spectral_connectivity(data, method, kernel, foi_idx, frequency. mt_bandwidth : float | None Multitaper time-bandwidth. + gc_n_lags : int + Number of lags to use for the vector autoregressive model when + computing Granger causality. + rank : tuple of array + Ranks to project the seed and target data to. decim : int Decimation factor after time-frequency decomposition. padding : float Amount of time to consider as padding at the beginning and end of each epoch in seconds. + multivariate_con : bool + Whether or not multivariate connectivity is to be computed. Returns ------- - this_conn : list of array - List of connectivity estimates corresponding to the metrics in - ``method``. Each element is an array of shape (n_pairs, n_freqs) or - (n_pairs, n_fbands) if ``faverage`` is `True`. + scores : dict + Dictionary containing the connectivity estimates corresponding to the + metrics in ``method``. Each element is an array of shape (n_pairs, + n_freqs) or (n_pairs, n_fbands) if ``faverage`` is `True`. + + patterns : dict + Dictionary containing the connectivity patterns (for reconstructing the + connectivity components in source-space) corresponding to the metrics + in ``method``, if multivariate methods are called, else an empty + dictionary. Each element is an array of shape (2, n_channels, n_freqs) + or (2, n_channels, 1) if ``faverage`` is `True`, where 2 corresponds to + the seed and target signals (respectively). """ n_pairs = len(source_idx) data = np.expand_dims(data, axis=0) @@ -500,13 +662,20 @@ def _spectral_connectivity(data, method, kernel, foi_idx, else None # compute for each connectivity method - this_conn = {} + scores = {} + patterns = {} conn = _parallel_con(out, method, kernel, foi_idx, source_idx, target_idx, - n_jobs, verbose, n_pairs, faverage, weights) + signals_use, gc_n_lags, rank, n_jobs, verbose, + n_pairs, faverage, weights, multivariate_con) for i, m in enumerate(method): - this_conn[m] = [out[i] for out in conn] + if multivariate_con: + scores[m] = conn[0][i] + patterns[m] = conn[1][i][:, 0] if conn[1][i] is not None else None + else: + scores[m] = [out[i] for out in conn] + patterns[m] = None - return this_conn + return scores, patterns ############################################################################### @@ -515,8 +684,9 @@ def _spectral_connectivity(data, method, kernel, foi_idx, ############################################################################### ############################################################################### -def _parallel_con(w, method, kernel, foi_idx, source_idx, target_idx, n_jobs, - verbose, total, faverage, weights): +def _parallel_con(w, method, kernel, foi_idx, source_idx, target_idx, + signals_use, gc_n_lags, rank, n_jobs, verbose, total, + faverage, weights, multivariate_con): """Compute spectral connectivity in parallel. Parameters @@ -533,6 +703,13 @@ def _parallel_con(w, method, kernel, foi_idx, source_idx, target_idx, n_jobs, Defines the signal pairs of interest together with ``target_idx``. target_idx : array_like, shape (n_pairs,) Defines the signal pairs of interest together with ``source_idx``. + signals_use : list of int + The unique signals on which connectivity is to be computed. + gc_n_lags : int + Number of lags to use for the vector autoregressive model when + computing Granger causality. + rank : tuple of array + Ranks to project the seed and target data to. n_jobs : int Number of parallel jobs. total : int @@ -541,12 +718,17 @@ def _parallel_con(w, method, kernel, foi_idx, source_idx, target_idx, n_jobs, Average over frequency bands. weights : array_like, shape (n_tapers, n_freqs, n_times) Multitaper weights. + multivariate_con : bool + Whether or not multivariate connectivity is being computed. Returns ------- - out : array_like, shape (n_pairs, n_methods, n_freqs_out) + out : tuple of list of array Connectivity estimates for each signal pair, method, and frequency or - frequency band. + frequency band. If bivariate methods are called, the output is a tuple + of a list of arrays containing the connectivity scores. If multivariate + methods are called, the output is a tuple of lists containing arrays + for the connectivity scores and patterns, respectively. """ if 'coh' in method: # psd @@ -564,18 +746,22 @@ def _parallel_con(w, method, kernel, foi_idx, source_idx, target_idx, n_jobs, else: psd = None - # only show progress if verbosity level is DEBUG - if verbose != 'DEBUG' and verbose != 'debug' and verbose != 10: - total = None + if not multivariate_con: + # only show progress if verbosity level is DEBUG + if verbose != 'DEBUG' and verbose != 'debug' and verbose != 10: + total = None + + # define the function to compute in parallel + parallel, my_pairwise_con, n_jobs = parallel_func( + _pairwise_con, n_jobs=n_jobs, verbose=verbose, total=total) - # define the function to compute in parallel - parallel, my_pairwise_con, n_jobs = parallel_func( - _pairwise_con, n_jobs=n_jobs, verbose=verbose, total=total) + return tuple(parallel( + my_pairwise_con(w, psd, s, t, method, kernel, foi_idx, faverage, + weights) for s, t in zip(source_idx, target_idx))) - return parallel( - my_pairwise_con(w, psd, s, t, method, kernel, - foi_idx, faverage, weights) - for s, t in zip(source_idx, target_idx)) + return _multivariate_con(w, source_idx, target_idx, signals_use, method, + kernel, foi_idx, faverage, weights, gc_n_lags, + rank, n_jobs) def _pairwise_con(w, psd, x, y, method, kernel, foi_idx, @@ -639,6 +825,96 @@ def _pairwise_con(w, psd, x, y, method, kernel, foi_idx, return out +def _multivariate_con(w, source_idx, target_idx, signals_use, method, kernel, + foi_idx, faverage, weights, gc_n_lags, rank, n_jobs): + """Compute spectral connectivity metrics between multiple signals. + + Parameters + ---------- + w : array_like, shape (n_chans, n_tapers, n_freqs, n_times) + Time-frequency data. + x : int + Channel index. + y : int + Channel index. + method : str + Connectivity method. + kernel : array_like, shape (n_sm_fres, n_sm_times) + Smoothing kernel. + foi_idx : array_like, shape (n_foi, 2) + Upper and lower bound indices of frequency bands. + faverage : bool + Average over frequency bands. + weights : array_like, shape (n_tapers, n_freqs, n_times) | None + Multitaper weights. + + Returns + ------- + scores : list + List of connectivity scores between seed and target signals for each + connectivity method. Each element is an array with shape (n_freqs,) or + (n_fbands) depending on ``faverage``. + + patterns : list + List of connectivity patterns between seed and target signals for each + connectivity method. Each element is an array of length 2 corresponding + to the seed and target patterns, respectively, each with shape + (n_channels, n_freqs,) or (n_channels, n_fbands) depending on + ``faverage``. + """ + csd = [] + for x in signals_use: + for y in signals_use: + w_x, w_y = w[x], w[y] + if weights is not None: + s_xy = np.sum(weights * w_x * np.conj(weights * w_y), axis=0) + s_xy = s_xy * 2 / (weights * np.conj(weights)).real.sum(axis=0) + else: + s_xy = w_x * np.conj(w_y) + s_xy = np.squeeze(s_xy, axis=0) + csd.append(_smooth_spectra(s_xy, kernel).mean(axis=-1)) + csd = np.array(csd) + + # initialise connectivity estimators and add CSD information + conn_class = {'mic': _MICEst, 'mim': _MIMEst, 'gc': _GCEst, + 'gc_tr': _GCTREst} + conn = [] + for m in method: + # N_CONS = 1 UNTIL RAGGED ARRAYS SUPPORTED + call_params = {'n_signals': len(signals_use), 'n_cons': 1, + 'n_freqs': csd.shape[1], 'n_times': 0, + 'n_jobs': n_jobs} + if m in _gc_methods: + call_params['n_lags'] = gc_n_lags + con_est = conn_class[m](**call_params) + for con_i, con_csd in enumerate(csd): + con_est.accumulate(con_i, con_csd) + conn.append(con_est) + + # compute connectivity + scores = [] + patterns = [] + for con_est in conn: + con_est.compute_con(np.array([source_idx, target_idx]), rank) + scores.append(con_est.con_scores[..., np.newaxis]) + patterns.append(con_est.patterns) + if patterns[-1] is not None: + patterns[-1] = patterns[-1][..., np.newaxis] + + for i, _ in enumerate(scores): + # mean inside frequency sliding window (if needed) + if isinstance(foi_idx, np.ndarray) and faverage: + scores[i] = _foi_average(scores[i], foi_idx) + if patterns[i] is not None: + patterns[i] = _foi_average(patterns[i], foi_idx) + # squeeze time dimension + scores[i] = scores[i].squeeze(axis=-1) + if patterns[i] is not None: + patterns[i] = patterns[i].squeeze(axis=-1) + + return scores, patterns + + def _plv(s_xy): """Compute phase-locking value given the cross power spectral density.