closes #2927 power divergence statistic (#2932)

* closes #2927 power divergence statistic

* add scipy to requirements

* add arkouda/akstats/_stats_py.pyi

* Fix F403 and F401 error codes on flake8 arkouda from arkouda/akmath/__init__.py and arkouda/akstats/__init__.py

* un-pin scipy from specific version

* add scipy license and minor changes in response to code review

* Update tests/akmath/akmath_test.py

---------

Co-authored-by: Amanda Potts <ajpotts@users.noreply.github.com>
Co-authored-by: pierce <48131946+pierce314159@users.noreply.github.com>

PROTO_tests/tests/akmath/akmath_tests.py

@@ -0,0 +1,28 @@
+import math
+
+import numpy as np
+
+import arkouda as ak
+from arkouda.akmath import xlogy
+from arkouda.pdarrayclass import pdarray
+
+
+class TestStats:
+    def test_xlogy(self):
+        from scipy.special import xlogy as scipy_xlogy
+
+        ys = [ak.array([1, 2, 3]), ak.array([10, 100, 100]), ak.array([-1, 0, np.nan])]
+        xs = [3, 5, np.float64(6), ak.array([1.0, 2.0, 4.5])]
+
+        for y in ys:
+            for x in xs:
+                ak_result = xlogy(x, y)
+
+                np_y = y.to_ndarray()
+                np_x = x
+                if isinstance(np_x, pdarray):
+                    np_x = np_x.to_ndarray()
+
+                scipy_result = scipy_xlogy(np_x, np_y)
+
+                assert np.allclose(ak_result.to_ndarray(), scipy_result, equal_nan=True)

PROTO_tests/tests/akstats/akstats_test.py

@@ -0,0 +1,73 @@
+import math
+
+import numpy as np
+from scipy.stats import power_divergence as scipy_power_divergence
+
+import arkouda as ak
+from arkouda.akstats import power_divergence as ak_power_divergence
+
+
+class TestStats:
+    @staticmethod
+    def create_stat_test_pairs():
+        pairs = [
+            (
+                ak.array([10000000, 20000000, 30000000, 40000000, 50000000, 60000000, 70000000]),
+                ak.array([10000000, 20000000, 30000000, 40000001, 50000000, 60000000, 70000000]),
+            ),
+            (ak.array([10000000, 20000000, 30000000, 40000000, 50000000, 60000000, 70000000]), None),
+            (ak.array([44, 24, 29, 3]) / 100 * 189, ak.array([43, 52, 54, 40])),
+        ]
+        return pairs
+
+    def test_power_divergence(self):
+        pairs = self.create_stat_test_pairs()
+
+        lambdas = [
+            "pearson",
+            "log-likelihood",
+            "freeman-tukey",
+            "mod-log-likelihood",
+            "neyman",
+            "cressie-read",
+        ]
+
+        ddofs = [0, 1, 2, 3, 4, 5]
+
+        for f_obs, f_exp in pairs:
+            for lambda0 in lambdas:
+                for ddof in ddofs:
+                    ak_power_div = ak_power_divergence(f_obs, f_exp, ddof=ddof, lambda_=lambda0)
+
+                    np_f_obs = f_obs.to_ndarray()
+                    np_f_exp = None
+                    if f_exp is not None:
+                        np_f_exp = f_exp.to_ndarray()
+
+                    scipy_power_div = scipy_power_divergence(
+                        np_f_obs, np_f_exp, ddof=ddof, axis=0, lambda_=lambda0
+                    )
+
+                    assert np.allclose(ak_power_div, scipy_power_div, equal_nan=True)
+
+    def test_chisquare(self):
+        from scipy.stats import chisquare as scipy_chisquare
+
+        from arkouda.akstats import chisquare as ak_chisquare
+
+        pairs = self.create_stat_test_pairs()
+
+        ddofs = [0, 1, 2, 3, 4, 5]
+
+        for f_obs, f_exp in pairs:
+            for ddof in ddofs:
+                ak_chisq = ak_chisquare(f_obs, f_exp, ddof=ddof)
+
+                np_f_obs = f_obs.to_ndarray()
+                np_f_exp = None
+                if f_exp is not None:
+                    np_f_exp = f_exp.to_ndarray()
+
+                scipy_chisq = scipy_chisquare(np_f_obs, np_f_exp, ddof=ddof, axis=0)
+
+                assert np.allclose(ak_chisq, scipy_chisq, equal_nan=True)

arkouda-env-dev.yml

@@ -20,6 +20,7 @@ dependencies:
   - libiconv
   - libidn2
   - jupyter
+  - scipy
 
   # Developer dependencies
   - pexpect
@@ -42,4 +43,4 @@ dependencies:
     - furo # sphinx theme
     - myst-parser
     - linkify-it-py
-
+

arkouda-env.yml

@@ -20,6 +20,7 @@ dependencies:
   - libiconv
   - libidn2
   - jupyter
-
+  - scipy
+
   - pip:
-      - typeguard==2.10.0
+      - typeguard==2.10.0

arkouda/__init__.py

@@ -39,3 +39,5 @@
     is_registered,
     broadcast_dims,
 )
+from arkouda.akmath import *
+from arkouda.akstats import *

arkouda/akmath/__init__.py

@@ -0,0 +1,5 @@
+from ._math import xlogy
+
+__all__ = [
+    "xlogy",
+]

arkouda/akmath/_math.py

@@ -0,0 +1,52 @@
+from typing import Union
+from warnings import warn
+
+import numpy as np
+
+from arkouda.numeric import log
+from arkouda.pdarrayclass import pdarray
+
+
+def xlogy(x: Union[pdarray, np.float64], y: pdarray):
+    """
+    Computes x * log(y).
+
+    Parameters
+    ----------
+    x : pdarray or np.float64
+        x must have a datatype that is castable to float64
+    y : pdarray
+
+    Returns
+    -------
+    arkouda.pdarrayclass.pdarray
+
+    Examples
+    --------
+
+    >>> import arkouda as ak
+    >>> ak.connect()
+    >>> from arkouda.akmath import xlogy
+    >>> xlogy( ak.array([1, 2, 3, 4]),  ak.array([5,6,7,8]))
+    array([1.6094379124341003 3.5835189384561099 5.8377304471659395 8.317766166719343])
+    >>> xlogy( 5.0, ak.array([1, 2, 3, 4]))
+    array([0.00000000000000000 3.4657359027997265 5.4930614433405491 6.9314718055994531])
+
+
+    """
+    if not isinstance(x, (np.float64, pdarray)) and np.can_cast(x, np.float64):
+        x = np.float64(x)
+
+    if isinstance(x, pdarray) and isinstance(y, pdarray):
+        if x.size == y.size:
+            return x * log(y)
+        else:
+            msg = "x and y must have the same size."
+            warn(msg, UserWarning)
+            return None
+    elif isinstance(x, np.float64) and isinstance(y, pdarray):
+        return x * log(y)
+    else:
+        msg = "x and y must both be pdarrays or x must be castable to float64 and y must be a pdarray."
+        warn(msg, UserWarning)
+        return None

arkouda/akstats/LICENSE.txt

@@ -0,0 +1,30 @@
+Copyright (c) 2001-2002 Enthought, Inc. 2003-2024, SciPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following
+   disclaimer in the documentation and/or other materials provided
+   with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

arkouda/akstats/__init__.py

@@ -0,0 +1,3 @@
+from ._stats_py import Power_divergenceResult, chisquare, power_divergence
+
+__all__ = ["power_divergence", "chisquare", "Power_divergenceResult"]