diff --git a/arkouda/array_api/utility_functions.py b/arkouda/array_api/utility_functions.py index 69875d1ef6..05eaf907d9 100644 --- a/arkouda/array_api/utility_functions.py +++ b/arkouda/array_api/utility_functions.py @@ -71,9 +71,9 @@ def clip(a: Array, a_min, a_max, /) -> Array: return Array._new( create_pdarray( generic_msg( - cmd=f"clip{a.ndim}D", + cmd=f"clip<{a.dtype},{a.ndim}>", args={ - "name": a._array, + "x": a._array, "min": a_min, "max": a_max, }, @@ -111,9 +111,9 @@ def diff(a: Array, /, n: int = 1, axis: int = -1, prepend=None, append=None) -> return Array._new( create_pdarray( generic_msg( - cmd=f"diff{a.ndim}D", + cmd=f"diff<{a.dtype},{a.ndim}>", args={ - "name": a_._array, + "x": a_._array, "n": n, "axis": axis, }, @@ -176,7 +176,7 @@ def pad( return Array._new( create_pdarray( generic_msg( - cmd=f"pad{array.ndim}D", + cmd=f"pad<{array.dtype},{array.ndim}>", args={ "name": array._array, "padWidthBefore": tuple(pad_widths_b), diff --git a/src/UtilMsg.chpl b/src/UtilMsg.chpl index 4c572c05fe..a23bf24a87 100644 --- a/src/UtilMsg.chpl +++ b/src/UtilMsg.chpl @@ -8,6 +8,8 @@ module UtilMsg { use Logging; use Message; use AryUtil; + use List; + use BigInteger; use MultiTypeSymEntry; use MultiTypeSymbolTable; @@ -21,50 +23,29 @@ module UtilMsg { see: https://numpy.org/doc/stable/reference/generated/numpy.clip.html */ - @arkouda.registerND - proc clipMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, param nd: int): MsgTuple throws { - param pn = Reflection.getRoutineName(); + @arkouda.registerCommand() + proc clip(const ref x: [?d] ?t, min: real, max: real): [] t throws + where (t == int) || (t == real) || (t == uint(8)) || (t == uint(64)) { - const name = msgArgs.getValueOf("name"), - min = msgArgs.get("min"), - max = msgArgs.get("max"), - rname = st.nextName(); + var y = makeDistArray(d, t); - var gEnt: borrowed GenSymEntry = getGenericTypedArrayEntry(name, st); + const minVal = min: t, + maxVal = max: t; - proc doClip(type t): MsgTuple throws { - const minVal = min.getScalarValue(t), - maxVal = max.getScalarValue(t); - - const e = toSymEntry(gEnt, t, nd); - var c = st.addEntry(rname, (...e.tupShape), t); - - forall i in e.a.domain { - if e.a[i] < minVal then - c.a[i] = minVal; - else if e.a[i] > maxVal then - c.a[i] = maxVal; + forall i in d { + if x[i] < minVal then + y[i] = minVal; + else if x[i] > maxVal then + y[i] = maxVal; else - c.a[i] = e.a[i]; + y[i] = x[i]; } + return y; + } - const repMsg = "created " + st.attrib(rname); - uLogger.debug(getModuleName(),pn,getLineNumber(),repMsg); - return new MsgTuple(repMsg, MsgType.NORMAL); - } - - select gEnt.dtype { - when DType.Int64 do return doClip(int); - when DType.UInt8 do return doClip(uint(8)); - when DType.UInt64 do return doClip(uint); - when DType.Float64 do return doClip(real); - when DType.Bool do return doClip(bool); - otherwise { - const errorMsg = notImplementedError(pn,gEnt.dtype); - uLogger.error(getModuleName(),pn,getLineNumber(),errorMsg); - return new MsgTuple(errorMsg, MsgType.ERROR); - } - } + proc clip(const ref x: [?d] ?t, min: real, max: real): [d] t throws + where (t != int) && (t != real) && (t != uint(8)) && (t != uint(64)){ + throw new Error("clip does not support dtype %s".format(t:string)); } /* @@ -72,76 +53,51 @@ module UtilMsg { see: https://numpy.org/doc/stable/reference/generated/numpy.diff.html */ - @arkouda.registerND - proc diffMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, param nd: int): MsgTuple throws { - param pn = Reflection.getRoutineName(); - - const name = msgArgs.getValueOf("name"), - n = msgArgs.get("n").getIntValue(), - axis = msgArgs.get("axis").getPositiveIntValue(nd), - rname = st.nextName(); - - var gEnt: borrowed GenSymEntry = getGenericTypedArrayEntry(name, st); - - proc doDiff(type t): MsgTuple throws { - const e = toSymEntry(gEnt, t, nd); - - if n == 1 { - // 1st order difference: requires no temporary storage - const outDom = subDomain(e.tupShape, axis, 1); - var d = st.addEntry(rname, (...outDom.shape), t); - - forall axisSliceIdx in domOffAxis(e.a.domain, axis) { - const slice = domOnAxis(outDom, tuplify(axisSliceIdx), axis); - for i in slice { - var idxp = tuplify(i); - idxp[axis] += 1; - d.a[i] = e.a[idxp] - e.a[i]; - } + + @arkouda.registerCommand() + proc diff(x: [?d] ?t, n: int, axis: int): [] t throws + where (t == real) || (t == int) || (t == uint(8)) || (t == uint(64)){ + + const outDom = subDomain(x.shape, axis, n); + if n == 1 { + // 1st order difference: requires no temporary storage + var y = makeDistArray(outDom, t); + for axisSliceIdx in domOffAxis(d, axis) { + const slice = domOnAxis(outDom, tuplify(axisSliceIdx), axis); + for i in slice { + var idxp = tuplify(i); + idxp[axis] += 1; + y[i] = x[idxp] - x[i]; } - } else { - // n'th order difference: requires 2 temporary arrays - var d1 = makeDistArray(e.a); - - { - var d2 = makeDistArray(e.a.domain, e.a.eltType); - for m in 1..n { - d1 <=> d2; - const diffSubDom = subDomain(e.tupShape, axis, m); - - forall axisSliceIdx in domOffAxis(e.a.domain, axis) { - const slice = domOnAxis(diffSubDom, tuplify(axisSliceIdx), axis); - - for i in slice { - var idxp = tuplify(i); - idxp[axis] += 1; - d1[i] = d2[idxp] - d2[i]; - } + } + return y; + } else { + // n'th order difference: requires 2 temporary arrays + var d1 = makeDistArray(x); + { + var d2 = makeDistArray(d, t); + for m in 1..n { + d1 <=> d2; + const diffSubDom = subDomain(x.shape, axis, m); + + forall axisSliceIdx in domOffAxis(d, axis) { + const slice = domOnAxis(diffSubDom, tuplify(axisSliceIdx), axis); + + for i in slice { + var idxp = tuplify(i); + idxp[axis] += 1; + d1[i] = d2[idxp] - d2[i]; } } - } // d2 deinit here - - const outDom = subDomain(e.tupShape, axis, n), - d = createSymEntry(d1[outDom]); - st.addEntry(rname, d); - } - - const repMsg = "created " + st.attrib(rname); - uLogger.debug(getModuleName(),pn,getLineNumber(),repMsg); - return new MsgTuple(repMsg, MsgType.NORMAL); + } + } // d2 deinit here + return d1[outDom]; } + } - select gEnt.dtype { - when DType.Int64 do return doDiff(int); - when DType.UInt8 do return doDiff(uint(8)); - when DType.UInt64 do return doDiff(uint); - when DType.Float64 do return doDiff(real); - otherwise { - const errorMsg = notImplementedError(pn,gEnt.dtype); - uLogger.error(getModuleName(),pn,getLineNumber(),errorMsg); - return new MsgTuple(errorMsg, MsgType.ERROR); - } - } + proc diff(x: [?d] ?t, n: int, axis: int): [d] t throws + where (t != real) && (t != int) && (t != uint(8)) && (t != uint(64)){ + throw new Error("diff does not support dtype %s".format(t:string)); } // helper to create a domain that's 'n' elements smaller in the 'axis' dimension @@ -165,71 +121,52 @@ module UtilMsg { Implements the 'constant' mode of numpy.pad: https://numpy.org/doc/stable/reference/generated/numpy.pad.html */ - @arkouda.registerND - proc padMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, param nd: int): MsgTuple throws { - param pn = Reflection.getRoutineName(); - - const name = msgArgs.getValueOf("name"), - padWidthBefore = msgArgs.get("padWidthBefore").getTuple(nd), - padWidthAfter = msgArgs.get("padWidthAfter").getTuple(nd), - padValsBefore = msgArgs.get("padValsBefore"), - padValsAfter = msgArgs.get("padValsAfter"), - rname = st.nextName(); - - var gEnt: borrowed GenSymEntry = getGenericTypedArrayEntry(name, st); - - proc doPad(type t): MsgTuple throws { - const e = toSymEntry(gEnt, t, nd); - - const pvb = padValsBefore.toScalarArray(t, nd), - pva = padValsAfter.toScalarArray(t, nd); - - // compute the padded shape - var outShape: nd*int; - for i in 0..', ark_cumSum_bigint_1, 'StatsMsg', 120); +import UtilMsg; + +proc ark_reg_clip_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var min = msgArgs['min'].toScalar(real); + var max = msgArgs['max'].toScalar(real); + var ark_result = UtilMsg.clip(x,min,max); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_clip_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('clip', ark_clip_int_1, 'UtilMsg', 27); + +proc ark_clip_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('clip', ark_clip_uint_1, 'UtilMsg', 27); + +proc ark_clip_uint8_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=uint(8), array_nd_0=1); +registerFunction('clip', ark_clip_uint8_1, 'UtilMsg', 27); + +proc ark_clip_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('clip', ark_clip_real_1, 'UtilMsg', 27); + +proc ark_clip_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('clip', ark_clip_bool_1, 'UtilMsg', 27); + +proc ark_clip_bigint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_clip_generic(cmd, msgArgs, st, array_dtype_0=bigint, array_nd_0=1); +registerFunction('clip', ark_clip_bigint_1, 'UtilMsg', 27); + +proc ark_reg_diff_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var n = msgArgs['n'].toScalar(int); + var axis = msgArgs['axis'].toScalar(int); + var ark_result = UtilMsg.diff(x,n,axis); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_diff_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('diff', ark_diff_int_1, 'UtilMsg', 58); + +proc ark_diff_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('diff', ark_diff_uint_1, 'UtilMsg', 58); + +proc ark_diff_uint8_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=uint(8), array_nd_0=1); +registerFunction('diff', ark_diff_uint8_1, 'UtilMsg', 58); + +proc ark_diff_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('diff', ark_diff_real_1, 'UtilMsg', 58); + +proc ark_diff_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('diff', ark_diff_bool_1, 'UtilMsg', 58); + +proc ark_diff_bigint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_diff_generic(cmd, msgArgs, st, array_dtype_0=bigint, array_nd_0=1); +registerFunction('diff', ark_diff_bigint_1, 'UtilMsg', 58); + +proc ark_pad_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('pad', ark_pad_int_1, 'UtilMsg', 125); + +proc ark_pad_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('pad', ark_pad_uint_1, 'UtilMsg', 125); + +proc ark_pad_uint8_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=uint(8), array_nd=1); +registerFunction('pad', ark_pad_uint8_1, 'UtilMsg', 125); + +proc ark_pad_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('pad', ark_pad_real_1, 'UtilMsg', 125); + +proc ark_pad_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('pad', ark_pad_bool_1, 'UtilMsg', 125); + +proc ark_pad_bigint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return UtilMsg.pad(cmd, msgArgs, st, array_dtype=bigint, array_nd=1); +registerFunction('pad', ark_pad_bigint_1, 'UtilMsg', 125); + import MsgProcessing; proc ark_create_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do diff --git a/tests/array_api/util_functions.py b/tests/array_api/util_functions.py index 009976af7a..21725a44da 100644 --- a/tests/array_api/util_functions.py +++ b/tests/array_api/util_functions.py @@ -9,17 +9,20 @@ SEED = 314159 s = SEED +DTYPES = [ak.int64, ak.float64, ak.uint64, ak.uint8] + def get_server_max_array_dims(): try: - return json.load(open('serverConfig.json', 'r'))['max_array_dims'] + return json.load(open("serverConfig.json", "r"))["max_array_dims"] except (ValueError, FileNotFoundError, TypeError, KeyError): return 1 -def randArr(shape): + +def randArr(shape, dtype): global s s += 2 - return xp.asarray(ak.randint(0, 100, shape, dtype=ak.int64, seed=s)) + return xp.asarray(ak.randint(0, 100, shape, dtype=ak.int64, seed=s), dtype=dtype) class TestUtilFunctions: @@ -45,9 +48,12 @@ def test_any(self): a[3, 4] = True assert xp.any(a) - @pytest.mark.skipif(get_server_max_array_dims() < 3, reason="test_clip requires server with 'max_array_dims' >= 3") - def test_clip(self): - a = randArr((5, 6, 7)) + @pytest.mark.parametrize("dtype", DTYPES) + @pytest.mark.skipif( + get_server_max_array_dims() < 3, reason="test_clip requires server with 'max_array_dims' >= 3" + ) + def test_clip(self, dtype): + a = randArr((5, 6, 7), dtype) anp = a.to_ndarray() a_c = xp.clip(a, 10, 90) @@ -58,8 +64,33 @@ def test_clip(self): get_server_max_array_dims() < 3, reason="test_diff requires server with 'max_array_dims' >= 3", ) - def test_diff(self): - a = randArr((5, 6, 7)) + def test_clip_errors(self): + # bool + a = xp.asarray(ak.randint(0, 100, (5, 6, 7), dtype=ak.bool_, seed=s), dtype=ak.bool_) + with pytest.raises( + RuntimeError, match="Error executing command: clip does not support dtype bool" + ): + xp.clip(a, 10, 90) + + # bigint + bi_arr = ak.array( + [0, 1, 2, 3, 4, 2**64 - 5, 2**64 - 4, 2**64 - 3, 2**64 - 2, 2**64 - 1], + dtype=ak.bigint, + max_bits=64, + ) + a = xp.asarray(bi_arr, dtype=ak.bool_) + with pytest.raises( + RuntimeError, match="Error executing command: clip does not support dtype bigint" + ): + xp.clip(a, 10, 90) + + @pytest.mark.parametrize("dtype", DTYPES) + @pytest.mark.skipif( + get_server_max_array_dims() < 3, + reason="test_diff requires server with 'max_array_dims' >= 3", + ) + def test_diff(self, dtype): + a = randArr((5, 6, 7), dtype) anp = a.to_ndarray() a_d = xp.diff(a, n=1, axis=1) @@ -71,6 +102,30 @@ def test_diff(self): assert a_d.tolist() == anp_d.tolist() + @pytest.mark.skipif( + get_server_max_array_dims() < 3, + reason="test_diff requires server with 'max_array_dims' >= 3", + ) + def test_diff_error(self): + # bool + a = xp.asarray(ak.randint(0, 100, (5, 6, 7), dtype=ak.bool_, seed=s), dtype=ak.bool_) + with pytest.raises( + RuntimeError, match="Error executing command: diff does not support dtype bool" + ): + xp.diff(a, n=2, axis=0) + + # bigint + bi_arr = ak.array( + [0, 1, 2, 3, 4, 2**64 - 5, 2**64 - 4, 2**64 - 3, 2**64 - 2, 2**64 - 1], + dtype=ak.bigint, + max_bits=64, + ) + a = xp.asarray(bi_arr, dtype=ak.bool_) + with pytest.raises( + RuntimeError, match="Error executing command: diff does not support dtype bigint" + ): + xp.diff(a, n=2, axis=0) + @pytest.mark.skipif( get_server_max_array_dims() < 3, reason="test_pad requires server with 'max_array_dims' >= 3", @@ -94,3 +149,19 @@ def test_pad(self): a_p = xp.pad(a, 2, constant_values=3) anp_p = np.pad(anp, 2, constant_values=3) assert a_p.tolist() == anp_p.tolist() + + def test_pad_error(self): + # bigint + bi_arr = ak.array( + [0, 1, 2, 3, 4, 2**64 - 5, 2**64 - 4, 2**64 - 3, 2**64 - 2, 2**64 - 1], + dtype=ak.bigint, + max_bits=64, + ) + a = xp.asarray(bi_arr, dtype=ak.bool_) + with pytest.raises( + RuntimeError, match="Error executing command: pad does not support dtype bigint" + ): + xp.pad( + a, ((1, 1)), mode="constant", constant_values=((-1, 1)) + ) + xp.diff(a, n=2, axis=0) \ No newline at end of file