diff --git a/doc/release/upcoming_changes/30803.compatibility.rst b/doc/release/upcoming_changes/30803.compatibility.rst new file mode 100644 index 000000000000..3fc75cae53ab --- /dev/null +++ b/doc/release/upcoming_changes/30803.compatibility.rst @@ -0,0 +1,8 @@ +``numpy.where`` no longer truncates Python integers +--------------------------------------------------- + +Previously, if the ``x`` or ``y`` argument of ``numpy.where`` was a Python integer +that was out of range of the output type, it would be silently truncated. +Now, an `OverflowError` will be raised instead. + +This change also applies to the underlying C API function ``PyArray_Where``. diff --git a/numpy/_core/multiarray.py b/numpy/_core/multiarray.py index 1757270deb62..c7f21b60c488 100644 --- a/numpy/_core/multiarray.py +++ b/numpy/_core/multiarray.py @@ -210,6 +210,9 @@ def concatenate(arrays, axis=0, out=None, *, dtype=None, casting="same_kind"): Join a sequence of arrays along an existing axis. + .. versionadded:: 2.0 + ``numpy.concat`` added as a shorthand for ``numpy.concatenate``. + Parameters ---------- a1, a2, ... : sequence of array_like diff --git a/numpy/_core/src/common/cblasfuncs.c b/numpy/_core/src/common/cblasfuncs.c index 66a215dfeb64..04ca81086bcf 100644 --- a/numpy/_core/src/common/cblasfuncs.c +++ b/numpy/_core/src/common/cblasfuncs.c @@ -225,10 +225,11 @@ _bad_strides(PyArrayObject *ap) * __array_ufunc__ nonsense is also assumed to have been taken care of. */ NPY_NO_EXPORT PyObject * -cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, +cblas_matrixproduct(PyArray_Descr *typec, PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject *out) { PyArrayObject *result = NULL, *out_buf = NULL; + int typenum = typec->type_num; npy_intp j, lda, ldb; npy_intp l; int nd; @@ -364,7 +365,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, } } - out_buf = new_array_for_sum(ap1, ap2, out, nd, dimensions, typenum, &result); + out_buf = new_array_for_sum(ap1, ap2, out, nd, dimensions, typec, &result); if (out_buf == NULL) { goto fail; } diff --git a/numpy/_core/src/common/cblasfuncs.h b/numpy/_core/src/common/cblasfuncs.h index 71c533f369a4..fb9c325dd4d9 100644 --- a/numpy/_core/src/common/cblasfuncs.h +++ b/numpy/_core/src/common/cblasfuncs.h @@ -2,6 +2,6 @@ #define NUMPY_CORE_SRC_COMMON_CBLASFUNCS_H_ NPY_NO_EXPORT PyObject * -cblas_matrixproduct(int, PyArrayObject *, PyArrayObject *, PyArrayObject *); +cblas_matrixproduct(PyArray_Descr *, PyArrayObject *, PyArrayObject *, PyArrayObject *); #endif /* NUMPY_CORE_SRC_COMMON_CBLASFUNCS_H_ */ diff --git a/numpy/_core/src/multiarray/common.c b/numpy/_core/src/multiarray/common.c index fd4f24151331..954179c66cb3 100644 --- a/numpy/_core/src/multiarray/common.c +++ b/numpy/_core/src/multiarray/common.c @@ -381,7 +381,7 @@ _may_have_objects(PyArray_Descr *dtype) */ NPY_NO_EXPORT PyArrayObject * new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, - int nd, npy_intp dimensions[], int typenum, PyArrayObject **result) + int nd, npy_intp dimensions[], PyArray_Descr *descr, PyArrayObject **result) { PyArrayObject *out_buf; @@ -390,7 +390,7 @@ new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, /* verify that out is usable */ if (PyArray_NDIM(out) != nd || - PyArray_TYPE(out) != typenum || + !PyArray_EquivTypes(PyArray_DESCR(out), descr) || !PyArray_ISCARRAY(out)) { PyErr_SetString(PyExc_ValueError, "output array is not acceptable (must have the right datatype, " @@ -452,10 +452,11 @@ new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, subtype = Py_TYPE(ap1); } - out_buf = (PyArrayObject *)PyArray_New(subtype, nd, dimensions, - typenum, NULL, NULL, 0, 0, - (PyObject *) - (prior2 > prior1 ? ap2 : ap1)); + Py_INCREF(descr); + out_buf = (PyArrayObject *)PyArray_NewFromDescr(subtype, descr, nd, dimensions, + NULL, NULL, 0, + (PyObject *) + (prior2 > prior1 ? ap2 : ap1)); if (out_buf != NULL && result) { Py_INCREF(out_buf); diff --git a/numpy/_core/src/multiarray/common.h b/numpy/_core/src/multiarray/common.h index c0b5c043c7a3..f4d0e595aaa5 100644 --- a/numpy/_core/src/multiarray/common.h +++ b/numpy/_core/src/multiarray/common.h @@ -339,7 +339,7 @@ check_is_convertible_to_scalar(PyArrayObject *v); */ NPY_NO_EXPORT PyArrayObject * new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, - int nd, npy_intp dimensions[], int typenum, PyArrayObject **result); + int nd, npy_intp dimensions[], PyArray_Descr *descr, PyArrayObject **result); /* diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 9587ea5753c7..a451e0099675 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -881,7 +881,6 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) { PyArrayObject *ap1 = NULL; PyArrayObject *ap2 = NULL; - int typenum; PyArray_Descr *typec = NULL; PyObject* ap2t = NULL; npy_intp dims[NPY_MAXDIMS]; @@ -889,23 +888,18 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) int i; PyObject* ret = NULL; - typenum = PyArray_ObjectType(op1, NPY_NOTYPE); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op1, NPY_MAXDIMS, &typec) < 0) { return NULL; } - typenum = PyArray_ObjectType(op2, typenum); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op2, NPY_MAXDIMS, &typec) < 0) { + Py_XDECREF(typec); return NULL; } - typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "Cannot find a common data type."); - } - goto fail; + typec = PyArray_DescrFromType(NPY_DEFAULT_TYPE); } + Py_SETREF(typec, NPY_DT_CALL_ensure_canonical(typec)); Py_INCREF(typec); ap1 = (PyArrayObject *)PyArray_FromAny(op1, typec, 0, 0, @@ -914,6 +908,8 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) Py_DECREF(typec); goto fail; } + + Py_INCREF(typec); ap2 = (PyArrayObject *)PyArray_FromAny(op2, typec, 0, 0, NPY_ARRAY_ALIGNED, NULL); if (ap2 == NULL) { @@ -947,6 +943,7 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) Py_DECREF(ap1); Py_DECREF(ap2); Py_DECREF(ap2t); + Py_DECREF(typec); return ret; fail: @@ -954,6 +951,7 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) Py_XDECREF(ap2); Py_XDECREF(ap2t); Py_XDECREF(ret); + Py_XDECREF(typec); return NULL; } @@ -985,23 +983,19 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) PyArray_Descr *typec = NULL; NPY_BEGIN_THREADS_DEF; - typenum = PyArray_ObjectType(op1, NPY_NOTYPE); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op1, NPY_MAXDIMS, &typec) < 0) { return NULL; } - typenum = PyArray_ObjectType(op2, typenum); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op2, NPY_MAXDIMS, &typec) < 0) { + Py_XDECREF(typec); return NULL; } - typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "Cannot find a common data type."); - } - return NULL; + typec = PyArray_DescrFromType(NPY_DEFAULT_TYPE); } + Py_SETREF(typec, NPY_DT_CALL_ensure_canonical(typec)); + typenum = typec->type_num; Py_INCREF(typec); ap1 = (PyArrayObject *)PyArray_FromAny(op1, typec, 0, 0, @@ -1010,6 +1004,8 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) Py_DECREF(typec); return NULL; } + + Py_INCREF(typec); ap2 = (PyArrayObject *)PyArray_FromAny(op2, typec, 0, 0, NPY_ARRAY_ALIGNED, NULL); if (ap2 == NULL) { @@ -1021,7 +1017,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) if (PyArray_NDIM(ap1) <= 2 && PyArray_NDIM(ap2) <= 2 && (NPY_DOUBLE == typenum || NPY_CDOUBLE == typenum || NPY_FLOAT == typenum || NPY_CFLOAT == typenum)) { - return cblas_matrixproduct(typenum, ap1, ap2, out); + return cblas_matrixproduct(typec, ap1, ap2, out); } #endif @@ -1062,7 +1058,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) is1 = PyArray_STRIDES(ap1)[PyArray_NDIM(ap1)-1]; is2 = PyArray_STRIDES(ap2)[matchDim]; /* Choose which subtype to return */ - out_buf = new_array_for_sum(ap1, ap2, out, nd, dimensions, typenum, &result); + out_buf = new_array_for_sum(ap1, ap2, out, nd, dimensions, typec, &result); if (out_buf == NULL) { goto fail; } @@ -1124,6 +1120,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) /* Trigger possible copy-back into `result` */ PyArray_ResolveWritebackIfCopy(out_buf); Py_DECREF(out_buf); + Py_DECREF(typec); return (PyObject *)result; @@ -1132,6 +1129,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) Py_XDECREF(ap2); Py_XDECREF(out_buf); Py_XDECREF(result); + Py_XDECREF(typec); return NULL; } @@ -1143,7 +1141,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) * inverted is set to 1 if computed correlate(ap2, ap1), 0 otherwise */ static PyArrayObject* -_pyarray_correlate(PyArrayObject *ap1, PyArrayObject *ap2, int typenum, +_pyarray_correlate(PyArrayObject *ap1, PyArrayObject *ap2, PyArray_Descr *typec, int mode, int *inverted) { PyArrayObject *ret; @@ -1203,7 +1201,7 @@ _pyarray_correlate(PyArrayObject *ap1, PyArrayObject *ap2, int typenum, * Need to choose an output array that can hold a sum * -- use priority to determine which subtype. */ - ret = new_array_for_sum(ap1, ap2, NULL, 1, &length, typenum, NULL); + ret = new_array_for_sum(ap1, ap2, NULL, 1, &length, typec, NULL); if (ret == NULL) { return NULL; } @@ -1323,21 +1321,23 @@ NPY_NO_EXPORT PyObject * PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) { PyArrayObject *ap1, *ap2, *ret = NULL; - int typenum; - PyArray_Descr *typec; + PyArray_Descr *typec = NULL; int inverted; int st; - typenum = PyArray_ObjectType(op1, NPY_NOTYPE); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op1, NPY_MAXDIMS, &typec) < 0) { return NULL; } - typenum = PyArray_ObjectType(op2, typenum); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op2, NPY_MAXDIMS, &typec) < 0) { + Py_XDECREF(typec); return NULL; } - typec = PyArray_DescrFromType(typenum); + if (typec == NULL) { + typec = PyArray_DescrFromType(NPY_DEFAULT_TYPE); + } + Py_SETREF(typec, NPY_DT_CALL_ensure_canonical(typec)); + Py_INCREF(typec); ap1 = (PyArrayObject *)PyArray_FromAny(op1, typec, 1, 1, NPY_ARRAY_DEFAULT, NULL); @@ -1345,6 +1345,8 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) Py_DECREF(typec); return NULL; } + + Py_INCREF(typec); ap2 = (PyArrayObject *)PyArray_FromAny(op2, typec, 1, 1, NPY_ARRAY_DEFAULT, NULL); if (ap2 == NULL) { @@ -1361,7 +1363,7 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) ap2 = cap2; } - ret = _pyarray_correlate(ap1, ap2, typenum, mode, &inverted); + ret = _pyarray_correlate(ap1, ap2, typec, mode, &inverted); if (ret == NULL) { goto clean_ap2; } @@ -1379,6 +1381,7 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) Py_DECREF(ap1); Py_DECREF(ap2); + Py_DECREF(typec); return (PyObject *)ret; clean_ret: @@ -1387,6 +1390,8 @@ PyArray_Correlate2(PyObject *op1, PyObject *op2, int mode) Py_DECREF(ap2); clean_ap1: Py_DECREF(ap1); + + Py_DECREF(typec); return NULL; } @@ -1397,20 +1402,22 @@ NPY_NO_EXPORT PyObject * PyArray_Correlate(PyObject *op1, PyObject *op2, int mode) { PyArrayObject *ap1, *ap2, *ret = NULL; - int typenum; int unused; - PyArray_Descr *typec; + PyArray_Descr *typec = NULL; - typenum = PyArray_ObjectType(op1, NPY_NOTYPE); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op1, NPY_MAXDIMS, &typec) < 0) { return NULL; } - typenum = PyArray_ObjectType(op2, typenum); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op2, NPY_MAXDIMS, &typec) < 0) { + Py_XDECREF(typec); return NULL; } - typec = PyArray_DescrFromType(typenum); + if (typec == NULL) { + typec = PyArray_DescrFromType(NPY_DEFAULT_TYPE); + } + Py_SETREF(typec, NPY_DT_CALL_ensure_canonical(typec)); + Py_INCREF(typec); ap1 = (PyArrayObject *)PyArray_FromAny(op1, typec, 1, 1, NPY_ARRAY_DEFAULT, NULL); @@ -1418,24 +1425,28 @@ PyArray_Correlate(PyObject *op1, PyObject *op2, int mode) Py_DECREF(typec); return NULL; } + + Py_INCREF(typec); ap2 = (PyArrayObject *)PyArray_FromAny(op2, typec, 1, 1, NPY_ARRAY_DEFAULT, NULL); if (ap2 == NULL) { goto fail; } - ret = _pyarray_correlate(ap1, ap2, typenum, mode, &unused); + ret = _pyarray_correlate(ap1, ap2, typec, mode, &unused); if (ret == NULL) { goto fail; } Py_DECREF(ap1); Py_DECREF(ap2); + Py_DECREF(typec); return (PyObject *)ret; fail: Py_XDECREF(ap1); Py_XDECREF(ap2); Py_XDECREF(ret); + Py_XDECREF(typec); return NULL; } @@ -2587,14 +2598,13 @@ array_matrixproduct(PyObject *NPY_UNUSED(dummy), static PyObject * array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_args) { - int typenum; char *ip1, *ip2, *op; npy_intp n, stride1, stride2; PyObject *op1, *op2; npy_intp newdimptr[1] = {-1}; PyArray_Dims newdims = {newdimptr, 1}; PyArrayObject *ap1 = NULL, *ap2 = NULL, *ret = NULL; - PyArray_Descr *type; + PyArray_Descr *type = NULL; PyArray_DotFunc *vdot; NPY_BEGIN_THREADS_DEF; @@ -2610,16 +2620,19 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_ar * Conjugating dot product using the BLAS for vectors. * Flattens both op1 and op2 before dotting. */ - typenum = PyArray_ObjectType(op1, NPY_NOTYPE); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op1, NPY_MAXDIMS, &type) < 0) { return NULL; } - typenum = PyArray_ObjectType(op2, typenum); - if (typenum == NPY_NOTYPE) { + if (PyArray_DTypeFromObject(op2, NPY_MAXDIMS, &type) < 0) { + Py_XDECREF(type); return NULL; } - type = PyArray_DescrFromType(typenum); + if (type == NULL) { + type = PyArray_DescrFromType(NPY_DEFAULT_TYPE); + } + Py_SETREF(type, NPY_DT_CALL_ensure_canonical(type)); + Py_INCREF(type); ap1 = (PyArrayObject *)PyArray_FromAny(op1, type, 0, 0, 0, NULL); if (ap1 == NULL) { @@ -2635,6 +2648,7 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_ar Py_DECREF(ap1); ap1 = (PyArrayObject *)op1; + Py_INCREF(type); ap2 = (PyArrayObject *)PyArray_FromAny(op2, type, 0, 0, 0, NULL); if (ap2 == NULL) { goto fail; @@ -2653,7 +2667,7 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_ar } /* array scalar output */ - ret = new_array_for_sum(ap1, ap2, NULL, 0, (npy_intp *)NULL, typenum, NULL); + ret = new_array_for_sum(ap1, ap2, NULL, 0, (npy_intp *)NULL, type, NULL); if (ret == NULL) { goto fail; } @@ -2665,7 +2679,7 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_ar ip2 = PyArray_DATA(ap2); op = PyArray_DATA(ret); - switch (typenum) { + switch (type->type_num) { case NPY_CFLOAT: vdot = (PyArray_DotFunc *)CFLOAT_vdot; break; @@ -2698,11 +2712,13 @@ array_vdot(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_ar Py_XDECREF(ap1); Py_XDECREF(ap2); + Py_XDECREF(type); return PyArray_Return(ret); fail: Py_XDECREF(ap1); Py_XDECREF(ap2); Py_XDECREF(ret); + Py_XDECREF(type); return NULL; } @@ -3281,6 +3297,21 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) if (common_dt == NULL) { goto fail; } + + if (PyArray_FLAGS(ax) & NPY_ARRAY_WAS_PYTHON_LITERAL) { + if (npy_update_operand_for_scalar(&ax, x, common_dt, NPY_SAFE_CASTING) < 0) { + goto fail; + } + op_in[2] = ax; + } + + if (PyArray_FLAGS(ay) & NPY_ARRAY_WAS_PYTHON_LITERAL) { + if (npy_update_operand_for_scalar(&ay, y, common_dt, NPY_SAFE_CASTING) < 0) { + goto fail; + } + op_in[3] = ay; + } + npy_intp itemsize = common_dt->elsize; // If x and y don't have references, we ask the iterator to create buffers diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index e485f4578910..7d01fb5ed441 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -7274,6 +7274,13 @@ def test_dot_array_order(self): assert_equal(np.dot(b, a), res) assert_equal(np.dot(b, b), res) + def test_dot_has_native_byteorder(self): + # gh-30931 + a = np.array([1, 2, 3], ">f8") + dot = a.dot([[], [], []]) + + assert_equal(dot.dtype, np.dtype("=f8")) + def test_accelerate_framework_sgemv_fix(self): def aligned_array(shape, align, dtype, order='C'): @@ -9633,6 +9640,13 @@ def test_error(self): assert_raises(ValueError, np.where, c, a, a) assert_raises(ValueError, np.where, c[0], a, b) + def test_scalar_overflow(self): + c = [True] + a = np.array([1], dtype=np.uint8) + b = 1000 + assert_raises(OverflowError, np.where, c, a, b) + assert_raises(OverflowError, np.where, c, b, a) + def test_string(self): # gh-4778 check strings are properly filled with nulls a = np.array("abc") diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 54d040a6ed3f..81e990bbca3b 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -744,7 +744,7 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='', ox, oy = x, y def isnumber(x): - return x.dtype.char in '?bhilqpBHILQPefdgFDG' + return type(x.dtype)._is_numeric def istime(x): return x.dtype.char in "Mm"