Fixed regression from cx_Oracle which ignored the value of the

"encoding_errors" parameter when creating variables by calling the
method Cursor.var() (#279).

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -32,6 +32,10 @@ Common Changes
 #)  Fixed regression which prevented a null value from being set on DbObject
     attributes or used as elements of collections
     (`issue 273 <https://github.com/oracle/python-oracledb/issues/273>`__).
+#)  Fixed regression from cx_Oracle which ignored the value of the
+    ``encoding_errors`` parameter when creating variables by calling the method
+    :meth:`Cursor.var()`
+    (`issue 279 <https://github.com/oracle/python-oracledb/issues/279>`__).
 #)  Corrected typing declarations.
 #)  Bumped minimum requirement of Cython to 3.0.
 

src/oracledb/base_impl.pxd

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -342,6 +342,7 @@ cdef class BaseVarImpl:
         readonly uint32_t size
         readonly uint32_t buffer_size
         readonly bint bypass_decode
+        readonly str encoding_errors
         readonly bint is_array
         readonly bint nulls_allowed
         readonly bint convert_nulls

src/oracledb/impl/base/cursor.pyx

@@ -419,6 +419,7 @@ cdef class BaseCursorImpl:
         var_impl.inconverter = inconverter
         var_impl.outconverter = outconverter
         var_impl.bypass_decode = bypass_decode
+        var_impl.encoding_errors = encoding_errors
         var_impl.is_array = is_array
         var_impl.convert_nulls = convert_nulls
         var_impl._finalize_init()

src/oracledb/impl/thick/utils.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -261,7 +261,8 @@ cdef object _convert_to_python(ThickConnImpl conn_impl, DbType dbtype,
                                ThickDbObjectTypeImpl obj_type_impl,
                                dpiDataBuffer *dbvalue,
                                int preferred_num_type=NUM_TYPE_FLOAT,
-                               bint bypass_decode=False):
+                               bint bypass_decode=False,
+                               const char* encoding_errors=NULL):
     cdef:
         uint32_t oracle_type = dbtype.num
         ThickDbObjectImpl obj_impl
@@ -282,7 +283,7 @@ cdef object _convert_to_python(ThickConnImpl conn_impl, DbType dbtype,
             or oracle_type == DPI_ORACLE_TYPE_LONG_NVARCHAR \
             or oracle_type == DPI_ORACLE_TYPE_XMLTYPE:
         as_bytes = &dbvalue.asBytes
-        return as_bytes.ptr[:as_bytes.length].decode()
+        return as_bytes.ptr[:as_bytes.length].decode("utf-8", encoding_errors)
     elif oracle_type == DPI_ORACLE_TYPE_NUMBER:
         as_bytes = &dbvalue.asBytes
         if preferred_num_type == NUM_TYPE_INT \

src/oracledb/impl/thick/var.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -264,15 +264,21 @@ cdef class ThickVarImpl(BaseVarImpl):
         Transforms a single element from the value supplied by ODPI-C to its
         equivalent Python value.
         """
-        cdef object value
+        cdef:
+            const char *encoding_errors = NULL
+            bytes encoding_errors_bytes
+            object value
         data = &data[pos]
         if not data.isNull:
             if self._native_type_num == DPI_NATIVE_TYPE_STMT:
                 return self._get_cursor_value(&data.value)
+            if self.encoding_errors is not None:
+                encoding_errors_bytes = self.encoding_errors.encode()
+                encoding_errors = encoding_errors_bytes
             value = _convert_to_python(self._conn_impl, self.dbtype,
                                        self.objtype, &data.value,
                                        self._preferred_num_type,
-                                       self.bypass_decode)
+                                       self.bypass_decode, encoding_errors)
             if self.outconverter is not None:
                 value = self.outconverter(value)
             return value

src/oracledb/impl/thin/buffer.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -765,7 +765,7 @@ cdef class Buffer:
         self._pos = end_pos + 1
         return self._data[start_pos:self._pos]
 
-    cdef object read_str(self, int csfrm):
+    cdef object read_str(self, int csfrm, const char* encoding_errors=NULL):
         """
         Reads bytes from the buffer and decodes them into a string following
         the supplied character set form.
@@ -776,8 +776,9 @@ cdef class Buffer:
         self.read_raw_bytes_and_length(&ptr, &num_bytes)
         if ptr != NULL:
             if csfrm == TNS_CS_IMPLICIT:
-                return ptr[:num_bytes].decode()
-            return ptr[:num_bytes].decode(TNS_ENCODING_UTF16)
+                return ptr[:num_bytes].decode(TNS_ENCODING_UTF8,
+                                              encoding_errors)
+            return ptr[:num_bytes].decode(TNS_ENCODING_UTF16, encoding_errors)
 
     cdef int read_ub1(self, uint8_t *value) except -1:
         """

src/oracledb/impl/thin/messages.pyx

@@ -496,6 +496,8 @@ cdef class MessageWithData(Message):
                                      ThinVarImpl var_impl, uint32_t pos):
         cdef:
             uint8_t num_bytes, ora_type_num, csfrm
+            const char* encoding_errors = NULL
+            bytes encoding_errors_bytes
             ThinDbObjectTypeImpl typ_impl
             BaseThinCursorImpl cursor_impl
             object column_value = None
@@ -525,7 +527,10 @@ cdef class MessageWithData(Message):
                 or ora_type_num == TNS_DATA_TYPE_LONG:
             if csfrm == TNS_CS_NCHAR:
                 buf._caps._check_ncharset_id()
-            column_value = buf.read_str(csfrm)
+            if var_impl.encoding_errors is not None:
+                encoding_errors_bytes = var_impl.encoding_errors.encode()
+                encoding_errors = encoding_errors_bytes
+            column_value = buf.read_str(csfrm, encoding_errors)
         elif ora_type_num == TNS_DATA_TYPE_RAW \
                 or ora_type_num == TNS_DATA_TYPE_LONG_RAW:
             column_value = buf.read_bytes()

tests/test_3800_typehandler.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------
-# Copyright (c) 2021, 2023, Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -281,6 +281,26 @@ def output_type_handler(cursor, metadata):
         self.cursor.execute("select * from TestJson")
         self.assertEqual(self.cursor.fetchall(), data_to_insert)
 
+    def test_3807(self):
+        "3807 - output type handler for encoding errors"
+
+        def output_type_handler(cursor, metadata):
+            if metadata.type_code is oracledb.DB_TYPE_VARCHAR:
+                return cursor.var(
+                    metadata.type_code,
+                    arraysize=cursor.arraysize,
+                    encoding_errors="replace",
+                )
+
+        self.cursor.outputtypehandler = output_type_handler
+        self.cursor.execute(
+            "select utl_raw.cast_to_varchar2('41ab42cd43ef') from dual"
+        )
+        (result,) = self.cursor.fetchone()
+        rc = chr(0xFFFD)
+        expected_result = f"A{rc}B{rc}C{rc}"
+        self.assertEqual(result, expected_result)
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()