feat(Connection, numeric_to_float): add connection option to convert numeric datatype to Python float

Author: Brooke-white

README.rst

@@ -314,6 +314,8 @@ Connection Parameters
 +-----------------------------------+------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------+
 | max_prepared_statements           | int  | The maximum number of prepared statements that can be open at once                                                                                                                                                                                                                                                                                                    | 1000                 | No       |
 +-----------------------------------+------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------+
+| numeric_to_float                  | bool | Specifies if NUMERIC datatype values will be converted from decimal.Decimal to float. By default NUMERIC values are received as decimal.Decimal                                                                                                                                                                                                                       | False                | No       |
++-----------------------------------+------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------+
 | partner_sp_id                     | str  | The Partner SP Id used for authentication with Ping                                                                                                                                                                                                                                                                                                                   | None                 | No       |
 +-----------------------------------+------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------+
 | password                          | str  | The password to use for authentication                                                                                                                                                                                                                                                                                                                                | None                 | No       |

redshift_connector/__init__.py

@@ -179,6 +179,7 @@ def connect(
     endpoint_url: typing.Optional[str] = None,
     provider_name: typing.Optional[str] = None,
     scope: typing.Optional[str] = None,
+    numeric_to_float: typing.Optional[bool] = False,
 ) -> Connection:
     """
     Establishes a :class:`Connection` to an Amazon Redshift cluster. This function validates user input, optionally authenticates using an identity provider plugin, then constructs a :class:`Connection` object.
@@ -272,6 +273,8 @@ def connect(
         The name of the Redshift Native Auth Provider.
     scope: Optional[str]
         Scope for BrowserAzureOauth2CredentialsProvider authentication.
+    numeric_to_float: Optional[str]
+        Specifies if NUMERIC datatype values will be converted from ``decimal.Decimal`` to ``float``. By default NUMERIC values are received as ``decimal.Decimal``.
     Returns
     -------
     A Connection object associated with the specified Amazon Redshift cluster: :class:`Connection`
@@ -304,6 +307,7 @@ def connect(
     info.put("listen_port", listen_port)
     info.put("login_url", login_url)
     info.put("max_prepared_statements", max_prepared_statements)
+    info.put("numeric_to_float", numeric_to_float)
     info.put("partner_sp_id", partner_sp_id)
     info.put("password", password)
     info.put("port", port)
@@ -382,6 +386,7 @@ def connect(
         credentials_provider=info.credentials_provider,
         provider_name=info.provider_name,
         web_identity_token=info.web_identity_token,
+        numeric_to_float=info.numeric_to_float,
     )
 
 

redshift_connector/core.py

@@ -75,6 +75,8 @@
     make_divider_block,
     numeric_in,
     numeric_in_binary,
+    numeric_to_float_binary,
+    numeric_to_float_in,
 )
 from redshift_connector.utils import pg_types as PG_TYPES
 from redshift_connector.utils import py_types as PY_TYPES
@@ -419,6 +421,7 @@ def __init__(
         credentials_provider: typing.Optional[str] = None,
         provider_name: typing.Optional[str] = None,
         web_identity_token: typing.Optional[str] = None,
+        numeric_to_float: bool = False,
     ):
         """
         Creates a :class:`Connection` to an Amazon Redshift cluster. For more information on establishing a connection to an Amazon Redshift cluster using `federated API access <https://aws.amazon.com/blogs/big-data/federated-api-access-to-amazon-redshift-using-an-amazon-redshift-connector-for-python/>`_ see our examples page.
@@ -461,6 +464,8 @@ def __init__(
             The name of the Redshift Native Auth Provider.
         web_identity_token: Optional[str]
             A web identity token used for authentication via Redshift Native IDP Integration
+        numeric_to_float: bool
+            Specifies if NUMERIC datatype values will be converted from ``decimal.Decimal`` to ``float``. By default NUMERIC values are received as ``decimal.Decimal``.
         """
         self.merge_socket_read = True
 
@@ -484,6 +489,7 @@ def __init__(
         self.py_types = deepcopy(PY_TYPES)
         self.pg_types = deepcopy(PG_TYPES)
         self._database_metadata_current_db_only: bool = database_metadata_current_db_only
+        self.numeric_to_float: bool = numeric_to_float
 
         # based on _client_protocol_version value, we must use different conversion functions
         # for receiving some datatypes
@@ -725,6 +731,10 @@ def _enable_protocol_based_conversion_funcs(self: "Connection"):
             self.pg_types[1028] = (FC_BINARY, array_recv_binary)  # OID[]
             self.pg_types[1034] = (FC_BINARY, array_recv_binary)  # ACLITEM[]
             self.pg_types[VARBYTE] = (FC_TEXT, text_recv)  # VARBYTE
+
+            if self.numeric_to_float:
+                self.pg_types[NUMERIC] = (FC_BINARY, numeric_to_float_binary)
+
         else:  # text protocol
             self.pg_types[NUMERIC] = (FC_TEXT, numeric_in)
             self.pg_types[TIME] = (FC_TEXT, time_in)
@@ -741,6 +751,9 @@ def _enable_protocol_based_conversion_funcs(self: "Connection"):
             self.pg_types[1034] = (FC_TEXT, array_recv_text)  # ACLITEM[]
             self.pg_types[VARBYTE] = (FC_TEXT, varbytehex_recv)  # VARBYTE
 
+            if self.numeric_to_float:
+                self.pg_types[NUMERIC] = (FC_TEXT, numeric_to_float_in)
+
     @property
     def _is_multi_databases_catalog_enable_in_server(self: "Connection") -> bool:
         if (b"datashare_enabled", str("on").encode()) in self.parameter_statuses:
@@ -1918,7 +1931,7 @@ def handle_DATA_ROW(self: "Connection", data: bytes, cursor: Cursor) -> None:
             data_idx += 4
             if vlen == -1:
                 row.append(None)
-            elif desc[0] == numeric_in_binary:
+            elif desc[0] in (numeric_in_binary, numeric_to_float_binary):
                 row.append(desc[0](data, data_idx, vlen, desc[1]))
                 data_idx += vlen
             else:

redshift_connector/redshift_property.py

@@ -113,6 +113,7 @@ def __init__(self: "RedshiftProperty", **kwargs):
             # The name of the Redshift Native Auth Provider
             self.provider_name: typing.Optional[str] = None
             self.scope: str = ""
+            self.numeric_to_float: bool = False
 
         else:
             for k, v in kwargs.items():

redshift_connector/utils/__init__.py

@@ -32,6 +32,8 @@
     int_array_recv,
     numeric_in,
     numeric_in_binary,
+    numeric_to_float_binary,
+    numeric_to_float_in,
     pg_types,
     py_types,
     q_pack,

redshift_connector/utils/type_utils.py

@@ -312,10 +312,28 @@ def numeric_in_binary(data: bytes, offset: int, length: int, scale: int) -> Deci
     return Decimal(raw_value).scaleb(-1 * scale)
 
 
+def numeric_to_float_binary(data: bytes, offset: int, length: int, scale: int) -> float:
+    raw_value: int
+
+    if length == 8:
+        raw_value = q_unpack(data, offset)[0]
+    elif length == 16:
+        temp: typing.Tuple[int, int] = qq_unpack(data, offset)
+        raw_value = (temp[0] << 64) | temp[1]
+    else:
+        raise Exception("Malformed column value of type numeric received")
+
+    return raw_value * 10 ** (-1 * scale)
+
+
 def numeric_in(data: bytes, offset: int, length: int) -> Decimal:
     return Decimal(data[offset : offset + length].decode(_client_encoding))
 
 
+def numeric_to_float_in(data: bytes, offset: int, length: int) -> float:
+    return float(data[offset : offset + length].decode(_client_encoding))
+
+
 # def uuid_recv(data: bytes, offset: int, length: int) -> UUID:
 #     return UUID(bytes=data[offset:offset+length])
 

test/integration/datatype/test_datatypes.py

@@ -181,3 +181,28 @@ def test_abstime(db_kwargs, _input, client_protocol):
             cursor.execute("select '{}'::abstime".format(insert_val))
             res = cursor.fetchone()
             assert res[0] == exp_val
+
+
+numeric_vals: typing.List[typing.Tuple[str, float]] = [
+    ("to_number('12,454.8-', 'S99G999D9')", -12454.8),
+    ("to_number('8.1-', '9D9S')", -8.1),
+    ("to_number('897.6', '999D9S')", 897.6),
+]
+
+
+@pytest.mark.parametrize("client_protocol", ClientProtocolVersion.list())
+@pytest.mark.parametrize("_input", numeric_vals)
+def test_numeric_to_float(db_kwargs, _input, client_protocol):
+    insert_val, exp_val = _input
+    db_kwargs["numeric_to_float"] = True
+    with redshift_connector.connect(**db_kwargs) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute("select {}".format(insert_val))
+            res = cursor.fetchone()
+            assert isinstance(res[0], float)
+            assert isclose(
+                typing.cast(float, res[0]),
+                typing.cast(float, exp_val),
+                rel_tol=1e-05,
+                abs_tol=1e-08,
+            )