feat: support timestamp_precision in table schema (#2333)

Thank you for opening a Pull Request! Before submitting your PR, there
are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a
[bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose)
before writing your code! That way we can discuss the change, evaluate
designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)

Fixes #<issue_number_goes_here> 🦕

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: Linchin

google/cloud/bigquery/enums.py

@@ -480,3 +480,18 @@ class SourceColumnMatch(str, enum.Enum):
     NAME = "NAME"
     """Matches by name. This reads the header row as column names and reorders
     columns to match the field names in the schema."""
+
+
+class TimestampPrecision(enum.Enum):
+    """Precision (maximum number of total digits in base 10) for seconds of
+    TIMESTAMP type."""
+
+    MICROSECOND = None
+    """
+    Default, for TIMESTAMP type with microsecond precision.
+    """
+
+    PICOSECOND = 12
+    """
+    For TIMESTAMP type with picosecond precision.
+    """

google/cloud/bigquery/schema.py

@@ -196,6 +196,14 @@ class SchemaField(object):
 
             Only valid for top-level schema fields (not nested fields).
             If the type is FOREIGN, this field is required.
+
+        timestamp_precision: Optional[enums.TimestampPrecision]
+            Precision (maximum number of total digits in base 10) for seconds
+            of TIMESTAMP type.
+
+            Defaults to `enums.TimestampPrecision.MICROSECOND` (`None`) for
+            microsecond precision. Use `enums.TimestampPrecision.PICOSECOND`
+            (`12`) for picosecond precision.
     """
 
     def __init__(
@@ -213,6 +221,7 @@ def __init__(
         range_element_type: Union[FieldElementType, str, None] = None,
         rounding_mode: Union[enums.RoundingMode, str, None] = None,
         foreign_type_definition: Optional[str] = None,
+        timestamp_precision: Optional[enums.TimestampPrecision] = None,
     ):
         self._properties: Dict[str, Any] = {
             "name": name,
@@ -237,6 +246,13 @@ def __init__(
                 if isinstance(policy_tags, PolicyTagList)
                 else None
             )
+        if isinstance(timestamp_precision, enums.TimestampPrecision):
+            self._properties["timestampPrecision"] = timestamp_precision.value
+        elif timestamp_precision is not None:
+            raise ValueError(
+                "timestamp_precision must be class enums.TimestampPrecision "
+                f"or None, got {type(timestamp_precision)} instead."
+            )
         if isinstance(range_element_type, str):
             self._properties["rangeElementType"] = {"type": range_element_type}
         if isinstance(range_element_type, FieldElementType):
@@ -254,15 +270,22 @@ def from_api_repr(cls, api_repr: dict) -> "SchemaField":
         """Return a ``SchemaField`` object deserialized from a dictionary.
 
         Args:
-            api_repr (Mapping[str, str]): The serialized representation
-                of the SchemaField, such as what is output by
-                :meth:`to_api_repr`.
+            api_repr (dict): The serialized representation of the SchemaField,
+                such as what is output by :meth:`to_api_repr`.
 
         Returns:
             google.cloud.bigquery.schema.SchemaField: The ``SchemaField`` object.
         """
         placeholder = cls("this_will_be_replaced", "PLACEHOLDER")
 
+        # The API would return a string despite we send an integer. To ensure
+        # success of resending received schema, we convert string to integer
+        # to ensure consistency.
+        try:
+            api_repr["timestampPrecision"] = int(api_repr["timestampPrecision"])
+        except (TypeError, KeyError):
+            pass
+
         # Note: we don't make a copy of api_repr because this can cause
         # unnecessary slowdowns, especially on deeply nested STRUCT / RECORD
         # fields. See https://github.com/googleapis/python-bigquery/issues/6
@@ -374,6 +397,16 @@ def policy_tags(self):
         resource = self._properties.get("policyTags")
         return PolicyTagList.from_api_repr(resource) if resource is not None else None
 
+    @property
+    def timestamp_precision(self) -> enums.TimestampPrecision:
+        """Precision (maximum number of total digits in base 10) for seconds of
+        TIMESTAMP type.
+
+        Returns:
+            enums.TimestampPrecision: value of TimestampPrecision.
+        """
+        return enums.TimestampPrecision(self._properties.get("timestampPrecision"))
+
     def to_api_repr(self) -> dict:
         """Return a dictionary representing this schema field.
 
@@ -408,6 +441,8 @@ def _key(self):
             None if self.policy_tags is None else tuple(sorted(self.policy_tags.names))
         )
 
+        timestamp_precision = self._properties.get("timestampPrecision")
+
         return (
             self.name,
             field_type,
@@ -417,6 +452,7 @@ def _key(self):
             self.description,
             self.fields,
             policy_tags,
+            timestamp_precision,
         )
 
     def to_standard_sql(self) -> standard_sql.StandardSqlField:
@@ -467,10 +503,9 @@ def __hash__(self):
         return hash(self._key())
 
     def __repr__(self):
-        key = self._key()
-        policy_tags = key[-1]
+        *initial_tags, policy_tags, timestamp_precision_tag = self._key()
         policy_tags_inst = None if policy_tags is None else PolicyTagList(policy_tags)
-        adjusted_key = key[:-1] + (policy_tags_inst,)
+        adjusted_key = (*initial_tags, policy_tags_inst, timestamp_precision_tag)
         return f"{self.__class__.__name__}{adjusted_key}"
 
 
@@ -530,9 +565,11 @@ def _to_schema_fields(schema):
     if isinstance(schema, Sequence):
         # Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
         return [
-            field
-            if isinstance(field, SchemaField)
-            else SchemaField.from_api_repr(field)
+            (
+                field
+                if isinstance(field, SchemaField)
+                else SchemaField.from_api_repr(field)
+            )
             for field in schema
         ]
 

tests/system/test_client.py

@@ -74,6 +74,16 @@
     bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
     bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
 ]
+SCHEMA_PICOSECOND = [
+    bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
+    bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
+    bigquery.SchemaField(
+        "time_pico",
+        "TIMESTAMP",
+        mode="REQUIRED",
+        timestamp_precision=enums.TimestampPrecision.PICOSECOND,
+    ),
+]
 CLUSTERING_SCHEMA = [
     bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
     bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
@@ -631,6 +641,19 @@ def test_create_table_w_time_partitioning_w_clustering_fields(self):
         self.assertEqual(time_partitioning.field, "transaction_time")
         self.assertEqual(table.clustering_fields, ["user_email", "store_code"])
 
+    def test_create_table_w_picosecond_timestamp(self):
+        dataset = self.temp_dataset(_make_dataset_id("create_table"))
+        table_id = "test_table"
+        table_arg = Table(dataset.table(table_id), schema=SCHEMA_PICOSECOND)
+        self.assertFalse(_table_exists(table_arg))
+
+        table = helpers.retry_403(Config.CLIENT.create_table)(table_arg)
+        self.to_delete.insert(0, table)
+
+        self.assertTrue(_table_exists(table))
+        self.assertEqual(table.table_id, table_id)
+        self.assertEqual(table.schema, SCHEMA_PICOSECOND)
+
     def test_delete_dataset_with_string(self):
         dataset_id = _make_dataset_id("delete_table_true_with_string")
         project = Config.CLIENT.project

tests/unit/test_schema.py

@@ -52,6 +52,9 @@ def test_constructor_defaults(self):
         self.assertIsNone(field.default_value_expression)
         self.assertEqual(field.rounding_mode, None)
         self.assertEqual(field.foreign_type_definition, None)
+        self.assertEqual(
+            field.timestamp_precision, enums.TimestampPrecision.MICROSECOND
+        )
 
     def test_constructor_explicit(self):
         FIELD_DEFAULT_VALUE_EXPRESSION = "This is the default value for this field"
@@ -69,6 +72,7 @@ def test_constructor_explicit(self):
             default_value_expression=FIELD_DEFAULT_VALUE_EXPRESSION,
             rounding_mode=enums.RoundingMode.ROUNDING_MODE_UNSPECIFIED,
             foreign_type_definition="INTEGER",
+            timestamp_precision=enums.TimestampPrecision.PICOSECOND,
         )
         self.assertEqual(field.name, "test")
         self.assertEqual(field.field_type, "STRING")
@@ -87,6 +91,10 @@ def test_constructor_explicit(self):
         )
         self.assertEqual(field.rounding_mode, "ROUNDING_MODE_UNSPECIFIED")
         self.assertEqual(field.foreign_type_definition, "INTEGER")
+        self.assertEqual(
+            field.timestamp_precision,
+            enums.TimestampPrecision.PICOSECOND,
+        )
 
     def test_constructor_explicit_none(self):
         field = self._make_one("test", "STRING", description=None, policy_tags=None)
@@ -189,6 +197,23 @@ def test_to_api_repr_with_subfield(self):
                 },
             )
 
+    def test_to_api_repr_w_timestamp_precision(self):
+        field = self._make_one(
+            "foo",
+            "TIMESTAMP",
+            "NULLABLE",
+            timestamp_precision=enums.TimestampPrecision.PICOSECOND,
+        )
+        self.assertEqual(
+            field.to_api_repr(),
+            {
+                "mode": "NULLABLE",
+                "name": "foo",
+                "type": "TIMESTAMP",
+                "timestampPrecision": 12,
+            },
+        )
+
     def test_from_api_repr(self):
         field = self._get_target_class().from_api_repr(
             {
@@ -198,6 +223,7 @@ def test_from_api_repr(self):
                 "name": "foo",
                 "type": "record",
                 "roundingMode": "ROUNDING_MODE_UNSPECIFIED",
+                "timestampPrecision": 12,
             }
         )
         self.assertEqual(field.name, "foo")
@@ -210,6 +236,10 @@ def test_from_api_repr(self):
         self.assertEqual(field.fields[0].mode, "NULLABLE")
         self.assertEqual(field.range_element_type, None)
         self.assertEqual(field.rounding_mode, "ROUNDING_MODE_UNSPECIFIED")
+        self.assertEqual(
+            field.timestamp_precision,
+            enums.TimestampPrecision.PICOSECOND,
+        )
 
     def test_from_api_repr_policy(self):
         field = self._get_target_class().from_api_repr(
@@ -264,6 +294,17 @@ def test_from_api_repr_defaults(self):
         self.assertNotIn("policyTags", field._properties)
         self.assertNotIn("rangeElementType", field._properties)
 
+    def test_from_api_repr_timestamp_precision_str(self):
+        # The backend would return timestampPrecision field as a string, even
+        # if we send over an integer. This test verifies we manually converted
+        # it into integer to ensure resending could succeed.
+        field = self._get_target_class().from_api_repr(
+            {
+                "timestampPrecision": "12",
+            }
+        )
+        self.assertEqual(field._properties["timestampPrecision"], 12)
+
     def test_name_property(self):
         name = "lemon-ness"
         schema_field = self._make_one(name, "INTEGER")
@@ -323,6 +364,22 @@ def test_foreign_type_definition_property_str(self):
         schema_field._properties["foreignTypeDefinition"] = FOREIGN_TYPE_DEFINITION
         self.assertEqual(schema_field.foreign_type_definition, FOREIGN_TYPE_DEFINITION)
 
+    def test_timestamp_precision_unsupported_type(self):
+        with pytest.raises(ValueError) as e:
+            self._make_one("test", "TIMESTAMP", timestamp_precision=12)
+
+        assert "timestamp_precision must be class enums.TimestampPrecision" in str(
+            e.value
+        )
+
+    def test_timestamp_precision_property(self):
+        TIMESTAMP_PRECISION = enums.TimestampPrecision.PICOSECOND
+        schema_field = self._make_one("test", "TIMESTAMP")
+        schema_field._properties[
+            "timestampPrecision"
+        ] = enums.TimestampPrecision.PICOSECOND.value
+        self.assertEqual(schema_field.timestamp_precision, TIMESTAMP_PRECISION)
+
     def test_to_standard_sql_simple_type(self):
         examples = (
             # a few legacy types
@@ -637,7 +694,9 @@ def test___hash__not_equals(self):
 
     def test___repr__(self):
         field1 = self._make_one("field1", "STRING")
-        expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, None, (), None)"
+        expected = (
+            "SchemaField('field1', 'STRING', 'NULLABLE', None, None, (), None, None)"
+        )
         self.assertEqual(repr(field1), expected)
 
     def test___repr__evaluable_no_policy_tags(self):