API: microsecond resolution for Timedelta strings

doc/source/whatsnew/v3.0.0.rst

@@ -378,6 +378,8 @@ In cases with mixed-resolution inputs, the highest resolution is used:
 
 .. warning:: Many users will now get "M8[us]" dtype data in cases when they used to get "M8[ns]". For most use cases they should not notice a difference. One big exception is converting to integers, which will give integers 1000x smaller.
 
+Similarly, the :class:`Timedelta` constructor and :func:`to_timedelta` with a string input now defaults to a microsecond unit, using nanosecond unit only in cases that actually have nanosecond precision.
+
 .. _whatsnew_300.api_breaking.concat_datetime_sorting:
 
 :func:`concat` no longer ignores ``sort`` when all objects have a :class:`DatetimeIndex`

pandas/_libs/tslibs/timedeltas.pyx

@@ -1,4 +1,5 @@
 import collections
+import re
 import warnings
 
 from pandas.util._decorators import set_module
@@ -679,6 +680,17 @@ cdef timedelta_from_spec(object number, object frac, object unit):
     return cast_from_unit(float(n), unit)
 
 
+cdef bint needs_nano_unit(int64_t ival, str item):
+    """
+    Check if a passed string `item` needs to be stored with nano unit or can
+    use microsecond instead.
+    """
+    # TODO: more performant way of doing this check?
+    if ival % 1000 != 0:
+        return True
+    return re.search(r"\.\d{7}", item) or "ns" in item or "nano" in item
+
+
 cpdef inline str parse_timedelta_unit(str unit):
     """
     Parameters
@@ -2078,10 +2090,17 @@ class Timedelta(_Timedelta):
             if (len(value) > 0 and value[0] == "P") or (
                 len(value) > 1 and value[:2] == "-P"
             ):
-                value = parse_iso_format_string(value)
+                ival = parse_iso_format_string(value)
+            else:
+                ival = parse_timedelta_string(value)
+
+            if not needs_nano_unit(ival, value):
+                # If we don't specifically need nanosecond resolution, default
+                #  to microsecond like we do for datetimes
+                value = np.timedelta64(ival // 1000, "us")
+                return cls(value)
             else:
-                value = parse_timedelta_string(value)
-            value = np.timedelta64(value)
+                value = np.timedelta64(ival, "ns")
         elif PyDelta_Check(value):
             # pytimedelta object -> microsecond resolution
             new_value = delta_to_nanoseconds(

pandas/tests/arithmetic/test_numeric.py

@@ -278,7 +278,9 @@ def test_numeric_arr_rdiv_tdscalar(self, three_days, numeric_idx, box_with_array
                 # i.e. resolution is lower -> use lowest supported resolution
                 dtype = np.dtype("m8[s]")
             expected = expected.astype(dtype)
-        elif type(three_days) is timedelta:
+        elif type(three_days) is timedelta or (
+            isinstance(three_days, Timedelta) and three_days.unit == "us"
+        ):
             expected = expected.astype("m8[us]")
         elif isinstance(
             three_days,

pandas/tests/groupby/test_groupby.py

@@ -147,7 +147,7 @@ def test_len_nan_group():
 
 def test_groupby_timedelta_median():
     # issue 57926
-    expected = Series(data=Timedelta("1D"), index=["foo"])
+    expected = Series(data=Timedelta("1D"), index=["foo"], dtype="m8[ns]")
     df = DataFrame({"label": ["foo", "foo"], "timedelta": [pd.NaT, Timedelta("1D")]})
     gb = df.groupby("label")["timedelta"]
     actual = gb.median()

pandas/tests/scalar/timedelta/test_arithmetic.py

@@ -440,7 +440,7 @@ def test_td_mul_td64_ndarray_invalid(self):
 
         msg = (
             "ufunc '?multiply'? cannot use operands with types "
-            rf"dtype\('{tm.ENDIAN}m8\[ns\]'\) and dtype\('{tm.ENDIAN}m8\[ns\]'\)"
+            rf"dtype\('{tm.ENDIAN}m8\[us\]'\) and dtype\('{tm.ENDIAN}m8\[us\]'\)"
         )
         with pytest.raises(TypeError, match=msg):
             td * other
@@ -1219,6 +1219,7 @@ def test_ops_str_deprecated(box):
                 "ufunc 'divide' cannot use operands",
                 "Invalid dtype object for __floordiv__",
                 r"unsupported operand type\(s\) for /: 'int' and 'str'",
+                r"unsupported operand type\(s\) for /: 'datetime.timedelta' and 'str'",
             ]
         )
         with pytest.raises(TypeError, match=msg):

pandas/tests/scalar/timedelta/test_constructors.py

@@ -271,12 +271,12 @@ def test_construction():
     expected = np.timedelta64(10, "D").astype("m8[ns]").view("i8")
     assert Timedelta(10, unit="D")._value == expected
     assert Timedelta(10.0, unit="D")._value == expected
-    assert Timedelta("10 days")._value == expected
+    assert Timedelta("10 days")._value == expected // 1000
     assert Timedelta(days=10)._value == expected
     assert Timedelta(days=10.0)._value == expected
 
     expected += np.timedelta64(10, "s").astype("m8[ns]").view("i8")
-    assert Timedelta("10 days 00:00:10")._value == expected
+    assert Timedelta("10 days 00:00:10")._value == expected // 1000
     assert Timedelta(days=10, seconds=10)._value == expected
     assert Timedelta(days=10, milliseconds=10 * 1000)._value == expected
     assert Timedelta(days=10, microseconds=10 * 1000 * 1000)._value == expected
@@ -434,7 +434,7 @@ def test_td_construction_with_np_dtypes(npdtype, item):
 def test_td_from_repr_roundtrip(val):
     # round-trip both for string and value
     td = Timedelta(val)
-    assert Timedelta(td._value) == td
+    assert Timedelta(td.value) == td
 
     assert Timedelta(str(td)) == td
     assert Timedelta(td._repr_base(format="all")) == td
@@ -443,7 +443,7 @@ def test_td_from_repr_roundtrip(val):
 
 def test_overflow_on_construction():
     # GH#3374
-    value = Timedelta("1day")._value * 20169940
+    value = Timedelta("1day").as_unit("ns")._value * 20169940
     msg = "Cannot cast 1742682816000000000000 from ns to 'ns' without overflow"
     with pytest.raises(OutOfBoundsTimedelta, match=msg):
         Timedelta(value)
@@ -705,3 +705,21 @@ def test_non_nano_value():
     # check that the suggested workaround actually works
     result = td.asm8.view("i8")
     assert result == 86400000000
+
+
+def test_parsed_unit():
+    td = Timedelta("1 Day")
+    assert td.unit == "us"
+
+    td = Timedelta("1 Day 2 hours 3 minutes 4 ns")
+    assert td.unit == "ns"
+
+    td = Timedelta("1 Day 2:03:04.012345")
+    assert td.unit == "us"
+
+    td = Timedelta("1 Day 2:03:04.012345000")
+    assert td.unit == "ns"
+
+    # 7 digits after the decimal
+    td = Timedelta("1 Day 2:03:04.0123450")
+    assert td.unit == "ns"

pandas/tests/scalar/timedelta/test_timedelta.py

@@ -338,6 +338,7 @@ def test_total_seconds_scalar(self):
 
     def test_conversion(self):
         for td in [Timedelta(10, unit="D"), Timedelta("1 days, 10:11:12.012345")]:
+            td = td.as_unit("ns")
             pydt = td.to_pytimedelta()
             assert td == Timedelta(pydt)
             assert td == pydt
@@ -385,8 +386,8 @@ def check(value):
         assert abs(td) == Timedelta("13:48:48")
         assert str(td) == "-1 days +10:11:12"
         assert -td == Timedelta("0 days 13:48:48")
-        assert -Timedelta("-1 days, 10:11:12")._value == 49728000000000
-        assert Timedelta("-1 days, 10:11:12")._value == -49728000000000
+        assert -Timedelta("-1 days, 10:11:12")._value == 49728000000
+        assert Timedelta("-1 days, 10:11:12")._value == -49728000000
 
         rng = to_timedelta("-1 days, 10:11:12.100123456")
         assert rng.days == -1

pandas/tests/scalar/timestamp/test_timestamp.py

@@ -367,11 +367,11 @@ def test_roundtrip(self):
         # further test accessors
         base = Timestamp("20140101 00:00:00").as_unit("ns")
 
-        result = Timestamp(base._value + Timedelta("5ms")._value)
+        result = Timestamp(base._value + Timedelta("5ms").value)
         assert result == Timestamp(f"{base}.005000")
         assert result.microsecond == 5000
 
-        result = Timestamp(base._value + Timedelta("5us")._value)
+        result = Timestamp(base._value + Timedelta("5us").value)
         assert result == Timestamp(f"{base}.000005")
         assert result.microsecond == 5
 
@@ -380,11 +380,11 @@ def test_roundtrip(self):
         assert result.nanosecond == 5
         assert result.microsecond == 0
 
-        result = Timestamp(base._value + Timedelta("6ms 5us")._value)
+        result = Timestamp(base._value + Timedelta("6ms 5us").value)
         assert result == Timestamp(f"{base}.006005")
         assert result.microsecond == 5 + 6 * 1000
 
-        result = Timestamp(base._value + Timedelta("200ms 5us")._value)
+        result = Timestamp(base._value + Timedelta("200ms 5us").value)
         assert result == Timestamp(f"{base}.200005")
         assert result.microsecond == 5 + 200 * 1000
 

pandas/tests/series/methods/test_fillna.py

@@ -611,7 +611,7 @@ def test_fillna_pytimedelta(self):
         ser = Series([np.nan, Timedelta("1 days")], index=["A", "B"])
 
         result = ser.fillna(timedelta(1))
-        expected = Series(Timedelta("1 days"), index=["A", "B"])
+        expected = Series(Timedelta("1 days"), index=["A", "B"], dtype="m8[ns]")
         tm.assert_series_equal(result, expected)
 
     def test_fillna_period(self):

pandas/tests/series/test_arithmetic.py

@@ -358,7 +358,7 @@ def test_arithmetic_with_duplicate_index(self):
         ser = Series(date_range("20130101 09:00:00", periods=5), index=index)
         other = Series(date_range("20130101", periods=5), index=index)
         result = ser - other
-        expected = Series(Timedelta("9 hours"), index=[2, 2, 3, 3, 4])
+        expected = Series(Timedelta("9 hours"), index=[2, 2, 3, 3, 4], dtype="m8[ns]")
         tm.assert_series_equal(result, expected)
 
     def test_masked_and_non_masked_propagate_na(self):