diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 38be038efcaa5..02653a63afa0f 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -45,6 +45,7 @@
     tzconversion,
 )
 from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
+from pandas._libs.tslibs.timedeltas import Timedelta
 from pandas.errors import PerformanceWarning
 from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_inclusive
@@ -93,7 +94,6 @@
 
     from pandas import (
         DataFrame,
-        Timedelta,
     )
     from pandas.core.arrays import PeriodArray
 
@@ -817,7 +817,22 @@ def _add_offset(self, offset: BaseOffset) -> Self:
             result = type(self)._from_sequence(res_values, dtype=self.dtype)
 
         else:
+            units = [
+                "ns",
+                "us",
+                "ms",
+                "s",
+            ]
+            res_unit = self.unit
+            if hasattr(offset, "offset"):
+                offset_td = Timedelta(offset.offset)
+                offset_unit = offset_td.unit
+                idx_self = units.index(self.unit)
+                idx_offset = units.index(offset_unit)
+                res_unit = units[min(idx_self, idx_offset)]
             result = type(self)._simple_new(res_values, dtype=res_values.dtype)
+            result = result.as_unit(res_unit)
+
             if offset.normalize:
                 result = result.normalize()
                 result._freq = None
diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py
index 199e3572732a0..505b49f7b7fb4 100644
--- a/pandas/tests/arrays/test_datetimes.py
+++ b/pandas/tests/arrays/test_datetimes.py
@@ -844,3 +844,20 @@ def test_factorize_sort_without_freq():
     tda = dta - dta[0]
     with pytest.raises(NotImplementedError, match=msg):
         tda.factorize(sort=True)
+
+
+def test_dt64_non_nano_offset_no_rounding():
+    # GH#56586
+    dti = pd.date_range("2016-01-01", periods=3, unit="s")
+    offset = pd.offsets.CustomBusinessDay(offset=pd.Timedelta("1ms"))
+    result = dti + offset
+
+    assert result.dtype == np.dtype("datetime64[ms]")
+    expected = pd.DatetimeIndex(
+        [
+            pd.Timestamp("2016-01-02 00:00:00.001"),
+            pd.Timestamp("2016-01-03 00:00:00.001"),
+            pd.Timestamp("2016-01-04 00:00:00.001"),
+        ]
+    )
+    tm.assert_index_equal(result, expected)