diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 38be038efcaa5..101757071f865 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -28,6 +28,7 @@
     NaT,
     NaTType,
     Resolution,
+    Timedelta,
     Timestamp,
     astype_overflowsafe,
     fields,
@@ -70,6 +71,7 @@
 
 from pandas.tseries.frequencies import get_period_alias
 from pandas.tseries.offsets import (
+    DateOffset,
     Day,
     Tick,
 )
@@ -93,7 +95,6 @@
 
     from pandas import (
         DataFrame,
-        Timedelta,
     )
     from pandas.core.arrays import PeriodArray
 
@@ -817,7 +818,34 @@ def _add_offset(self, offset: BaseOffset) -> Self:
             result = type(self)._from_sequence(res_values, dtype=self.dtype)
 
         else:
+            units = [
+                "ns",
+                "us",
+                "ms",
+                "s",
+            ]
+            res_unit = self.unit
+            if type(offset) is DateOffset:
+                nano = offset.kwds.get("nanoseconds", 0)
+                micro = offset.kwds.get("microseconds", 0)
+                if nano:
+                    res_unit = "ns"
+                elif micro and self.unit != "ns":
+                    res_unit = "us"
+            if (
+                hasattr(offset, "offset")
+                and offset.offset is not None
+                and not isinstance(offset, Tick)
+            ):
+                offset_td = Timedelta(offset.offset)
+                if offset_td.value != 0:
+                    offset_unit = offset_td.unit
+                    idx_self = units.index(self.unit)
+                    idx_offset = units.index(offset_unit)
+                    res_unit = units[min(idx_self, idx_offset)]
             result = type(self)._simple_new(res_values, dtype=res_values.dtype)
+            result = result.as_unit(res_unit)
+
             if offset.normalize:
                 result = result.normalize()
                 result._freq = None
diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py
index 199e3572732a0..27d694e1d7813 100644
--- a/pandas/tests/arrays/test_datetimes.py
+++ b/pandas/tests/arrays/test_datetimes.py
@@ -844,3 +844,23 @@ def test_factorize_sort_without_freq():
     tda = dta - dta[0]
     with pytest.raises(NotImplementedError, match=msg):
         tda.factorize(sort=True)
+
+
+@pytest.mark.filterwarnings(
+    "ignore:Non-vectorized DateOffset being applied to Series or DatetimeIndex"
+)
+def test_dt64_non_nano_offset_no_rounding():
+    # GH#56586
+    dti = pd.date_range("2016-01-01", periods=3, unit="s")
+    offset = pd.offsets.CustomBusinessDay(offset=pd.Timedelta("1ms"))
+    result = dti + offset
+
+    assert result.dtype == np.dtype("datetime64[ms]")
+    expected = pd.DatetimeIndex(
+        [
+            pd.Timestamp("2016-01-02 00:00:00.001"),
+            pd.Timestamp("2016-01-03 00:00:00.001"),
+            pd.Timestamp("2016-01-04 00:00:00.001"),
+        ]
+    )
+    tm.assert_index_equal(result, expected)