diff --git a/druid/src/widget/button.rs b/druid/src/widget/button.rs
index 52294ebc0a..8874045741 100644
--- a/druid/src/widget/button.rs
+++ b/druid/src/widget/button.rs
@@ -155,13 +155,16 @@ impl<T: Data> Widget<T> for Button<T> {
         // HACK: to make sure we look okay at default sizes when beside a textbox,
         // we make sure we will have at least the same height as the default textbox.
         let min_height = env.get(theme::BORDERED_WIDGET_HEIGHT);
-        let baseline = self.label.baseline_offset();
-        ctx.set_baseline_offset(baseline + LABEL_INSETS.y1);
 
         let button_size = bc.constrain(Size::new(
             self.label_size.width + padding.width,
             (self.label_size.height + padding.height).max(min_height),
         ));
+
+        let extra_height = (button_size.height - self.label_size.height).max(0.0);
+        let baseline = self.label.baseline_offset() + extra_height / 2.0;
+        ctx.set_baseline_offset(baseline);
+
         trace!("Computed button size: {}", button_size);
         button_size
     }
diff --git a/druid/src/widget/label.rs b/druid/src/widget/label.rs
index 29e5df56d6..391094783f 100644
--- a/druid/src/widget/label.rs
+++ b/druid/src/widget/label.rs
@@ -99,6 +99,7 @@ pub struct RawLabel<T> {
 
     disabled: bool,
     default_text_color: KeyOrValue<Color>,
+    baseline_offset: f64,
 }
 
 /// Options for handling lines that are too wide for the label.
@@ -160,6 +161,7 @@ impl<T: TextStorage> RawLabel<T> {
             line_break_mode: LineBreaking::Overflow,
             disabled: false,
             default_text_color: crate::theme::TEXT_COLOR.into(),
+            baseline_offset: 0f64,
         }
     }
 
@@ -287,8 +289,7 @@ impl<T: TextStorage> RawLabel<T> {
 
     /// Return the offset of the first baseline relative to the bottom of the widget.
     pub fn baseline_offset(&self) -> f64 {
-        let text_metrics = self.layout.layout_metrics();
-        text_metrics.size.height - text_metrics.first_baseline
+        self.baseline_offset
     }
 }
 
@@ -615,11 +616,12 @@ impl<T: TextStorage> Widget<T> for RawLabel<T> {
         self.layout.rebuild_if_needed(ctx.text(), env);
 
         let text_metrics = self.layout.layout_metrics();
-        ctx.set_baseline_offset(text_metrics.size.height - text_metrics.first_baseline);
         let size = bc.constrain(Size::new(
             text_metrics.size.width + 2. * LABEL_X_PADDING,
             text_metrics.size.height,
         ));
+        self.baseline_offset = size.height - text_metrics.first_baseline;
+        ctx.set_baseline_offset(self.baseline_offset);
         trace!("Computed size: {}", size);
         size
     }
diff --git a/druid/src/widget/textbox.rs b/druid/src/widget/textbox.rs
index 6907543c63..36097947af 100644
--- a/druid/src/widget/textbox.rs
+++ b/druid/src/widget/textbox.rs
@@ -580,11 +580,8 @@ impl<T: TextStorage + EditableText> Widget<T> for TextBox<T> {
             self.text().borrow().layout.layout_metrics()
         };
 
-        let layout_baseline = text_metrics.size.height - text_metrics.first_baseline;
-        let baseline_off = layout_baseline
-            - (self.inner.child_size().height - self.inner.viewport_rect().height())
-            + textbox_insets.y1;
-        ctx.set_baseline_offset(baseline_off);
+        let baseline_offset = size.height - text_metrics.first_baseline - textbox_insets.y1;
+        ctx.set_baseline_offset(baseline_offset);
         if self.scroll_to_selection_after_layout {
             self.scroll_to_selection_end();
             self.scroll_to_selection_after_layout = false;
@@ -593,7 +590,7 @@ impl<T: TextStorage + EditableText> Widget<T> for TextBox<T> {
         trace!(
             "Computed layout: size={}, baseline_offset={:?}",
             size,
-            baseline_off
+            baseline_offset
         );
         size
     }