diff --git a/CHANGELOG.md b/CHANGELOG.md
index 014726d00..1dc818639 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 # Unreleased
 
+  * Avoid sending VLA on every packet #866
   * Add `abs-capture-time` RTP header extension #864
 
 # 0.16.2
diff --git a/src/packet/payload.rs b/src/packet/payload.rs
index 27d5a57ef..f3e6da2b1 100644
--- a/src/packet/payload.rs
+++ b/src/packet/payload.rs
@@ -1,5 +1,6 @@
 use crate::format::CodecSpec;
 use crate::media::ToPayload;
+use crate::rtp::vla::VideoLayersAllocation;
 use crate::rtp_::Frequency;
 use crate::streams::StreamTx;
 
@@ -42,6 +43,7 @@ impl Payloader {
 
         for (idx, data) in chunks.into_iter().enumerate() {
             let last = idx == len - 1;
+            let first = idx == 0;
 
             let previous_data = stream.last_packet();
             let marker = self.pack.is_marker(data.as_slice(), previous_data, last)
@@ -52,13 +54,18 @@ impl Payloader {
             // TODO: delegate to self.pack to decide whether this packet is nackable.
             let nackable = !is_audio;
 
+            let mut pkt_ext_vals = ext_vals.clone();
+            if !first {
+                pkt_ext_vals.user_values.remove::<VideoLayersAllocation>();
+            }
+
             stream.write_rtp(
                 pt,
                 seq_no,
                 rtp_time.rebase(self.clock_rate).numer() as u32,
                 wallclock,
                 marker,
-                ext_vals.clone(),
+                pkt_ext_vals,
                 nackable,
                 data,
             )?;
diff --git a/src/rtp/ext.rs b/src/rtp/ext.rs
index 1a704b127..2375fce72 100644
--- a/src/rtp/ext.rs
+++ b/src/rtp/ext.rs
@@ -1071,6 +1071,13 @@ impl UserExtensionValues {
             .downcast()
             .ok()
     }
+
+    /// Remove a user extension value (by type).
+    pub fn remove<T: Send + Sync + 'static>(&mut self) {
+        if let Some(map) = &mut self.map {
+            map.remove(&TypeId::of::<T>());
+        }
+    }
 }
 
 impl UnwindSafe for UserExtensionValues {}