diff --git a/BREAKING_CHANGES.md b/BREAKING_CHANGES.md
index ebbd994d..5a6a86b3 100644
--- a/BREAKING_CHANGES.md
+++ b/BREAKING_CHANGES.md
@@ -8,6 +8,12 @@ code then it will not be listed here.
 
 ## [2022-07-06]
 
+- The block smoothing API has been reworked. Instead of `Smoother`s having their
+  own built in block buffer, you now need to provide your own mutable slice for
+  the smoother to fill. This makes the API easier to understand, more flexible,
+  and it allows cloning smoothers without worrying about allocations for use
+  with polyphonic modulation. In addition, the new implementation is much more
+  efficient when the smoothing period has ended before or during the block.
 - There are new `NoteEvent::PolyModulation` and `NoteEvent::MonoAutomation` as
   part of polyphonic modulation support for CLAP plugins.
 
diff --git a/src/buffer.rs b/src/buffer.rs
index a7e49831..bfdc06ce 100644
--- a/src/buffer.rs
+++ b/src/buffer.rs
@@ -83,10 +83,7 @@ impl<'a> Buffer<'a> {
     /// SIMD.
     ///
     /// The parameter smoothers can also produce smoothed values for an entire block using
-    /// [`Smoother::next_block()`][crate::prelude::Smoother::next_block()]. Before using this, you
-    /// will need to call
-    /// [`Plugin::initialize_block_smoothers()`][crate::prelude::Plugin::initialize_block_smoothers()]
-    /// with the same `max_block_size` in your initialization function first.
+    /// [`Smoother::next_block()`][crate::prelude::Smoother::next_block()].
     ///
     /// You can use this to obtain block-slices from a buffer so you can pass them to a library:
     ///
diff --git a/src/lib.rs b/src/lib.rs
index 684e9c44..30938e40 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -75,9 +75,7 @@
 //!    plugins](https://github.com/robbert-vdh/nih-plug/tree/master/plugins).
 //!  - After calling `.with_smoother()` during an integer or floating point parameter's creation,
 //!    you can use `param.smoothed` to access smoothed values for that parameter. Be sure to check
-//!    out the [`Smoother`][prelude::Smoother] API for more details. If you want to generate entire
-//!    blocks of smoothed values, be sure to call the predefined
-//!    `[Plugin::initialize_block_smoothers()]` method from your plugin's `initialize()` function.
+//!    out the [`Smoother`][prelude::Smoother] API for more details.
 //!
 //! There's a whole lot more to discuss, but once you understand the above you should be able to
 //! figure out the rest by reading through the examples and the API documetnation. Good luck!
diff --git a/src/param.rs b/src/param.rs
index 1214b929..777011de 100644
--- a/src/param.rs
+++ b/src/param.rs
@@ -147,11 +147,6 @@ pub trait Param: Display {
     /// wrappers. This **does** snap to step sizes for continuous parameters (i.e. [`FloatParam`]).
     fn preview_plain(&self, normalized: f32) -> Self::Plain;
 
-    /// Allocate memory for block-based smoothing. The
-    /// [`Plugin::initialize_block_smoothers()`][crate::prelude::Plugin::initialize_block_smoothers()] method
-    /// will do this for every smoother.
-    fn initialize_block_smoother(&mut self, max_block_size: usize);
-
     /// Flags to control the parameter's behavior. See [`ParamFlags`].
     fn flags(&self) -> ParamFlags;
 
diff --git a/src/param/boolean.rs b/src/param/boolean.rs
index 95d7cc87..3759598b 100644
--- a/src/param/boolean.rs
+++ b/src/param/boolean.rs
@@ -143,8 +143,6 @@ impl Param for BoolParam {
         normalized > 0.5
     }
 
-    fn initialize_block_smoother(&mut self, _max_block_size: usize) {}
-
     fn flags(&self) -> ParamFlags {
         self.flags
     }
diff --git a/src/param/enums.rs b/src/param/enums.rs
index 884e6477..f430340d 100644
--- a/src/param/enums.rs
+++ b/src/param/enums.rs
@@ -175,10 +175,6 @@ impl<T: Enum + PartialEq> Param for EnumParam<T> {
         T::from_index(self.inner.preview_plain(normalized) as usize)
     }
 
-    fn initialize_block_smoother(&mut self, max_block_size: usize) {
-        self.inner.initialize_block_smoother(max_block_size)
-    }
-
     fn flags(&self) -> ParamFlags {
         self.inner.flags()
     }
@@ -261,10 +257,6 @@ impl Param for EnumParamInner {
         self.inner.preview_plain(normalized)
     }
 
-    fn initialize_block_smoother(&mut self, max_block_size: usize) {
-        self.inner.initialize_block_smoother(max_block_size)
-    }
-
     fn flags(&self) -> ParamFlags {
         self.inner.flags()
     }
diff --git a/src/param/float.rs b/src/param/float.rs
index 07863c28..e013bc8a 100644
--- a/src/param/float.rs
+++ b/src/param/float.rs
@@ -201,10 +201,6 @@ impl Param for FloatParam {
         }
     }
 
-    fn initialize_block_smoother(&mut self, max_block_size: usize) {
-        self.smoothed.initialize_block_smoother(max_block_size);
-    }
-
     fn flags(&self) -> ParamFlags {
         self.flags
     }
diff --git a/src/param/integer.rs b/src/param/integer.rs
index bb0d12b3..fac104cc 100644
--- a/src/param/integer.rs
+++ b/src/param/integer.rs
@@ -166,10 +166,6 @@ impl Param for IntParam {
         self.range.unnormalize(normalized)
     }
 
-    fn initialize_block_smoother(&mut self, max_block_size: usize) {
-        self.smoothed.initialize_block_smoother(max_block_size);
-    }
-
     fn flags(&self) -> ParamFlags {
         self.flags
     }
diff --git a/src/param/internals.rs b/src/param/internals.rs
index 6ec84f0a..bfeca153 100644
--- a/src/param/internals.rs
+++ b/src/param/internals.rs
@@ -161,7 +161,6 @@ impl ParamPtr {
     param_ptr_forward!(pub unsafe fn step_count(&self) -> Option<usize>);
     param_ptr_forward!(pub unsafe fn previous_normalized_step(&self, from: f32) -> f32);
     param_ptr_forward!(pub unsafe fn next_normalized_step(&self, from: f32) -> f32);
-    param_ptr_forward!(pub unsafe fn initialize_block_smoother(&mut self, max_block_size: usize));
     param_ptr_forward!(pub unsafe fn normalized_value_to_string(&self, normalized: f32, include_unit: bool) -> String);
     param_ptr_forward!(pub unsafe fn string_to_normalized_value(&self, string: &str) -> Option<f32>);
     param_ptr_forward!(pub unsafe fn flags(&self) -> ParamFlags);
diff --git a/src/param/smoothing.rs b/src/param/smoothing.rs
index 512de8c2..f969657d 100644
--- a/src/param/smoothing.rs
+++ b/src/param/smoothing.rs
@@ -1,11 +1,8 @@
 //! Utilities to handle smoothing parameter changes over time.
 
 use atomic_float::AtomicF32;
-use atomic_refcell::{AtomicRefCell, AtomicRefMut};
 use std::sync::atomic::{AtomicI32, Ordering};
 
-use crate::buffer::Block;
-
 /// Controls if and how parameters gets smoothed.
 #[derive(Debug, Clone, Copy)]
 pub enum SmoothingStyle {
@@ -55,15 +52,10 @@ pub struct Smoother<T> {
     current: AtomicF32,
     /// The value we're smoothing towards
     target: T,
-
-    /// A dense buffer containing smoothed values for an entire block of audio. Useful when using
-    /// [`Buffer::iter_blocks()`][crate::prelude::Buffer::iter_blocks()] to process small blocks of audio
-    /// multiple times.
-    block_values: AtomicRefCell<Vec<T>>,
 }
 
 /// An iterator that continuously produces smoothed values. Can be used as an alternative to the
-/// built-in block-based smoothing API. Since the iterator itself is infinite, you can use
+/// block-based smoothing API. Since the iterator itself is infinite, you can use
 /// [`Smoother::is_smoothing()`] and [`Smoother::steps_left()`] to get information on the current
 /// smoothing status.
 pub struct SmootherIter<'a, T> {
@@ -85,8 +77,6 @@ impl<T: Smoothable> Default for Smoother<T> {
             step_size: Default::default(),
             current: AtomicF32::new(0.0),
             target: Default::default(),
-
-            block_values: AtomicRefCell::new(Vec::new()),
         }
     }
 }
@@ -143,15 +133,6 @@ impl<T: Smoothable> Smoother<T> {
         SmootherIter { smoother: self }
     }
 
-    /// Allocate memory to store smoothed values for an entire block of audio. Call this in
-    /// [`Plugin::initialize()`][crate::prelude::Plugin::initialize()] with the same max block size you are
-    /// going to pass to [`Buffer::iter_blocks()`][crate::prelude::Buffer::iter_blocks()].
-    pub fn initialize_block_smoother(&mut self, max_block_size: usize) {
-        self.block_values
-            .borrow_mut()
-            .resize_with(max_block_size, || T::default());
-    }
-
     /// Reset the smoother the specified value.
     pub fn reset(&mut self, value: T) {
         self.target = value;
@@ -248,43 +229,44 @@ impl<T: Smoothable> Smoother<T> {
         T::from_f32(self.current.load(Ordering::Relaxed))
     }
 
-    /// Produce smoothed values for an entire block of audio. Used in conjunction with
-    /// [`Buffer::iter_blocks()`][crate::prelude::Buffer::iter_blocks()]. Make sure to call
-    /// [`Plugin::initialize_block_smoothers()`][crate::prelude::Plugin::initialize_block_smoothers()] with
-    /// the same maximum buffer block size as the one passed to `iter_blocks()` in your
-    /// [`Plugin::initialize()`][crate::prelude::Plugin::initialize()] function first to allocate memory for
-    /// the block smoothing.
-    ///
-    /// Returns a `None` value if the block length exceed's the allocated capacity.
+    /// Produce smoothed values for an entire block of audio. This is useful when iterating the same
+    /// block of audio multiple times. For instance when summing voices for a synthesizer.
+    /// `block_values[..block_len]` will be filled with the smoothed values. This is simply a
+    /// convenient function for [`next_block_exact()`][Self::next_block_exact()] when iterating over
+    /// variable length blocks with a known maximum size.
     ///
     /// # Panics
     ///
-    /// Panics if this function is called again while another block value slice is still alive.
-    pub fn next_block(&self, block: &Block) -> Option<AtomicRefMut<[T]>> {
-        self.next_block_mapped(block, |x| x)
+    /// Panics if `block_len > block_values.len()`.
+    pub fn next_block(&self, block_values: &mut [T], block_len: usize) {
+        self.next_block_exact_mapped(&mut block_values[..block_len], |x| x)
+    }
+
+    /// The same as [`next_block()`][Self::next_block()], but filling the entire slice.
+    pub fn next_block_exact(&self, block_values: &mut [T]) {
+        self.next_block_exact_mapped(block_values, |x| x)
     }
 
     /// The same as [`next_block()`][Self::next_block()], but with a function applied to each
     /// produced value. Useful when applying modulation to a smoothed parameter.
-    pub fn next_block_mapped(
-        &self,
-        block: &Block,
-        mut f: impl FnMut(T) -> T,
-    ) -> Option<AtomicRefMut<[T]>> {
-        let mut block_values = self.block_values.borrow_mut();
-        if block_values.len() < block.len() {
-            return None;
-        }
+    pub fn next_block_mapped(&self, block_values: &mut [T], block_len: usize, f: impl Fn(T) -> T) {
+        self.next_block_exact_mapped(&mut block_values[..block_len], f)
+    }
 
-        // TODO: As a small optimization we could split this up into two loops for the smoothed and
-        //       unsmoothed parts. Another worthwhile optimization would be to remember if the
-        //       buffer is already filled with the target value and [Self::is_smoothing()] is false.
-        //       In that case we wouldn't need to do anything ehre.
-        (&mut block_values[..block.len()]).fill_with(|| f(self.next()));
+    /// The same as [`next_block_exact()`][Self::next_block()], but with a function applied to each
+    /// produced value. Useful when applying modulation to a smoothed parameter.
+    pub fn next_block_exact_mapped(&self, block_values: &mut [T], f: impl Fn(T) -> T) {
+        // `self.next()` will yield the current value if the parameter is no longer smoothing, but
+        // it's a bit of a waste to continuesly call that if only the first couple or none of the
+        // values in `block_values` would require smoothing and the rest don't. Instead, we'll just
+        // smooth the values as necessary, and then reuse the target value for the rest of the
+        // block.
+        let num_smoothed_values = block_values
+            .len()
+            .min(self.steps_left.load(Ordering::Relaxed) as usize);
 
-        Some(AtomicRefMut::map(block_values, |values| {
-            &mut values[..block.len()]
-        }))
+        block_values[..num_smoothed_values].fill_with(|| f(self.next()));
+        block_values[num_smoothed_values..].fill(self.target);
     }
 }
 
diff --git a/src/plugin.rs b/src/plugin.rs
index 371f4b01..5fc6a00a 100644
--- a/src/plugin.rs
+++ b/src/plugin.rs
@@ -152,10 +152,7 @@ pub trait Plugin: Default + Send + Sync + 'static {
     /// per-sample SIMD or excessive branching. The parameter smoothers can also work in both modes:
     /// use [`Smoother::next()`][crate::prelude::Smoother::next()] for per-sample processing, and
     /// [`Smoother::next_block()`][crate::prelude::Smoother::next_block()] for block-based
-    /// processing. In order to use block-based smoothing, you will need to call
-    /// [`initialize_block_smoothers()`][Self::initialize_block_smoothers()] in your
-    /// [`initialize()`][Self::initialize()] function first to reserve enough capacity in the
-    /// smoothers.
+    /// processing.
     ///
     /// The `context` object contains context information as well as callbacks for working with note
     /// events. The [`AuxiliaryBuffers`] contain the plugin's sidechain input buffers and
@@ -179,17 +176,6 @@ pub trait Plugin: Default + Send + Sync + 'static {
     /// `initialize()` may be called more than once before `deactivate()` is called, for instance
     /// when restoring state while the plugin is still activate.
     fn deactivate(&mut self) {}
-
-    /// Convenience function provided to allocate memory for block-based smoothing for this plugin.
-    /// Since this allocates memory, this should be called in [`initialize()`][Self::initialize()].
-    /// If you are going to use [`Buffer::iter_blocks()`] and want to use parameter smoothing in
-    /// those blocks, then call this function with the same maximum block size first before calling
-    /// [`Smoother::next_block()`][crate::prelude::Smoother::next_block()].
-    fn initialize_block_smoothers(&mut self, max_block_size: usize) {
-        for (_, mut param, _) in self.params().param_map() {
-            unsafe { param.initialize_block_smoother(max_block_size) };
-        }
-    }
 }
 
 /// Provides auxiliary metadata needed for a CLAP plugin.