From 02622e6e4f7584781411e2b2ce55828e273af93a Mon Sep 17 00:00:00 2001 From: Robbert van der Helm Date: Thu, 10 Mar 2022 18:15:55 +0100 Subject: [PATCH] Split up the buffer module --- src/buffer.rs | 573 +----------------------------------------- src/buffer/blocks.rs | 330 ++++++++++++++++++++++++ src/buffer/samples.rs | 314 +++++++++++++++++++++++ 3 files changed, 649 insertions(+), 568 deletions(-) create mode 100644 src/buffer/blocks.rs create mode 100644 src/buffer/samples.rs diff --git a/src/buffer.rs b/src/buffer.rs index 39332dcf..52919244 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -2,8 +2,11 @@ use std::marker::PhantomData; -#[cfg(feature = "simd")] -use std::simd::{LaneCount, Simd, SupportedLaneCount}; +mod blocks; +mod samples; + +pub use blocks::{Block, BlockChannelsIter, BlocksIter}; +pub use samples::{ChannelSamples, ChannelSamplesIter, SamplesIter}; /// The audio buffers used during processing. This contains the output audio output buffers with the /// inputs already copied to the outputs. You can either use the iterator adapters to conveniently @@ -24,234 +27,6 @@ pub struct Buffer<'a> { output_slices: Vec<&'a mut [f32]>, } -// Per-sample per-channel iterators - -/// An iterator over all samples in a buffer or block, yielding iterators over each channel for -/// every sample. This iteration order offers good cache locality for per-sample access. -pub struct SamplesIter<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) current_sample: usize, - /// The last sample index to iterate over plus one. Would be equal to `buffers.len()` when - /// iterating over an entire buffer, but this can also be used to iterate over smaller blocks in - /// a similar fashion. - pub(self) samples_end: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -/// Can construct iterators over actual iterator over the channel data for a sample, yielded by -/// [`SamplesIter`]. Can be turned into an iterator, or [`Channels::iter_mut()`] can be used to -/// iterate over the channel data multiple times, or more efficiently you can use -/// [`Channels::get_unchecked_mut()`] to do the same thing. -pub struct ChannelSamples<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) current_sample: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -/// The actual iterator over the channel data for a sample, yielded by [`Channels`]. -pub struct ChannelSamplesIter<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) current_sample: usize, - pub(self) current_channel: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -// Per-block per-channel per-sample iterators - -/// An iterator over all samples in the buffer, slicing over the sample-dimension with a maximum -/// size of `max_block_size`. See [`Buffer::iter_blocks()`]. Yields both the block and the offset -/// from the start of the buffer. -pub struct BlocksIter<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) max_block_size: usize, - pub(self) current_block_start: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -/// A block yielded by [`BlocksIter`]. Can be iterated over once or multiple times, and also -/// supports direct access to the block's samples if needed. -pub struct Block<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) current_block_start: usize, - /// The index of the last sample in the block plus one. - pub(self) current_block_end: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -/// An iterator over all channels in a block yielded by [`Block`], returning an entire channel slice -/// at a time. -pub struct BlockChannelsIter<'slice, 'sample: 'slice> { - /// The raw output buffers. - pub(self) buffers: *mut [&'sample mut [f32]], - pub(self) current_block_start: usize, - pub(self) current_block_end: usize, - pub(self) current_channel: usize, - pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, -} - -impl<'slice, 'sample> Iterator for SamplesIter<'slice, 'sample> { - type Item = ChannelSamples<'slice, 'sample>; - - #[inline] - fn next(&mut self) -> Option { - // This can iterate over both the entire buffer or over a smaller sample slice of it - if self.current_sample < self.samples_end { - let channels = ChannelSamples { - buffers: self.buffers, - current_sample: self.current_sample, - _marker: self._marker, - }; - - self.current_sample += 1; - - Some(channels) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let remaining = unsafe { (*self.buffers)[0].len() } - self.current_sample; - (remaining, Some(remaining)) - } -} - -impl<'slice, 'sample> Iterator for BlockChannelsIter<'slice, 'sample> { - type Item = &'sample mut [f32]; - - #[inline] - fn next(&mut self) -> Option { - if self.current_channel < unsafe { (*self.buffers).len() } { - // SAFETY: These bounds have already been checked - // SAFETY: It is also not possible to have multiple mutable references to the same - // sample at the same time - let slice = unsafe { - (*self.buffers) - .get_unchecked_mut(self.current_channel) - .get_unchecked_mut(self.current_block_start..self.current_block_end) - }; - - self.current_channel += 1; - - Some(slice) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let remaining = unsafe { (*self.buffers).len() } - self.current_channel; - (remaining, Some(remaining)) - } -} - -impl<'slice, 'sample> Iterator for BlocksIter<'slice, 'sample> { - type Item = (usize, Block<'slice, 'sample>); - - #[inline] - fn next(&mut self) -> Option { - let buffer_len = unsafe { (*self.buffers)[0].len() }; - if self.current_block_start < buffer_len { - let current_block_start = self.current_block_start; - let current_block_end = - (self.current_block_start + self.max_block_size).min(buffer_len); - let block = Block { - buffers: self.buffers, - current_block_start, - current_block_end, - _marker: self._marker, - }; - - self.current_block_start += self.max_block_size; - - Some((current_block_start, block)) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let remaining = ((unsafe { (*self.buffers)[0].len() } - self.current_block_start) as f32 - / self.max_block_size as f32) - .ceil() as usize; - (remaining, Some(remaining)) - } -} - -impl<'slice, 'sample> Iterator for ChannelSamplesIter<'slice, 'sample> { - type Item = &'sample mut f32; - - #[inline] - fn next(&mut self) -> Option { - if self.current_channel < unsafe { (*self.buffers).len() } { - // SAFETY: These bounds have already been checked - // SAFETY: It is also not possible to have multiple mutable references to the same - // sample at the same time - let sample = unsafe { - (*self.buffers) - .get_unchecked_mut(self.current_channel) - .get_unchecked_mut(self.current_sample) - }; - - self.current_channel += 1; - - Some(sample) - } else { - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let remaining = unsafe { (*self.buffers).len() } - self.current_channel; - (remaining, Some(remaining)) - } -} - -impl<'slice, 'sample> IntoIterator for ChannelSamples<'slice, 'sample> { - type Item = &'sample mut f32; - type IntoIter = ChannelSamplesIter<'slice, 'sample>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - ChannelSamplesIter { - buffers: self.buffers, - current_sample: self.current_sample, - current_channel: 0, - _marker: self._marker, - } - } -} - -impl<'slice, 'sample> IntoIterator for Block<'slice, 'sample> { - type Item = &'sample mut [f32]; - type IntoIter = BlockChannelsIter<'slice, 'sample>; - - #[inline] - fn into_iter(self) -> Self::IntoIter { - BlockChannelsIter { - buffers: self.buffers, - current_block_start: self.current_block_start, - current_block_end: self.current_block_end, - current_channel: 0, - _marker: self._marker, - } - } -} - -impl ExactSizeIterator for SamplesIter<'_, '_> {} -impl ExactSizeIterator for ChannelSamplesIter<'_, '_> {} -impl ExactSizeIterator for BlocksIter<'_, '_> {} -impl ExactSizeIterator for BlockChannelsIter<'_, '_> {} - impl<'a> Buffer<'a> { /// Return the numer of samples in this buffer. #[inline] @@ -350,344 +125,6 @@ impl<'a> Buffer<'a> { } } -impl<'slice, 'sample> ChannelSamples<'slice, 'sample> { - /// Get the number of channels. - #[allow(clippy::len_without_is_empty)] - #[inline] - pub fn len(&self) -> usize { - unsafe { (*self.buffers).len() } - } - - /// A resetting iterator. This lets you iterate over the same channels multiple times. Otherwise - /// you don't need to use this function as [`Channels`] already implements [Iterator]. - #[inline] - pub fn iter_mut(&mut self) -> ChannelSamplesIter<'slice, 'sample> { - ChannelSamplesIter { - buffers: self.buffers, - current_sample: self.current_sample, - current_channel: 0, - _marker: self._marker, - } - } - - /// Access a sample by index. Useful when you would otherwise iterate over this 'Channels' - /// iterator multiple times. - #[inline] - pub fn get_mut(&mut self, channel_index: usize) -> Option<&mut f32> { - // SAFETY: The sample bound has already been checked - unsafe { - Some( - (*self.buffers) - .get_mut(channel_index)? - .get_unchecked_mut(self.current_sample), - ) - } - } - - /// The same as [`get_mut()`][Self::get_mut()], but without any bounds checking. - /// - /// # Safety - /// - /// `channel_index` must be in the range `0..Self::len()`. - #[inline] - pub unsafe fn get_unchecked_mut(&mut self, channel_index: usize) -> &mut f32 { - (*self.buffers) - .get_unchecked_mut(channel_index) - .get_unchecked_mut(self.current_sample) - } - - /// Get a SIMD vector containing the channel data for this buffer. If `LANES > channels.len()` - /// then this will be padded with zeroes. If `LANES < channels.len()` then this won't contain - /// all values. - #[cfg(feature = "simd")] - #[inline] - pub fn to_simd(&self) -> Simd - where - LaneCount: SupportedLaneCount, - { - let used_lanes = self.len().max(LANES); - let mut values = [0.0; LANES]; - for (channel_idx, value) in values.iter_mut().enumerate().take(used_lanes) { - *value = unsafe { - *(*self.buffers) - .get_unchecked(channel_idx) - .get_unchecked(self.current_sample) - }; - } - - Simd::from_array(values) - } - - /// Get a SIMD vector containing the channel data for this buffer. Will always read exactly - /// `LANES` channels. - /// - /// # Safety - /// - /// Undefined behavior if `LANES > channels.len()`. - #[cfg(feature = "simd")] - #[inline] - pub unsafe fn to_simd_unchecked(&self) -> Simd - where - LaneCount: SupportedLaneCount, - { - let mut values = [0.0; LANES]; - for (channel_idx, value) in values.iter_mut().enumerate() { - *value = *(*self.buffers) - .get_unchecked(channel_idx) - .get_unchecked(self.current_sample); - } - - Simd::from_array(values) - } - - /// Write data from a SIMD vector to this sample's channel data. This takes the padding added by - /// [`to_simd()`][Self::to_simd()] into account. - #[cfg(feature = "simd")] - #[allow(clippy::wrong_self_convention)] - #[inline] - pub fn from_simd(&mut self, vector: Simd) - where - LaneCount: SupportedLaneCount, - { - let used_lanes = self.len().max(LANES); - let values = vector.to_array(); - for (channel_idx, value) in values.into_iter().enumerate().take(used_lanes) { - *unsafe { - (*self.buffers) - .get_unchecked_mut(channel_idx) - .get_unchecked_mut(self.current_sample) - } = value; - } - } - - /// Write data from a SIMD vector to this sample's channel data. This assumes `LANES` matches - /// exactly with the number of channels in the buffer. - /// - /// # Safety - /// - /// Undefined behavior if `LANES > channels.len()`. - #[cfg(feature = "simd")] - #[allow(clippy::wrong_self_convention)] - #[inline] - pub unsafe fn from_simd_unchecked(&mut self, vector: Simd) - where - LaneCount: SupportedLaneCount, - { - let values = vector.to_array(); - for (channel_idx, value) in values.into_iter().enumerate() { - *(*self.buffers) - .get_unchecked_mut(channel_idx) - .get_unchecked_mut(self.current_sample) = value; - } - } -} - -impl<'slice, 'sample> Block<'slice, 'sample> { - /// Get the number of samples (not channels) in the block. - #[allow(clippy::len_without_is_empty)] - #[inline] - pub fn len(&self) -> usize { - self.current_block_end - self.current_block_start - } - - /// Return the numer of channels in this buffer. - #[inline] - pub fn channels(&self) -> usize { - unsafe { (*self.buffers).len() } - } - - /// A resetting iterator. This lets you iterate over the same block multiple times. Otherwise - /// you don't need to use this function as [`Block`] already implements [`Iterator`]. You can - /// also use the direct accessor functions on this block instead. - #[inline] - pub fn iter_mut(&mut self) -> BlockChannelsIter<'slice, 'sample> { - BlockChannelsIter { - buffers: self.buffers, - current_block_start: self.current_block_start, - current_block_end: self.current_block_end, - current_channel: 0, - _marker: self._marker, - } - } - - /// Iterate over this block on a per-sample per-channel basis. This is identical to - /// [`Buffer::iter_samples()`] but for a smaller block instead of the entire buffer - #[inline] - pub fn iter_samples(&mut self) -> SamplesIter<'slice, 'sample> { - SamplesIter { - buffers: self.buffers, - current_sample: self.current_block_start, - samples_end: self.current_block_end, - _marker: self._marker, - } - } - - /// Access a channel by index. Useful when you would otherwise iterate over this [`Block`] - /// multiple times. - #[inline] - pub fn get(&self, channel_index: usize) -> Option<&[f32]> { - // SAFETY: The block bound has already been checked - unsafe { - Some( - (*self.buffers) - .get(channel_index)? - .get_unchecked(self.current_block_start..self.current_block_end), - ) - } - } - - /// The same as [`get()`][Self::get], but without any bounds checking. - /// - /// # Safety - /// - /// `channel_index` must be in the range `0..Self::len()`. - #[inline] - pub unsafe fn get_unchecked(&self, channel_index: usize) -> &[f32] { - (*self.buffers) - .get_unchecked(channel_index) - .get_unchecked(self.current_block_start..self.current_block_end) - } - - /// Access a mutable channel by index. Useful when you would otherwise iterate over this - /// [`Block`] multiple times. - #[inline] - pub fn get_mut(&mut self, channel_index: usize) -> Option<&mut [f32]> { - // SAFETY: The block bound has already been checked - unsafe { - Some( - (*self.buffers) - .get_mut(channel_index)? - .get_unchecked_mut(self.current_block_start..self.current_block_end), - ) - } - } - - /// The same as [`get_mut()`][Self::get_mut], but without any bounds checking. - /// - /// # Safety - /// - /// `channel_index` must be in the range `0..Self::len()`. - #[inline] - pub unsafe fn get_unchecked_mut(&mut self, channel_index: usize) -> &mut [f32] { - (*self.buffers) - .get_unchecked_mut(channel_index) - .get_unchecked_mut(self.current_block_start..self.current_block_end) - } - - /// Get a SIMD vector containing the channel data for a specific sample in this block. If `LANES - /// > channels.len()` then this will be padded with zeroes. If `LANES < channels.len()` then - /// this won't contain all values. - /// - /// Returns a `None` value if `sample_index` is out of bounds. - #[cfg(feature = "simd")] - #[inline] - pub fn to_channel_simd( - &self, - sample_index: usize, - ) -> Option> - where - LaneCount: SupportedLaneCount, - { - if sample_index > self.len() { - return None; - } - - let used_lanes = self.len().max(LANES); - let mut values = [0.0; LANES]; - for (channel_idx, value) in values.iter_mut().enumerate().take(used_lanes) { - *value = unsafe { - *(*self.buffers) - .get_unchecked(channel_idx) - .get_unchecked(self.current_block_start + sample_index) - }; - } - - Some(Simd::from_array(values)) - } - - /// Get a SIMD vector containing the channel data for a specific sample in this block. Will - /// always read exactly `LANES` channels, and does not perform bounds checks on `sample_index`. - /// - /// # Safety - /// - /// Undefined behavior if `LANES > block.len()` or if `sample_index > block.len()`. - #[cfg(feature = "simd")] - #[inline] - pub unsafe fn to_channel_simd_unchecked( - &self, - sample_index: usize, - ) -> Simd - where - LaneCount: SupportedLaneCount, - { - let mut values = [0.0; LANES]; - for (channel_idx, value) in values.iter_mut().enumerate() { - *value = *(*self.buffers) - .get_unchecked(channel_idx) - .get_unchecked(self.current_block_start + sample_index); - } - - Simd::from_array(values) - } - - /// Write data from a SIMD vector to this sample's channel data for a specific sample in this - /// block. This takes the padding added by [`to_simd()`][Self::to_simd()] into account. - /// - /// Returns `false` if `sample_index` is out of bounds. - #[cfg(feature = "simd")] - #[allow(clippy::wrong_self_convention)] - #[inline] - pub fn from_channel_simd( - &mut self, - sample_index: usize, - vector: Simd, - ) -> bool - where - LaneCount: SupportedLaneCount, - { - if sample_index > self.len() { - return false; - } - - let used_lanes = self.len().max(LANES); - let values = vector.to_array(); - for (channel_idx, value) in values.into_iter().enumerate().take(used_lanes) { - *unsafe { - (*self.buffers) - .get_unchecked_mut(channel_idx) - .get_unchecked_mut(self.current_block_start + sample_index) - } = value; - } - - true - } - - /// Write data from a SIMD vector to this sample's channel data for a specific sample in this - /// block.. This assumes `LANES` matches exactly with the number of channels in the buffer, and - /// does not perform bounds checks on `sample_index`. - /// - /// # Safety - /// - /// Undefined behavior if `LANES > block.len()` or if `sample_index > block.len()`. - #[cfg(feature = "simd")] - #[allow(clippy::wrong_self_convention)] - #[inline] - pub unsafe fn from_channel_simd_unchecked( - &mut self, - sample_index: usize, - vector: Simd, - ) where - LaneCount: SupportedLaneCount, - { - let values = vector.to_array(); - for (channel_idx, value) in values.into_iter().enumerate() { - *(*self.buffers) - .get_unchecked_mut(channel_idx) - .get_unchecked_mut(self.current_block_start + sample_index) = value; - } - } -} - #[cfg(any(miri, test))] mod miri { use super::*; diff --git a/src/buffer/blocks.rs b/src/buffer/blocks.rs new file mode 100644 index 00000000..25201d26 --- /dev/null +++ b/src/buffer/blocks.rs @@ -0,0 +1,330 @@ +//! Per-block per-channel per-sample iterators. + +use std::marker::PhantomData; + +#[cfg(feature = "simd")] +use std::simd::{LaneCount, Simd, SupportedLaneCount}; + +use super::SamplesIter; + +/// An iterator over all samples in the buffer, slicing over the sample-dimension with a maximum +/// size of `max_block_size`. See [`Buffer::iter_blocks()`]. Yields both the block and the offset +/// from the start of the buffer. +pub struct BlocksIter<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(super) buffers: *mut [&'sample mut [f32]], + pub(super) max_block_size: usize, + pub(super) current_block_start: usize, + pub(super) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +/// A block yielded by [`BlocksIter`]. Can be iterated over once or multiple times, and also +/// supports direct access to the block's samples if needed. +pub struct Block<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(self) buffers: *mut [&'sample mut [f32]], + pub(self) current_block_start: usize, + /// The index of the last sample in the block plus one. + pub(self) current_block_end: usize, + pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +/// An iterator over all channels in a block yielded by [`Block`], returning an entire channel slice +/// at a time. +pub struct BlockChannelsIter<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(self) buffers: *mut [&'sample mut [f32]], + pub(self) current_block_start: usize, + pub(self) current_block_end: usize, + pub(self) current_channel: usize, + pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +impl<'slice, 'sample> Iterator for BlocksIter<'slice, 'sample> { + type Item = (usize, Block<'slice, 'sample>); + + #[inline] + fn next(&mut self) -> Option { + let buffer_len = unsafe { (*self.buffers)[0].len() }; + if self.current_block_start < buffer_len { + let current_block_start = self.current_block_start; + let current_block_end = + (self.current_block_start + self.max_block_size).min(buffer_len); + let block = Block { + buffers: self.buffers, + current_block_start, + current_block_end, + _marker: self._marker, + }; + + self.current_block_start += self.max_block_size; + + Some((current_block_start, block)) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let remaining = ((unsafe { (*self.buffers)[0].len() } - self.current_block_start) as f32 + / self.max_block_size as f32) + .ceil() as usize; + (remaining, Some(remaining)) + } +} + +impl<'slice, 'sample> IntoIterator for Block<'slice, 'sample> { + type Item = &'sample mut [f32]; + type IntoIter = BlockChannelsIter<'slice, 'sample>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + BlockChannelsIter { + buffers: self.buffers, + current_block_start: self.current_block_start, + current_block_end: self.current_block_end, + current_channel: 0, + _marker: self._marker, + } + } +} + +impl<'slice, 'sample> Iterator for BlockChannelsIter<'slice, 'sample> { + type Item = &'sample mut [f32]; + + #[inline] + fn next(&mut self) -> Option { + if self.current_channel < unsafe { (*self.buffers).len() } { + // SAFETY: These bounds have already been checked + // SAFETY: It is also not possible to have multiple mutable references to the same + // sample at the same time + let slice = unsafe { + (*self.buffers) + .get_unchecked_mut(self.current_channel) + .get_unchecked_mut(self.current_block_start..self.current_block_end) + }; + + self.current_channel += 1; + + Some(slice) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let remaining = unsafe { (*self.buffers).len() } - self.current_channel; + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for BlocksIter<'_, '_> {} +impl ExactSizeIterator for BlockChannelsIter<'_, '_> {} + +impl<'slice, 'sample> Block<'slice, 'sample> { + /// Get the number of samples (not channels) in the block. + #[allow(clippy::len_without_is_empty)] + #[inline] + pub fn len(&self) -> usize { + self.current_block_end - self.current_block_start + } + + /// Return the numer of channels in this buffer. + #[inline] + pub fn channels(&self) -> usize { + unsafe { (*self.buffers).len() } + } + + /// A resetting iterator. This lets you iterate over the same block multiple times. Otherwise + /// you don't need to use this function as [`Block`] already implements [`Iterator`]. You can + /// also use the direct accessor functions on this block instead. + #[inline] + pub fn iter_mut(&mut self) -> BlockChannelsIter<'slice, 'sample> { + BlockChannelsIter { + buffers: self.buffers, + current_block_start: self.current_block_start, + current_block_end: self.current_block_end, + current_channel: 0, + _marker: self._marker, + } + } + + /// Iterate over this block on a per-sample per-channel basis. This is identical to + /// [`Buffer::iter_samples()`] but for a smaller block instead of the entire buffer + #[inline] + pub fn iter_samples(&mut self) -> SamplesIter<'slice, 'sample> { + SamplesIter { + buffers: self.buffers, + current_sample: self.current_block_start, + samples_end: self.current_block_end, + _marker: self._marker, + } + } + + /// Access a channel by index. Useful when you would otherwise iterate over this [`Block`] + /// multiple times. + #[inline] + pub fn get(&self, channel_index: usize) -> Option<&[f32]> { + // SAFETY: The block bound has already been checked + unsafe { + Some( + (*self.buffers) + .get(channel_index)? + .get_unchecked(self.current_block_start..self.current_block_end), + ) + } + } + + /// The same as [`get()`][Self::get], but without any bounds checking. + /// + /// # Safety + /// + /// `channel_index` must be in the range `0..Self::len()`. + #[inline] + pub unsafe fn get_unchecked(&self, channel_index: usize) -> &[f32] { + (*self.buffers) + .get_unchecked(channel_index) + .get_unchecked(self.current_block_start..self.current_block_end) + } + + /// Access a mutable channel by index. Useful when you would otherwise iterate over this + /// [`Block`] multiple times. + #[inline] + pub fn get_mut(&mut self, channel_index: usize) -> Option<&mut [f32]> { + // SAFETY: The block bound has already been checked + unsafe { + Some( + (*self.buffers) + .get_mut(channel_index)? + .get_unchecked_mut(self.current_block_start..self.current_block_end), + ) + } + } + + /// The same as [`get_mut()`][Self::get_mut], but without any bounds checking. + /// + /// # Safety + /// + /// `channel_index` must be in the range `0..Self::len()`. + #[inline] + pub unsafe fn get_unchecked_mut(&mut self, channel_index: usize) -> &mut [f32] { + (*self.buffers) + .get_unchecked_mut(channel_index) + .get_unchecked_mut(self.current_block_start..self.current_block_end) + } + + /// Get a SIMD vector containing the channel data for a specific sample in this block. If `LANES + /// > channels.len()` then this will be padded with zeroes. If `LANES < channels.len()` then + /// this won't contain all values. + /// + /// Returns a `None` value if `sample_index` is out of bounds. + #[cfg(feature = "simd")] + #[inline] + pub fn to_channel_simd( + &self, + sample_index: usize, + ) -> Option> + where + LaneCount: SupportedLaneCount, + { + if sample_index > self.len() { + return None; + } + + let used_lanes = self.len().max(LANES); + let mut values = [0.0; LANES]; + for (channel_idx, value) in values.iter_mut().enumerate().take(used_lanes) { + *value = unsafe { + *(*self.buffers) + .get_unchecked(channel_idx) + .get_unchecked(self.current_block_start + sample_index) + }; + } + + Some(Simd::from_array(values)) + } + + /// Get a SIMD vector containing the channel data for a specific sample in this block. Will + /// always read exactly `LANES` channels, and does not perform bounds checks on `sample_index`. + /// + /// # Safety + /// + /// Undefined behavior if `LANES > block.len()` or if `sample_index > block.len()`. + #[cfg(feature = "simd")] + #[inline] + pub unsafe fn to_channel_simd_unchecked( + &self, + sample_index: usize, + ) -> Simd + where + LaneCount: SupportedLaneCount, + { + let mut values = [0.0; LANES]; + for (channel_idx, value) in values.iter_mut().enumerate() { + *value = *(*self.buffers) + .get_unchecked(channel_idx) + .get_unchecked(self.current_block_start + sample_index); + } + + Simd::from_array(values) + } + + /// Write data from a SIMD vector to this sample's channel data for a specific sample in this + /// block. This takes the padding added by [`to_simd()`][Self::to_simd()] into account. + /// + /// Returns `false` if `sample_index` is out of bounds. + #[cfg(feature = "simd")] + #[allow(clippy::wrong_self_convention)] + #[inline] + pub fn from_channel_simd( + &mut self, + sample_index: usize, + vector: Simd, + ) -> bool + where + LaneCount: SupportedLaneCount, + { + if sample_index > self.len() { + return false; + } + + let used_lanes = self.len().max(LANES); + let values = vector.to_array(); + for (channel_idx, value) in values.into_iter().enumerate().take(used_lanes) { + *unsafe { + (*self.buffers) + .get_unchecked_mut(channel_idx) + .get_unchecked_mut(self.current_block_start + sample_index) + } = value; + } + + true + } + + /// Write data from a SIMD vector to this sample's channel data for a specific sample in this + /// block.. This assumes `LANES` matches exactly with the number of channels in the buffer, and + /// does not perform bounds checks on `sample_index`. + /// + /// # Safety + /// + /// Undefined behavior if `LANES > block.len()` or if `sample_index > block.len()`. + #[cfg(feature = "simd")] + #[allow(clippy::wrong_self_convention)] + #[inline] + pub unsafe fn from_channel_simd_unchecked( + &mut self, + sample_index: usize, + vector: Simd, + ) where + LaneCount: SupportedLaneCount, + { + let values = vector.to_array(); + for (channel_idx, value) in values.into_iter().enumerate() { + *(*self.buffers) + .get_unchecked_mut(channel_idx) + .get_unchecked_mut(self.current_block_start + sample_index) = value; + } + } +} diff --git a/src/buffer/samples.rs b/src/buffer/samples.rs new file mode 100644 index 00000000..3a0c02eb --- /dev/null +++ b/src/buffer/samples.rs @@ -0,0 +1,314 @@ +//! Per-sample per-channel iterators. + +use std::marker::PhantomData; + +#[cfg(feature = "simd")] +use std::simd::{LaneCount, Simd, SupportedLaneCount}; + +/// An iterator over all samples in a buffer or block, yielding iterators over each channel for +/// every sample. This iteration order offers good cache locality for per-sample access. +pub struct SamplesIter<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(super) buffers: *mut [&'sample mut [f32]], + pub(super) current_sample: usize, + /// The last sample index to iterate over plus one. Would be equal to `buffers.len()` when + /// iterating over an entire buffer, but this can also be used to iterate over smaller blocks in + /// a similar fashion. + pub(super) samples_end: usize, + pub(super) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +/// Can construct iterators over actual iterator over the channel data for a sample, yielded by +/// [`SamplesIter`]. Can be turned into an iterator, or [`Channels::iter_mut()`] can be used to +/// iterate over the channel data multiple times, or more efficiently you can use +/// [`Channels::get_unchecked_mut()`] to do the same thing. +pub struct ChannelSamples<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(self) buffers: *mut [&'sample mut [f32]], + pub(self) current_sample: usize, + pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +/// The actual iterator over the channel data for a sample, yielded by [`Channels`]. +pub struct ChannelSamplesIter<'slice, 'sample: 'slice> { + /// The raw output buffers. + pub(self) buffers: *mut [&'sample mut [f32]], + pub(self) current_sample: usize, + pub(self) current_channel: usize, + pub(self) _marker: PhantomData<&'slice mut [&'sample mut [f32]]>, +} + +impl<'slice, 'sample> Iterator for SamplesIter<'slice, 'sample> { + type Item = ChannelSamples<'slice, 'sample>; + + #[inline] + fn next(&mut self) -> Option { + // This can iterate over both the entire buffer or over a smaller sample slice of it + if self.current_sample < self.samples_end { + let channels = ChannelSamples { + buffers: self.buffers, + current_sample: self.current_sample, + _marker: self._marker, + }; + + self.current_sample += 1; + + Some(channels) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let remaining = unsafe { (*self.buffers)[0].len() } - self.current_sample; + (remaining, Some(remaining)) + } +} + +impl<'slice, 'sample> IntoIterator for ChannelSamples<'slice, 'sample> { + type Item = &'sample mut f32; + type IntoIter = ChannelSamplesIter<'slice, 'sample>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + ChannelSamplesIter { + buffers: self.buffers, + current_sample: self.current_sample, + current_channel: 0, + _marker: self._marker, + } + } +} + +impl<'slice, 'sample> Iterator for ChannelSamplesIter<'slice, 'sample> { + type Item = &'sample mut f32; + + #[inline] + fn next(&mut self) -> Option { + if self.current_channel < unsafe { (*self.buffers).len() } { + // SAFETY: These bounds have already been checked + // SAFETY: It is also not possible to have multiple mutable references to the same + // sample at the same time + let sample = unsafe { + (*self.buffers) + .get_unchecked_mut(self.current_channel) + .get_unchecked_mut(self.current_sample) + }; + + self.current_channel += 1; + + Some(sample) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let remaining = unsafe { (*self.buffers).len() } - self.current_channel; + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for SamplesIter<'_, '_> {} +impl ExactSizeIterator for ChannelSamplesIter<'_, '_> {} + +impl<'slice, 'sample> ChannelSamples<'slice, 'sample> { + /// Get the number of channels. + #[allow(clippy::len_without_is_empty)] + #[inline] + pub fn len(&self) -> usize { + unsafe { (*self.buffers).len() } + } + + /// A resetting iterator. This lets you iterate over the same channels multiple times. Otherwise + /// you don't need to use this function as [`Channels`] already implements [Iterator]. + #[inline] + pub fn iter_mut(&mut self) -> ChannelSamplesIter<'slice, 'sample> { + ChannelSamplesIter { + buffers: self.buffers, + current_sample: self.current_sample, + current_channel: 0, + _marker: self._marker, + } + } + + /// Access a sample by index. Useful when you would otherwise iterate over this 'Channels' + /// iterator multiple times. + #[inline] + pub fn get_mut(&mut self, channel_index: usize) -> Option<&mut f32> { + // SAFETY: The sample bound has already been checked + unsafe { + Some( + (*self.buffers) + .get_mut(channel_index)? + .get_unchecked_mut(self.current_sample), + ) + } + } + + /// The same as [`get_mut()`][Self::get_mut()], but without any bounds checking. + /// + /// # Safety + /// + /// `channel_index` must be in the range `0..Self::len()`. + #[inline] + pub unsafe fn get_unchecked_mut(&mut self, channel_index: usize) -> &mut f32 { + (*self.buffers) + .get_unchecked_mut(channel_index) + .get_unchecked_mut(self.current_sample) + } + + /// Get a SIMD vector containing the channel data for this buffer. If `LANES > channels.len()` + /// then this will be padded with zeroes. If `LANES < channels.len()` then this won't contain + /// all values. + #[cfg(feature = "simd")] + #[inline] + pub fn to_simd(&self) -> Simd + where + LaneCount: SupportedLaneCount, + { + let used_lanes = self.len().max(LANES); + let mut values = [0.0; LANES]; + for (channel_idx, value) in values.iter_mut().enumerate().take(used_lanes) { + *value = unsafe { + *(*self.buffers) + .get_unchecked(channel_idx) + .get_unchecked(self.current_sample) + }; + } + + Simd::from_array(values) + } + + /// Get a SIMD vector containing the channel data for this buffer. Will always read exactly + /// `LANES` channels. + /// + /// # Safety + /// + /// Undefined behavior if `LANES > channels.len()`. + #[cfg(feature = "simd")] + #[inline] + pub unsafe fn to_simd_unchecked(&self) -> Simd + where + LaneCount: SupportedLaneCount, + { + let mut values = [0.0; LANES]; + for (channel_idx, value) in values.iter_mut().enumerate() { + *value = *(*self.buffers) + .get_unchecked(channel_idx) + .get_unchecked(self.current_sample); + } + + Simd::from_array(values) + } + + /// Write data from a SIMD vector to this sample's channel data. This takes the padding added by + /// [`to_simd()`][Self::to_simd()] into account. + #[cfg(feature = "simd")] + #[allow(clippy::wrong_self_convention)] + #[inline] + pub fn from_simd(&mut self, vector: Simd) + where + LaneCount: SupportedLaneCount, + { + let used_lanes = self.len().max(LANES); + let values = vector.to_array(); + for (channel_idx, value) in values.into_iter().enumerate().take(used_lanes) { + *unsafe { + (*self.buffers) + .get_unchecked_mut(channel_idx) + .get_unchecked_mut(self.current_sample) + } = value; + } + } + + /// Write data from a SIMD vector to this sample's channel data. This assumes `LANES` matches + /// exactly with the number of channels in the buffer. + /// + /// # Safety + /// + /// Undefined behavior if `LANES > channels.len()`. + #[cfg(feature = "simd")] + #[allow(clippy::wrong_self_convention)] + #[inline] + pub unsafe fn from_simd_unchecked(&mut self, vector: Simd) + where + LaneCount: SupportedLaneCount, + { + let values = vector.to_array(); + for (channel_idx, value) in values.into_iter().enumerate() { + *(*self.buffers) + .get_unchecked_mut(channel_idx) + .get_unchecked_mut(self.current_sample) = value; + } + } +} + +#[cfg(any(miri, test))] +mod miri { + use super::*; + + #[test] + fn repeated_access() { + let mut real_buffers = vec![vec![0.0; 512]; 2]; + let mut buffer = Buffer::default(); + unsafe { + buffer.with_raw_vec(|output_slices| { + let (first_channel, other_channels) = real_buffers.split_at_mut(1); + *output_slices = vec![&mut first_channel[0], &mut other_channels[0]]; + }) + }; + + for samples in buffer.iter_samples() { + for sample in samples { + *sample += 0.001; + } + } + + for mut samples in buffer.iter_samples() { + for _ in 0..2 { + for sample in samples.iter_mut() { + *sample += 0.001; + } + } + } + + assert_eq!(real_buffers[0][0], 0.003); + } + + #[test] + fn repeated_slices() { + let mut real_buffers = vec![vec![0.0; 512]; 2]; + let mut buffer = Buffer::default(); + unsafe { + buffer.with_raw_vec(|output_slices| { + let (first_channel, other_channels) = real_buffers.split_at_mut(1); + *output_slices = vec![&mut first_channel[0], &mut other_channels[0]]; + }) + }; + + // These iterators should not alias + let mut blocks = buffer.iter_blocks(16); + let (_block1_offset, block1) = blocks.next().unwrap(); + let (_block2_offset, block2) = blocks.next().unwrap(); + for channel in block1 { + for sample in channel.iter_mut() { + *sample += 0.001; + } + } + for channel in block2 { + for sample in channel.iter_mut() { + *sample += 0.001; + } + } + + for i in 0..32 { + assert_eq!(real_buffers[0][i], 0.001); + } + for i in 32..48 { + assert_eq!(real_buffers[0][i], 0.0); + } + } +}