From 1bb1cde9135d9e5814e87e7995dd3f03a8f264be Mon Sep 17 00:00:00 2001 From: Robbert van der Helm Date: Thu, 18 Aug 2022 13:55:31 +0200 Subject: [PATCH] Add optional Zstandard compression for state This can be particularly useful when using the persistent fields feature to store JSON or other large textual documents. --- .github/workflows/docs.yml | 2 +- .github/workflows/test.yml | 2 +- Cargo.lock | 42 ++++++++++++++++++++++++ Cargo.toml | 13 ++++++-- README.md | 2 ++ plugins/examples/gain/Cargo.toml | 2 +- src/wrapper/clap/wrapper.rs | 2 +- src/wrapper/state.rs | 55 +++++++++++++++++++++++++++++--- src/wrapper/vst3/wrapper.rs | 2 +- 9 files changed, 110 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 505d8d6a..5932d58d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,7 +51,7 @@ jobs: # conflicting iced features. We also don't want to use `--workspace` # here because that would also document our plugins and binary crates. args: >- - --features docs,simd,standalone --no-deps + --features docs,simd,standalone,zstd --no-deps -p nih_plug -p nih_plug_derive -p nih_plug_egui diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a498bf0..b1c9525b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: command: build # Don't use --all-features as that will enable a whole bunch of # conflicting iced features - args: --workspace --features "simd,standalone" + args: --workspace --features "simd,standalone,zstd" - name: Run the tests uses: actions-rs/cargo@v1 with: diff --git a/Cargo.lock b/Cargo.lock index b0372c8b..f1599648 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,6 +449,9 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -1992,6 +1995,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "jpeg-decoder" version = "0.1.22" @@ -2397,6 +2409,7 @@ dependencies = [ "widestring", "win_dbg_logger", "windows", + "zstd", ] [[package]] @@ -4686,3 +4699,32 @@ version = "0.1.0" dependencies = [ "nih_plug_xtask", ] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 4dd10a9e..af8c7d57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ assert_process_allocs = ["dep:assert_no_alloc"] # Enables an export target for standalone binaries through the # `nih_export_standalone()` function. Disabled by default as this requires # building additional dependencies for audio and MIDI handling. -standalone = ["dep:anyhow", "dep:baseview", "dep:clap", "dep:jack"] +standalone = ["dep:baseview", "dep:clap", "dep:jack"] # Enables the `nih_export_vst3!()` macro. Enabled by default. This feature # exists mostly for GPL-compliance reasons, since even if you don't use the VST3 # wrapper you might otherwise still include a couple (unused) symbols from the @@ -58,12 +58,19 @@ vst3 = ["dep:vst3-sys"] # Add adapters to the Buffer object for reading the channel data to and from # `std::simd` vectors. Requires a nightly compiler. simd = [] +# Compress plugin state using the Zstandard algorithm. Loading uncompressed +# state is still supported so existing state will still load after enabling this +# feature for a plugin, but it can not be disabled again without losing state +# compatibility. +zstd = ["dep:zstd"] + # Only relevant when generating docs, adds the `doc_auto_cfg` nightly feature docs = [] [dependencies] nih_plug_derive = { path = "nih_plug_derive" } +anyhow = "1.0" atomic_float = "0.1" atomic_refcell = "0.1" backtrace = "0.3.65" @@ -85,7 +92,6 @@ widestring = "1.0.0-beta.1" assert_no_alloc = { version = "1.1", optional = true } # Used for the `standalone` feature -anyhow = { version = "1.0", optional = true } # NOTE: OpenGL support is not needed here, but rust-analyzer gets confused when # some crates do use it and others don't baseview = { git = "https://github.com/robbert-vdh/baseview.git", branch = "feature/resize", features = ["opengl"], optional = true } @@ -97,6 +103,9 @@ jack = { git = "https://github.com/robbert-vdh/rust-jack.git", branch = "feature # Used for the `vst3` feature vst3-sys = { git = "https://github.com/robbert-vdh/vst3-sys.git", branch = "fix/note-off-event", optional = true } +# Used for the `zstd` feature +zstd = { version = "0.11.2", optional = true } + [target.'cfg(all(target_family = "unix", not(target_os = "macos")))'.dependencies] libc = "0.2.124" diff --git a/README.md b/README.md index 3f68f89a..80ceae60 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ Scroll down for more information on the plugin framework. enabled by setting the `Plugin::SAMPLE_ACCURATE_AUTOMATION` constant to `true`. - Support for CLAP's polyphonic modulation on a per-parameter basis. +- Optional support for compressing the human readable JSON state files using + [Zstandard](https://en.wikipedia.org/wiki/Zstd). - Comes with adapters for popular Rust GUI frameworks as well as some basic widgets for them that integrate with NIH-plug's parameter system. Currently there's support for [egui](nih_plug_egui), [iced](nih_plug_iced) and diff --git a/plugins/examples/gain/Cargo.toml b/plugins/examples/gain/Cargo.toml index 7317a843..568df3bb 100644 --- a/plugins/examples/gain/Cargo.toml +++ b/plugins/examples/gain/Cargo.toml @@ -9,6 +9,6 @@ license = "ISC" crate-type = ["cdylib"] [dependencies] -nih_plug = { path = "../../../", features = ["assert_process_allocs"] } +nih_plug = { path = "../../../", features = ["assert_process_allocs", "zstd"] } parking_lot = "0.12" diff --git a/src/wrapper/clap/wrapper.rs b/src/wrapper/clap/wrapper.rs index 5f237c34..6b020c16 100644 --- a/src/wrapper/clap/wrapper.rs +++ b/src/wrapper/clap/wrapper.rs @@ -3139,7 +3139,7 @@ impl Wrapper

{ true } Err(err) => { - nih_debug_assert_failure!("Could not save state: {}", err); + nih_debug_assert_failure!("Could not save state: {:#}", err); false } } diff --git a/src/wrapper/state.rs b/src/wrapper/state.rs index 8deec79c..6419ce93 100644 --- a/src/wrapper/state.rs +++ b/src/wrapper/state.rs @@ -1,6 +1,7 @@ //! Utilities for saving a [crate::plugin::Plugin]'s state. The actual state object is also exposed //! to plugins through the [`GuiContext`][crate::prelude::GuiContext]. +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; @@ -115,13 +116,24 @@ pub(crate) unsafe fn serialize_object<'a>( } /// Serialize a plugin's state to a vector containing JSON data. This can (and should) be shared -/// across plugin formats. +/// across plugin formats. If the `zstd` feature is enabled, then the state will be compressed using +/// Zstandard. pub(crate) unsafe fn serialize_json<'a>( plugin_params: Arc, params_iter: impl IntoIterator, -) -> serde_json::Result> { +) -> Result> { let plugin_state = serialize_object(plugin_params, params_iter); - serde_json::to_vec(&plugin_state) + let json = serde_json::to_vec(&plugin_state).context("Could not format as JSON")?; + + #[cfg(feature = "zstd")] + { + zstd::encode_all(json.as_slice(), zstd::DEFAULT_COMPRESSION_LEVEL) + .context("Could not compress state") + } + #[cfg(not(feature = "zstd"))] + { + Ok(json) + } } /// Deserialize a plugin's state from a [`PluginState`] object. This is used to allow the plugin to @@ -191,8 +203,9 @@ pub(crate) unsafe fn deserialize_object( true } -/// Deserialize a plugin's state from a vector containing JSON data. This can (and should) be shared -/// across plugin formats. Returns `false` and logs an error if the state could not be deserialized. +/// Deserialize a plugin's state from a vector containing (compressed) JSON data. This can (and +/// should) be shared across plugin formats. Returns `false` and logs an error if the state could +/// not be deserialized. If the `zstd` feature is enabled, then this can /// /// Make sure to reinitialize plugin after deserializing the state so it can react to the new /// parameter values. The smoothers have already been reset by this function. @@ -202,6 +215,38 @@ pub(crate) unsafe fn deserialize_json( params_getter: impl Fn(&str) -> Option, current_buffer_config: Option<&BufferConfig>, ) -> bool { + #[cfg(feature = "zstd")] + let state: PluginState = match zstd::decode_all(state) { + Ok(decompressed) => match serde_json::from_slice(decompressed.as_slice()) { + Ok(s) => { + nih_log!("Deserialized compressed"); + s + } + Err(err) => { + nih_debug_assert_failure!("Error while deserializing state: {}", err); + return false; + } + }, + // Uncompressed state files can still be loaded after enabling this feature to prevent + // breaking existing plugin instances + Err(zstd_err) => match serde_json::from_slice(state) { + Ok(s) => { + nih_log!("Deserialized uncompressed"); + s + } + Err(json_err) => { + nih_debug_assert_failure!( + "Error while deserializing state as either compressed or uncompressed state: \ + {}, {}", + zstd_err, + json_err + ); + return false; + } + }, + }; + + #[cfg(not(feature = "zstd"))] let state: PluginState = match serde_json::from_slice(state) { Ok(s) => s, Err(err) => { diff --git a/src/wrapper/vst3/wrapper.rs b/src/wrapper/vst3/wrapper.rs index f713db6d..bc918c70 100644 --- a/src/wrapper/vst3/wrapper.rs +++ b/src/wrapper/vst3/wrapper.rs @@ -555,7 +555,7 @@ impl IComponent for Wrapper

{ kResultOk } Err(err) => { - nih_debug_assert_failure!("Could not save state: {}", err); + nih_debug_assert_failure!("Could not save state: {:#}", err); kResultFalse } }