alex/librashader
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

pack: implement shader pack library to preload resources from disk

Browse source
This commit is contained in:
chyyran 2024-10-01 01:38:14 -04:00 committed by Ronny Chan
parent 1bbbd784d8
commit c19593e289
6 changed files with 304 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

238
Cargo.lock generated
View file

@ -90,12 +90,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04"
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
@ -473,7 +467,7 @@ checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb"
dependencies = [
"clap",
"heck",
"indexmap 2.5.0",
"indexmap",
"log",
"proc-macro2",
"quote",
@ -513,19 +507,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
[[package]]
name = "chrono"
version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
"serde",
"windows-targets 0.52.6",
]
[[package]]
name = "clap"
version = "4.3.0"
@ -547,7 +528,7 @@ dependencies = [
"anstyle",
"bitflags 1.3.2",
"clap_lex",
"strsim 0.10.0",
"strsim",
]
[[package]]
@ -858,57 +839,12 @@ dependencies = [
"windows 0.58.0",
]
[[package]]
name = "darling"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim 0.11.1",
"syn 2.0.79",
]
[[package]]
name = "darling_macro"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
"syn 2.0.79",
]
[[package]]
name = "data-encoding"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
[[package]]
name = "deranged"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
dependencies = [
"powerfmt",
"serde",
]
[[package]]
name = "digest"
version = "0.10.7"
@ -1377,12 +1313,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hexf-parse"
version = "0.2.1"
@ -1395,29 +1325,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "iana-time-zone"
version = "0.1.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core 0.52.0",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "icrate"
version = "0.0.4"
@ -1429,12 +1336,6 @@ dependencies = [
"objc2 0.4.1",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "image"
version = "0.25.2"
@ -1475,17 +1376,6 @@ dependencies = [
"quick-error",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]]
name = "indexmap"
version = "2.5.0"
@ -1494,7 +1384,6 @@ checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
dependencies = [
"equivalent",
"hashbrown 0.14.5",
"serde",
]
[[package]]
@ -1634,6 +1523,7 @@ dependencies = [
"halfbrown",
"librashader-cache",
"librashader-common",
"librashader-pack",
"librashader-preprocess",
"librashader-presets",
"librashader-reflect",
@ -1746,14 +1636,16 @@ name = "librashader-pack"
version = "0.4.5"
dependencies = [
"anyhow",
"base64 0.22.1",
"image",
"librashader-preprocess",
"librashader-presets",
"librashader-reflect",
"rayon",
"rmp-serde",
"serde",
"serde_bytes",
"serde_json",
"serde_with",
"thiserror",
]
@ -2100,7 +1992,7 @@ dependencies = [
"cfg_aliases",
"codespan-reporting",
"hexf-parse",
"indexmap 2.5.0",
"indexmap",
"log",
"petgraph",
"pp-rs",
@ -2195,12 +2087,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-derive"
version = "0.4.2"
@ -2567,7 +2453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset",
"indexmap 2.5.0",
"indexmap",
]
[[package]]
@ -2631,12 +2517,6 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "pp-rs"
version = "0.2.1"
@ -2862,6 +2742,28 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832"
[[package]]
name = "rmp"
version = "0.8.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
name = "rmp-serde"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "ron"
version = "0.7.1"
@ -2967,6 +2869,15 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_bytes"
version = "0.11.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a"
dependencies = [
"serde",
]
[[package]]
name = "serde_derive"
version = "1.0.210"
@ -2999,36 +2910,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_with"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.5.0",
"serde",
"serde_derive",
"serde_json",
"serde_with_macros",
"time",
]
[[package]]
name = "serde_with_macros"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "sha2"
version = "0.10.8"
@ -3262,12 +3143,6 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "1.0.109"
@ -3349,37 +3224,6 @@ dependencies = [
"weezl",
]
[[package]]
name = "time"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
dependencies = [
"deranged",
"itoa",
"num-conv",
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
dependencies = [
"num-conv",
"time-core",
]
[[package]]
name = "tiny-skia"
version = "0.11.4"
@ -3441,7 +3285,7 @@ version = "0.22.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
dependencies = [
"indexmap 2.5.0",
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
@ -3800,7 +3644,7 @@ dependencies = [
"bitflags 2.6.0",
"cfg_aliases",
"document-features",
"indexmap 2.5.0",
"indexmap",
"log",
"naga",
"once_cell",

15
librashader-pack/Cargo.toml
View file

@ -9,13 +9,20 @@ librashader-preprocess = { path = "../librashader-preprocess", version = "0.4.5"
librashader-reflect= { path = "../librashader-reflect", version = "0.4.5" }
thiserror = "1.0.64"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.128"
serde = { version = "1.0", features = ["derive"], optional = true }
rayon = { workspace = true }
image = { workspace = true}
image = {workspace = true}
base64 = { version = "0.22.1", optional = true }
serde_bytes = { version = "0.11.15", optional = true }
[features]
parse_legacy_glsl = ["librashader-presets/parse_legacy_glsl"]
serde = ["dep:serde", "dep:base64", "dep:serde_bytes"]
[target.'cfg(not(target_arch="wasm32"))'.dependencies]
rayon = { workspace = true}
rayon = { workspace = true }
[dev-dependencies]
anyhow = "1.0.89"
serde_json = "1.0.128"
rmp-serde = "1.3.0"

247
librashader-pack/src/lib.rs
View file

@ -1 +1,248 @@
//! Shader preset resource handling for librashader.
//!
//! This crate contains facilities to load shader preset resources from a [`ShaderPreset`].
//!
//! Also defines abstractly the `.slangpack` shader format implemented via serde derives on [`ShaderPresetPack`].
//!
use image::{ImageError, RgbaImage};
use librashader_preprocess::{PreprocessError, ShaderSource};
use librashader_presets::{ParameterMeta, ShaderPassMeta, ShaderPreset, TextureMeta};
use std::path::Path;
#[cfg(not(target_arch = "wasm32"))]
use rayon::prelude::*;
/// A buffer holding RGBA image bytes.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct TextureBuffer {
#[cfg_attr(feature = "serde", serde(with = "serde_base64_or_bytes"))]
image: Vec<u8>,
width: u32,
height: u32,
}
impl From<TextureBuffer> for Option<RgbaImage> {
fn from(value: TextureBuffer) -> Self {
RgbaImage::from_raw(value.width, value.height, value.image)
}
}
impl AsRef<[u8]> for TextureBuffer {
fn as_ref(&self) -> &[u8] {
self.image.as_ref()
}
}
impl From<RgbaImage> for TextureBuffer {
fn from(value: RgbaImage) -> Self {
let width = value.width();
let height = value.height();
TextureBuffer {
image: value.into_raw(),
width,
height,
}
}
}
/// A resource for a shader preset, fully loaded into memory.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ShaderPresetResource<M: LoadableResource> {
/// The fully qualified path to the texture.
pub data: M::ResourceType,
/// Meta information about the texture.
pub meta: M,
}
/// Trait for a resource that is loadable from disk.
pub trait LoadableResource {
/// The type of the resource.
type ResourceType;
/// The error type when loading the resource.
type Error;
/// Load the resource from the path.
fn load(path: &Path) -> Result<Self::ResourceType, Self::Error>;
}
impl LoadableResource for ShaderPassMeta {
type ResourceType = ShaderSource;
type Error = PreprocessError;
fn load(path: &Path) -> Result<Self::ResourceType, Self::Error> {
ShaderSource::load(path)
}
}
impl LoadableResource for TextureMeta {
type ResourceType = TextureBuffer;
type Error = ImageError;
fn load(path: &Path) -> Result<Self::ResourceType, Self::Error> {
image::open(path).map(|img| TextureBuffer::from(img.to_rgba8()))
}
}
/// The configuration for a single shader pass.
pub type ShaderPassData = ShaderPresetResource<ShaderPassMeta>;
pub type TextureData = ShaderPresetResource<TextureMeta>;
/// A shader preset, not reliant on disk, with all information needed.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ShaderPresetPack {
/// Used in legacy GLSL shader semantics. If < 0, no feedback pass is used.
/// Otherwise, the FBO after pass #N is passed a texture to next frame
#[cfg(feature = "parse_legacy_glsl")]
pub feedback_pass: i32,
/// The number of shaders enabled in the filter chain.
pub shader_count: i32,
// Everything is in Vecs because the expect number of values is well below 64.
/// Preset information for each shader.
pub shaders: Vec<ShaderPassData>,
/// Preset information for each texture.
pub textures: Vec<TextureData>,
/// Preset information for each user parameter.
pub parameters: Vec<ParameterMeta>,
}
impl ShaderPresetPack {
/// Load a `ShaderPack` from a [`ShaderPreset`].
pub fn load_from_preset<E>(preset: ShaderPreset) -> Result<ShaderPresetPack, E>
where
E: From<PreprocessError>,
E: From<ImageError>,
E: Send,
{
#[cfg(not(target_arch = "wasm32"))]
let shaders_iter = preset.shaders.into_par_iter();
#[cfg(target_arch = "wasm32")]
let shaders_iter = preset.shaders.into_iter();
#[cfg(not(target_arch = "wasm32"))]
let textures_iter = preset.textures.into_par_iter();
#[cfg(target_arch = "wasm32")]
let textures_iter = preset.textures.into_iter();
Ok(ShaderPresetPack {
#[cfg(feature = "parse_legacy_glsl")]
feedback_pass: preset.feedback_pass,
shader_count: preset.shader_count,
shaders: shaders_iter
.map(|v| {
Ok::<_, E>(ShaderPassData {
data: ShaderPassMeta::load(v.path.as_path())?,
meta: v.meta,
})
})
.collect::<Result<Vec<_>, _>>()?,
textures: textures_iter
.into_par_iter()
.map(|t| {
Ok::<_, E>(TextureData {
data: TextureMeta::load(t.path.as_path())?,
meta: t.meta,
})
})
.collect::<Result<Vec<_>, _>>()?,
parameters: preset.parameters,
})
}
}
#[cfg(feature = "serde")]
mod serde_base64_or_bytes {
use base64::display::Base64Display;
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use serde::{Deserializer, Serializer};
#[allow(clippy::ptr_arg)]
pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
if s.is_human_readable() {
s.collect_str(&Base64Display::new(v, &STANDARD))
} else {
serde_bytes::serialize(v, s)
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
if d.is_human_readable() {
struct Base64Visitor;
impl<'de> serde::de::Visitor<'de> for Base64Visitor {
type Value = Vec<u8>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a base64 encoded string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
self.visit_bytes(v.as_ref())
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
self.visit_bytes(v.as_ref())
}
fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
STANDARD.decode(v).map_err(serde::de::Error::custom)
}
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
self.visit_bytes(v.as_ref())
}
}
d.deserialize_str(Base64Visitor)
} else {
serde_bytes::deserialize(d)
}
}
}
#[cfg(test)]
mod test {
use crate::ShaderPresetPack;
use librashader_presets::ShaderPreset;
use std::fs::File;
use std::io::Write;
#[test]
fn test() {
let preset =
ShaderPreset::try_parse("../test/shaders_slang/crt/crt-royale.slangp").unwrap();
let resolved = ShaderPresetPack::load_from_preset::<anyhow::Error>(preset).unwrap();
let mut file = File::create("crt-royale.slangpack.json").unwrap();
file.write_all(serde_json::to_vec_pretty(&resolved).unwrap().as_ref())
.unwrap();
}
#[test]
fn test_rmp() {
let preset =
ShaderPreset::try_parse("../test/shaders_slang/crt/crt-royale.slangp").unwrap();
let resolved = ShaderPresetPack::load_from_preset::<anyhow::Error>(preset).unwrap();
let mut file = File::create("crt-royale.slangpack").unwrap();
file.write_all(rmp_serde::to_vec(&resolved).unwrap().as_ref())
.unwrap();
}
}

1
librashader-runtime/src/image.rs
View file

@ -2,7 +2,6 @@ pub use image::ImageError;
use librashader_common::Size;
use std::marker::PhantomData;
use image::RgbaImage;
use std::path::Path;
/// An uncompressed raw image ready to upload to GPU buffers.

3
librashader/Cargo.toml
View file

@ -19,6 +19,7 @@ librashader-preprocess = { path = "../librashader-preprocess", version = "0.4.5"
librashader-reflect = { path = "../librashader-reflect", version = "0.4.5" }
librashader-cache = { path = "../librashader-cache", version = "0.4.5" }
librashader-runtime = { path = "../librashader-runtime", version = "0.4.5" }
librashader-pack = { path = "../librashader-pack", version = "0.4.5" }
librashader-runtime-d3d11 = { path = "../librashader-runtime-d3d11", version = "0.4.5", optional = true }
librashader-runtime-d3d12 = { path = "../librashader-runtime-d3d12", version = "0.4.5", optional = true }
librashader-runtime-d3d9 = { path = "../librashader-runtime-d3d9", version = "0.4.5", optional = true }
@ -84,7 +85,7 @@ full = ["runtime-all", "reflect-all", "preprocess", "presets"]
# cache hack
docsrs = ["librashader-cache/docsrs"]
serde = ["librashader-presets/serde", "librashader-preprocess/serde", "librashader-reflect/serde"]
serde = ["librashader-presets/serde", "librashader-preprocess/serde", "librashader-reflect/serde", "librashader-pack/serde"]
# emits warning messages in tests
github-ci = []

3
librashader/src/lib.rs
View file

@ -69,6 +69,9 @@ pub use librashader_common::map::ShortString;
pub mod presets {
use librashader_preprocess::{PreprocessError, ShaderParameter, ShaderSource};
pub use librashader_presets::*;
pub use librashader_pack::*;
/// Get full parameter metadata from a shader preset.
pub fn get_parameter_meta(
preset: &ShaderPreset,