From 5573f1322706fc7f00e95cc63d0188ef52297114 Mon Sep 17 00:00:00 2001 From: chyyran <ronny@ronnychan.ca> Date: Thu, 26 Sep 2024 02:54:25 -0400 Subject: [PATCH] test: add CLI with multiple functions --- Cargo.lock | 107 +++++++- librashader-test/Cargo.toml | 4 +- librashader-test/src/cli/main.rs | 445 +++++++++++++++++++++++++++++++ librashader/Cargo.toml | 2 +- librashader/src/lib.rs | 1 + 5 files changed, 555 insertions(+), 4 deletions(-) create mode 100644 librashader-test/src/cli/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1737fce..2bc2c0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -251,6 +251,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -296,6 +302,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitvec" @@ -678,7 +687,7 @@ dependencies = [ "lazy_static", "nom", "pathdiff", - "ron", + "ron 0.7.1", "rust-ini", "serde", "serde_json", @@ -790,6 +799,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -977,6 +992,12 @@ dependencies = [ "miniz_oxide 0.8.0", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foreign-types" version = "0.5.0" @@ -1230,6 +1251,17 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + [[package]] name = "halfbrown" version = "0.2.5" @@ -1481,6 +1513,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "librashader" version = "0.4.5" @@ -1577,6 +1615,7 @@ dependencies = [ "librashader-common", "nom", "rayon", + "serde", "thiserror", ] @@ -1800,8 +1839,10 @@ dependencies = [ "objc2-metal", "parking_lot", "pollster", + "ron 0.8.1", "serde", "serde_json", + "spq-spvasm", "wgpu", "wgpu-types", "windows 0.58.0", @@ -2091,6 +2132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2267,6 +2309,15 @@ dependencies = [ "libredox 0.0.2", ] +[[package]] +name = "ordered-float" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2684,11 +2735,23 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" dependencies = [ - "base64", + "base64 0.13.1", "bitflags 1.3.2", "serde", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.6.0", + "serde", + "serde_derive", +] + [[package]] name = "rspirv" version = "0.12.0+sdk-1.3.268.0" @@ -2908,6 +2971,19 @@ dependencies = [ "serde", ] +[[package]] +name = "spirq" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5ab05ab7b72dbb729fe1831a7e4b717b0d18f552c8b2dc9fb06b85878a017a6" +dependencies = [ + "fnv", + "num-derive", + "num-traits", + "ordered-float", + "spq-core", +] + [[package]] name = "spirv" version = "0.3.0+sdk-1.3.268.0" @@ -2978,6 +3054,33 @@ dependencies = [ "cc", ] +[[package]] +name = "spq-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605fb8ae60065f7a9d21d1ae1e89f7a5ff93ca7755df88ada937ac06ba0a0a43" +dependencies = [ + "anyhow", + "bytemuck", + "fnv", + "half", + "num-traits", + "ordered-float", + "spirv", +] + +[[package]] +name = "spq-spvasm" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e703e41f4ae2b3081129430559e5a0a0420a4de624c050a6d58b76e9b632b742" +dependencies = [ + "anyhow", + "half", + "num-traits", + "spirq", +] + [[package]] name = "sptr" version = "0.3.2" diff --git a/librashader-test/Cargo.toml b/librashader-test/Cargo.toml index 4fc0a35..778f9ce 100644 --- a/librashader-test/Cargo.toml +++ b/librashader-test/Cargo.toml @@ -11,7 +11,7 @@ name = "librashader-cli" path = "src/cli/main.rs" [dependencies] -librashader = { version = "0.4.5", path = "../librashader", features = ["presets", "serde"], default-features = false } +librashader = { version = "0.4.5", path = "../librashader", features = ["presets", "preprocess", "serde"], default-features = false } librashader-runtime = { version = "0.4.5", path = "../librashader-runtime"} wgpu = { version = "22", default-features = false, optional = true } wgpu-types = { version = "22", optional = true } @@ -32,6 +32,8 @@ ash = { workspace = true, optional = true } clap = { workspace = true } serde = "1.0" serde_json = "1.0" +spq-spvasm = "0.1.4" +ron = "0.8.1" [features] default = ["full"] diff --git a/librashader-test/src/cli/main.rs b/librashader-test/src/cli/main.rs new file mode 100644 index 0000000..c7aa3a5 --- /dev/null +++ b/librashader-test/src/cli/main.rs @@ -0,0 +1,445 @@ +use anyhow::anyhow; +use clap::{Parser, Subcommand}; +use image::codecs::png::PngEncoder; +use librashader::presets::context::ContextItem; +use librashader::presets::{ShaderPreset, WildcardContext}; +use librashader::reflect::cross::{GlslVersion, HlslShaderModel, MslVersion, SpirvCross}; +use librashader::reflect::naga::{Naga, NagaLoweringOptions}; +use librashader::reflect::semantics::{ ShaderSemantics}; +use librashader::reflect::{CompileShader, FromCompilation, ReflectShader, SpirvCompilation}; +use librashader_test::render::RenderTest; +use std::path::{Path, PathBuf}; +use ron::ser::PrettyConfig; + +#[derive(Parser, Debug)] +#[command(version, about)] +struct Args { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Render a shader preset against an image + Render { + /// The frame to render. + #[arg(short, long, default_value_t = 60)] + frame: usize, + /// The path to the shader preset to load. + #[arg(short, long)] + preset: PathBuf, + /// The path to the input image. + #[arg(short, long)] + image: PathBuf, + /// The path to the output image. + /// + /// If `-`, writes the image in PNG format to stdout. + #[arg(short, long)] + out: PathBuf, + /// The runtime to use to render the shader preset. + #[arg(value_enum, short, long)] + runtime: Runtime, + }, + /// Compare two runtimes and get a similarity score between the two + /// runtimes rendering the same frame + Compare { + /// The frame to render. + #[arg(short, long, default_value_t = 60)] + frame: usize, + /// The path to the shader preset to load. + #[arg(short, long)] + preset: PathBuf, + /// The path to the input image. + #[arg(short, long)] + image: PathBuf, + /// The runtime to compare against + #[arg(value_enum, short, long)] + left: Runtime, + /// The runtime to compare to + #[arg(value_enum, short, long)] + right: Runtime, + /// The path to write the similarity image. + /// + /// If `-`, writes the image to stdout. + #[arg(short, long)] + out: Option<PathBuf>, + }, + /// Parse a preset and get a JSON representation of the data. + Parse { + /// The path to the shader preset to load. + #[arg(short, long)] + preset: PathBuf, + /// Additional wildcard options, comma separated with equals signs. The PRESET and PRESET_DIR + /// wildcards are always added to the preset parsing context. + /// + /// For example, CONTENT-DIR=MyVerticalGames,GAME=mspacman + #[arg(short, long, value_delimiter = ',', num_args = 1..)] + wildcards: Option<Vec<String>>, + }, + /// Get the raw GLSL output of a preprocessed shader. + Preprocess { + /// The path to the slang shader. + #[arg(short, long)] + shader: PathBuf, + /// The item to output. + /// + /// `json` will print a JSON representation of the preprocessed shader. + #[arg(value_enum, short, long)] + output: PreprocessOutput, + }, + /// Transpile a shader in a given preset to the given format. + Transpile { + /// The path to the slang shader. + #[arg(short, long)] + shader: PathBuf, + + /// The shader stage to output. + #[arg(value_enum, short = 'o', long)] + stage: TranspileStage, + + /// The output format. + #[arg(value_enum, short, long)] + format: TranspileFormat, + + /// The version of the output format to parse as. + /// This could be a GLSL version, a shader model, or an MSL version. + #[arg(short, long)] + version: Option<String>, + }, + /// Reflect the shader relative to a preset, giving information about semantics used in a slang shader. + /// + /// Due to limitations + Reflect { + /// The path to the shader preset to load. + #[arg(short, long)] + preset: PathBuf, + /// Additional wildcard options, comma separated with equals signs. The PRESET and PRESET_DIR + /// wildcards are always added to the preset parsing context. + /// + /// For example, CONTENT-DIR=MyVerticalGames,GAME=mspacman + #[arg(short, long, value_delimiter = ',', num_args = 1..)] + wildcards: Option<Vec<String>>, + + /// The pass index to use. + #[arg(short, long)] + index: usize, + + #[arg(value_enum, short, long, default_value = "cross")] + backend: ReflectionBackend, + }, +} + +#[derive(clap::ValueEnum, Clone, Debug)] +enum PreprocessOutput { + #[clap(name = "fragment")] + Fragment, + #[clap(name = "vertex")] + Vertex, + #[clap(name = "params")] + Params, + #[clap(name = "passformat")] + Format, + #[clap(name = "json")] + Json, +} + +#[derive(clap::ValueEnum, Clone, Debug)] +enum TranspileStage { + #[clap(name = "fragment")] + Fragment, + #[clap(name = "vertex")] + Vertex, +} + +#[derive(clap::ValueEnum, Clone, Debug)] +enum TranspileFormat { + #[clap(name = "glsl")] + GLSL, + #[clap(name = "hlsl")] + HLSL, + #[clap(name = "wgsl")] + WGSL, + #[clap(name = "msl")] + MSL, + #[clap(name = "spirv")] + SPIRV, +} + +#[derive(clap::ValueEnum, Clone, Debug)] +enum Runtime { + #[cfg(feature = "opengl")] + #[clap(name = "opengl3")] + OpenGL3, + #[cfg(feature = "opengl")] + #[clap(name = "opengl4")] + OpenGL4, + #[cfg(feature = "vulkan")] + #[clap(name = "vulkan")] + Vulkan, + #[cfg(feature = "wgpu")] + #[clap(name = "wgpu")] + Wgpu, + #[cfg(all(windows, feature = "d3d9"))] + #[clap(name = "d3d9")] + Direct3D9, + #[cfg(all(windows, feature = "d3d11"))] + #[clap(name = "d3d11")] + Direct3D11, + #[cfg(all(windows, feature = "d3d12"))] + #[clap(name = "d3d12")] + Direct3D12, + #[cfg(all(target_vendor = "apple", feature = "metal"))] + #[clap(name = "metal")] + Metal, +} + +#[derive(clap::ValueEnum, Clone, Debug)] +enum ReflectionBackend { + #[clap(name = "cross")] + SpirvCross, + #[clap(name = "naga")] + Naga, +} + +macro_rules! get_runtime { + ($rt:ident, $image:ident) => { + match $rt { + #[cfg(feature = "opengl")] + Runtime::OpenGL3 => &mut librashader_test::render::gl::OpenGl3::new($image.as_path())?, + #[cfg(feature = "opengl")] + Runtime::OpenGL4 => &mut librashader_test::render::gl::OpenGl4::new($image.as_path())?, + #[cfg(feature = "vulkan")] + Runtime::Vulkan => &mut librashader_test::render::vk::Vulkan::new($image.as_path())?, + #[cfg(feature = "wgpu")] + Runtime::Wgpu => &mut librashader_test::render::wgpu::Wgpu::new($image.as_path())?, + #[cfg(all(windows, feature = "d3d9"))] + Runtime::Direct3D9 => { + &mut librashader_test::render::d3d9::Direct3D9::new($image.as_path())? + } + #[cfg(all(windows, feature = "d3d11"))] + Runtime::Direct3D11 => { + &mut librashader_test::render::d3d11::Direct3D11::new($image.as_path())? + } + #[cfg(all(windows, feature = "d3d12"))] + Runtime::Direct3D12 => { + &mut librashader_test::render::d3d12::Direct3D12::new($image.as_path())? + } + #[cfg(all(target_vendor = "apple", feature = "metal"))] + Runtime::Metal => &mut librashader_test::render::mtl::Metal::new($image.as_path())?, + } + }; +} +pub fn main() -> Result<(), anyhow::Error> { + let args = Args::parse(); + + match args.command { + Commands::Render { + frame, + preset, + image, + out, + runtime, + } => { + let test: &mut dyn RenderTest = get_runtime!(runtime, image); + let image = test.render(preset.as_path(), frame)?; + + if out.as_path() == Path::new("-") { + let out = std::io::stdout(); + image.write_with_encoder(PngEncoder::new(out))?; + } else { + image.save(out)?; + } + } + Commands::Compare { + frame, + preset, + image, + left, + right, + out, + } => { + let left: &mut dyn RenderTest = get_runtime!(left, image); + let right: &mut dyn RenderTest = get_runtime!(right, image); + + let left_image = left.render(preset.as_path(), frame)?; + let right_image = right.render(preset.as_path(), frame)?; + let similarity = image_compare::rgba_hybrid_compare(&left_image, &right_image)?; + print!("{}", similarity.score); + + if let Some(out) = out { + let image = similarity.image.to_color_map(); + if out.as_path() == Path::new("-") { + let out = std::io::stdout(); + image.write_with_encoder(PngEncoder::new(out))?; + } else { + image.save(out)?; + } + } + } + Commands::Parse { preset, wildcards } => { + let preset = get_shader_preset(preset, wildcards)?; + let out = serde_json::to_string_pretty(&preset)?; + print!("{out:}"); + } + Commands::Preprocess { shader, output } => { + let source = librashader::preprocess::ShaderSource::load(shader.as_path())?; + match output { + PreprocessOutput::Fragment => print!("{}", source.fragment), + PreprocessOutput::Vertex => print!("{}", source.vertex), + PreprocessOutput::Params => { + print!("{}", serde_json::to_string_pretty(&source.parameters)?) + } + PreprocessOutput::Format => print!("{:?}", source.format), + PreprocessOutput::Json => print!("{}", serde_json::to_string_pretty(&source)?), + } + } + Commands::Transpile { + shader, + stage, + format, + version, + } => { + let source = librashader::preprocess::ShaderSource::load(shader.as_path())?; + let compilation = SpirvCompilation::try_from(&source)?; + let output = match format { + TranspileFormat::GLSL => { + let compilation = + librashader::reflect::targets::GLSL::from_compilation(compilation)?; + let output = compilation.compile(GlslVersion::Glsl330)?; + TranspileOutput { + vertex: output.vertex, + fragment: output.fragment, + } + } + TranspileFormat::HLSL => { + let compilation = + librashader::reflect::targets::HLSL::from_compilation(compilation)?; + let output = compilation.compile(Some(HlslShaderModel::ShaderModel5_0))?; + TranspileOutput { + vertex: output.vertex, + fragment: output.fragment, + } + } + TranspileFormat::WGSL => { + let compilation = + librashader::reflect::targets::WGSL::from_compilation(compilation)?; + let output = compilation.compile(NagaLoweringOptions { + write_pcb_as_ubo: true, + sampler_bind_group: 1, + })?; + TranspileOutput { + vertex: output.vertex, + fragment: output.fragment, + } + } + TranspileFormat::MSL => { + let compilation = <librashader::reflect::targets::MSL as FromCompilation< + SpirvCompilation, + SpirvCross, + >>::from_compilation(compilation)?; + let output = compilation.compile(Some(MslVersion::new(1, 2, 0)))?; + TranspileOutput { + vertex: output.vertex, + fragment: output.fragment, + } + } + TranspileFormat::SPIRV => { + let compilation = <librashader::reflect::targets::SPIRV as FromCompilation< + SpirvCompilation, + SpirvCross, + >>::from_compilation(compilation)?; + let output = compilation.compile(None)?; + + TranspileOutput { + vertex: spirv_to_dis(output.vertex)?, + fragment: spirv_to_dis(output.fragment)?, + } + } + }; + + let print = match stage { + TranspileStage::Fragment => output.fragment, + TranspileStage::Vertex => output.vertex, + }; + + print!("{print}") + } + Commands::Reflect { + preset, + wildcards, + index, + backend, + } => { + let preset = get_shader_preset(preset, wildcards)?; + let Some(shader) = preset.shaders.get(index) else { + return Err(anyhow!("Invalid pass index for the preset")); + }; + + let source = librashader::preprocess::ShaderSource::load(shader.name.as_path())?; + let compilation = SpirvCompilation::try_from(&source)?; + + let semantics = ShaderSemantics::create_pass_semantics::<anyhow::Error>(&preset, index)?; + + let reflection = match backend { + ReflectionBackend::SpirvCross => { + let mut compilation = + <librashader::reflect::targets::SPIRV as FromCompilation< + SpirvCompilation, + SpirvCross, + >>::from_compilation(compilation)?; + compilation.reflect(index, &semantics)? + } + ReflectionBackend::Naga => { + let mut compilation = + <librashader::reflect::targets::SPIRV as FromCompilation< + SpirvCompilation, + Naga, + >>::from_compilation(compilation)?; + compilation.reflect(index, &semantics)? + } + }; + + print!("{}", ron::ser::to_string_pretty(&reflection, PrettyConfig::new())?); + } + } + + Ok(()) +} + +struct TranspileOutput { + vertex: String, + fragment: String, +} + +fn get_shader_preset( + preset: PathBuf, + wildcards: Option<Vec<String>>, +) -> anyhow::Result<ShaderPreset> { + let mut context = WildcardContext::new(); + context.add_path_defaults(preset.as_path()); + if let Some(wildcards) = wildcards { + for string in wildcards { + let Some((left, right)) = string.split_once("=") else { + return Err(anyhow!("Encountered invalid context string {string}")); + }; + + context.append_item(ContextItem::ExternContext( + left.to_string(), + right.to_string(), + )) + } + } + let preset = ShaderPreset::try_parse_with_context(preset, context)?; + Ok(preset) +} + +fn spirv_to_dis(spirv: Vec<u32>) -> anyhow::Result<String> { + let binary = spq_spvasm::SpirvBinary::from(spirv); + spq_spvasm::Disassembler::new() + .print_header(true) + .name_ids(true) + .name_type_ids(true) + .name_const_ids(true) + .indent(true) + .disassemble(&binary) +} diff --git a/librashader/Cargo.toml b/librashader/Cargo.toml index 58d3d21..22bdaf1 100644 --- a/librashader/Cargo.toml +++ b/librashader/Cargo.toml @@ -84,7 +84,7 @@ full = ["runtime-all", "reflect-all", "preprocess", "presets"] # cache hack docsrs = ["librashader-cache/docsrs"] -serde = ["librashader-presets/serde"] +serde = ["librashader-presets/serde", "librashader-preprocess/serde", "librashader-reflect/serde"] # emits warning messages in tests github-ci = [] diff --git a/librashader/src/lib.rs b/librashader/src/lib.rs index 29a74dd..3d12993 100644 --- a/librashader/src/lib.rs +++ b/librashader/src/lib.rs @@ -236,6 +236,7 @@ pub mod reflect { pub use librashader_reflect::reflect::presets::{CompilePresetTarget, ShaderPassArtifact}; pub use librashader_reflect::front::ShaderInputCompiler; + #[doc(hidden)] #[cfg(feature = "internal")] /// Helper methods for runtimes.