hlsl: implement hlsl reflection

This commit is contained in:
chyyran 2022-11-07 00:25:11 -05:00
parent d6f47f83ca
commit f1280202e5
16 changed files with 358 additions and 96 deletions

View file

@ -9,6 +9,7 @@
<sourceFolder url="file://$MODULE_DIR$/naga/benches" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/naga/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/naga/tests" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/librashader-runtime-dx11/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />

12
Cargo.lock generated
View file

@ -439,6 +439,18 @@ dependencies = [
"thiserror",
]
[[package]]
name = "librashader-runtime-dx11"
version = "0.1.0"
dependencies = [
"librashader",
"librashader-preprocess",
"librashader-presets",
"librashader-reflect",
"rustc-hash",
"spirv_cross",
]
[[package]]
name = "log"
version = "0.4.17"

View file

@ -4,5 +4,6 @@ members = [
"librashader-presets",
"librashader-preprocess",
"librashader-reflect",
"librashader-runtime-dx11",
"naga"
]

View file

@ -0,0 +1,5 @@
use crate::back::ShaderCompiler;
use crate::back::targets::GLSL;
use crate::error::ShaderCompileError;
use crate::reflect::ShaderReflection;

View file

@ -0,0 +1,11 @@
pub mod targets;
mod cross;
use std::fmt::Debug;
pub use targets::ShaderCompiler;
#[derive(Debug)]
pub struct CompiledShader<T> {
pub vertex: T,
pub fragment: T,
}

View file

@ -0,0 +1,22 @@
use crate::back::CompiledShader;
use crate::error::ShaderCompileError;
use crate::reflect::ShaderReflection;
pub trait OutputTarget { }
pub struct GLSL;
pub struct HLSL;
pub struct SpirV;
pub struct MSL;
impl OutputTarget for GLSL {}
impl OutputTarget for HLSL {}
impl OutputTarget for SpirV {}
impl OutputTarget for MSL {}
pub trait ShaderCompiler<T: OutputTarget> {
type Output;
type Options;
fn compile(&mut self, options: &Self::Options, reflection: &ShaderReflection) -> Result<CompiledShader<Self::Output>, ShaderCompileError>;
}

View file

@ -11,6 +11,9 @@ pub enum ShaderCompileError {
#[error("shaderc init")]
ShaderCInitError,
#[error("cross")]
SpirvCrossCompileError(#[from] spirv_cross::ErrorCode),
}
#[derive(Debug)]

View file

@ -1,5 +1,6 @@
#![feature(let_else)]
mod error;
mod front;
mod reflect;
pub mod error;
pub mod front;
pub mod reflect;
pub mod back;

View file

@ -1,17 +1,19 @@
use crate::error::{SemanticsErrorKind, ShaderReflectError};
use crate::error::{SemanticsErrorKind, ShaderCompileError, ShaderReflectError};
use crate::front::shaderc::GlslangCompilation;
use crate::reflect::semantics::{
BindingStage, MemberOffset, PushReflection, SemanticMap, ShaderReflection, TextureImage,
TextureSemantics, TextureSizeMeta, TypeInfo, UboReflection, ValidateTypeSemantics,
VariableMeta, VariableSemantics, MAX_BINDINGS_COUNT, MAX_PUSH_BUFFER_SIZE,
};
use crate::reflect::{ReflectMeta, ReflectOptions, ReflectShader, UniformSemantic};
use crate::reflect::{ReflectMeta, ReflectSemantics, ReflectShader, UniformSemantic};
use rustc_hash::FxHashMap;
use spirv_cross::hlsl::{CompilerOptions, ShaderModel};
use spirv_cross::hlsl::{ShaderModel};
use spirv_cross::spirv::{Ast, Decoration, Module, Resource, ShaderResources, Type};
use spirv_cross::{hlsl, ErrorCode};
use spirv_cross::{hlsl, ErrorCode, glsl};
use std::str::FromStr;
use spirv_cross::glsl::Version;
use crate::back::{CompiledShader, ShaderCompiler};
pub struct CrossReflect<T>
where
@ -21,6 +23,7 @@ where
fragment: Ast<T>,
}
pub type HlslReflect = CrossReflect<hlsl::Target>;
impl ValidateTypeSemantics<Type> for VariableSemantics {
fn validate_type(&self, ty: &Type) -> Option<TypeInfo> {
let (Type::Float { ref array, vecsize, columns } | Type::Int { ref array, vecsize, columns } | Type::UInt { ref array, vecsize, columns }) = *ty else {
@ -89,7 +92,7 @@ impl TryFrom<GlslangCompilation> for CrossReflect<hlsl::Target> {
let mut vertex = Ast::parse(&vertex_module)?;
let mut fragment = Ast::parse(&fragment_module)?;
let mut options = CompilerOptions::default();
let mut options = hlsl::CompilerOptions::default();
options.shader_model = ShaderModel::V5_0;
fragment.set_compiler_options(&options)?;
vertex.set_compiler_options(&options)?;
@ -98,6 +101,25 @@ impl TryFrom<GlslangCompilation> for CrossReflect<hlsl::Target> {
}
}
impl TryFrom<GlslangCompilation> for CrossReflect<glsl::Target> {
type Error = ShaderReflectError;
fn try_from(value: GlslangCompilation) -> Result<Self, Self::Error> {
let vertex_module = Module::from_words(value.vertex.as_binary());
let fragment_module = Module::from_words(value.fragment.as_binary());
let mut vertex = Ast::parse(&vertex_module)?;
let mut fragment = Ast::parse(&fragment_module)?;
let mut options = glsl::CompilerOptions::default();
options.version = Version::V3_30;
fragment.set_compiler_options(&options)?;
vertex.set_compiler_options(&options)?;
Ok(CrossReflect { vertex, fragment })
}
}
impl<T> CrossReflect<T>
where
T: spirv_cross::spirv::Target,
@ -371,7 +393,8 @@ where
fn reflect_buffer_range_metas(
ast: &Ast<T>,
resource: &Resource,
options: &ReflectOptions,
pass_number: u32,
semantics: &ReflectSemantics,
meta: &mut ReflectMeta,
offset_type: impl Fn(usize) -> MemberOffset,
blame: SemanticErrorBlame,
@ -392,7 +415,7 @@ where
_ => return Err(blame.error(SemanticsErrorKind::InvalidResourceType)),
};
if let Some(parameter) = options.uniform_semantics.get_variable_semantic(&name) {
if let Some(parameter) = semantics.uniform_semantics.get_variable_semantic(&name) {
let Some(typeinfo) = parameter.semantics.validate_type(&range_type) else {
return Err(blame.error(SemanticsErrorKind::InvalidTypeForSemantic(name)))
};
@ -453,15 +476,15 @@ where
}
}
}
} else if let Some(texture) = options.uniform_semantics.get_texture_semantic(&name) {
} else if let Some(texture) = semantics.uniform_semantics.get_texture_semantic(&name) {
let Some(_typeinfo) = texture.semantics.validate_type(&range_type) else {
return Err(blame.error(SemanticsErrorKind::InvalidTypeForSemantic(name)))
};
if let TextureSemantics::PassOutput = texture.semantics {
if texture.index >= options.pass_number {
if texture.index >= pass_number {
return Err(ShaderReflectError::NonCausalFilterChain {
pass: options.pass_number,
pass: pass_number,
target: texture.index,
});
}
@ -486,7 +509,7 @@ where
texture,
TextureSizeMeta {
offset,
// todo: fix this.
// todo: fix this. to allow both
stage_mask: match blame {
SemanticErrorBlame::Vertex => BindingStage::VERTEX,
SemanticErrorBlame::Fragment => BindingStage::FRAGMENT,
@ -502,10 +525,18 @@ where
}
fn reflect_ubos(
&self,
&mut self,
vertex_ubo: Option<&Resource>,
fragment_ubo: Option<&Resource>,
) -> Result<Option<UboReflection>, ShaderReflectError> {
if let Some(vertex_ubo) = vertex_ubo {
self.vertex.set_decoration(vertex_ubo.id, Decoration::Binding, 0)?;
}
if let Some(fragment_ubo) = fragment_ubo {
self.fragment.set_decoration(fragment_ubo.id, Decoration::Binding, 0)?;
}
match (vertex_ubo, fragment_ubo) {
(None, None) => Ok(None),
(Some(vertex_ubo), Some(fragment_ubo)) => {
@ -551,18 +582,19 @@ where
fn reflect_texture_metas(
&self,
texture: TextureData,
options: &ReflectOptions,
pass_number: u32,
semantics: &ReflectSemantics,
meta: &mut ReflectMeta,
) -> Result<(), ShaderReflectError> {
let Some(semantic) = options.non_uniform_semantics.get_texture_semantic(texture.name) else {
let Some(semantic) = semantics.non_uniform_semantics.get_texture_semantic(texture.name) else {
return Err(SemanticErrorBlame::Fragment.error(SemanticsErrorKind::UnknownSemantics(texture.name.to_string())))
};
if semantic.semantics == TextureSemantics::PassOutput
&& semantic.index >= options.pass_number
&& semantic.index >= pass_number
{
return Err(ShaderReflectError::NonCausalFilterChain {
pass: options.pass_number,
pass: pass_number,
target: semantic.index,
});
}
@ -605,10 +637,19 @@ where
}
fn reflect_push_constant_buffer(
&self,
&mut self,
vertex_pcb: Option<&Resource>,
fragment_pcb: Option<&Resource>,
) -> Result<Option<PushReflection>, ShaderReflectError> {
if let Some(vertex_pcb) = vertex_pcb {
self.vertex.set_decoration(vertex_pcb.id, Decoration::Binding, 1)?;
}
if let Some(fragment_pcb) = fragment_pcb {
self.fragment.set_decoration(fragment_pcb.id, Decoration::Binding, 1)?;
}
match (vertex_pcb, fragment_pcb) {
(None, None) => Ok(None),
(Some(vertex_push), Some(fragment_push)) => {
@ -656,7 +697,7 @@ where
Ast<T>: spirv_cross::spirv::Compile<T>,
Ast<T>: spirv_cross::spirv::Parse<T>,
{
fn reflect(&self, options: &ReflectOptions) -> Result<ShaderReflection, ShaderReflectError> {
fn reflect(&mut self, pass_number: u32, semantics: &ReflectSemantics) -> Result<ShaderReflection, ShaderReflectError> {
let vertex_res = self.vertex.get_shader_resources()?;
let fragment_res = self.fragment.get_shader_resources()?;
self.validate(&vertex_res, &fragment_res)?;
@ -677,7 +718,8 @@ where
Self::reflect_buffer_range_metas(
&self.vertex,
ubo,
options,
pass_number,
semantics,
&mut meta,
MemberOffset::Ubo,
SemanticErrorBlame::Vertex,
@ -688,7 +730,8 @@ where
Self::reflect_buffer_range_metas(
&self.fragment,
ubo,
options,
pass_number,
semantics,
&mut meta,
MemberOffset::Ubo,
SemanticErrorBlame::Fragment,
@ -699,7 +742,8 @@ where
Self::reflect_buffer_range_metas(
&self.vertex,
push,
options,
pass_number,
semantics,
&mut meta,
MemberOffset::PushConstant,
SemanticErrorBlame::Vertex,
@ -710,14 +754,14 @@ where
Self::reflect_buffer_range_metas(
&self.fragment,
push,
options,
pass_number,
semantics,
&mut meta,
MemberOffset::PushConstant,
SemanticErrorBlame::Fragment,
)?;
}
// todo: slang_reflection: 556
let mut ubo_bindings = 0u16;
if vertex_ubo.is_some() || fragment_ubo.is_some() {
ubo_bindings = 1 << ubo.as_ref().expect("UBOs should be present").binding;
@ -730,11 +774,9 @@ where
}
ubo_bindings |= 1 << texture_data.binding;
self.reflect_texture_metas(texture_data, options, &mut meta)?;
self.reflect_texture_metas(texture_data, pass_number, semantics, &mut meta)?;
}
// slang-reflection:611
Ok(ShaderReflection {
ubo,
push_constant,
@ -743,28 +785,76 @@ where
}
}
impl ShaderCompiler<crate::back::targets::GLSL> for CrossReflect<glsl::Target> {
type Output = String;
type Options = glsl::CompilerOptions;
fn compile(&mut self, options: &Self::Options, reflection: &ShaderReflection) -> Result<CompiledShader<Self::Output>, ShaderCompileError> {
self.vertex.set_compiler_options(options)?;
self.fragment.set_compiler_options(options)?;
Ok(CompiledShader {
vertex: self.vertex.compile()?,
fragment: self.fragment.compile()?
})
}
}
pub type HlslOptions = hlsl::CompilerOptions;
impl ShaderCompiler<crate::back::targets::HLSL> for CrossReflect<hlsl::Target> {
type Output = String;
type Options = hlsl::CompilerOptions;
fn compile(&mut self, options: &Self::Options, reflection: &ShaderReflection) -> Result<CompiledShader<Self::Output>, ShaderCompileError> {
self.vertex.set_compiler_options(options)?;
self.fragment.set_compiler_options(options)?;
Ok(CompiledShader {
vertex: self.vertex.compile()?,
fragment: self.fragment.compile()?
})
}
}
#[cfg(test)]
mod test {
use rustc_hash::FxHashMap;
use crate::reflect::cross::CrossReflect;
use crate::reflect::{ReflectOptions, ReflectShader};
use crate::reflect::{ReflectSemantics, ReflectShader, UniformSemantic};
use spirv_cross::{hlsl};
use spirv_cross::{glsl, hlsl};
use spirv_cross::glsl::{CompilerOptions, Version};
use crate::back::ShaderCompiler;
use crate::reflect::semantics::{SemanticMap, VariableSemantics};
#[test]
pub fn test_into() {
let result = librashader_preprocess::load_shader_source("../test/basic.slang").unwrap();
let mut uniform_semantics: FxHashMap<String, UniformSemantic> = Default::default();
for (index, param) in result.parameters.iter().enumerate() {
uniform_semantics.insert(param.id.clone(), UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::FloatParameter,
index: index as u32
}));
}
let spirv = crate::front::shaderc::compile_spirv(&result).unwrap();
let mut reflect = CrossReflect::<hlsl::Target>::try_from(spirv).unwrap();
let huh = reflect
.reflect(&ReflectOptions {
pass_number: 0,
uniform_semantics: Default::default(),
let mut reflect = CrossReflect::<glsl::Target>::try_from(spirv).unwrap();
let shader_reflection = reflect
.reflect(0, &ReflectSemantics {
uniform_semantics,
non_uniform_semantics: Default::default(),
})
.unwrap();
eprintln!("{huh:#?}");
eprintln!("{:#}", reflect.fragment.compile().unwrap())
let mut opts = CompilerOptions::default();
opts.version = Version::V4_60;
opts.enable_420_pack_extension = false;
let compiled = reflect.compile(&opts, &shader_reflection)
.unwrap();
// eprintln!("{shader_reflection:#?}");
eprintln!("{:#}", compiled.fragment)
// let mut loader = rspirv::dr::Loader::new();
// rspirv::binary::parse_words(spirv.fragment.as_binary(), &mut loader).unwrap();
// let module = loader.module();

View file

@ -1,17 +1,17 @@
use crate::error::{ShaderReflectError};
use crate::reflect::semantics::{
SemanticMap, ShaderReflection, TextureImage, TextureSemantics, TextureSizeMeta, VariableMeta,
SemanticMap, TextureImage, TextureSemantics, TextureSizeMeta, VariableMeta,
VariableSemantics,
};
use rustc_hash::FxHashMap;
mod cross;
pub mod cross;
mod naga;
mod rspirv;
pub mod semantics;
pub trait ReflectShader {
fn reflect(&self, options: &ReflectOptions) -> Result<ShaderReflection, ShaderReflectError>;
fn reflect(&mut self, pass_number: u32, semantics: &ReflectSemantics) -> Result<ShaderReflection, ShaderReflectError>;
}
#[derive(Debug)]
@ -21,8 +21,7 @@ pub enum UniformSemantic {
}
#[derive(Debug)]
pub struct ReflectOptions {
pub pass_number: u32,
pub struct ReflectSemantics {
pub uniform_semantics: FxHashMap<String, UniformSemantic>,
pub non_uniform_semantics: FxHashMap<String, SemanticMap<TextureSemantics>>,
}
@ -35,47 +34,4 @@ pub struct ReflectMeta {
pub texture_size_meta: FxHashMap<SemanticMap<TextureSemantics>, TextureSizeMeta>,
}
pub fn builtin_uniform_semantics() -> FxHashMap<String, UniformSemantic> {
let mut map = FxHashMap::default();
map.insert(
"MVP".into(),
UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::MVP,
index: 0,
}),
);
map.insert(
"OutputSize".into(),
UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::Output,
index: 0,
}),
);
map.insert(
"FinalViewportSize".into(),
UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::FinalViewport,
index: 0,
}),
);
map.insert(
"FrameCount".into(),
UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::FrameCount,
index: 0,
}),
);
map.insert(
"FrameDirection".into(),
UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::FrameDirection,
index: 0,
}),
);
map
}
pub use semantics::ShaderReflection;

View file

@ -81,8 +81,8 @@ pub trait ValidateTypeSemantics<T> {
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct SemanticMap<T> {
pub(crate) semantics: T,
pub(crate) index: u32,
pub semantics: T,
pub index: u32,
}
bitflags! {

View file

@ -0,0 +1,14 @@
[package]
name = "librashader-runtime-dx11"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
"librashader" = { path = "../librashader" }
"librashader-presets" = { path = "../librashader-presets" }
"librashader-preprocess" = { path = "../librashader-preprocess" }
"librashader-reflect" = { path = "../librashader-reflect" }
spirv_cross = { version = "0.23.1", features = [ "hlsl" ] }
rustc-hash = "1.1.0"

View file

@ -0,0 +1,137 @@
use std::collections::HashMap;
use std::error::Error;
use std::path::Path;
use rustc_hash::FxHashMap;
use spirv_cross::hlsl::ShaderModel;
use librashader::ShaderSource;
use librashader_presets::ShaderPassConfig;
use librashader_reflect::back::ShaderCompiler;
use librashader_reflect::front::shaderc::GlslangCompilation;
use librashader_reflect::reflect::cross::{CrossReflect, HlslOptions, HlslReflect};
use librashader_reflect::reflect::{ReflectSemantics, ReflectShader, UniformSemantic};
use librashader_reflect::reflect::semantics::{SemanticMap, TextureSemantics, VariableSemantics};
pub fn load_pass_semantics(uniform_semantics: &mut FxHashMap<String, UniformSemantic>, texture_semantics: &mut FxHashMap<String, SemanticMap<TextureSemantics>>,
config: &ShaderPassConfig) {
let Some(alias) = &config.alias else {
return;
};
// Ignore empty aliases
if alias.trim().is_empty() {
return;
}
let index = config.id as u32;
// PassOutput
texture_semantics.insert(alias.clone(), SemanticMap {
semantics: TextureSemantics::PassOutput,
index
});
uniform_semantics.insert(format!("{alias}Size"), UniformSemantic::Texture(SemanticMap {
semantics: TextureSemantics::PassOutput,
index
}));
// PassFeedback
texture_semantics.insert(format!("{alias}Feedback"), SemanticMap {
semantics: TextureSemantics::PassFeedback,
index
});
uniform_semantics.insert(format!("{alias}FeedbackSize"), UniformSemantic::Texture(SemanticMap {
semantics: TextureSemantics::PassFeedback,
index
}));
}
pub fn load(path: impl AsRef<Path>) -> Result<(), Box<dyn Error>>{
let preset = librashader_presets::ShaderPreset::try_parse(path)?;
let mut passes: Vec<(&ShaderPassConfig, ShaderSource, HlslReflect)> = preset.shaders.iter()
.map(|shader| {
let source = librashader_preprocess::load_shader_source(&shader.name)
.unwrap();
let spirv = librashader_reflect::front::shaderc::compile_spirv(&source)
.unwrap();
let mut reflect = librashader_reflect::reflect::cross::HlslReflect::try_from(spirv).unwrap();
(shader, source, reflect)
}).collect();
// todo: this can probably be extracted out.
let mut uniform_semantics: FxHashMap<String, UniformSemantic> = Default::default();
let mut texture_semantics: FxHashMap<String, SemanticMap<TextureSemantics>> = Default::default();
for details in &passes {
load_pass_semantics(&mut uniform_semantics, &mut texture_semantics, details.0)
}
// add float params
for (index, parameter) in preset.parameters.iter().enumerate() {
uniform_semantics.insert(parameter.name.clone(), UniformSemantic::Variable(SemanticMap {
semantics: VariableSemantics::FloatParameter,
index: index as u32
}));
}
// add lut params
for (index, texture) in preset.textures.iter().enumerate() {
texture_semantics.insert(texture.name.clone(), SemanticMap {
semantics: TextureSemantics::User,
index: index as u32
});
}
let semantics = ReflectSemantics {
uniform_semantics,
non_uniform_semantics: texture_semantics
};
let mut reflections = Vec::new();
let mut compiled = Vec::new();
for (index, (_, _, reflect)) in passes.iter_mut().enumerate() {
let reflection = reflect.reflect(index as u32, &semantics)
.unwrap();
let mut options: HlslOptions = Default::default();
options.shader_model = ShaderModel::V5_0;
let hlsl = reflect.compile(&options, &reflection)
.unwrap();
eprintln!("{:#}", hlsl.vertex);
eprintln!("{:#}", hlsl.fragment);
compiled.push(hlsl);
reflections.push(reflection);
}
eprintln!("{:#?}", reflections);
// //todo: add the semantics for other shit (slang_process:68)
// eprintln!("{:?}", preset);
// eprintln!("{:?}", reflect.reflect(&ReflectOptions {
// pass_number: i as u32,
// uniform_semantics,
// non_uniform_semantics: Default::default(),
// }));
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn load_preset() {
load("../test/basic.slangp")
.unwrap();
}
}

BIN
test/agb.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

View file

@ -4,10 +4,12 @@
layout(set = 0, binding = 0, std140) uniform UBO
{
mat4 MVP;
vec4 SourceSize; // Not used here, but doesn't hurt
float ColorMod;
};
layout(push_constant) uniform Push {
float ColorMod;
} params;
#pragma name StockShader
#pragma format R8G8B8A8_UNORM
#pragma parameter ColorMod "Color intensity" 1.0 0.1 2.0 0.1
@ -26,10 +28,7 @@ void main()
layout(location = 0) in vec2 vTexCoord;
layout(location = 0) out vec4 FragColor;
layout(binding = 1) uniform sampler2D Source;
// layout(binding = 1) uniform texture2D Source;
// layout(binding = 2) uniform sampler _Source_sampler;
void main()
{
// FragColor = texture(sampler2D(Source, _Source_sampler), vTexCoord) * ColorMod;
FragColor = texture(Source, vTexCoord) * SourceSize[0];
FragColor = texture(Source, vTexCoord) * params.ColorMod;
}

10
test/basic.slangp Normal file
View file

@ -0,0 +1,10 @@
shaders = "1"
shader0 = "basic.slang"
wrap_mode0 = "clamp_to_border"
mipmap_input0 = "false"
alias0 = ""
float_framebuffer0 = "false"
srgb_framebuffer0 = "false"
ColorMod = "1.700000"
textures = "BORDER;"
BORDER = "agb.png"