cache: implement transparent shader object cache

supported objects:

- SPIR-V artifacts
- Validated DXIL artifacts
- DXBC artifacts
- OpenGL Program Binaries
- Vulkan Pipeline caches
- D3D12 CACHED_PIPELINE_STATE
This commit is contained in:
chyyran 2023-02-12 22:36:50 -05:00 committed by Ronny Chan
parent 4978e1f24f
commit 604edfb78f
38 changed files with 1207 additions and 160 deletions

1
.gitignore vendored
View file

@ -11,3 +11,4 @@ librashader_runtime_*.exe
/test/capi-tests/librashader-capi-tests/**/x64/
/test/capi-tests/librashader-capi-tests/**/*.so
/test/capi-tests/librashader-capi-tests/**/*.out
/test/Mega_Bezel_Packs/Duimon-Mega-Bezel/

245
Cargo.lock generated
View file

@ -17,6 +17,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "array-concat"
version = "0.5.2"
@ -41,6 +52,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "arrayvec"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "ash"
version = "0.37.2+1.3.238"
@ -99,6 +116,25 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "bincode"
version = "2.0.0-rc.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bb50c5a2ef4b9b1e7ae73e3a73b52ea24b20312d629f9c4df28260b7ad2c3c4"
dependencies = [
"bincode_derive",
"serde",
]
[[package]]
name = "bincode_derive"
version = "2.0.0-rc.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a45a23389446d2dd25dc8e73a7a3b3c43522b630cac068927f0649d43d719d2"
dependencies = [
"virtue",
]
[[package]]
name = "bit-set"
version = "0.5.3"
@ -132,12 +168,36 @@ dependencies = [
"wyz",
]
[[package]]
name = "blake3"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef"
dependencies = [
"arrayref",
"arrayvec 0.7.2",
"cc",
"cfg-if",
"constant_time_eq",
"digest",
"rayon",
]
[[package]]
name = "block"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
[[package]]
name = "block-buffer"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.12.0"
@ -329,6 +389,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "constant_time_eq"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279"
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -457,6 +523,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "cty"
version = "0.2.2"
@ -498,6 +574,38 @@ dependencies = [
"syn",
]
[[package]]
name = "digest"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "dirs-next"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf36e65a80337bea855cd4ef9b8401ffce06a7baedf2e85ec467b1ac3f6e82b6"
dependencies = [
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "dispatch"
version = "0.2.0"
@ -579,6 +687,18 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "1.8.0"
@ -689,6 +809,27 @@ dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "gfx-maths"
version = "0.2.8"
@ -792,6 +933,18 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
dependencies = [
"hashbrown",
]
[[package]]
name = "heck"
@ -979,6 +1132,22 @@ dependencies = [
"clap 4.1.4",
]
[[package]]
name = "librashader-cache"
version = "0.1.0-rc.3"
dependencies = [
"bincode",
"blake3",
"bytemuck",
"librashader-preprocess",
"librashader-reflect",
"platform-dirs",
"rusqlite",
"serde",
"thiserror",
"windows",
]
[[package]]
name = "librashader-capi"
version = "0.1.0-rc.3"
@ -1064,6 +1233,7 @@ dependencies = [
"naga",
"rspirv",
"rustc-hash",
"serde",
"shaderc",
"spirv-to-dxil",
"thiserror",
@ -1090,6 +1260,7 @@ dependencies = [
"array-concat",
"bytemuck",
"gfx-maths",
"librashader-cache",
"librashader-common 0.1.0-rc.3",
"librashader-preprocess",
"librashader-presets 0.1.0-rc.3",
@ -1111,6 +1282,7 @@ dependencies = [
"bitvec",
"bytemuck",
"gfx-maths",
"librashader-cache",
"librashader-common 0.1.0-rc.3",
"librashader-preprocess",
"librashader-presets 0.1.0-rc.3",
@ -1132,6 +1304,7 @@ dependencies = [
"bytemuck",
"gl",
"glfw 0.47.0",
"librashader-cache",
"librashader-common 0.1.0-rc.3",
"librashader-preprocess",
"librashader-presets 0.1.0-rc.3",
@ -1152,6 +1325,7 @@ dependencies = [
"bytemuck",
"glfw 0.49.1",
"gpu-allocator",
"librashader-cache",
"librashader-common 0.1.0-rc.3",
"librashader-preprocess",
"librashader-presets 0.1.0-rc.3",
@ -1178,6 +1352,17 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "libsqlite3-sys"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.1.4"
@ -1591,6 +1776,15 @@ version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "platform-dirs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e188d043c1a692985f78b5464853a263f1a27e5bd6322bad3a4078ee3c998a38"
dependencies = [
"dirs-next",
]
[[package]]
name = "png"
version = "0.17.7"
@ -1731,6 +1925,17 @@ dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]]
name = "remove_dir_all"
version = "0.5.3"
@ -1760,6 +1965,20 @@ dependencies = [
"spirv",
]
[[package]]
name = "rusqlite"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
@ -1969,6 +2188,12 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.107"
@ -2053,7 +2278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "642680569bb895b16e4b9d181c60be1ed136fa0c9c7f11d004daf053ba89bf82"
dependencies = [
"arrayref",
"arrayvec",
"arrayvec 0.5.2",
"bytemuck",
"cfg-if",
"png",
@ -2097,6 +2322,12 @@ dependencies = [
"toml_datetime",
]
[[package]]
name = "typenum"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "unicode-ident"
version = "1.0.6"
@ -2109,6 +2340,12 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
@ -2121,6 +2358,12 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "virtue"
version = "0.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b60dcd6a64dd45abf9bd426970c9843726da7fc08f44cd6fcebf68c21220a63"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

View file

@ -10,6 +10,7 @@ members = [
"librashader-runtime-d3d12",
"librashader-runtime-gl",
"librashader-runtime-vk",
"librashader-cache",
"librashader-capi",
"librashader-build-script"
]

View file

@ -1,14 +1,14 @@
use std::{env, fs};
use clap::Parser;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
use clap::Parser;
use std::{env, fs};
#[derive(Parser, Debug)]
#[command(version, about)]
struct Args {
#[arg(long, default_value="debug", global = true)]
#[arg(long, default_value = "debug", global = true)]
profile: String,
}
@ -28,11 +28,15 @@ pub fn main() {
let mut cmd = Command::new("cargo");
cmd.arg("build");
cmd.args(["--package", "librashader-capi"]);
cmd.arg(format!("--profile={}", if profile == "debug" { "dev" } else { &profile }));
cmd.arg(format!(
"--profile={}",
if profile == "debug" { "dev" } else { &profile }
));
Some(cmd.status().expect("Failed to build librashader-capi"));
let output_dir = PathBuf::from(format!("target/{}", profile))
.canonicalize().expect("Could not find output directory.");
.canonicalize()
.expect("Could not find output directory.");
println!("Generating C headers...");
@ -60,7 +64,14 @@ pub fn main() {
}
if cfg!(target_os = "windows") {
let artifacts = &["librashader_capi.dll", "librashader_capi.lib", "librashader_capi.d", "librashader_capi.dll.exp", "librashader_capi.dll.lib", "librashader_capi.pdb"];
let artifacts = &[
"librashader_capi.dll",
"librashader_capi.lib",
"librashader_capi.d",
"librashader_capi.dll.exp",
"librashader_capi.dll.lib",
"librashader_capi.pdb",
];
for artifact in artifacts {
let ext = artifact.replace("_capi", "");
fs::rename(output_dir.join(artifact), output_dir.join(ext)).unwrap();

View file

@ -0,0 +1,36 @@
[package]
name = "librashader-cache"
edition = "2021"
license = "MPL-2.0 OR GPL-3.0-only"
version = "0.1.0-rc.3"
authors = ["Ronny Chan <ronny@ronnychan.ca>"]
repository = "https://github.com/SnowflakePowered/librashader"
readme = "../README.md"
categories = ["emulators", "compilers", "graphics"]
keywords = ["shader", "retroarch", "SPIR-V"]
description = "RetroArch shaders for all."
[dependencies]
serde = { version = "1.0" }
librashader-reflect = { path = "../librashader-reflect", version = "0.1.0-rc.2", features = ["serialize", "dxil"] }
librashader-preprocess = { path = "../librashader-preprocess", version = "0.1.0-rc.2" }
platform-dirs = "0.3.0"
blake3 = { version = "1.3.3", features = ["rayon"] }
thiserror = "1.0.38"
bincode = { version = "2.0.0-rc.2", features = ["serde"] }
rusqlite = { version = "0.28.0", features = ["bundled"] }
bytemuck = "1.13.0"
[target.'cfg(windows)'.dependencies.windows]
version = "0.44.0"
features = [
"Win32_Graphics_Direct3D",
"Win32_Graphics_Direct3D_Fxc",
"Win32_Graphics_Direct3D_Dxc",
]
optional = true
[features]
d3d = ["windows"]

View file

@ -0,0 +1,196 @@
use crate::key::CacheKey;
use crate::cacheable::Cacheable;
use platform_dirs::AppDirs;
use rusqlite::{params, Connection, DatabaseName};
use std::error::Error;
use std::path::PathBuf;
pub fn get_cache_dir() -> Result<PathBuf, Box<dyn Error>> {
let cache_dir =
if let Some(cache_dir) = AppDirs::new(Some("librashader"), false).map(|a| a.cache_dir) {
cache_dir
} else {
let mut current_dir = std::env::current_dir()?;
current_dir.push("librashader");
current_dir
};
std::fs::create_dir_all(&cache_dir)?;
Ok(cache_dir)
}
pub fn get_cache() -> Result<Connection, Box<dyn Error>> {
let cache_dir = get_cache_dir()?;
let mut conn = Connection::open(&cache_dir.join("librashader.db"))?;
let tx = conn.transaction()?;
tx.pragma_update(Some(DatabaseName::Main), "journal_mode", "wal2")?;
tx.execute(
r#"create table if not exists cache (
type text not null,
id blob not null,
value blob not null unique,
primary key (id, type)
)"#,
[],
)?;
tx.commit()?;
Ok(conn)
}
pub(crate) fn get_blob(
conn: &Connection,
index: &str,
key: &[u8],
) -> Result<Vec<u8>, Box<dyn Error>> {
let value = conn.query_row(
&*format!("select value from cache where (type = (?1) and id = (?2))"),
params![index, key],
|row| row.get(0),
)?;
Ok(value)
}
pub(crate) fn set_blob(conn: &Connection, index: &str, key: &[u8], value: &[u8]) {
match conn.execute(
&*format!("insert or replace into cache (type, id, value) values (?1, ?2, ?3)"),
params![index, key, value],
) {
Ok(_) => return,
Err(e) => println!("err: {:?}", e),
}
}
pub fn get_cached_blob<T, H, const KEY_SIZE: usize>(
index: &str,
key: &[H; KEY_SIZE],
transform: impl FnOnce(Vec<u8>) -> T,
) -> Option<T>
where
H: CacheKey,
{
let cache = get_cache();
let Ok(cache) = cache else {
return None
};
let key = {
let mut hasher = blake3::Hasher::new();
for subkeys in key {
hasher.update(subkeys.hash_bytes());
}
let hash = hasher.finalize();
hash
};
let Ok(blob) = get_blob(&cache, index, key.as_bytes()) else {
return None;
};
Some(transform(blob))
}
pub fn cache_object<E, T, R, H, const KEY_SIZE: usize>(
index: &str,
keys: &[H; KEY_SIZE],
factory: impl FnOnce(&[H; KEY_SIZE]) -> Result<T, E>,
attempt: impl Fn(T) -> Result<R, E>,
do_cache: bool,
) -> Result<R, E>
where
H: CacheKey,
T: Cacheable,
{
if !do_cache {
return Ok(attempt(factory(keys)?)?);
}
let cache = get_cache();
let Ok(cache) = cache else {
return Ok(attempt(factory(keys)?)?);
};
let hashkey = {
let mut hasher = blake3::Hasher::new();
for subkeys in keys {
hasher.update(subkeys.hash_bytes());
}
let hash = hasher.finalize();
hash
};
'attempt: {
if let Ok(blob) = get_blob(&cache, index, hashkey.as_bytes()) {
let cached = T::from_bytes(&blob).map(&attempt);
match cached {
None => break 'attempt,
Some(Err(_)) => break 'attempt,
Some(Ok(res)) => return Ok(res),
}
}
};
let blob = factory(keys)?;
if let Some(slice) = T::to_bytes(&blob) {
set_blob(&cache, index, hashkey.as_bytes(), &slice);
}
Ok(attempt(blob)?)
}
pub fn cache_pipeline<E, T, R, const KEY_SIZE: usize>(
index: &str,
keys: &[&dyn CacheKey; KEY_SIZE],
attempt: impl Fn(Option<Vec<u8>>) -> Result<R, E>,
factory: impl FnOnce(&R) -> Result<T, E>,
do_cache: bool,
) -> Result<R, E>
where
T: Cacheable,
{
if !do_cache {
return Ok(attempt(None)?);
}
let cache = get_cache();
let Ok(cache) = cache else {
return Ok(attempt(None)?);
};
let hashkey = {
let mut hasher = blake3::Hasher::new();
for subkeys in keys {
hasher.update(subkeys.hash_bytes());
}
let hash = hasher.finalize();
hash
};
let pipeline = 'attempt: {
if let Ok(blob) = get_blob(&cache, index, hashkey.as_bytes()) {
let cached = attempt(Some(blob));
match cached {
Ok(res) => {
break 'attempt res;
}
_ => (),
}
}
attempt(None)?
};
// update the pso every time just in case.
if let Ok(state) = factory(&pipeline) {
if let Some(slice) = T::to_bytes(&state) {
set_blob(&cache, index, hashkey.as_bytes(), &slice);
}
}
Ok(pipeline)
}

View file

@ -0,0 +1,17 @@
pub trait Cacheable {
fn from_bytes(bytes: &[u8]) -> Option<Self>
where
Self: Sized;
fn to_bytes(&self) -> Option<Vec<u8>>;
}
impl Cacheable for Vec<u8> {
fn from_bytes(bytes: &[u8]) -> Option<Self> {
Some(Vec::from(bytes))
}
fn to_bytes(&self) -> Option<Vec<u8>> {
Some(self.to_vec())
}
}

View file

@ -0,0 +1,109 @@
use crate::cache::{get_blob, get_cache, set_blob};
use librashader_preprocess::ShaderSource;
use librashader_reflect::back::targets::{DXIL, GLSL, HLSL, SPIRV};
use librashader_reflect::back::{CompilerBackend, FromCompilation};
use librashader_reflect::error::{ShaderCompileError, ShaderReflectError};
use librashader_reflect::front::{GlslangCompilation, ShaderCompilation};
pub struct CachedCompilation<T> {
compilation: T,
}
impl<T: ShaderCompilation + for<'de> serde::Deserialize<'de> + serde::Serialize + Clone>
ShaderCompilation for CachedCompilation<T>
{
fn compile(source: &ShaderSource) -> Result<Self, ShaderCompileError> {
let cache = get_cache();
let Ok(cache) = cache else {
return Ok(CachedCompilation {
compilation: T::compile(source)?
})
};
let key = {
let mut hasher = blake3::Hasher::new();
hasher.update(source.vertex.as_bytes());
hasher.update(source.fragment.as_bytes());
let hash = hasher.finalize();
hash
};
let compilation = 'cached: {
if let Ok(cached) = get_blob(&cache, "spirv", key.as_bytes()) {
let decoded =
bincode::serde::decode_from_slice(&cached, bincode::config::standard())
.map(|(compilation, _)| CachedCompilation { compilation })
.ok();
if let Some(compilation) = decoded {
break 'cached compilation;
}
}
CachedCompilation {
compilation: T::compile(source)?,
}
};
if let Ok(updated) =
bincode::serde::encode_to_vec(&compilation.compilation, bincode::config::standard())
{
set_blob(&cache, "spirv", key.as_bytes(), &updated)
}
Ok(compilation)
}
}
impl FromCompilation<CachedCompilation<GlslangCompilation>> for DXIL {
type Target = <DXIL as FromCompilation<GlslangCompilation>>::Target;
type Options = <DXIL as FromCompilation<GlslangCompilation>>::Options;
type Context = <DXIL as FromCompilation<GlslangCompilation>>::Context;
type Output = <DXIL as FromCompilation<GlslangCompilation>>::Output;
fn from_compilation(
compile: CachedCompilation<GlslangCompilation>,
) -> Result<CompilerBackend<Self::Output>, ShaderReflectError> {
DXIL::from_compilation(compile.compilation)
}
}
impl FromCompilation<CachedCompilation<GlslangCompilation>> for HLSL {
type Target = <HLSL as FromCompilation<GlslangCompilation>>::Target;
type Options = <HLSL as FromCompilation<GlslangCompilation>>::Options;
type Context = <HLSL as FromCompilation<GlslangCompilation>>::Context;
type Output = <HLSL as FromCompilation<GlslangCompilation>>::Output;
fn from_compilation(
compile: CachedCompilation<GlslangCompilation>,
) -> Result<CompilerBackend<Self::Output>, ShaderReflectError> {
HLSL::from_compilation(compile.compilation)
}
}
impl FromCompilation<CachedCompilation<GlslangCompilation>> for GLSL {
type Target = <GLSL as FromCompilation<GlslangCompilation>>::Target;
type Options = <GLSL as FromCompilation<GlslangCompilation>>::Options;
type Context = <GLSL as FromCompilation<GlslangCompilation>>::Context;
type Output = <GLSL as FromCompilation<GlslangCompilation>>::Output;
fn from_compilation(
compile: CachedCompilation<GlslangCompilation>,
) -> Result<CompilerBackend<Self::Output>, ShaderReflectError> {
GLSL::from_compilation(compile.compilation)
}
}
impl FromCompilation<CachedCompilation<GlslangCompilation>> for SPIRV {
type Target = <SPIRV as FromCompilation<GlslangCompilation>>::Target;
type Options = <SPIRV as FromCompilation<GlslangCompilation>>::Options;
type Context = <SPIRV as FromCompilation<GlslangCompilation>>::Context;
type Output = <SPIRV as FromCompilation<GlslangCompilation>>::Output;
fn from_compilation(
compile: CachedCompilation<GlslangCompilation>,
) -> Result<CompilerBackend<Self::Output>, ShaderReflectError> {
SPIRV::from_compilation(compile.compilation)
}
}

View file

@ -0,0 +1,66 @@
//! Cache implementations for D3D blob types that need to live
//! here because of the orphan rule.
use crate::{Cacheable, CacheKey};
impl CacheKey for windows::Win32::Graphics::Direct3D::ID3DBlob {
fn hash_bytes(&self) -> &[u8] {
unsafe { std::slice::from_raw_parts(self.GetBufferPointer().cast(), self.GetBufferSize()) }
}
}
impl CacheKey for windows::Win32::Graphics::Direct3D::Dxc::IDxcBlob {
fn hash_bytes(&self) -> &[u8] {
unsafe { std::slice::from_raw_parts(self.GetBufferPointer().cast(), self.GetBufferSize()) }
}
}
impl Cacheable for windows::Win32::Graphics::Direct3D::ID3DBlob {
fn from_bytes(bytes: &[u8]) -> Option<Self> {
let Some(blob) = (unsafe { windows::Win32::Graphics::Direct3D::Fxc::D3DCreateBlob(bytes.len()).ok() }) else {
return None;
};
let slice = unsafe {
std::slice::from_raw_parts_mut(blob.GetBufferPointer().cast(), blob.GetBufferSize())
};
slice.copy_from_slice(bytes);
Some(blob)
}
fn to_bytes(&self) -> Option<Vec<u8>> {
let slice = unsafe {
std::slice::from_raw_parts(self.GetBufferPointer().cast(), self.GetBufferSize())
};
Some(Vec::from(slice))
}
}
impl Cacheable for windows::Win32::Graphics::Direct3D::Dxc::IDxcBlob {
fn from_bytes(bytes: &[u8]) -> Option<Self> {
let Some(blob) = (unsafe {
windows::Win32::Graphics::Direct3D::Dxc::DxcCreateInstance(
&windows::Win32::Graphics::Direct3D::Dxc::CLSID_DxcLibrary)
.and_then(|library: windows::Win32::Graphics::Direct3D::Dxc::IDxcUtils| {
library.CreateBlob(bytes.as_ptr().cast(), bytes.len() as u32,
windows::Win32::Graphics::Direct3D::Dxc::DXC_CP(0))
}).ok()
}) else {
return None;
};
Some(blob.into())
}
fn to_bytes(&self) -> Option<Vec<u8>> {
let slice = unsafe {
std::slice::from_raw_parts(self.GetBufferPointer().cast(), self.GetBufferSize())
};
Some(Vec::from(slice))
}
}

View file

@ -0,0 +1,10 @@
use std::error::Error;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum CacheError {
#[error("serde error")]
SerdeError,
#[error("unknown error")]
UnknownError(#[from] Box<dyn Error>),
}

View file

@ -0,0 +1,43 @@
/// Trait for objects that can be used as part of a key for a cached object.
pub trait CacheKey {
/// Get a byte representation of the object that
/// will be fed into the hash.
fn hash_bytes(&self) -> &[u8];
}
impl CacheKey for u32 {
fn hash_bytes(&self) -> &[u8] {
&bytemuck::bytes_of(&*self)
}
}
impl CacheKey for i32 {
fn hash_bytes(&self) -> &[u8] {
&bytemuck::bytes_of(&*self)
}
}
impl CacheKey for &[u8] {
fn hash_bytes(&self) -> &[u8] {
self
}
}
impl CacheKey for Vec<u8> {
fn hash_bytes(&self) -> &[u8] {
&self
}
}
impl CacheKey for Vec<u32> {
fn hash_bytes(&self) -> &[u8] {
bytemuck::cast_slice(&self)
}
}
impl CacheKey for &str {
fn hash_bytes(&self) -> &[u8] {
// need to be explicit
self.as_bytes()
}
}

View file

@ -0,0 +1,21 @@
//! This crate implements the librashader transparent cache.
//!
//! This crate is exempt from semantic versioning guarantees and is an implementation
//! detail of librashader runtimes.
#![feature(try_blocks)]
#![feature(once_cell)]
pub mod cache;
pub mod compilation;
pub mod error;
mod key;
mod cacheable;
#[cfg(test)]
mod tests {}
pub use cacheable::Cacheable;
pub use key::CacheKey;
#[cfg(all(target_os = "windows", feature = "d3d"))]
mod d3d;

View file

@ -64,7 +64,11 @@ fn multiline_comment(i: Span) -> IResult<Span, Span> {
}
fn single_comment(i: Span) -> IResult<Span, Span> {
delimited(alt((tag("//"), tag("#"))), not_line_ending, alt((line_ending, eof)))(i)
delimited(
alt((tag("//"), tag("#"))),
not_line_ending,
alt((line_ending, eof)),
)(i)
}
fn whitespace(i: Span) -> IResult<Span, ()> {
@ -704,6 +708,4 @@ SHARPEN = 0"#;
// PRESET END
// ----------------------------------------------------------------------------------------------------------------
"#;
}

View file

@ -1,10 +1,10 @@
use std::collections::VecDeque;
use crate::error::{ParseErrorKind, ParsePresetError};
use crate::parse::{remove_if, Span, Token};
use crate::{ScaleFactor, ScaleType};
use nom::bytes::complete::tag;
use nom::character::complete::digit1;
use nom::combinator::{eof, map_res};
use std::collections::VecDeque;
use nom::IResult;
use num_traits::cast::ToPrimitive;

View file

@ -29,9 +29,11 @@ naga = { version = "0.11.0", features = ["glsl-in", "spv-in", "spv-out", "glsl-o
rspirv = { version = "0.11.0+1.5.4", optional = true }
serde = { version = "1.0", features = ["derive"], optional = true }
[features]
default = ["cross"]
default = ["cross", "serialize"]
unstable-naga = [ "naga", "rspirv" ]
standalone = ["shaderc/build-from-source"]
dxil = ["cross", "spirv-to-dxil"]
cross = [ "spirv_cross", "spirv_cross/glsl", "spirv_cross/hlsl" ]
serialize = [ "serde" ]

View file

@ -2,7 +2,12 @@ use crate::error::ShaderCompileError;
use librashader_preprocess::ShaderSource;
use shaderc::{CompileOptions, Limit, ShaderKind};
#[cfg(feature = "serialize")]
use serde::{Deserialize, Serialize};
/// A reflectable shader compilation via glslang (shaderc).
#[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))]
#[derive(Debug, Clone)]
pub struct GlslangCompilation {
pub(crate) vertex: Vec<u32>,
pub(crate) fragment: Vec<u32>,

View file

@ -17,9 +17,10 @@ librashader-presets = { path = "../librashader-presets", version = "0.1.0-rc.3"
librashader-preprocess = { path = "../librashader-preprocess", version = "0.1.0-rc.3" }
librashader-reflect = { path = "../librashader-reflect", version = "0.1.0-rc.3", features = ["standalone"] }
librashader-runtime = { path = "../librashader-runtime", version = "0.1.0-rc.3" }
thiserror = "1.0.37"
spirv_cross = { package = "librashader-spirv-cross", version = "0.23" }
librashader-cache = { path = "../librashader-cache", version = "0.1.0-rc.3", features = ["d3d"]}
thiserror = "1.0.37"
rustc-hash = "1.1.0"
bytemuck = "1.12.3"
rayon = "1.6.1"

View file

@ -22,6 +22,8 @@ use crate::options::{FilterChainOptionsD3D11, FrameOptionsD3D11};
use crate::samplers::SamplerSet;
use crate::util::d3d11_compile_bound_shader;
use crate::{error, util, D3D11OutputView};
use librashader_cache::cache::cache_object;
use librashader_cache::compilation::CachedCompilation;
use librashader_reflect::reflect::presets::{CompilePresetTarget, ShaderPassArtifact};
use librashader_runtime::binding::{BindingUtil, TextureInput};
use librashader_runtime::framebuffer::FramebufferInit;
@ -117,7 +119,7 @@ impl FilterChainD3D11 {
options: Option<&FilterChainOptionsD3D11>,
) -> error::Result<FilterChainD3D11> {
let (passes, semantics) = HLSL::compile_preset_passes::<
GlslangCompilation,
CachedCompilation<GlslangCompilation>,
FilterChainError,
>(preset.shaders, &preset.textures)?;
@ -125,6 +127,7 @@ impl FilterChainD3D11 {
// initialize passes
let filters = FilterChainD3D11::init_passes(device, passes, &semantics)?;
println!("passes loded");
let immediate_context = unsafe { device.GetImmediateContext()? };
@ -152,6 +155,7 @@ impl FilterChainD3D11 {
let draw_quad = DrawQuad::new(device)?;
let state = D3D11State::new(device)?;
println!("ready");
Ok(FilterChainD3D11 {
passes: filters,
output_framebuffers,
@ -216,25 +220,35 @@ impl FilterChainD3D11 {
let reflection = reflect.reflect(index, semantics)?;
let hlsl = reflect.compile(None)?;
let vertex_dxbc =
util::d3d_compile_shader(hlsl.vertex.as_bytes(), b"main\0", b"vs_5_0\0")?;
let vs = d3d11_compile_bound_shader(
device,
&vertex_dxbc,
None,
ID3D11Device::CreateVertexShader,
let (vs, vertex_dxbc) = cache_object(
"dxbc",
&[hlsl.vertex.as_bytes()],
|&[bytes]| util::d3d_compile_shader(bytes, b"main\0", b"vs_5_0\0"),
|blob| {
Ok((
d3d11_compile_bound_shader(
device,
&blob,
None,
ID3D11Device::CreateVertexShader,
)?,
blob,
))
},
true,
)?;
let ia_desc = DrawQuad::get_spirv_cross_vbo_desc();
let vao = util::d3d11_create_input_layout(device, &ia_desc, &vertex_dxbc)?;
let fragment_dxbc =
util::d3d_compile_shader(hlsl.fragment.as_bytes(), b"main\0", b"ps_5_0\0")?;
let ps = d3d11_compile_bound_shader(
device,
&fragment_dxbc,
None,
ID3D11Device::CreatePixelShader,
let ps = cache_object(
"dxbc",
&[hlsl.fragment.as_bytes()],
|&[bytes]| util::d3d_compile_shader(bytes, b"main\0", b"ps_5_0\0"),
|blob| {
d3d11_compile_bound_shader(device, &blob, None, ID3D11Device::CreatePixelShader)
},
true,
)?;
let ubo_cbuffer = if let Some(ubo) = &reflection.ubo && ubo.size != 0 {
@ -317,7 +331,8 @@ impl FilterChainD3D11 {
textures: &[TextureConfig],
) -> error::Result<FxHashMap<usize, LutTexture>> {
let mut luts = FxHashMap::default();
let images = textures.par_iter()
let images = textures
.par_iter()
.map(|texture| Image::load(&texture.path, UVDirection::TopLeft))
.collect::<Result<Vec<Image>, ImageError>>()?;

View file

@ -38,7 +38,7 @@ mod tests {
// "../test/slang-shaders/handheld/console-border/gbc-lcd-grid-v2.slangp";
// "../test/null.slangp",
const FILTER_PATH: &str =
"../test/shaders_slang/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV-GLASS.slangp";
"../test/Mega_Bezel_Packs/Duimon-Mega-Bezel/Presets/Advanced/Nintendo_GBA_SP/GBA_SP-[ADV]-[LCD-GRID].slangp";
// const FILTER_PATH: &str = "../test/slang-shaders/test/history.slangp";
// const FILTER_PATH: &str = "../test/slang-shaders/test/feedback.slangp";

View file

@ -17,6 +17,8 @@ librashader-presets = { path = "../librashader-presets", version = "0.1.0-rc.3"
librashader-preprocess = { path = "../librashader-preprocess", version = "0.1.0-rc.3" }
librashader-reflect = { path = "../librashader-reflect", version = "0.1.0-rc.3", features = ["dxil", "standalone"] }
librashader-runtime = { path = "../librashader-runtime", version = "0.1.0-rc.3" }
librashader-cache = { path = "../librashader-cache", version = "0.1.0-rc.3", features = ["d3d"] }
thiserror = "1.0.37"
spirv_cross = { package = "librashader-spirv-cross", version = "0.23" }
parking_lot = "0.12.1"

View file

@ -46,6 +46,7 @@ use windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_UNKNOWN;
use windows::Win32::System::Threading::{CreateEventA, WaitForSingleObject};
use windows::Win32::System::WindowsProgramming::INFINITE;
use librashader_cache::compilation::CachedCompilation;
use librashader_runtime::framebuffer::FramebufferInit;
use librashader_runtime::render_target::RenderTarget;
use librashader_runtime::scaling::ScaleFramebuffer;
@ -208,14 +209,14 @@ impl FilterChainD3D12 {
let shader_copy = preset.shaders.clone();
let (passes, semantics) = DXIL::compile_preset_passes::<
GlslangCompilation,
CachedCompilation<GlslangCompilation>,
FilterChainError,
>(preset.shaders, &preset.textures)?;
let (hlsl_passes, _) = HLSL::compile_preset_passes::<GlslangCompilation, FilterChainError>(
shader_copy,
&preset.textures,
)?;
let (hlsl_passes, _) = HLSL::compile_preset_passes::<
CachedCompilation<GlslangCompilation>,
FilterChainError,
>(shader_copy, &preset.textures)?;
let samplers = SamplerSet::new(device)?;
let mipmap_gen = D3D12MipmapGen::new(device, false)?;
@ -327,7 +328,8 @@ impl FilterChainD3D12 {
let mipmap_gen = D3D12MipmapGen::new(device, true)?;
let mut luts = FxHashMap::default();
let images = textures.par_iter()
let images = textures
.par_iter()
.map(|texture| Image::load(&texture.path, UVDirection::TopLeft))
.collect::<Result<Vec<Image>, ImageError>>()?;
@ -420,6 +422,7 @@ impl FilterChainD3D12 {
ImageFormat::R8G8B8A8Unorm
}.into();
// incredibly cursed.
let (reflection, graphics_pipeline) = if !force_hlsl &&
let Ok(graphics_pipeline) =

View file

@ -2,9 +2,11 @@ use crate::error::assume_d3d12_init;
use crate::error::FilterChainError::Direct3DOperationError;
use crate::quad_render::DrawQuad;
use crate::{error, util};
use librashader_cache::cache::{cache_object, cache_pipeline};
use librashader_reflect::back::cross::CrossHlslContext;
use librashader_reflect::back::dxil::DxilObject;
use librashader_reflect::back::ShaderCompilerOutput;
use std::ops::Deref;
use widestring::u16cstr;
use windows::Win32::Foundation::BOOL;
use windows::Win32::Graphics::Direct3D::Dxc::{
@ -13,14 +15,14 @@ use windows::Win32::Graphics::Direct3D::Dxc::{
use windows::Win32::Graphics::Direct3D12::{
D3D12SerializeVersionedRootSignature, ID3D12Device, ID3D12PipelineState, ID3D12RootSignature,
D3D12_BLEND_DESC, D3D12_BLEND_INV_SRC_ALPHA, D3D12_BLEND_OP_ADD, D3D12_BLEND_SRC_ALPHA,
D3D12_COLOR_WRITE_ENABLE_ALL, D3D12_CULL_MODE_NONE, D3D12_DESCRIPTOR_RANGE1,
D3D12_DESCRIPTOR_RANGE_FLAGS, D3D12_DESCRIPTOR_RANGE_FLAG_DATA_VOLATILE,
D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE, D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER,
D3D12_DESCRIPTOR_RANGE_TYPE_SRV, D3D12_FILL_MODE_SOLID, D3D12_GRAPHICS_PIPELINE_STATE_DESC,
D3D12_INPUT_LAYOUT_DESC, D3D12_LOGIC_OP_NOOP, D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE,
D3D12_RASTERIZER_DESC, D3D12_RENDER_TARGET_BLEND_DESC, D3D12_ROOT_DESCRIPTOR1,
D3D12_ROOT_DESCRIPTOR_FLAG_NONE, D3D12_ROOT_DESCRIPTOR_TABLE1, D3D12_ROOT_PARAMETER1,
D3D12_ROOT_PARAMETER1_0, D3D12_ROOT_PARAMETER_TYPE_CBV,
D3D12_CACHED_PIPELINE_STATE, D3D12_COLOR_WRITE_ENABLE_ALL, D3D12_CULL_MODE_NONE,
D3D12_DESCRIPTOR_RANGE1, D3D12_DESCRIPTOR_RANGE_FLAGS,
D3D12_DESCRIPTOR_RANGE_FLAG_DATA_VOLATILE, D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE,
D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, D3D12_DESCRIPTOR_RANGE_TYPE_SRV, D3D12_FILL_MODE_SOLID,
D3D12_GRAPHICS_PIPELINE_STATE_DESC, D3D12_INPUT_LAYOUT_DESC, D3D12_LOGIC_OP_NOOP,
D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE, D3D12_RASTERIZER_DESC, D3D12_RENDER_TARGET_BLEND_DESC,
D3D12_ROOT_DESCRIPTOR1, D3D12_ROOT_DESCRIPTOR_FLAG_NONE, D3D12_ROOT_DESCRIPTOR_TABLE1,
D3D12_ROOT_PARAMETER1, D3D12_ROOT_PARAMETER1_0, D3D12_ROOT_PARAMETER_TYPE_CBV,
D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE, D3D12_ROOT_SIGNATURE_DESC1,
D3D12_ROOT_SIGNATURE_FLAG_ALLOW_INPUT_ASSEMBLER_INPUT_LAYOUT, D3D12_SHADER_BYTECODE,
D3D12_SHADER_VISIBILITY_ALL, D3D12_SHADER_VISIBILITY_PIXEL,
@ -144,7 +146,7 @@ impl D3D12RootSignature {
}
}
impl D3D12GraphicsPipeline {
fn new_from_blobs(
pub fn new_from_blobs(
device: &ID3D12Device,
vertex_dxil: IDxcBlob,
fragment_dxil: IDxcBlob,
@ -220,7 +222,32 @@ impl D3D12GraphicsPipeline {
..Default::default()
};
device.CreateGraphicsPipelineState(&pipeline_desc)?
cache_pipeline(
"d3d12",
&[&vertex_dxil, &fragment_dxil, &render_format.0],
|cached: Option<Vec<u8>>| {
if let Some(cached) = cached {
let pipeline_desc = D3D12_GRAPHICS_PIPELINE_STATE_DESC {
CachedPSO: D3D12_CACHED_PIPELINE_STATE {
pCachedBlob: cached.as_ptr().cast(),
CachedBlobSizeInBytes: cached.len(),
},
pRootSignature: windows::core::ManuallyDrop::new(
&root_signature.handle,
),
..pipeline_desc
};
device.CreateGraphicsPipelineState(&pipeline_desc)
} else {
device.CreateGraphicsPipelineState(&pipeline_desc)
}
},
|pso: &ID3D12PipelineState| {
let cached_pso = pso.GetCachedBlob()?;
Ok(cached_pso)
},
true,
)?
};
unsafe {
@ -286,9 +313,22 @@ impl D3D12GraphicsPipeline {
"Compiled DXIL fragment shader needs unexpected runtime data",
));
}
let vertex_dxil = util::dxc_validate_shader(library, validator, &shader_assembly.vertex)?;
let fragment_dxil =
util::dxc_validate_shader(library, validator, &shader_assembly.fragment)?;
let vertex_dxil = cache_object(
"dxil",
&[shader_assembly.vertex.deref()],
|&[source]| util::dxc_validate_shader(library, validator, source),
|f| Ok(f),
true,
)?;
let fragment_dxil = cache_object(
"dxil",
&[shader_assembly.fragment.deref()],
|&[source]| util::dxc_validate_shader(library, validator, source),
|f| Ok(f),
true,
)?;
Self::new_from_blobs(
device,
@ -307,10 +347,21 @@ impl D3D12GraphicsPipeline {
root_signature: &D3D12RootSignature,
render_format: DXGI_FORMAT,
) -> error::Result<D3D12GraphicsPipeline> {
let vertex_dxil =
util::dxc_compile_shader(library, dxc, &shader_assembly.vertex, u16cstr!("vs_6_0"))?;
let fragment_dxil =
util::dxc_compile_shader(library, dxc, &shader_assembly.fragment, u16cstr!("ps_6_0"))?;
let vertex_dxil = cache_object(
"dxil",
&[shader_assembly.vertex.as_bytes()],
|&[source]| util::dxc_compile_shader(library, dxc, source, u16cstr!("vs_6_0")),
|f| Ok(f),
true,
)?;
let fragment_dxil = cache_object(
"dxil",
&[shader_assembly.fragment.as_bytes()],
|&[source]| util::dxc_compile_shader(library, dxc, source, u16cstr!("ps_6_0")),
|f| Ok(f),
true,
)?;
Self::new_from_blobs(
device,

View file

@ -36,10 +36,10 @@ mod tests {
// "../test/slang-shaders/crt/crt-lottes.slangp",
// "../test/basic.slangp",
// "../test/slang-shaders/handheld/console-border/gbc-lcd-grid-v2.slangp",
// "../test/slang-shaders/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV.slangp",
"../test/Mega_Bezel_Packs/Duimon-Mega-Bezel/Presets/Advanced/Nintendo_GBA_SP/GBA_SP-[ADV]-[LCD-GRID]-[Night].slangp",
// "../test/slang-shaders/test/feedback.slangp",
// "../test/slang-shaders/test/history.slangp",
"../test/slang-shaders/crt/crt-royale.slangp",
// "../test/slang-shaders/crt/crt-royale.slangp",
// "../test/slang-shaders/vhs/VHSPro.slangp",
&SampleCommandLine {
use_warp_device: false,

View file

@ -17,6 +17,8 @@ librashader-presets = { path = "../librashader-presets", version = "0.1.0-rc.3"
librashader-preprocess = { path = "../librashader-preprocess", version = "0.1.0-rc.3" }
librashader-reflect = { path = "../librashader-reflect", version = "0.1.0-rc.3", features = ["standalone"] }
librashader-runtime = { path = "../librashader-runtime" , version = "0.1.0-rc.3" }
librashader-cache = { path = "../librashader-cache", version = "0.1.0-rc.3" }
spirv_cross = { package = "librashader-spirv-cross", version = "0.23" }
rustc-hash = "1.1.0"
gl = "0.14.0"

View file

@ -1,13 +1,15 @@
use crate::binding::{GlUniformStorage, UniformLocation, VariableLocation};
use crate::error::FilterChainError;
use crate::filter_pass::{FilterPass, UniformOffset};
use crate::gl::{DrawQuad, FramebufferInterface, GLFramebuffer, GLInterface, LoadLut, UboRing};
use crate::gl::{
CompileProgram, DrawQuad, FramebufferInterface, GLFramebuffer, GLInterface, LoadLut, UboRing,
};
use crate::options::{FilterChainOptionsGL, FrameOptionsGL};
use crate::samplers::SamplerSet;
use crate::texture::InputTexture;
use crate::util::{gl_get_version, gl_u16_to_version};
use crate::{error, util, GLImage};
use gl::types::{GLint, GLuint};
use crate::{error, GLImage};
use gl::types::GLuint;
use librashader_common::Viewport;
use librashader_presets::ShaderPreset;
@ -24,7 +26,6 @@ use librashader_runtime::framebuffer::FramebufferInit;
use librashader_runtime::render_target::RenderTarget;
use librashader_runtime::scaling::ScaleFramebuffer;
use rustc_hash::FxHashMap;
use spirv_cross::spirv::Decoration;
use std::collections::VecDeque;
#[rustfmt::skip]
@ -188,64 +189,7 @@ impl<T: GLInterface> FilterChainImpl<T> {
let reflection = reflect.reflect(index, semantics)?;
let glsl = reflect.compile(version)?;
let vertex_resources = glsl.context.artifact.vertex.get_shader_resources()?;
// todo: split this out.
let (program, ubo_location) = unsafe {
let vertex = util::gl_compile_shader(gl::VERTEX_SHADER, glsl.vertex.as_str())?;
let fragment =
util::gl_compile_shader(gl::FRAGMENT_SHADER, glsl.fragment.as_str())?;
let program = gl::CreateProgram();
gl::AttachShader(program, vertex);
gl::AttachShader(program, fragment);
for res in vertex_resources.stage_inputs {
let loc = glsl
.context
.artifact
.vertex
.get_decoration(res.id, Decoration::Location)?;
let mut name = res.name;
name.push('\0');
gl::BindAttribLocation(program, loc, name.as_str().as_ptr().cast())
}
gl::LinkProgram(program);
gl::DeleteShader(vertex);
gl::DeleteShader(fragment);
let mut status = 0;
gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);
if status != 1 {
return Err(FilterChainError::GLLinkError);
}
gl::UseProgram(program);
for (name, binding) in &glsl.context.sampler_bindings {
let location = gl::GetUniformLocation(program, name.as_str().as_ptr().cast());
if location >= 0 {
// eprintln!("setting sampler {location} to sample from {binding}");
gl::Uniform1i(location, *binding as GLint);
}
}
gl::UseProgram(0);
(
program,
UniformLocation {
vertex: gl::GetUniformBlockIndex(
program,
b"LIBRA_UBO_VERTEX\0".as_ptr().cast(),
),
fragment: gl::GetUniformBlockIndex(
program,
b"LIBRA_UBO_FRAGMENT\0".as_ptr().cast(),
),
},
)
};
let (program, ubo_location) = T::CompileShader::compile_program(glsl, true)?;
let ubo_ring = if let Some(ubo) = &reflection.ubo {
let ring = UboRing::new(ubo.size);

View file

@ -0,0 +1,75 @@
use crate::binding::UniformLocation;
use crate::error::FilterChainError;
use crate::gl::CompileProgram;
use crate::util;
use gl::types::{GLint, GLuint};
use librashader_reflect::back::cross::CrossGlslContext;
use librashader_reflect::back::ShaderCompilerOutput;
use spirv_cross::spirv::Decoration;
pub struct Gl3CompileProgram;
impl CompileProgram for Gl3CompileProgram {
fn compile_program(
glsl: ShaderCompilerOutput<String, CrossGlslContext>,
_cache: bool,
) -> crate::error::Result<(GLuint, UniformLocation<GLuint>)> {
let vertex_resources = glsl.context.artifact.vertex.get_shader_resources()?;
let (program, ubo_location) = unsafe {
let vertex = util::gl_compile_shader(gl::VERTEX_SHADER, glsl.vertex.as_str())?;
let fragment = util::gl_compile_shader(gl::FRAGMENT_SHADER, glsl.fragment.as_str())?;
let program = gl::CreateProgram();
gl::AttachShader(program, vertex);
gl::AttachShader(program, fragment);
for res in vertex_resources.stage_inputs {
let loc = glsl
.context
.artifact
.vertex
.get_decoration(res.id, Decoration::Location)?;
let mut name = res.name;
name.push('\0');
gl::BindAttribLocation(program, loc, name.as_str().as_ptr().cast())
}
gl::LinkProgram(program);
gl::DeleteShader(vertex);
gl::DeleteShader(fragment);
let mut status = 0;
gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);
if status != 1 {
return Err(FilterChainError::GLLinkError);
}
gl::UseProgram(program);
for (name, binding) in &glsl.context.sampler_bindings {
let location = gl::GetUniformLocation(program, name.as_str().as_ptr().cast());
if location >= 0 {
// eprintln!("setting sampler {location} to sample from {binding}");
gl::Uniform1i(location, *binding as GLint);
}
}
gl::UseProgram(0);
(
program,
UniformLocation {
vertex: gl::GetUniformBlockIndex(
program,
b"LIBRA_UBO_VERTEX\0".as_ptr().cast(),
),
fragment: gl::GetUniformBlockIndex(
program,
b"LIBRA_UBO_FRAGMENT\0".as_ptr().cast(),
),
},
)
};
Ok((program, ubo_location))
}
}

View file

@ -6,8 +6,8 @@ use gl::types::{GLsizei, GLuint};
use librashader_presets::TextureConfig;
use librashader_runtime::image::{Image, ImageError, UVDirection};
use librashader_runtime::scaling::MipmapSize;
use rustc_hash::FxHashMap;
use rayon::prelude::*;
use rustc_hash::FxHashMap;
pub struct Gl3LutLoad;
impl LoadLut for Gl3LutLoad {
@ -19,7 +19,8 @@ impl LoadLut for Gl3LutLoad {
binding
};
let images = textures.par_iter()
let images = textures
.par_iter()
.map(|texture| Image::load(&texture.path, UVDirection::BottomLeft))
.collect::<std::result::Result<Vec<Image>, ImageError>>()?;

View file

@ -1,3 +1,4 @@
mod compile_program;
mod draw_quad;
mod framebuffer;
#[cfg(test)]
@ -7,6 +8,7 @@ mod texture_bind;
mod ubo_ring;
use crate::gl::GLInterface;
use compile_program::*;
use draw_quad::*;
use framebuffer::*;
use lut_load::*;
@ -20,4 +22,5 @@ impl GLInterface for CompatibilityGL {
type DrawQuad = Gl3DrawQuad;
type LoadLut = Gl3LutLoad;
type BindTexture = Gl3BindTexture;
type CompileShader = Gl3CompileProgram;
}

View file

@ -0,0 +1,145 @@
use crate::binding::UniformLocation;
use crate::error::FilterChainError;
use crate::gl::CompileProgram;
use crate::util;
use gl::types::{GLint, GLsizei, GLuint};
use librashader_cache::Cacheable;
use librashader_reflect::back::cross::CrossGlslContext;
use librashader_reflect::back::ShaderCompilerOutput;
use spirv_cross::spirv::Decoration;
pub struct Gl4CompileProgram;
struct GlProgramBinary {
program: Vec<u8>,
format: GLuint,
}
impl Cacheable for GlProgramBinary {
fn from_bytes(cached: &[u8]) -> Option<Self>
where
Self: Sized,
{
let mut cached = Vec::from(cached);
let format = cached.split_off(cached.len() - std::mem::size_of::<u32>());
let format: Option<&GLuint> = bytemuck::try_from_bytes(&format).ok();
let Some(format) = format else {
return None;
};
return Some(GlProgramBinary {
program: cached,
format: *format,
});
}
fn to_bytes(&self) -> Option<Vec<u8>> {
let mut slice = self.program.clone();
slice.extend(bytemuck::bytes_of(&self.format));
Some(slice)
}
}
impl CompileProgram for Gl4CompileProgram {
fn compile_program(
glsl: ShaderCompilerOutput<String, CrossGlslContext>,
cache: bool,
) -> crate::error::Result<(GLuint, UniformLocation<GLuint>)> {
let vertex_resources = glsl.context.artifact.vertex.get_shader_resources()?;
let program = librashader_cache::cache::cache_object(
"opengl4",
&[glsl.vertex.as_str(), glsl.fragment.as_str()],
|&[vertex, fragment]| unsafe {
let vertex = util::gl_compile_shader(gl::VERTEX_SHADER, vertex)?;
let fragment = util::gl_compile_shader(gl::FRAGMENT_SHADER, fragment)?;
let program = gl::CreateProgram();
gl::AttachShader(program, vertex);
gl::AttachShader(program, fragment);
for res in &vertex_resources.stage_inputs {
let loc = glsl
.context
.artifact
.vertex
.get_decoration(res.id, Decoration::Location)?;
let mut name = res.name.clone();
name.push('\0');
gl::BindAttribLocation(program, loc, name.as_str().as_ptr().cast())
}
gl::LinkProgram(program);
gl::DeleteShader(vertex);
gl::DeleteShader(fragment);
let mut status = 0;
gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);
if status != 1 {
return Err(FilterChainError::GLLinkError);
}
let mut length = 0;
gl::GetProgramiv(program, gl::PROGRAM_BINARY_LENGTH, &mut length);
let mut binary = vec![0; length as usize];
let mut format = 0;
gl::GetProgramBinary(
program,
length,
std::ptr::null_mut(),
&mut format,
binary.as_mut_ptr().cast(),
);
gl::DeleteProgram(program);
Ok(GlProgramBinary {
program: binary,
format,
})
},
|GlProgramBinary {
program: blob,
format,
}| {
let program = unsafe {
let program = gl::CreateProgram();
gl::ProgramBinary(program, format, blob.as_ptr().cast(), blob.len() as GLsizei);
program
};
unsafe {
let mut status = 0;
gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);
if status != 1 {
return Err(FilterChainError::GLLinkError);
}
if gl::GetError() == gl::INVALID_ENUM {
return Err(FilterChainError::GLLinkError);
}
}
return Ok(program);
},
cache,
)?;
let ubo_location = unsafe {
for (name, binding) in &glsl.context.sampler_bindings {
let location = gl::GetUniformLocation(program, name.as_str().as_ptr().cast());
if location >= 0 {
gl::ProgramUniform1i(program, location, *binding as GLint);
}
}
UniformLocation {
vertex: gl::GetUniformBlockIndex(program, b"LIBRA_UBO_VERTEX\0".as_ptr().cast()),
fragment: gl::GetUniformBlockIndex(
program,
b"LIBRA_UBO_FRAGMENT\0".as_ptr().cast(),
),
}
};
Ok((program, ubo_location))
}
}

View file

@ -6,8 +6,8 @@ use gl::types::{GLsizei, GLuint};
use librashader_presets::TextureConfig;
use librashader_runtime::image::{Image, ImageError, UVDirection};
use librashader_runtime::scaling::MipmapSize;
use rustc_hash::FxHashMap;
use rayon::prelude::*;
use rustc_hash::FxHashMap;
pub struct Gl46LutLoad;
impl LoadLut for Gl46LutLoad {
@ -23,7 +23,8 @@ impl LoadLut for Gl46LutLoad {
gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, 0);
}
let images = textures.par_iter()
let images = textures
.par_iter()
.map(|texture| Image::load(&texture.path, UVDirection::BottomLeft))
.collect::<std::result::Result<Vec<Image>, ImageError>>()?;

View file

@ -4,9 +4,11 @@ mod lut_load;
mod texture_bind;
mod ubo_ring;
mod compile_program;
#[cfg(test)]
pub mod hello_triangle;
use crate::gl::gl46::compile_program::Gl4CompileProgram;
use crate::gl::GLInterface;
use draw_quad::*;
use framebuffer::*;
@ -21,4 +23,5 @@ impl GLInterface for DirectStateAccessGL {
type DrawQuad = Gl46DrawQuad;
type LoadLut = Gl46LutLoad;
type BindTexture = Gl46BindTexture;
type CompileShader = Gl4CompileProgram;
}

View file

@ -11,6 +11,8 @@ pub use framebuffer::GLFramebuffer;
use gl::types::{GLenum, GLuint};
use librashader_common::{ImageFormat, Size};
use librashader_presets::{Scale2D, TextureConfig};
use librashader_reflect::back::cross::CrossGlslContext;
use librashader_reflect::back::ShaderCompilerOutput;
use librashader_reflect::reflect::semantics::{TextureBinding, UboReflection};
use librashader_runtime::uniforms::UniformStorageAccess;
use rustc_hash::FxHashMap;
@ -19,6 +21,13 @@ pub(crate) trait LoadLut {
fn load_luts(textures: &[TextureConfig]) -> Result<FxHashMap<usize, InputTexture>>;
}
pub(crate) trait CompileProgram {
fn compile_program(
shader: ShaderCompilerOutput<String, CrossGlslContext>,
cache: bool,
) -> Result<(GLuint, UniformLocation<GLuint>)>;
}
pub(crate) trait DrawQuad {
fn new() -> Self;
fn bind_vertices(&self);
@ -61,4 +70,5 @@ pub(crate) trait GLInterface {
type DrawQuad: DrawQuad;
type LoadLut: LoadLut;
type BindTexture: BindTexture;
type CompileShader: CompileProgram;
}

View file

@ -35,7 +35,7 @@ mod tests {
fn triangle_gl() {
let (glfw, window, events, shader, vao) = gl::gl3::hello_triangle::setup();
let mut filter = FilterChainGL::load_from_path(
"../test/slang-shaders/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV.slangp",
"../test/shaders_slang/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV.slangp",
Some(&FilterChainOptionsGL {
glsl_version: 0,
use_dsa: false,
@ -54,7 +54,7 @@ mod tests {
// "../test/slang-shaders/vhs/VHSPro.slangp",
// "../test/slang-shaders/test/history.slangp",
// "../test/slang-shaders/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV.slangp",
"../test/slang-shaders/test/feedback.slangp",
"../test/shaders_slang/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV.slangp",
Some(&FilterChainOptionsGL {
glsl_version: 0,
use_dsa: true,

View file

@ -19,6 +19,8 @@ librashader-presets = { path = "../librashader-presets", version = "0.1.0-rc.3"
librashader-preprocess = { path = "../librashader-preprocess", version = "0.1.0-rc.3" }
librashader-reflect = { path = "../librashader-reflect", version = "0.1.0-rc.3", features = [] }
librashader-runtime = { path = "../librashader-runtime" , version = "0.1.0-rc.3" }
librashader-cache = { path = "../librashader-cache", version = "0.1.0-rc.3" }
spirv_cross = { package = "librashader-spirv-cross", version = "0.23" }
rustc-hash = "1.1.0"
bytemuck = "1.12.3"

View file

@ -22,7 +22,7 @@ use librashader_reflect::reflect::presets::{CompilePresetTarget, ShaderPassArtif
use librashader_reflect::reflect::semantics::ShaderSemantics;
use librashader_reflect::reflect::ReflectShader;
use librashader_runtime::binding::BindingUtil;
use librashader_runtime::image::{BGRA8, Image, ImageError, UVDirection};
use librashader_runtime::image::{Image, ImageError, UVDirection, BGRA8};
use librashader_runtime::quad::QuadType;
use librashader_runtime::uniforms::UniformStorage;
use parking_lot::RwLock;
@ -32,6 +32,7 @@ use std::convert::Infallible;
use std::path::Path;
use std::sync::Arc;
use librashader_cache::compilation::CachedCompilation;
use librashader_runtime::framebuffer::FramebufferInit;
use librashader_runtime::render_target::RenderTarget;
use librashader_runtime::scaling::ScaleFramebuffer;
@ -42,7 +43,6 @@ pub struct VulkanObjects {
pub(crate) device: Arc<ash::Device>,
pub(crate) alloc: Arc<RwLock<Allocator>>,
queue: vk::Queue,
pipeline_cache: vk::PipelineCache,
// pub(crate) memory_properties: vk::PhysicalDeviceMemoryProperties,
}
@ -76,9 +76,6 @@ impl TryFrom<VulkanInstance> for VulkanObjects {
let device = ash::Device::load(instance.fp_v1_0(), vulkan.device);
let pipeline_cache =
device.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None)?;
let queue = get_graphics_queue(&instance, &device, vulkan.physical_device);
// let memory_properties =
@ -90,7 +87,6 @@ impl TryFrom<VulkanInstance> for VulkanObjects {
device: Arc::new(device),
alloc,
queue,
pipeline_cache,
// memory_properties,
// debug,
})
@ -102,27 +98,21 @@ impl TryFrom<(vk::PhysicalDevice, ash::Instance, ash::Device)> for VulkanObjects
type Error = FilterChainError;
fn try_from(value: (vk::PhysicalDevice, ash::Instance, ash::Device)) -> error::Result<Self> {
unsafe {
let device = value.2;
let device = value.2;
let pipeline_cache =
device.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None)?;
let queue = get_graphics_queue(&value.1, &device, value.0);
let queue = get_graphics_queue(&value.1, &device, value.0);
// let memory_properties = value.1.get_physical_device_memory_properties(value.0);
// let memory_properties = value.1.get_physical_device_memory_properties(value.0);
let alloc = util::create_allocator(device.clone(), value.1, value.0)?;
let alloc = util::create_allocator(device.clone(), value.1, value.0)?;
Ok(VulkanObjects {
alloc,
device: Arc::new(device),
queue,
pipeline_cache,
// memory_properties,
// debug: value.3,
})
}
Ok(VulkanObjects {
alloc,
device: Arc::new(device),
queue,
// memory_properties,
// debug: value.3,
})
}
}
@ -318,7 +308,7 @@ impl FilterChainVulkan {
FilterChainError: From<E>,
{
let (passes, semantics) = SPIRV::compile_preset_passes::<
GlslangCompilation,
CachedCompilation<GlslangCompilation>,
FilterChainError,
>(preset.shaders, &preset.textures)?;
let device = vulkan.try_into().map_err(From::from)?;
@ -436,7 +426,6 @@ impl FilterChainVulkan {
let graphics_pipeline = VulkanGraphicsPipeline::new(
&vulkan.device,
&vulkan.pipeline_cache,
&spirv_words,
&reflection,
frames_in_flight,
@ -470,7 +459,8 @@ impl FilterChainVulkan {
textures: &[TextureConfig],
) -> error::Result<FxHashMap<usize, LutTexture>> {
let mut luts = FxHashMap::default();
let images = textures.par_iter()
let images = textures
.par_iter()
.map(|texture| Image::load(&texture.path, UVDirection::TopLeft))
.collect::<Result<Vec<Image<BGRA8>>, ImageError>>()?;
for (index, (texture, image)) in textures.iter().zip(images).enumerate() {

View file

@ -1,8 +1,10 @@
use crate::{error, util};
use ash::vk;
use crate::error::FilterChainError;
use crate::framebuffer::OutputImage;
use crate::render_pass::VulkanRenderPass;
use librashader_cache::cache::cache_pipeline;
use librashader_reflect::back::ShaderCompilerOutput;
use librashader_reflect::reflect::semantics::{TextureBinding, UboReflection};
use librashader_reflect::reflect::ShaderReflection;
@ -308,7 +310,6 @@ impl VulkanGraphicsPipeline {
pub fn new(
device: &Arc<ash::Device>,
cache: &vk::PipelineCache,
shader_assembly: &ShaderCompilerOutput<Vec<u32>>,
reflection: &ShaderReflection,
replicas: u32,
@ -334,13 +335,30 @@ impl VulkanGraphicsPipeline {
)?);
}
let pipeline = Self::create_pipeline(
&device,
&cache,
&pipeline_layout,
&vertex_module,
&fragment_module,
render_pass.as_ref(),
let (pipeline, pipeline_cache) = cache_pipeline(
"vulkan",
&[&shader_assembly.vertex, &shader_assembly.fragment],
|pipeline_data| {
let mut cache_info = vk::PipelineCacheCreateInfo::builder();
if let Some(pipeline_data) = pipeline_data.as_ref() {
cache_info = cache_info.initial_data(pipeline_data);
}
let cache_info = cache_info.build();
let pipeline_cache = unsafe { device.create_pipeline_cache(&cache_info, None)? };
let pipeline = Self::create_pipeline(
&device,
&pipeline_cache,
&pipeline_layout,
&vertex_module,
&fragment_module,
render_pass.as_ref(),
)?;
Ok::<_, FilterChainError>((pipeline, pipeline_cache))
},
|(_pipeline, cache)| unsafe { Ok(device.get_pipeline_cache_data(*cache)?) },
true,
)?;
Ok(VulkanGraphicsPipeline {
@ -350,7 +368,7 @@ impl VulkanGraphicsPipeline {
render_pass,
vertex: vertex_module,
fragment: fragment_module,
cache: *cache,
cache: pipeline_cache,
})
}
@ -373,7 +391,11 @@ impl VulkanGraphicsPipeline {
std::mem::swap(&mut self.render_pass, &mut new_renderpass);
std::mem::swap(&mut self.pipeline, &mut new_pipeline);
unsafe { self.device.destroy_pipeline(new_pipeline, None) }
unsafe {
if new_pipeline != vk::Pipeline::null() {
self.device.destroy_pipeline(new_pipeline, None)
}
}
Ok(())
}
#[inline(always)]
@ -452,3 +474,17 @@ impl VulkanGraphicsPipeline {
}
}
}
impl Drop for VulkanGraphicsPipeline {
fn drop(&mut self) {
unsafe {
if self.pipeline != vk::Pipeline::null() {
self.device.destroy_pipeline(self.pipeline, None)
}
if self.cache != vk::PipelineCache::null() {
self.device.destroy_pipeline_cache(self.cache, None)
}
}
}
}

View file

@ -43,7 +43,7 @@ mod tests {
let base = VulkanBase::new(entry).unwrap();
let filter = FilterChainVulkan::load_from_path(
// "../test/slang-shaders/crt/crt-royale.slangp",
"../test/slang-shaders/bezel/Mega_Bezel/Presets/MBZ__0__SMOOTH-ADV-GLASS.slangp",
"../test/Mega_Bezel_Packs/Duimon-Mega-Bezel/Presets/Advanced/Nintendo_GBA_SP/GBA_SP-[ADV]-[LCD-GRID]-[Night].slangp",
&base,
// "../test/slang-shaders/test/feedback.slancargogp",
// "../test/basic.slangp",

BIN
test/advwars.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 76 KiB