From d5ef5904f3d667811217585ce0282a7505360f58 Mon Sep 17 00:00:00 2001 From: chyyran Date: Wed, 14 Feb 2024 00:46:15 -0500 Subject: [PATCH] cache: get rid of rusqlite to avoid a C dependency --- Cargo.lock | 184 +++++++++++++++++---------- librashader-cache/Cargo.toml | 5 +- librashader-cache/src/cache.rs | 84 +++++++----- librashader-cache/src/compilation.rs | 6 +- 4 files changed, 171 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0516662..8f7cf86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,10 +705,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.3.2" +name = "crc" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -777,6 +792,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + [[package]] name = "digest" version = "0.10.7" @@ -895,18 +916,6 @@ dependencies = [ "zune-inflate", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.0.1" @@ -974,6 +983,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -1233,15 +1252,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.3", -] - [[package]] name = "hassle-rs" version = "0.11.0" @@ -1274,9 +1284,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hexf-parse" @@ -1331,9 +1341,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -1345,7 +1355,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.5", + "hermit-abi 0.3.6", "libc", "windows-sys 0.52.0", ] @@ -1509,8 +1519,8 @@ dependencies = [ "bytemuck", "librashader-preprocess", "librashader-reflect", + "persy", "platform-dirs", - "rusqlite", "serde", "thiserror", "windows 0.48.0", @@ -1601,7 +1611,7 @@ dependencies = [ "bitflags 2.4.2", "bytemuck", "glslang", - "indexmap 2.2.2", + "indexmap 2.2.3", "librashader-common 0.2.0-beta.9", "librashader-preprocess", "librashader-presets 0.2.0-beta.9", @@ -1776,17 +1786,6 @@ dependencies = [ "redox_syscall 0.4.1", ] -[[package]] -name = "libsqlite3-sys" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1886,7 +1885,7 @@ dependencies = [ "bitflags 2.4.2", "codespan-reporting", "hexf-parse", - "indexmap 2.2.2", + "indexmap 2.2.3", "log", "num-traits", "petgraph", @@ -2179,6 +2178,22 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "persy" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd38c602b23c2f451842d89f27cd5e0d4b292176daf40feeda859c658dcdc76" +dependencies = [ + "crc", + "data-encoding", + "fs2", + "linked-hash-map", + "rand", + "thiserror", + "unsigned-varint", + "zigzag", +] + [[package]] name = "pest" version = "2.7.7" @@ -2231,7 +2246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.2", + "indexmap 2.2.3", ] [[package]] @@ -2288,6 +2303,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "presser" version = "0.3.1" @@ -2375,6 +2396,36 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "range-alloc" version = "0.1.3" @@ -2519,20 +2570,6 @@ dependencies = [ "spirv", ] -[[package]] -name = "rusqlite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" -dependencies = [ - "bitflags 1.3.2", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rust-ini" version = "0.18.0" @@ -2841,18 +2878,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -2916,7 +2953,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.2", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -2980,10 +3017,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] -name = "vcpkg" -version = "0.2.15" +name = "unsigned-varint" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" [[package]] name = "version_check" @@ -3250,7 +3287,7 @@ dependencies = [ "bitflags 2.4.2", "cfg_aliases", "codespan-reporting", - "indexmap 2.2.2", + "indexmap 2.2.3", "log", "naga", "once_cell", @@ -3634,9 +3671,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.5.39" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5389a154b01683d28c77f8f68f49dea75f0a4da32557a58f68ee51ebba472d29" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -3742,6 +3779,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "zigzag" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf" +dependencies = [ + "num-traits", +] + [[package]] name = "zune-inflate" version = "0.2.54" diff --git a/librashader-cache/Cargo.toml b/librashader-cache/Cargo.toml index df26e3c..8ed2c12 100644 --- a/librashader-cache/Cargo.toml +++ b/librashader-cache/Cargo.toml @@ -18,7 +18,8 @@ platform-dirs = "0.3.0" blake3 = { version = "1.3.3" } thiserror = "1.0.38" bincode = { version = "2.0.0-rc.2", features = ["serde"] } -rusqlite = { version = "0.28.0", features = ["bundled"] } + +persy = "1.4.7" bytemuck = "1.13.0" @@ -35,7 +36,7 @@ optional = true d3d = ["windows", "librashader-reflect/dxil"] # hack to get building on docsrs -docsrs = ["blake3/pure", "rusqlite/in_gecko"] +docsrs = ["blake3/pure"] [package.metadata.docs.rs] features = ["docsrs"] diff --git a/librashader-cache/src/cache.rs b/librashader-cache/src/cache.rs index 55c0114..7ea11f2 100644 --- a/librashader-cache/src/cache.rs +++ b/librashader-cache/src/cache.rs @@ -3,10 +3,11 @@ use crate::key::CacheKey; pub(crate) mod internal { use platform_dirs::AppDirs; - use rusqlite::{Connection, DatabaseName}; use std::error::Error; use std::path::PathBuf; + use persy::{Config, Persy, ByteVec, ValueMode}; + pub(crate) fn get_cache_dir() -> Result> { let cache_dir = if let Some(cache_dir) = AppDirs::new(Some("librashader"), false).map(|a| a.cache_dir) @@ -23,46 +24,58 @@ pub(crate) mod internal { Ok(cache_dir) } - pub(crate) fn get_cache() -> Result> { - let cache_dir = get_cache_dir()?; - let mut conn = Connection::open(&cache_dir.join("librashader.db"))?; + // pub(crate) fn get_cache() -> Result> { + // let cache_dir = get_cache_dir()?; + // let mut conn = Connection::open(&cache_dir.join("librashader.db"))?; + // + // let tx = conn.transaction()?; + // tx.pragma_update(Some(DatabaseName::Main), "journal_mode", "wal2")?; + // tx.execute( + // r#"create table if not exists cache ( + // type text not null, + // id blob not null, + // value blob not null unique, + // primary key (id, type) + // )"#, + // [], + // )?; + // tx.commit()?; + // Ok(conn) + // } - let tx = conn.transaction()?; - tx.pragma_update(Some(DatabaseName::Main), "journal_mode", "wal2")?; - tx.execute( - r#"create table if not exists cache ( - type text not null, - id blob not null, - value blob not null unique, - primary key (id, type) - )"#, - [], - )?; - tx.commit()?; + pub(crate) fn get_cache() -> Result> { + let cache_dir = get_cache_dir()?; + let conn = Persy::open_or_create_with(&cache_dir.join("librashader.db.1"), Config::new(), |persy| { + let mut tx = persy.begin()?; + tx.commit()?; + Ok(()) + })?; Ok(conn) } pub(crate) fn get_blob( - conn: &Connection, + conn: &Persy, index: &str, key: &[u8], - ) -> Result, Box> { - let value = conn.query_row( - &*format!("select value from cache where (type = (?1) and id = (?2))"), - rusqlite::params![index, key], - |row| row.get(0), - )?; - Ok(value) + ) -> Result>, Box> { + if !conn.exists_index(index)? { + return Ok(None); + } + + let value = conn.get::<_, ByteVec>(index, &ByteVec::from(key))?.next(); + Ok(value.map(|v| v.to_vec())) } - pub(crate) fn set_blob(conn: &Connection, index: &str, key: &[u8], value: &[u8]) { - match conn.execute( - &*format!("insert or replace into cache (type, id, value) values (?1, ?2, ?3)"), - rusqlite::params![index, key, value], - ) { - Ok(_) => return, - Err(e) => println!("err: {:?}", e), + pub(crate) fn set_blob(conn: &Persy, index: &str, key: &[u8], value: &[u8]) -> Result<(), Box> { + let mut tx = conn.begin()?; + if !tx.exists_index(index)? { + tx.create_index::(index, ValueMode::Replace)?; } + + tx.put(index, ByteVec::from(key), ByteVec::from(value))?; + tx.commit()?; + + Ok(()) } } @@ -101,7 +114,7 @@ where }; 'attempt: { - if let Ok(blob) = internal::get_blob(&cache, index, hashkey.as_bytes()) { + if let Ok(Some(blob)) = internal::get_blob(&cache, index, hashkey.as_bytes()) { let cached = T::from_bytes(&blob).map(&load); match cached { @@ -115,7 +128,7 @@ where let blob = factory(keys)?; if let Some(slice) = T::to_bytes(&blob) { - internal::set_blob(&cache, index, hashkey.as_bytes(), &slice); + let _ = internal::set_blob(&cache, index, hashkey.as_bytes(), &slice); } Ok(load(blob)?) } @@ -157,7 +170,7 @@ where }; let pipeline = 'attempt: { - if let Ok(blob) = internal::get_blob(&cache, index, hashkey.as_bytes()) { + if let Ok(Some(blob)) = internal::get_blob(&cache, index, hashkey.as_bytes()) { let cached = restore_pipeline(Some(blob)); match cached { Ok(res) => { @@ -173,7 +186,8 @@ where // update the pso every time just in case. if let Ok(state) = fetch_pipeline_state(&pipeline) { if let Some(slice) = T::to_bytes(&state) { - internal::set_blob(&cache, index, hashkey.as_bytes(), &slice); + // We don't really care if the transaction fails, just try again next time. + let _ = internal::set_blob(&cache, index, hashkey.as_bytes(), &slice); } } diff --git a/librashader-cache/src/compilation.rs b/librashader-cache/src/compilation.rs index 54a7c7b..b5d1b95 100644 --- a/librashader-cache/src/compilation.rs +++ b/librashader-cache/src/compilation.rs @@ -39,7 +39,7 @@ where }; let compilation = 'cached: { - if let Ok(cached) = crate::cache::internal::get_blob(&cache, "spirv", key.as_bytes()) { + if let Ok(Some(cached)) = crate::cache::internal::get_blob(&cache, "spirv", key.as_bytes()) { let decoded = bincode::serde::decode_from_slice(&cached, bincode::config::standard()) .map(|(compilation, _)| CachedCompilation { compilation }) @@ -58,7 +58,9 @@ where if let Ok(updated) = bincode::serde::encode_to_vec(&compilation.compilation, bincode::config::standard()) { - crate::cache::internal::set_blob(&cache, "spirv", key.as_bytes(), &updated) + let Ok(()) = crate::cache::internal::set_blob(&cache, "spirv", key.as_bytes(), &updated) else { + return Ok(compilation); + }; } Ok(compilation)