diff --git a/tests/src/config.rs b/tests/src/config.rs new file mode 100644 index 0000000..50bd3be --- /dev/null +++ b/tests/src/config.rs @@ -0,0 +1,72 @@ +// Copyright 2021 The piet-gpu authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Also licensed under MIT license, at your choice. + +//! Test config parameters. + +use clap::ArgMatches; + +pub struct Config { + pub groups: Groups, + pub size: Size, +} + +pub struct Groups(String); + +pub enum Size { + Small, + Medium, + Large, +} + +impl Config { + pub fn from_matches(matches: &ArgMatches) -> Config { + let groups = Groups::from_str(matches.value_of("groups").unwrap_or("all")); + let size = Size::from_str(matches.value_of("size").unwrap_or("m")); + Config { + groups, size + } + } +} + +impl Groups { + pub fn from_str(s: &str) -> Groups { + Groups(s.to_string()) + } + + pub fn matches(&self, group_name: &str) -> bool { + self.0 == "all" || self.0 == group_name + } +} + +impl Size { + fn from_str(s: &str) -> Size { + if s == "small" || s == "s" { + Size::Small + } else if s == "large" || s == "l" { + Size::Large + } else { + Size::Medium + } + } + + pub fn choose(&self, small: T, medium: T, large: T) -> T { + match self { + Size::Small => small, + Size::Medium => medium, + Size::Large => large, + } + } +} diff --git a/tests/src/main.rs b/tests/src/main.rs index 85d8a66..b7bc1d9 100644 --- a/tests/src/main.rs +++ b/tests/src/main.rs @@ -16,16 +16,62 @@ //! Tests for piet-gpu shaders and GPU capabilities. +mod config; mod prefix; mod prefix_tree; mod runner; +mod test_result; -use runner::Runner; +use clap::{App, Arg}; + +use crate::config::Config; +use crate::runner::Runner; +use crate::test_result::{ReportStyle, TestResult}; fn main() { + let matches = App::new("piet-gpu-tests") + .arg( + Arg::with_name("verbose") + .short("v") + .long("verbose") + .help("Verbose reporting of results"), + ) + .arg( + Arg::with_name("groups") + .short("g") + .long("groups") + .help("Groups to run") + .takes_value(true) + ) + .arg( + Arg::with_name("size") + .short("s") + .long("size") + .help("Size of tests") + .takes_value(true) + ) + .arg( + Arg::with_name("n_iter") + .short("n") + .long("n_iter") + .help("Number of iterations") + .takes_value(true) + ) + .get_matches(); + let style = if matches.is_present("verbose") { + ReportStyle::Verbose + } else { + ReportStyle::Short + }; + let config = Config::from_matches(&matches); unsafe { + let report = |test_result: &TestResult| { + test_result.report(style); + }; let mut runner = Runner::new(); - prefix::run_prefix_test(&mut runner); - prefix_tree::run_prefix_test(&mut runner); + if config.groups.matches("prefix") { + report(&prefix::run_prefix_test(&mut runner, &config)); + report(&prefix_tree::run_prefix_test(&mut runner, &config)); + } } } diff --git a/tests/src/prefix.rs b/tests/src/prefix.rs index f95470a..adc58b4 100644 --- a/tests/src/prefix.rs +++ b/tests/src/prefix.rs @@ -17,7 +17,9 @@ use piet_gpu_hal::{include_shader, BufferUsage, DescriptorSet}; use piet_gpu_hal::{Buffer, Pipeline}; +use crate::config::Config; use crate::runner::{Commands, Runner}; +use crate::test_result::TestResult; const WG_SIZE: u64 = 512; const N_ROWS: u64 = 16; @@ -46,9 +48,10 @@ struct PrefixBinding { descriptor_set: DescriptorSet, } -pub unsafe fn run_prefix_test(runner: &mut Runner) { +pub unsafe fn run_prefix_test(runner: &mut Runner, config: &Config) -> TestResult { + let mut result = TestResult::new("prefix sum, decoupled look-back"); // This will be configurable. - let n_elements: u64 = 1 << 23; + let n_elements: u64 = config.size.choose(1 << 12, 1 << 24, 1 << 25); let data: Vec = (0..n_elements as u32).collect(); let data_buf = runner .session @@ -74,15 +77,13 @@ pub unsafe fn run_prefix_test(runner: &mut Runner) { if i == 0 { let mut dst: Vec = Default::default(); out_buf.read(&mut dst); - println!("failures: {:?}", verify(&dst)); + if let Some(failure) = verify(&dst) { + result.fail(format!("failure at {}", failure)); + } } } - let throughput = (n_elements * n_iter) as f64 / total_elapsed; - println!( - "total {:?}ms, throughput = {}G el/s", - total_elapsed * 1e3, - throughput * 1e-9 - ); + result.timing(total_elapsed, n_elements * n_iter); + result } impl PrefixCode { diff --git a/tests/src/prefix_tree.rs b/tests/src/prefix_tree.rs index 7b9743a..1f78202 100644 --- a/tests/src/prefix_tree.rs +++ b/tests/src/prefix_tree.rs @@ -17,7 +17,9 @@ use piet_gpu_hal::{include_shader, BufferUsage, DescriptorSet}; use piet_gpu_hal::{Buffer, Pipeline}; +use crate::config::Config; use crate::runner::{Commands, Runner}; +use crate::test_result::TestResult; const WG_SIZE: u64 = 512; const N_ROWS: u64 = 8; @@ -39,11 +41,12 @@ struct PrefixTreeBinding { descriptor_sets: Vec, } -pub unsafe fn run_prefix_test(runner: &mut Runner) { +pub unsafe fn run_prefix_test(runner: &mut Runner, config: &Config) -> TestResult { + let mut result = TestResult::new("prefix sum, tree reduction"); // This will be configurable. Note though that the current code is // prone to reading and writing past the end of buffers if this is // not a power of the number of elements processed in a workgroup. - let n_elements: u64 = 1 << 24; + let n_elements: u64 = config.size.choose(1 << 12, 1 << 24, 1 << 24); let data: Vec = (0..n_elements as u32).collect(); let data_buf = runner .session @@ -71,15 +74,13 @@ pub unsafe fn run_prefix_test(runner: &mut Runner) { if i == 0 { let mut dst: Vec = Default::default(); out_buf.read(&mut dst); - println!("failures: {:?}", verify(&dst)); + if let Some(failure) = verify(&dst) { + result.fail(format!("failure at {}", failure)); + } } } - let throughput = (n_elements * n_iter) as f64 / total_elapsed; - println!( - "total {:?}ms, throughput = {}G el/s", - total_elapsed * 1e3, - throughput * 1e-9 - ); + result.timing(total_elapsed, n_elements * n_iter); + result } impl PrefixTreeCode { diff --git a/tests/src/test_result.rs b/tests/src/test_result.rs new file mode 100644 index 0000000..84bbc85 --- /dev/null +++ b/tests/src/test_result.rs @@ -0,0 +1,110 @@ +// Copyright 2021 The piet-gpu authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Also licensed under MIT license, at your choice. + +//! Recording of results from tests. + +pub struct TestResult { + name: String, + // TODO: statistics. We're lean and mean for now. + total_time: f64, + n_elements: u64, + failure: Option, +} + +#[derive(Clone, Copy)] +pub enum ReportStyle { + Short, + Verbose, +} + +impl TestResult { + pub fn new(name: &str) -> TestResult { + TestResult { + name: name.to_string(), + total_time: 0.0, + n_elements: 0, + failure: None, + } + } + + pub fn report(&self, style: ReportStyle) { + let fail_string = match &self.failure { + None => "pass".into(), + Some(s) => format!("fail ({})", s), + }; + match style { + ReportStyle::Short => { + let mut timing_string = String::new(); + if self.total_time > 0.0 { + if self.n_elements > 0 { + let throughput = self.n_elements as f64 / self.total_time; + timing_string = format!(" {} elements/s", format_nice(throughput, 1)); + } else { + timing_string = format!(" {}s", format_nice(self.total_time, 1)); + } + } + println!("{}: {}{}", self.name, fail_string, timing_string) + } + ReportStyle::Verbose => { + println!("test {}", self.name); + println!(" {}", fail_string); + if self.total_time > 0.0 { + println!(" {}s total time", format_nice(self.total_time, 1)); + if self.n_elements > 0 { + println!(" {} elements", self.n_elements); + let throughput = self.n_elements as f64 / self.total_time; + println!(" {} elements/s", format_nice(throughput, 1)); + } + } + } + } + } + + pub fn fail(&mut self, explanation: String) { + self.failure = Some(explanation); + } + + pub fn timing(&mut self, total_time: f64, n_elements: u64) { + self.total_time = total_time; + self.n_elements = n_elements; + } +} + +fn format_nice(x: f64, precision: usize) -> String { + // Precision should probably scale; later + let (scale, suffix) = if x >= 1e12 && x < 1e15 { + (1e-12, "T") + } else if x >= 1e9 { + (1e-9, "G") + } else if x >= 1e6 { + (1e-6, "M") + } else if x >= 1e3 { + (1e-3, "k") + } else if x >= 1.0 { + (1.0, "") + } else if x >= 1e-3 { + (1e3, "m") + } else if x >= 1e-6 { + (1e6, "\u{00b5}") + } else if x >= 1e-9 { + (1e9, "n") + } else if x >= 1e-12 { + (1e12, "p") + } else { + return format!("{:.*e}", precision, x); + }; + format!("{:.*}{}", precision, scale * x, suffix) +}