Improve collection and reporting of test results

Have a structured way of gathering test results, rather than the
existing ad hoc approach of just printing stuff.

The details are still pretty primitive, but there's room to grow.

tests/src/config.rs

@@ -0,0 +1,72 @@
+// Copyright 2021 The piet-gpu authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Also licensed under MIT license, at your choice.
+
+//! Test config parameters.
+
+use clap::ArgMatches;
+
+pub struct Config {
+    pub groups: Groups,
+    pub size: Size,
+}
+
+pub struct Groups(String);
+
+pub enum Size {
+    Small,
+    Medium,
+    Large,
+}
+
+impl Config {
+    pub fn from_matches(matches: &ArgMatches) -> Config {
+        let groups = Groups::from_str(matches.value_of("groups").unwrap_or("all"));
+        let size = Size::from_str(matches.value_of("size").unwrap_or("m"));
+        Config {
+            groups, size
+        }
+    }
+}
+
+impl Groups {
+    pub fn from_str(s: &str) -> Groups {
+        Groups(s.to_string())
+    }
+
+    pub fn matches(&self, group_name: &str) -> bool {
+        self.0 == "all" || self.0 == group_name
+    }
+}
+
+impl Size {
+    fn from_str(s: &str) -> Size {
+        if s == "small" || s == "s" {
+            Size::Small
+        } else if s == "large" || s == "l" {
+            Size::Large
+        } else {
+            Size::Medium
+        }
+    }
+
+    pub fn choose<T>(&self, small: T, medium: T, large: T) -> T {
+        match self {
+            Size::Small => small,
+            Size::Medium => medium,
+            Size::Large => large,
+        }
+    }
+}

tests/src/main.rs

@@ -16,16 +16,62 @@
 
 //! Tests for piet-gpu shaders and GPU capabilities.
 
+mod config;
 mod prefix;
 mod prefix_tree;
 mod runner;
+mod test_result;
 
-use runner::Runner;
+use clap::{App, Arg};
+
+use crate::config::Config;
+use crate::runner::Runner;
+use crate::test_result::{ReportStyle, TestResult};
 
 fn main() {
+    let matches = App::new("piet-gpu-tests")
+        .arg(
+            Arg::with_name("verbose")
+                .short("v")
+                .long("verbose")
+                .help("Verbose reporting of results"),
+        )
+        .arg(
+            Arg::with_name("groups")
+                .short("g")
+                .long("groups")
+                .help("Groups to run")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("size")
+                .short("s")
+                .long("size")
+                .help("Size of tests")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("n_iter")
+                .short("n")
+                .long("n_iter")
+                .help("Number of iterations")
+                .takes_value(true)
+        )
+        .get_matches();
+    let style = if matches.is_present("verbose") {
+        ReportStyle::Verbose
+    } else {
+        ReportStyle::Short
+    };
+    let config = Config::from_matches(&matches);
     unsafe {
+        let report = |test_result: &TestResult| {
+            test_result.report(style);
+        };
         let mut runner = Runner::new();
-        prefix::run_prefix_test(&mut runner);
+        if config.groups.matches("prefix") {
-        prefix_tree::run_prefix_test(&mut runner);
+            report(&prefix::run_prefix_test(&mut runner, &config));
+            report(&prefix_tree::run_prefix_test(&mut runner, &config));
+        }
     }
 }

tests/src/prefix.rs

@@ -17,7 +17,9 @@
 use piet_gpu_hal::{include_shader, BufferUsage, DescriptorSet};
 use piet_gpu_hal::{Buffer, Pipeline};
 
+use crate::config::Config;
 use crate::runner::{Commands, Runner};
+use crate::test_result::TestResult;
 
 const WG_SIZE: u64 = 512;
 const N_ROWS: u64 = 16;
@@ -46,9 +48,10 @@ struct PrefixBinding {
     descriptor_set: DescriptorSet,
 }
 
-pub unsafe fn run_prefix_test(runner: &mut Runner) {
+pub unsafe fn run_prefix_test(runner: &mut Runner, config: &Config) -> TestResult {
+    let mut result = TestResult::new("prefix sum, decoupled look-back");
     // This will be configurable.
-    let n_elements: u64 = 1 << 23;
+    let n_elements: u64 = config.size.choose(1 << 12, 1 << 24, 1 << 25);
     let data: Vec<u32> = (0..n_elements as u32).collect();
     let data_buf = runner
         .session
@@ -74,15 +77,13 @@ pub unsafe fn run_prefix_test(runner: &mut Runner) {
         if i == 0 {
             let mut dst: Vec<u32> = Default::default();
             out_buf.read(&mut dst);
-            println!("failures: {:?}", verify(&dst));
+            if let Some(failure) = verify(&dst) {
+                result.fail(format!("failure at {}", failure));
             }
         }
-    let throughput = (n_elements * n_iter) as f64 / total_elapsed;
+    }
-    println!(
+    result.timing(total_elapsed, n_elements * n_iter);
-        "total {:?}ms, throughput = {}G el/s",
+    result
-        total_elapsed * 1e3,
-        throughput * 1e-9
-    );
 }
 
 impl PrefixCode {

tests/src/prefix_tree.rs

@@ -17,7 +17,9 @@
 use piet_gpu_hal::{include_shader, BufferUsage, DescriptorSet};
 use piet_gpu_hal::{Buffer, Pipeline};
 
+use crate::config::Config;
 use crate::runner::{Commands, Runner};
+use crate::test_result::TestResult;
 
 const WG_SIZE: u64 = 512;
 const N_ROWS: u64 = 8;
@@ -39,11 +41,12 @@ struct PrefixTreeBinding {
     descriptor_sets: Vec<DescriptorSet>,
 }
 
-pub unsafe fn run_prefix_test(runner: &mut Runner) {
+pub unsafe fn run_prefix_test(runner: &mut Runner, config: &Config) -> TestResult {
+    let mut result = TestResult::new("prefix sum, tree reduction");
     // This will be configurable. Note though that the current code is
     // prone to reading and writing past the end of buffers if this is
     // not a power of the number of elements processed in a workgroup.
-    let n_elements: u64 = 1 << 24;
+    let n_elements: u64 = config.size.choose(1 << 12, 1 << 24, 1 << 24);
     let data: Vec<u32> = (0..n_elements as u32).collect();
     let data_buf = runner
         .session
@@ -71,15 +74,13 @@ pub unsafe fn run_prefix_test(runner: &mut Runner) {
         if i == 0 {
             let mut dst: Vec<u32> = Default::default();
             out_buf.read(&mut dst);
-            println!("failures: {:?}", verify(&dst));
+            if let Some(failure) = verify(&dst) {
+                result.fail(format!("failure at {}", failure));
             }
         }
-    let throughput = (n_elements * n_iter) as f64 / total_elapsed;
+    }
-    println!(
+    result.timing(total_elapsed, n_elements * n_iter);
-        "total {:?}ms, throughput = {}G el/s",
+    result
-        total_elapsed * 1e3,
-        throughput * 1e-9
-    );
 }
 
 impl PrefixTreeCode {

tests/src/test_result.rs

@@ -0,0 +1,110 @@
+// Copyright 2021 The piet-gpu authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Also licensed under MIT license, at your choice.
+
+//! Recording of results from tests.
+
+pub struct TestResult {
+    name: String,
+    // TODO: statistics. We're lean and mean for now.
+    total_time: f64,
+    n_elements: u64,
+    failure: Option<String>,
+}
+
+#[derive(Clone, Copy)]
+pub enum ReportStyle {
+    Short,
+    Verbose,
+}
+
+impl TestResult {
+    pub fn new(name: &str) -> TestResult {
+        TestResult {
+            name: name.to_string(),
+            total_time: 0.0,
+            n_elements: 0,
+            failure: None,
+        }
+    }
+
+    pub fn report(&self, style: ReportStyle) {
+        let fail_string = match &self.failure {
+            None => "pass".into(),
+            Some(s) => format!("fail ({})", s),
+        };
+        match style {
+            ReportStyle::Short => {
+                let mut timing_string = String::new();
+                if self.total_time > 0.0 {
+                    if self.n_elements > 0 {
+                        let throughput = self.n_elements as f64 / self.total_time;
+                        timing_string = format!(" {} elements/s", format_nice(throughput, 1));
+                    } else {
+                        timing_string = format!(" {}s", format_nice(self.total_time, 1));
+                    }
+                }
+                println!("{}: {}{}", self.name, fail_string, timing_string)
+            }
+            ReportStyle::Verbose => {
+                println!("test {}", self.name);
+                println!("  {}", fail_string);
+                if self.total_time > 0.0 {
+                    println!("  {}s total time", format_nice(self.total_time, 1));
+                    if self.n_elements > 0 {
+                        println!("  {} elements", self.n_elements);
+                        let throughput = self.n_elements as f64 / self.total_time;
+                        println!("  {} elements/s", format_nice(throughput, 1));
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn fail(&mut self, explanation: String) {
+        self.failure = Some(explanation);
+    }
+
+    pub fn timing(&mut self, total_time: f64, n_elements: u64) {
+        self.total_time = total_time;
+        self.n_elements = n_elements;
+    }
+}
+
+fn format_nice(x: f64, precision: usize) -> String {
+    // Precision should probably scale; later
+    let (scale, suffix) = if x >= 1e12 && x < 1e15 {
+        (1e-12, "T")
+    } else if x >= 1e9 {
+        (1e-9, "G")
+    } else if x >= 1e6 {
+        (1e-6, "M")
+    } else if x >= 1e3 {
+        (1e-3, "k")
+    } else if x >= 1.0 {
+        (1.0, "")
+    } else if x >= 1e-3 {
+        (1e3, "m")
+    } else if x >= 1e-6 {
+        (1e6, "\u{00b5}")
+    } else if x >= 1e-9 {
+        (1e9, "n")
+    } else if x >= 1e-12 {
+        (1e12, "p")
+    } else {
+        return format!("{:.*e}", precision, x);
+    };
+    format!("{:.*}{}", precision, scale * x, suffix)
+}