From 70979fc7da247b46553d01d911268447f986e396 Mon Sep 17 00:00:00 2001 From: Gwilym Inzani Date: Tue, 11 Apr 2023 21:54:48 +0100 Subject: [PATCH 1/4] Publish gbafix --- tools/src/publish.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/src/publish.rs b/tools/src/publish.rs index da637e1e..1a352332 100644 --- a/tools/src/publish.rs +++ b/tools/src/publish.rs @@ -39,7 +39,9 @@ pub fn publish(matches: &ArgMatches) -> Result<(), Error> { let mut published_crates: HashSet = HashSet::new(); let dependencies = build_dependency_graph(&root_directory)?; - let crates_to_publish: Vec<_> = dependencies.keys().collect(); + let mut crates_to_publish: Vec<_> = dependencies.keys().collect(); + let agb_gbafix = "agb-gbafix".to_owned(); + crates_to_publish.push(&agb_gbafix); while published_crates.len() != crates_to_publish.len() { // find all crates which can be published now but haven't @@ -47,10 +49,13 @@ pub fn publish(matches: &ArgMatches) -> Result<(), Error> { .iter() .filter(|&&crate_to_publish| !published_crates.contains(crate_to_publish)) .filter(|&&crate_to_publish| { - let dependencies_of_crate = &dependencies[crate_to_publish]; - for dependency_of_crate in dependencies_of_crate { - if !fully_published_crates.contains(dependency_of_crate) { - return false; + let dependencies_of_crate = &dependencies.get(crate_to_publish); + + if let Some(dependencies_of_crate) = dependencies_of_crate { + for dependency_of_crate in *dependencies_of_crate { + if !fully_published_crates.contains(dependency_of_crate) { + return false; + } } } From 63995995ac740f02c12206144d3532c96a65b4a6 Mon Sep 17 00:00:00 2001 From: Gwilym Inzani Date: Tue, 11 Apr 2023 22:29:23 +0100 Subject: [PATCH 2/4] Make publishing a bit smarter --- tools/Cargo.lock | 26 +++++++ tools/Cargo.toml | 1 + tools/src/publish.rs | 172 +++++++++++++++++++------------------------ 3 files changed, 101 insertions(+), 98 deletions(-) diff --git a/tools/Cargo.lock b/tools/Cargo.lock index a2508c2c..6c804fdc 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -138,6 +138,15 @@ dependencies = [ "syn 2.0.8", ] +[[package]] +name = "dependency-graph" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143247629540606d0888beae9ca0e0b9a81a32151bfecd0b2be4a961155c24d" +dependencies = [ + "petgraph", +] + [[package]] name = "errno" version = "0.2.8" @@ -159,6 +168,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "glob" version = "0.3.1" @@ -310,6 +325,16 @@ version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +[[package]] +name = "petgraph" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "proc-macro2" version = "1.0.53" @@ -419,6 +444,7 @@ version = "0.1.0" dependencies = [ "chrono", "clap", + "dependency-graph", "glob", "toml_edit", ] diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 8c1095ac..30f662e6 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -10,3 +10,4 @@ clap = "4" toml_edit = "0.19" glob = "0.3" chrono = "0.4" +dependency-graph = "0.1.5" diff --git a/tools/src/publish.rs b/tools/src/publish.rs index 1a352332..8bbe20c7 100644 --- a/tools/src/publish.rs +++ b/tools/src/publish.rs @@ -1,10 +1,11 @@ use clap::{Arg, ArgAction, ArgMatches}; -use std::collections::{HashMap, HashSet}; +use dependency_graph::DependencyGraph; +use std::borrow::BorrowMut; +use std::cell::RefCell; +use std::collections::HashMap; use std::fs; use std::path::Path; use std::process::Command; -use std::thread; -use std::time::Duration; use toml_edit::Document; use crate::utils::*; @@ -19,6 +20,23 @@ pub enum Error { CargoToml, } +struct Package { + name: String, + dependencies: Vec, +} + +impl dependency_graph::Node for Package { + type DependencyType = String; + + fn dependencies(&self) -> &[Self::DependencyType] { + &self.dependencies + } + + fn matches(&self, dependency: &Self::DependencyType) -> bool { + &self.name == dependency + } +} + pub fn command() -> clap::Command { clap::Command::new("publish") .about("Publishes agb and all subcrates") @@ -32,96 +50,51 @@ pub fn command() -> clap::Command { pub fn publish(matches: &ArgMatches) -> Result<(), Error> { let dry_run = matches.get_one::("Dry run").expect("defined by clap"); + let dry_run = if *dry_run { vec!["--dry-run"] } else { vec![] }; let root_directory = find_agb_root_directory().map_err(|_| Error::FindRootDirectory)?; - let mut fully_published_crates: HashSet = HashSet::new(); - let mut published_crates: HashSet = HashSet::new(); + let mut in_progress: HashMap<_, RefCell> = HashMap::new(); let dependencies = build_dependency_graph(&root_directory)?; - let mut crates_to_publish: Vec<_> = dependencies.keys().collect(); - let agb_gbafix = "agb-gbafix".to_owned(); - crates_to_publish.push(&agb_gbafix); + let graph = DependencyGraph::from(&dependencies[..]); - while published_crates.len() != crates_to_publish.len() { - // find all crates which can be published now but haven't - let publishable_crates: Vec<_> = crates_to_publish - .iter() - .filter(|&&crate_to_publish| !published_crates.contains(crate_to_publish)) - .filter(|&&crate_to_publish| { - let dependencies_of_crate = &dependencies.get(crate_to_publish); + for package in graph { + let package = package.as_resolved().unwrap(); - if let Some(dependencies_of_crate) = dependencies_of_crate { - for dependency_of_crate in *dependencies_of_crate { - if !fully_published_crates.contains(dependency_of_crate) { - return false; - } - } - } - - true - }) - .collect(); - - for publishable_crate in publishable_crates { - if *dry_run { - println!("Would execute cargo publish for {publishable_crate}"); - } else { - assert!(Command::new("cargo") - .arg("publish") - .current_dir(&root_directory.join(publishable_crate)) - .status() - .map_err(|_| Error::PublishCrate)? - .success()); - } - - published_crates.insert(publishable_crate.to_string()); + for dep in &package.dependencies { + assert!(in_progress + .get(dep) + .unwrap() + .borrow_mut() + .wait() + .map_err(|_| Error::PublishCrate)? + .success()); } - for published_crate in published_crates.iter() { - if !fully_published_crates.contains(published_crate) { - let expected_version = - read_cargo_toml_version(&root_directory.join(published_crate))?; - if check_if_released(published_crate, &expected_version)? { - fully_published_crates.insert(published_crate.clone()); - } - } - } + println!("Publishing {}", package.name); - thread::sleep(Duration::from_secs(10)); + let publish_cmd = Command::new("cargo") + .arg("publish") + .args(&dry_run) + .current_dir(root_directory.join(&package.name)) + .spawn() + .map_err(|_| Error::PublishCrate)?; + + in_progress.insert(package.name.clone(), RefCell::new(publish_cmd)); + } + + for (_, in_progress) in in_progress { + assert!(in_progress + .borrow_mut() + .wait() + .map_err(|_| Error::PublishCrate)? + .success()); } Ok(()) } -fn check_if_released(crate_to_publish: &str, expected_version: &str) -> Result { - let url_to_poll = &get_url_to_poll(crate_to_publish); - - println!("Polling crates.io with URL {url_to_poll} for {crate_to_publish} hoping for version {expected_version}."); - - let curl_result = Command::new("curl") - .arg(url_to_poll) - .output() - .map_err(|_| Error::Poll)?; - - Ok(String::from_utf8_lossy(&curl_result.stdout).contains(expected_version)) -} - -fn get_url_to_poll(crate_name: &str) -> String { - let crate_name_with_underscores = crate_name.replace('-', "_"); - - let crate_folder = if crate_name_with_underscores.len() == 3 { - format!("3/{}", crate_name_with_underscores.chars().next().unwrap()) - } else { - let first_two_characters = &crate_name_with_underscores[0..2]; - let second_two_characters = &crate_name_with_underscores[2..4]; - - format!("{first_two_characters}/{second_two_characters}") - }; - - format!("https://raw.githubusercontent.com/rust-lang/crates.io-index/master/{crate_folder}/{crate_name_with_underscores}") -} - fn read_cargo_toml_version(folder: &Path) -> Result { let cargo_toml = read_cargo_toml(folder)?; @@ -134,30 +107,33 @@ fn read_cargo_toml_version(folder: &Path) -> Result { Ok(version_value.to_owned()) } -fn build_dependency_graph(root: &Path) -> Result>, Error> { - let mut result = HashMap::new(); - result.insert("agb".to_owned(), get_agb_dependencies(&root.join("agb"))?); +fn build_dependency_graph(root: &Path) -> Result, Error> { + let dirs = fs::read_dir(root).map_err(|_| Error::ReadingDependencies)?; + let mut packages = vec![]; - let mut added_new_crates = true; - while added_new_crates { - added_new_crates = false; - - let all_crates: HashSet = HashSet::from_iter(result.values().flatten().cloned()); - - for dep_crate in all_crates { - if result.contains_key(&dep_crate) { - continue; - } - - added_new_crates = true; - result.insert( - dep_crate.to_owned(), - get_agb_dependencies(&root.join(dep_crate))?, - ); + for dir in dirs { + let dir = dir.map_err(|_| Error::ReadingDependencies)?; + if !dir + .file_type() + .map_err(|_| Error::ReadingDependencies)? + .is_dir() + { + continue; } + + if !dir.file_name().to_string_lossy().starts_with("agb") { + continue; + } + + let crate_path = root.join(dir.path()); + + packages.push(Package { + name: dir.file_name().to_string_lossy().to_string(), + dependencies: get_agb_dependencies(&crate_path)?, + }); } - Ok(result) + Ok(packages) } fn get_agb_dependencies(folder: &Path) -> Result, Error> { From f677ff1840b93f0679b7a967d4a8dbc317b23aa8 Mon Sep 17 00:00:00 2001 From: Gwilym Inzani Date: Tue, 11 Apr 2023 22:30:47 +0100 Subject: [PATCH 3/4] Fix clippy lints --- tools/src/publish.rs | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/tools/src/publish.rs b/tools/src/publish.rs index 8bbe20c7..be68209f 100644 --- a/tools/src/publish.rs +++ b/tools/src/publish.rs @@ -1,6 +1,5 @@ use clap::{Arg, ArgAction, ArgMatches}; use dependency_graph::DependencyGraph; -use std::borrow::BorrowMut; use std::cell::RefCell; use std::collections::HashMap; use std::fs; @@ -95,18 +94,6 @@ pub fn publish(matches: &ArgMatches) -> Result<(), Error> { Ok(()) } -fn read_cargo_toml_version(folder: &Path) -> Result { - let cargo_toml = read_cargo_toml(folder)?; - - let version_value = cargo_toml["package"]["version"] - .as_value() - .ok_or(Error::CrateVersion)? - .as_str() - .ok_or(Error::CrateVersion)?; - - Ok(version_value.to_owned()) -} - fn build_dependency_graph(root: &Path) -> Result, Error> { let dirs = fs::read_dir(root).map_err(|_| Error::ReadingDependencies)?; let mut packages = vec![]; @@ -172,34 +159,6 @@ mod test { command().debug_assert(); } - #[test] - fn url_to_poll_should_return_correct_url() { - let test_cases = [ - ["agb", "3/a/agb"], - ["agb-image-converter", "ag/b_/agb_image_converter"], - ["agb-fixnum", "ag/b_/agb_fixnum"], - ]; - - for [name, result] in test_cases { - let url = get_url_to_poll(name); - assert_eq!( - url, - format!( - "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/{result}", - ) - ) - } - } - - #[test] - fn should_read_version() -> Result<(), Error> { - let root_directory = crate::utils::find_agb_root_directory().unwrap(); - let my_version = read_cargo_toml_version(&root_directory.join("tools"))?; - - assert_eq!(my_version, "0.1.0"); - Ok(()) - } - #[test] fn should_detect_dependencies() -> Result<(), Error> { let root_directory = crate::utils::find_agb_root_directory().unwrap(); From 0e006bd3d5795783be3e4335c35b04cd0a235ae5 Mon Sep 17 00:00:00 2001 From: Gwilym Inzani Date: Wed, 12 Apr 2023 15:12:00 +0100 Subject: [PATCH 4/4] Update publish justfile command to accept arguments --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 032849cb..4a0b9e7d 100644 --- a/justfile +++ b/justfile @@ -81,7 +81,7 @@ update-linker-scripts: find -type f -name gba.ld | grep -v ./agb/gba.ld | xargs -n1 cp -v -- agb/gba.ld find -type f -name gba_mb.ld | grep -v ./agb/gba_mb.ld | xargs -n1 cp -v -- agb/gba_mb.ld -publish: (_run-tool "publish") +publish *args: (_run-tool "publish" args) release +args: (_run-tool "release" args)