Merge pull request #285 from gwilymk/rewrite-shell-scripts-in-rust

Write release script in rust
This commit is contained in:
Gwilym Kuiper 2022-08-04 23:03:50 +01:00 committed by GitHub
commit 66e4e3759b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 486 additions and 46 deletions

View file

@ -1,44 +0,0 @@
#!/usr/bin/env bash
set -e # Fail if any command fails
function wait_for_release() {
local package="$1"
local package_with_underscores="${package/-/_}"
local first_two_characters="${package_with_underscores:0:2}"
local second_two_characters="${package_with_underscores:2:2}"
local path="$first_two_characters/$second_two_characters"
if [ "$package" == "agb" ]; then
path="3/a"
fi
local url_to_poll="https://raw.githubusercontent.com/rust-lang/crates.io-index/master/$path/$package_with_underscores"
local expected_version
expected_version=$(grep -E '^version' Cargo.toml | grep -oE '[0-9.]+')
local attempts=1
while [ $attempts -le 15 ]; do
echo "Polling crates.io with URL $url_to_poll to see if the version has updated (attempt $attempts)"
if curl "$url_to_poll" | grep "$expected_version"; then
return
fi
sleep 30s
attempts=$((attempts + 1))
done
}
PROJECTS_TO_RELEASE_IN_ORDER="agb-macros agb-fixnum agb-image-converter agb-sound-converter agb"
for PROJECT in $PROJECTS_TO_RELEASE_IN_ORDER; do
pushd "$PROJECT"
echo "Publishing $PROJECT"
cargo publish
wait_for_release "$PROJECT"
popd
done

View file

@ -19,8 +19,10 @@ jobs:
- name: Login to crates.io - name: Login to crates.io
run: cargo login ${{ secrets.CRATE_API }} run: cargo login ${{ secrets.CRATE_API }}
- uses: extractions/setup-just@v1
- name: Publish crates - name: Publish crates
run: bash .github/scripts/publish-crate.sh run: just publish
- name: Update template repo - name: Update template repo
env: env:
@ -30,7 +32,6 @@ jobs:
- name: Install gbafix - name: Install gbafix
run: cargo install gbafix run: cargo install gbafix
- uses: extractions/setup-just@v1
- name: Build the examples - name: Build the examples
run: just build-roms run: just build-roms
- name: Upload examples to the release - name: Upload examples to the release

View file

@ -9,11 +9,13 @@ build-release:
just _build-release agb just _build-release agb
clippy: clippy:
just _all-crates _clippy just _all-crates _clippy
just _clippy tools
test: test:
just _test-debug agb just _test-debug agb
just _test-debug agb-fixnum just _test-debug agb-fixnum
just _test-debug-arm agb just _test-debug-arm agb
just _test-debug tools
test-release: test-release:
just _test-release agb just _test-release agb
@ -64,6 +66,11 @@ update-linker-scripts:
find -type f -name gba.ld | grep -v ./agb/gba.ld | xargs -n1 cp -v -- agb/gba.ld find -type f -name gba.ld | grep -v ./agb/gba.ld | xargs -n1 cp -v -- agb/gba.ld
find -type f -name gba_mb.ld | grep -v ./agb/gba_mb.ld | xargs -n1 cp -v -- agb/gba_mb.ld find -type f -name gba_mb.ld | grep -v ./agb/gba_mb.ld | xargs -n1 cp -v -- agb/gba_mb.ld
publish: (_run-tool "publish")
_run-tool +tool:
cargo run --manifest-path "{{justfile_directory() + "/tools/Cargo.toml"}}" -- {{tool}}
_build-rom folder name: _build-rom folder name:
#!/usr/bin/env bash #!/usr/bin/env bash
set -euxo pipefail set -euxo pipefail

195
tools/Cargo.lock generated Normal file
View file

@ -0,0 +1,195 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bytes"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
[[package]]
name = "clap"
version = "3.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9"
dependencies = [
"atty",
"bitflags",
"clap_lex",
"indexmap",
"strsim",
"termcolor",
"textwrap",
]
[[package]]
name = "clap_lex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "combine"
version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948"
dependencies = [
"bytes",
"memchr",
]
[[package]]
name = "either"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "indexmap"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "itertools"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
dependencies = [
"either",
]
[[package]]
name = "libc"
version = "0.2.127"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "os_str_bytes"
version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4"
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "termcolor"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "toml_edit"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5376256e44f2443f8896ac012507c19a012df0fe8758b55246ae51a2279db51f"
dependencies = [
"combine",
"indexmap",
"itertools",
]
[[package]]
name = "tools"
version = "0.1.0"
dependencies = [
"clap",
"toml_edit",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

10
tools/Cargo.toml Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "tools"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
clap = "3.2.16"
toml_edit = "0.14.4"

21
tools/src/main.rs Normal file
View file

@ -0,0 +1,21 @@
#![deny(clippy::all)]
use clap::Command;
mod publish;
fn main() {
let matches = Command::new("Agb tools")
.subcommand_required(true)
.arg_required_else_help(true)
.subcommand(publish::command())
.get_matches();
let result = match matches.subcommand() {
Some(("publish", arg_matches)) => publish::publish(arg_matches),
_ => unreachable!("Exhausted list of subcommands and subcommand_required prevents `None`"),
};
if let Err(e) = result {
eprintln!("Error: {:?}", e);
}
}

250
tools/src/publish.rs Normal file
View file

@ -0,0 +1,250 @@
use clap::{Arg, ArgAction, ArgMatches};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;
use std::{env, thread};
use toml_edit::Document;
#[derive(Debug)]
pub enum Error {
FindRootDirectory,
PublishCrate,
Poll,
CrateVersion,
ReadingDependencies,
CargoToml,
}
pub fn command() -> clap::Command<'static> {
clap::Command::new("publish")
.about("Publishes agb and all subcrates")
.arg(
Arg::new("Dry run")
.long("dry-run")
.help("Don't actually publish")
.action(ArgAction::SetTrue),
)
}
pub fn publish(matches: &ArgMatches) -> Result<(), Error> {
let dry_run = matches.get_one::<bool>("Dry run").expect("defined by clap");
let root_directory = find_agb_root_directory()?;
let mut fully_published_crates: HashSet<String> = HashSet::new();
let mut published_crates: HashSet<String> = HashSet::new();
let dependencies = build_dependency_graph(&root_directory)?;
let crates_to_publish: Vec<_> = dependencies.keys().collect();
while published_crates.len() != crates_to_publish.len() {
// find all crates which can be published now but haven't
let publishable_crates: Vec<_> = crates_to_publish
.iter()
.filter(|&&crate_to_publish| !published_crates.contains(crate_to_publish))
.filter(|&&crate_to_publish| {
let dependencies_of_crate = &dependencies[crate_to_publish];
for dependency_of_crate in dependencies_of_crate {
if !fully_published_crates.contains(dependency_of_crate) {
return false;
}
}
true
})
.collect();
for publishable_crate in publishable_crates {
if *dry_run {
println!("Would execute cargo publish for {publishable_crate}");
} else {
Command::new("cargo")
.arg("publish")
.current_dir(&root_directory.join(publishable_crate))
.spawn()
.map_err(|_| Error::PublishCrate)?;
}
published_crates.insert(publishable_crate.to_string());
}
for published_crate in published_crates.iter() {
if !fully_published_crates.contains(published_crate) {
let expected_version =
read_cargo_toml_version(&root_directory.join(published_crate))?;
if check_if_released(published_crate, &expected_version)? {
fully_published_crates.insert(published_crate.clone());
}
}
}
thread::sleep(Duration::from_secs(10));
}
Ok(())
}
fn find_agb_root_directory() -> Result<PathBuf, Error> {
let mut current_path = env::current_dir().map_err(|_| Error::FindRootDirectory)?;
while !current_path.clone().join("justfile").exists() {
current_path = current_path
.parent()
.ok_or(Error::FindRootDirectory)?
.to_owned();
}
Ok(current_path)
}
fn check_if_released(crate_to_publish: &str, expected_version: &str) -> Result<bool, Error> {
let url_to_poll = &get_url_to_poll(crate_to_publish);
println!("Polling crates.io with URL {url_to_poll} for {crate_to_publish} hoping for version {expected_version}.");
let curl_result = Command::new("curl")
.arg(url_to_poll)
.output()
.map_err(|_| Error::Poll)?;
Ok(String::from_utf8_lossy(&curl_result.stdout).contains(expected_version))
}
fn get_url_to_poll(crate_name: &str) -> String {
let crate_name_with_underscores = crate_name.replace('-', "_");
let crate_folder = if crate_name_with_underscores.len() == 3 {
format!("3/{}", crate_name_with_underscores.chars().next().unwrap())
} else {
let first_two_characters = &crate_name_with_underscores[0..2];
let second_two_characters = &crate_name_with_underscores[2..4];
format!("{first_two_characters}/{second_two_characters}")
};
format!("https://raw.githubusercontent.com/rust-lang/crates.io-index/master/{crate_folder}/{crate_name_with_underscores}")
}
fn read_cargo_toml_version(folder: &Path) -> Result<String, Error> {
let cargo_toml = read_cargo_toml(folder)?;
let version_value = cargo_toml["package"]["version"]
.as_value()
.ok_or(Error::CrateVersion)?
.as_str()
.ok_or(Error::CrateVersion)?;
Ok(version_value.to_owned())
}
fn build_dependency_graph(root: &Path) -> Result<HashMap<String, Vec<String>>, Error> {
let mut result = HashMap::new();
result.insert("agb".to_owned(), get_agb_dependencies(&root.join("agb"))?);
let mut added_new_crates = true;
while added_new_crates {
added_new_crates = false;
let all_crates: HashSet<String> = HashSet::from_iter(result.values().flatten().cloned());
for dep_crate in all_crates {
if result.contains_key(&dep_crate) {
continue;
}
added_new_crates = true;
result.insert(
dep_crate.to_owned(),
get_agb_dependencies(&root.join(dep_crate))?,
);
}
}
Ok(result)
}
fn get_agb_dependencies(folder: &Path) -> Result<Vec<String>, Error> {
let cargo_toml = read_cargo_toml(folder)?;
let dependencies = cargo_toml["dependencies"]
.as_table()
.ok_or(Error::ReadingDependencies)?
.get_values();
let mut result = vec![];
for (key, _) in dependencies {
let dep = key[0].get();
if dep.starts_with("agb") {
result.push(dep.replace('_', "-"))
}
}
Ok(result)
}
fn read_cargo_toml(folder: &Path) -> Result<Document, Error> {
let cargo_toml_contents =
fs::read_to_string(folder.join("Cargo.toml")).map_err(|_| Error::CargoToml)?;
let cargo_toml: Document = cargo_toml_contents.parse().map_err(|_| Error::CargoToml)?;
Ok(cargo_toml)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn url_to_poll_should_return_correct_url() {
let test_cases = [
["agb", "3/a/agb"],
["agb-image-converter", "ag/b_/agb_image_converter"],
["agb-fixnum", "ag/b_/agb_fixnum"],
];
for [name, result] in test_cases {
let url = get_url_to_poll(name);
assert_eq!(
url,
format!(
"https://raw.githubusercontent.com/rust-lang/crates.io-index/master/{result}",
)
)
}
}
#[test]
fn should_find_root_directory() -> Result<(), Error> {
assert_ne!(find_agb_root_directory()?.to_string_lossy(), "");
Ok(())
}
#[test]
fn should_read_version() -> Result<(), Error> {
let root_directory = find_agb_root_directory()?;
let my_version = read_cargo_toml_version(&root_directory.join("tools"))?;
assert_eq!(my_version, "0.1.0");
Ok(())
}
#[test]
fn should_detect_dependencies() -> Result<(), Error> {
let root_directory = find_agb_root_directory()?;
let deps = get_agb_dependencies(&root_directory.join("agb"))?;
assert_eq!(
deps,
&[
"agb-image-converter",
"agb-sound-converter",
"agb-macros",
"agb-fixnum"
]
);
Ok(())
}
}