diff --git a/src/publish.rs b/src/publish.rs index a1b7d3f..124588e 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -1,22 +1,22 @@ #![allow(unused_labels)] -use std::path::{Path, PathBuf}; use std::collections::{BTreeMap, HashMap}; use std::io::{self, prelude::*}; +use std::path::{Path, PathBuf}; use std::time::*; -use serde::{Serialize, Deserialize}; +use anyhow::{anyhow, bail, Context, Error}; +use chrono::prelude::*; use clap::Parser; -use tracing::{debug, error, info, trace, warn}; -use tracing_subscriber::filter::EnvFilter; -use anyhow::{anyhow, bail, Error, Context}; -use semver::Version; +use convert_case::{Case, Casing}; use futures::stream::StreamExt; -use tokio::io::AsyncBufReadExt; -use tempfile::TempDir; use rayon::prelude::*; -use chrono::prelude::*; -use convert_case::{Case, Casing}; +use semver::Version; +use serde::{Deserialize, Serialize}; +use tempfile::TempDir; +use tokio::io::AsyncBufReadExt; +use tracing::{debug, error, info, trace, warn}; +use tracing_subscriber::filter::EnvFilter; #[derive(Parser, Debug)] #[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] @@ -130,10 +130,10 @@ fn csv_setup(path: &Path) -> Result { let file = std::fs::File::open(path)?; let buf = std::io::BufReader::new(file); let mut rdr = csv::Reader::from_reader(buf); - let headers = - rdr.byte_headers() - .map_err(|e| anyhow!("failed to parse csv headers: {}", e))? - .clone(); + let headers = rdr + .byte_headers() + .map_err(|e| anyhow!("failed to parse csv headers: {}", e))? + .clone(); let row = csv::ByteRecord::new(); Ok(CsvSetup { rdr, headers, row }) } @@ -258,17 +258,14 @@ fn extract_manifest_files_from_tar(rdr: R) -> Result Result { bail!("path does not exist: {:?}", opt.config_file); } let toml = std::fs::read_to_string(&opt.config_file)?; - let mut config: Config = toml::from_str(&toml) - .context("read config file, but unable to parse toml - check \ - format against example config")?; + let mut config: Config = toml::from_str(&toml).context( + "read config file, but unable to parse toml - check \ + format against example config", + )?; // augment using command line opts config.filter_crates = config.filter_crates.or_else(|| opt.filter_crates.clone()); config.dry_run |= opt.dry_run; @@ -313,9 +311,7 @@ fn is_hidden(entry: &walkdir::DirEntry) -> bool { .unwrap_or(false) } -async fn get_index_metas( - config: &Config, -) -> Result>, Error> { +async fn get_index_metas(config: &Config) -> Result>, Error> { let filter = config.compile_filter()?; let mut n_excl = 0; @@ -362,29 +358,26 @@ async fn get_index_metas( } let crate_versions: Vec), Error>> = - futures::stream::iter(files.into_iter().map(|(crate_name, path)| { - async move { - let file = tokio::fs::File::open(&path).await.map_err(|e| { - error!(err = ?e, ?path, "failed to open file"); + futures::stream::iter(files.into_iter().map(|(crate_name, path)| async move { + let file = tokio::fs::File::open(&path).await.map_err(|e| { + error!(err = ?e, ?path, "failed to open file"); + e + })?; + let buf = tokio::io::BufReader::new(file); + let mut out = Vec::new(); + let mut lines = buf.lines(); + 'lines: while let Some(line) = lines.next_line().await? { + let index_meta: IndexMeta = serde_json::from_str(&line).map_err(|e| { + error!(err = ?e, ?path, "failed to parse line"); e })?; - let buf = tokio::io::BufReader::new(file); - let mut out = Vec::new(); - let mut lines = buf.lines(); - 'lines: while let Some(line) = lines.next_line().await? { - let index_meta: IndexMeta = serde_json::from_str(&line) - .map_err(|e| { - error!(err = ?e, ?path, "failed to parse line"); - e - })?; - out.push(index_meta); - } - debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"), - "parsed {} crate versions from metadata file", out.len() - ); - - Ok((crate_name, out)) + out.push(index_meta); } + debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"), + "parsed {} crate versions from metadata file", out.len() + ); + + Ok((crate_name, out)) })) .buffer_unordered(num_cpus::get()) .collect() @@ -452,7 +445,10 @@ struct VersionMeta { impl VersionMeta { pub fn source_dir(&self) -> PathBuf { - self.tmp.path().join(&format!("{}-{}", self.index_meta.name, self.index_meta.vers)) + self.tmp.path().join(&format!( + "{}-{}", + self.index_meta.name, self.index_meta.vers + )) } } @@ -469,30 +465,32 @@ fn parse_one_manifest( ) -> Result { let version = index_meta.vers.clone(); trace!(%crate_name, %version, "processing crate version"); - let dot_crate_path = config.src.crate_files_dir + let dot_crate_path = config + .src + .crate_files_dir .join(&format!("{}/{}/download", crate_name, index_meta.vers)); verify_file_exists(&dot_crate_path)?; trace!(path = ?dot_crate_path, "reading .crate file"); - let dot_crate_bytes = std::fs::read(&dot_crate_path) - .with_context(|| { - format!("failed to read .crate file for \ + let dot_crate_bytes = std::fs::read(&dot_crate_path).with_context(|| { + format!( + "failed to read .crate file for \ {crate_name} v{0} with path {dot_crate_path:?}", - index_meta.vers, - ) - })?; + index_meta.vers, + ) + })?; trace!("extracting Cargo.toml from .crate targz archive"); let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - let manifest_files = extract_manifest_files_from_tar(decoder) - .map_err(|err| { - error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); - err - })?; + let manifest_files = extract_manifest_files_from_tar(decoder).map_err(|err| { + error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); + err + })?; let tmp = TempDir::new()?; let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - tar::Archive::new(decoder).unpack(tmp.path()) + tar::Archive::new(decoder) + .unpack(tmp.path()) .map_err(|err| { error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir"); err @@ -559,10 +557,7 @@ fn parse_manifests( /// [target.'cfg(not(target_env = "msvc"))'.dependencies] /// dep-one = { version = "0.1.0", registry = "old-registry" } /// ``` -fn edit_deps( - manifest: &mut toml_edit::Document, - config: &Config, -) { +fn edit_deps(manifest: &mut toml_edit::Document, config: &Config) { use toml_edit::{visit_mut::VisitMut, TableLike}; struct DepsVisitor<'a>(&'a Config); @@ -609,21 +604,30 @@ fn edit_publish_registry( src_registry_name: &str, dst_registry_name: &str, ) -> Result<(), Error> { - let Some(package) = manifest.get_mut("package").and_then(|item| item.as_table_like_mut()) else { + let Some(package) = manifest + .get_mut("package") + .and_then(|item| item.as_table_like_mut()) + else { anyhow::bail!("package key not found in manifest toml"); }; let Some(publish_item) = package.get_mut("publish") else { trace!("no 'publish' key in Cargo.toml package section"); - return Ok(()) + return Ok(()); }; let Some(publish_array) = publish_item.as_array_mut() else { anyhow::bail!("failed to cast publish item as array"); }; - let Some(i) = publish_array.iter().position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false)) else { - anyhow::bail!("publish key exists, but source registry name does not appear in it! (`{}`)", publish_array.to_string()); + let Some(i) = publish_array + .iter() + .position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false)) + else { + anyhow::bail!( + "publish key exists, but source registry name does not appear in it! (`{}`)", + publish_array.to_string() + ); }; let item_i = publish_array.get_mut(i).unwrap(); @@ -634,10 +638,17 @@ fn edit_publish_registry( fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Result<(), Error> { let source_dir = meta.source_dir(); - let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::()?; + let mut modified_manifest = meta + .manifest_files + .cargo_toml_orig + .parse::()?; edit_deps(&mut modified_manifest, &config); - edit_publish_registry(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; + edit_publish_registry( + &mut modified_manifest, + &config.src.registry_name, + &config.dst.registry_name, + )?; // write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is) let modified_manifest_toml = modified_manifest.to_string(); @@ -675,7 +686,10 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re Ok(()) } -fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap>) -> Result<(), Error> { +fn prepare_source_dirs_for_publish( + config: &Config, + manifests: &mut HashMap>, +) -> Result<(), Error> { let begin = Instant::now(); manifests.par_iter_mut() .map(|(name, versions)| -> Result<(), Error> { @@ -695,8 +709,14 @@ fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap Result<(), Error> { let begin = Instant::now(); info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "publishing crate version"); - let index_env_key = format!("CARGO_REGISTRIES_{}_INDEX", config.dst.registry_name.to_case(Case::ScreamingSnake)); - let token_env_key = format!("CARGO_REGISTRIES_{}_TOKEN", config.dst.registry_name.to_case(Case::ScreamingSnake)); + let index_env_key = format!( + "CARGO_REGISTRIES_{}_INDEX", + config.dst.registry_name.to_case(Case::ScreamingSnake) + ); + let token_env_key = format!( + "CARGO_REGISTRIES_{}_TOKEN", + config.dst.registry_name.to_case(Case::ScreamingSnake) + ); let source_dir = meta.source_dir(); let manifest_path = source_dir.join("Cargo.toml"); @@ -723,8 +743,14 @@ fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Res let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err"); error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr); if !stderr.contains("already exists") { - debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig); - debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap()); + debug!( + "cargo publish error - original Cargo.toml:\n***\n{}\n***", + meta.manifest_files.cargo_toml_orig + ); + debug!( + "cargo publish error - modified Cargo.toml:\n***\n{}\n***", + meta.modified_manifest_toml.as_ref().unwrap() + ); } } @@ -762,15 +788,18 @@ fn verify_file_exists>(path: P) -> Result<(), Error> { fn read_publish_log_csv(path: &Path) -> Result, Error> { let begin = Instant::now(); - let CsvSetup { mut rdr, headers, mut row } = csv_setup(path)?; + let CsvSetup { + mut rdr, + headers, + mut row, + } = csv_setup(path)?; let mut out = Vec::new(); while rdr.read_byte_record(&mut row)? { // only partially deserialized after this - let parsed: PublishLogRow = row.deserialize(Some(&headers)) - .map_err(|err| { - error!(?row, ?headers, ?err, "deserializing row failed"); - err - })?; + let parsed: PublishLogRow = row.deserialize(Some(&headers)).map_err(|err| { + error!(?row, ?headers, ?err, "deserializing row failed"); + err + })?; out.push(parsed); } info!(?path, "parsed publish log csv in {:?}", begin.elapsed()); @@ -796,7 +825,7 @@ fn main() -> Result<(), Error> { if opt.validate { println!("{:#?}", config); - return Ok(()) + return Ok(()); } let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?; @@ -805,9 +834,7 @@ fn main() -> Result<(), Error> { info!(n_rows = publish_log.len(), "parsed publish log csv"); if let Some(filter) = config.compile_filter()? { - publish_log.retain(|x| { - filter.is_match(&x.crate_name) - }); + publish_log.retain(|x| filter.is_match(&x.crate_name)); info!(n_filtered_rows = publish_log.len(), "filtered publish log"); } @@ -817,16 +844,18 @@ fn main() -> Result<(), Error> { prepare_source_dirs_for_publish(&config, &mut manifests)?; - let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests.iter() - .flat_map(|(k, v)| { - v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m)) - }).collect(); - + let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests + .iter() + .flat_map(|(k, v)| v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m))) + .collect(); for row in publish_log.iter() { let Some(meta) = by_name_vers.remove(&(row.crate_name.as_str(), &row.version)) else { - warn!(?row, "crate version in publish log not found in index versions"); - continue + warn!( + ?row, + "crate version in publish log not found in index versions" + ); + continue; }; if let Err(err) = cargo_publish_modified_source_dir(&config, meta) {