jstrong
/
registry-backup
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity
Browse Source

rustfmt

feat/mass-publish-tool
Jonathan Strong 1 year ago
parent
dd4eb957ad
commit
a9c8906096
1 changed files with 123 additions and 94 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 217
      src/publish.rs

217
src/publish.rs
Unescape View File

@ -1,22 +1,22 @@
#![allow(unused_labels)] #![allow(unused_labels)]
use std::path::{Path, PathBuf};
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::io::{self, prelude::*}; use std::io::{self, prelude::*};
use std::path::{Path, PathBuf};
use std::time::*; use std::time::*;
use serde::{Serialize, Deserialize}; use anyhow::{anyhow, bail, Context, Error};
use chrono::prelude::*;
use clap::Parser; use clap::Parser;
use tracing::{debug, error, info, trace, warn}; use convert_case::{Case, Casing};
use tracing_subscriber::filter::EnvFilter;
use anyhow::{anyhow, bail, Error, Context};
use semver::Version;
use futures::stream::StreamExt; use futures::stream::StreamExt;
use tokio::io::AsyncBufReadExt;
use tempfile::TempDir;
use rayon::prelude::*; use rayon::prelude::*;
use chrono::prelude::*; use semver::Version;
use convert_case::{Case, Casing}; use serde::{Deserialize, Serialize};
use tempfile::TempDir;
use tokio::io::AsyncBufReadExt;
use tracing::{debug, error, info, trace, warn};
use tracing_subscriber::filter::EnvFilter;
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] #[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))]
@ -130,10 +130,10 @@ fn csv_setup(path: &Path) -> Result<CsvSetup, Error> {
let file = std::fs::File::open(path)?; let file = std::fs::File::open(path)?;
let buf = std::io::BufReader::new(file); let buf = std::io::BufReader::new(file);
let mut rdr = csv::Reader::from_reader(buf); let mut rdr = csv::Reader::from_reader(buf);
let headers = let headers = rdr
rdr.byte_headers() .byte_headers()
.map_err(|e| anyhow!("failed to parse csv headers: {}", e))? .map_err(|e| anyhow!("failed to parse csv headers: {}", e))?
.clone(); .clone();
let row = csv::ByteRecord::new(); let row = csv::ByteRecord::new();
Ok(CsvSetup { rdr, headers, row }) Ok(CsvSetup { rdr, headers, row })
} }
@ -258,17 +258,14 @@ fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Err
cargo_lock = Some(data); cargo_lock = Some(data);
} }
if cargo_toml.is_some() if cargo_toml.is_some() && cargo_toml_orig.is_some() && cargo_lock.is_some() {
&& cargo_toml_orig.is_some() break;
&& cargo_lock.is_some()
{
break
} }
} }
if !(cargo_toml.is_some() && cargo_toml_orig.is_some()) if !(cargo_toml.is_some() && cargo_toml_orig.is_some()) {
{ anyhow::bail!(
anyhow::bail!("some required manifest files missing in .crate archive \ "some required manifest files missing in .crate archive \
(cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})", (cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})",
cargo_toml.is_some(), cargo_toml.is_some(),
cargo_toml_orig.is_some(), cargo_toml_orig.is_some(),
@ -296,9 +293,10 @@ fn load_config_file(opt: &Opt) -> Result<Config, Error> {
bail!("path does not exist: {:?}", opt.config_file); bail!("path does not exist: {:?}", opt.config_file);
} }
let toml = std::fs::read_to_string(&opt.config_file)?; let toml = std::fs::read_to_string(&opt.config_file)?;
let mut config: Config = toml::from_str(&toml) let mut config: Config = toml::from_str(&toml).context(
.context("read config file, but unable to parse toml - check \ "read config file, but unable to parse toml - check \
format against example config")?; format against example config",
)?;
// augment using command line opts // augment using command line opts
config.filter_crates = config.filter_crates.or_else(|| opt.filter_crates.clone()); config.filter_crates = config.filter_crates.or_else(|| opt.filter_crates.clone());
config.dry_run |= opt.dry_run; config.dry_run |= opt.dry_run;
@ -313,9 +311,7 @@ fn is_hidden(entry: &walkdir::DirEntry) -> bool {
.unwrap_or(false) .unwrap_or(false)
} }
async fn get_index_metas( async fn get_index_metas(config: &Config) -> Result<HashMap<String, Vec<IndexMeta>>, Error> {
config: &Config,
) -> Result<HashMap<String, Vec<IndexMeta>>, Error> {
let filter = config.compile_filter()?; let filter = config.compile_filter()?;
let mut n_excl = 0; let mut n_excl = 0;
@ -362,29 +358,26 @@ async fn get_index_metas(
} }
let crate_versions: Vec<Result<(String, Vec<IndexMeta>), Error>> = let crate_versions: Vec<Result<(String, Vec<IndexMeta>), Error>> =
futures::stream::iter(files.into_iter().map(|(crate_name, path)| { futures::stream::iter(files.into_iter().map(|(crate_name, path)| async move {
async move { let file = tokio::fs::File::open(&path).await.map_err(|e| {
let file = tokio::fs::File::open(&path).await.map_err(|e| { error!(err = ?e, ?path, "failed to open file");
error!(err = ?e, ?path, "failed to open file"); e
})?;
let buf = tokio::io::BufReader::new(file);
let mut out = Vec::new();
let mut lines = buf.lines();
'lines: while let Some(line) = lines.next_line().await? {
let index_meta: IndexMeta = serde_json::from_str(&line).map_err(|e| {
error!(err = ?e, ?path, "failed to parse line");
e e
})?; })?;
let buf = tokio::io::BufReader::new(file); out.push(index_meta);
let mut out = Vec::new();
let mut lines = buf.lines();
'lines: while let Some(line) = lines.next_line().await? {
let index_meta: IndexMeta = serde_json::from_str(&line)
.map_err(|e| {
error!(err = ?e, ?path, "failed to parse line");
e
})?;
out.push(index_meta);
}
debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"),
"parsed {} crate versions from metadata file", out.len()
);
Ok((crate_name, out))
} }
debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"),
"parsed {} crate versions from metadata file", out.len()
);
Ok((crate_name, out))
})) }))
.buffer_unordered(num_cpus::get()) .buffer_unordered(num_cpus::get())
.collect() .collect()
@ -452,7 +445,10 @@ struct VersionMeta {
impl VersionMeta { impl VersionMeta {
pub fn source_dir(&self) -> PathBuf { pub fn source_dir(&self) -> PathBuf {
self.tmp.path().join(&format!("{}-{}", self.index_meta.name, self.index_meta.vers)) self.tmp.path().join(&format!(
"{}-{}",
self.index_meta.name, self.index_meta.vers
))
} }
} }
@ -469,30 +465,32 @@ fn parse_one_manifest(
) -> Result<VersionMeta, Error> { ) -> Result<VersionMeta, Error> {
let version = index_meta.vers.clone(); let version = index_meta.vers.clone();
trace!(%crate_name, %version, "processing crate version"); trace!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir let dot_crate_path = config
.src
.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers)); .join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path)?; verify_file_exists(&dot_crate_path)?;
trace!(path = ?dot_crate_path, "reading .crate file"); trace!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = std::fs::read(&dot_crate_path) let dot_crate_bytes = std::fs::read(&dot_crate_path).with_context(|| {
.with_context(|| { format!(
format!("failed to read .crate file for \ "failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}", {crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers, index_meta.vers,
) )
})?; })?;
trace!("extracting Cargo.toml from .crate targz archive"); trace!("extracting Cargo.toml from .crate targz archive");
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_files = extract_manifest_files_from_tar(decoder) let manifest_files = extract_manifest_files_from_tar(decoder).map_err(|err| {
.map_err(|err| { error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); err
err })?;
})?;
let tmp = TempDir::new()?; let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path()) tar::Archive::new(decoder)
.unpack(tmp.path())
.map_err(|err| { .map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir"); error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir");
err err
@ -559,10 +557,7 @@ fn parse_manifests(
/// [target.'cfg(not(target_env = "msvc"))'.dependencies] /// [target.'cfg(not(target_env = "msvc"))'.dependencies]
/// dep-one = { version = "0.1.0", registry = "old-registry" } /// dep-one = { version = "0.1.0", registry = "old-registry" }
/// ``` /// ```
fn edit_deps( fn edit_deps(manifest: &mut toml_edit::Document, config: &Config) {
manifest: &mut toml_edit::Document,
config: &Config,
) {
use toml_edit::{visit_mut::VisitMut, TableLike}; use toml_edit::{visit_mut::VisitMut, TableLike};
struct DepsVisitor<'a>(&'a Config); struct DepsVisitor<'a>(&'a Config);
@ -609,21 +604,30 @@ fn edit_publish_registry(
src_registry_name: &str, src_registry_name: &str,
dst_registry_name: &str, dst_registry_name: &str,
) -> Result<(), Error> { ) -> Result<(), Error> {
let Some(package) = manifest.get_mut("package").and_then(|item| item.as_table_like_mut()) else { let Some(package) = manifest
.get_mut("package")
.and_then(|item| item.as_table_like_mut())
else {
anyhow::bail!("package key not found in manifest toml"); anyhow::bail!("package key not found in manifest toml");
}; };
let Some(publish_item) = package.get_mut("publish") else { let Some(publish_item) = package.get_mut("publish") else {
trace!("no 'publish' key in Cargo.toml package section"); trace!("no 'publish' key in Cargo.toml package section");
return Ok(()) return Ok(());
}; };
let Some(publish_array) = publish_item.as_array_mut() else { let Some(publish_array) = publish_item.as_array_mut() else {
anyhow::bail!("failed to cast publish item as array"); anyhow::bail!("failed to cast publish item as array");
}; };
let Some(i) = publish_array.iter().position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false)) else { let Some(i) = publish_array
anyhow::bail!("publish key exists, but source registry name does not appear in it! (`{}`)", publish_array.to_string()); .iter()
.position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false))
else {
anyhow::bail!(
"publish key exists, but source registry name does not appear in it! (`{}`)",
publish_array.to_string()
);
}; };
let item_i = publish_array.get_mut(i).unwrap(); let item_i = publish_array.get_mut(i).unwrap();
@ -634,10 +638,17 @@ fn edit_publish_registry(
fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Result<(), Error> { fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Result<(), Error> {
let source_dir = meta.source_dir(); let source_dir = meta.source_dir();
let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?; let mut modified_manifest = meta
.manifest_files
.cargo_toml_orig
.parse::<toml_edit::Document>()?;
edit_deps(&mut modified_manifest, &config); edit_deps(&mut modified_manifest, &config);
edit_publish_registry(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; edit_publish_registry(
&mut modified_manifest,
&config.src.registry_name,
&config.dst.registry_name,
)?;
// write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is) // write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is)
let modified_manifest_toml = modified_manifest.to_string(); let modified_manifest_toml = modified_manifest.to_string();
@ -675,7 +686,10 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
Ok(()) Ok(())
} }
fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap<String, Vec<VersionMeta>>) -> Result<(), Error> { fn prepare_source_dirs_for_publish(
config: &Config,
manifests: &mut HashMap<String, Vec<VersionMeta>>,
) -> Result<(), Error> {
let begin = Instant::now(); let begin = Instant::now();
manifests.par_iter_mut() manifests.par_iter_mut()
.map(|(name, versions)| -> Result<(), Error> { .map(|(name, versions)| -> Result<(), Error> {
@ -695,8 +709,14 @@ fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap<Stri
fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Result<(), Error> { fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Result<(), Error> {
let begin = Instant::now(); let begin = Instant::now();
info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "publishing crate version"); info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "publishing crate version");
let index_env_key = format!("CARGO_REGISTRIES_{}_INDEX", config.dst.registry_name.to_case(Case::ScreamingSnake)); let index_env_key = format!(
let token_env_key = format!("CARGO_REGISTRIES_{}_TOKEN", config.dst.registry_name.to_case(Case::ScreamingSnake)); "CARGO_REGISTRIES_{}_INDEX",
config.dst.registry_name.to_case(Case::ScreamingSnake)
);
let token_env_key = format!(
"CARGO_REGISTRIES_{}_TOKEN",
config.dst.registry_name.to_case(Case::ScreamingSnake)
);
let source_dir = meta.source_dir(); let source_dir = meta.source_dir();
let manifest_path = source_dir.join("Cargo.toml"); let manifest_path = source_dir.join("Cargo.toml");
@ -723,8 +743,14 @@ fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Res
let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err"); let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err");
error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr); error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr);
if !stderr.contains("already exists") { if !stderr.contains("already exists") {
debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig); debug!(
debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap()); "cargo publish error - original Cargo.toml:\n***\n{}\n***",
meta.manifest_files.cargo_toml_orig
);
debug!(
"cargo publish error - modified Cargo.toml:\n***\n{}\n***",
meta.modified_manifest_toml.as_ref().unwrap()
);
} }
} }
@ -762,15 +788,18 @@ fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> {
fn read_publish_log_csv(path: &Path) -> Result<Vec<PublishLogRow>, Error> { fn read_publish_log_csv(path: &Path) -> Result<Vec<PublishLogRow>, Error> {
let begin = Instant::now(); let begin = Instant::now();
let CsvSetup { mut rdr, headers, mut row } = csv_setup(path)?; let CsvSetup {
mut rdr,
headers,
mut row,
} = csv_setup(path)?;
let mut out = Vec::new(); let mut out = Vec::new();
while rdr.read_byte_record(&mut row)? { while rdr.read_byte_record(&mut row)? {
// only partially deserialized after this // only partially deserialized after this
let parsed: PublishLogRow = row.deserialize(Some(&headers)) let parsed: PublishLogRow = row.deserialize(Some(&headers)).map_err(|err| {
.map_err(|err| { error!(?row, ?headers, ?err, "deserializing row failed");
error!(?row, ?headers, ?err, "deserializing row failed"); err
err })?;
})?;
out.push(parsed); out.push(parsed);
} }
info!(?path, "parsed publish log csv in {:?}", begin.elapsed()); info!(?path, "parsed publish log csv in {:?}", begin.elapsed());
@ -796,7 +825,7 @@ fn main() -> Result<(), Error> {
if opt.validate { if opt.validate {
println!("{:#?}", config); println!("{:#?}", config);
return Ok(()) return Ok(());
} }
let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?; let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?;
@ -805,9 +834,7 @@ fn main() -> Result<(), Error> {
info!(n_rows = publish_log.len(), "parsed publish log csv"); info!(n_rows = publish_log.len(), "parsed publish log csv");
if let Some(filter) = config.compile_filter()? { if let Some(filter) = config.compile_filter()? {
publish_log.retain(|x| { publish_log.retain(|x| filter.is_match(&x.crate_name));
filter.is_match(&x.crate_name)
});
info!(n_filtered_rows = publish_log.len(), "filtered publish log"); info!(n_filtered_rows = publish_log.len(), "filtered publish log");
} }
@ -817,16 +844,18 @@ fn main() -> Result<(), Error> {
prepare_source_dirs_for_publish(&config, &mut manifests)?; prepare_source_dirs_for_publish(&config, &mut manifests)?;
let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests.iter() let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests
.flat_map(|(k, v)| { .iter()
v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m)) .flat_map(|(k, v)| v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m)))
}).collect(); .collect();
for row in publish_log.iter() { for row in publish_log.iter() {
let Some(meta) = by_name_vers.remove(&(row.crate_name.as_str(), &row.version)) else { let Some(meta) = by_name_vers.remove(&(row.crate_name.as_str(), &row.version)) else {
warn!(?row, "crate version in publish log not found in index versions"); warn!(
continue ?row,
"crate version in publish log not found in index versions"
);
continue;
}; };
if let Err(err) = cargo_publish_modified_source_dir(&config, meta) { if let Err(err) = cargo_publish_modified_source_dir(&config, meta) {

Write Preview
Loading…
Cancel
Save