jstrong
/
registry-backup
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity
Browse Source

rustfmt

feat/mass-publish-tool
Jonathan Strong 1 year ago
parent
dd4eb957ad
commit
a9c8906096
1 changed files with 123 additions and 94 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 217
      src/publish.rs

217
src/publish.rs
Unescape View File

@ -1,22 +1,22 @@
#![allow(unused_labels)]
use std::path::{Path, PathBuf};
use std::collections::{BTreeMap, HashMap};
use std::io::{self, prelude::*};
use std::path::{Path, PathBuf};
use std::time::*;
use serde::{Serialize, Deserialize};
use anyhow::{anyhow, bail, Context, Error};
use chrono::prelude::*;
use clap::Parser;
use tracing::{debug, error, info, trace, warn};
use tracing_subscriber::filter::EnvFilter;
use anyhow::{anyhow, bail, Error, Context};
use semver::Version;
use convert_case::{Case, Casing};
use futures::stream::StreamExt;
use tokio::io::AsyncBufReadExt;
use tempfile::TempDir;
use rayon::prelude::*;
use chrono::prelude::*;
use convert_case::{Case, Casing};
use semver::Version;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
use tokio::io::AsyncBufReadExt;
use tracing::{debug, error, info, trace, warn};
use tracing_subscriber::filter::EnvFilter;
#[derive(Parser, Debug)]
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))]
@ -130,10 +130,10 @@ fn csv_setup(path: &Path) -> Result<CsvSetup, Error> {
let file = std::fs::File::open(path)?;
let buf = std::io::BufReader::new(file);
let mut rdr = csv::Reader::from_reader(buf);
let headers =
rdr.byte_headers()
.map_err(|e| anyhow!("failed to parse csv headers: {}", e))?
.clone();
let headers = rdr
.byte_headers()
.map_err(|e| anyhow!("failed to parse csv headers: {}", e))?
.clone();
let row = csv::ByteRecord::new();
Ok(CsvSetup { rdr, headers, row })
}
@ -258,17 +258,14 @@ fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Err
cargo_lock = Some(data);
}
if cargo_toml.is_some()
&& cargo_toml_orig.is_some()
&& cargo_lock.is_some()
{
break
if cargo_toml.is_some() && cargo_toml_orig.is_some() && cargo_lock.is_some() {
break;
}
}
if !(cargo_toml.is_some() && cargo_toml_orig.is_some())
{
anyhow::bail!("some required manifest files missing in .crate archive \
if !(cargo_toml.is_some() && cargo_toml_orig.is_some()) {
anyhow::bail!(
"some required manifest files missing in .crate archive \
(cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})",
cargo_toml.is_some(),
cargo_toml_orig.is_some(),
@ -296,9 +293,10 @@ fn load_config_file(opt: &Opt) -> Result<Config, Error> {
bail!("path does not exist: {:?}", opt.config_file);
}
let toml = std::fs::read_to_string(&opt.config_file)?;
let mut config: Config = toml::from_str(&toml)
.context("read config file, but unable to parse toml - check \
format against example config")?;
let mut config: Config = toml::from_str(&toml).context(
"read config file, but unable to parse toml - check \
format against example config",
)?;
// augment using command line opts
config.filter_crates = config.filter_crates.or_else(|| opt.filter_crates.clone());
config.dry_run |= opt.dry_run;
@ -313,9 +311,7 @@ fn is_hidden(entry: &walkdir::DirEntry) -> bool {
.unwrap_or(false)
}
async fn get_index_metas(
config: &Config,
) -> Result<HashMap<String, Vec<IndexMeta>>, Error> {
async fn get_index_metas(config: &Config) -> Result<HashMap<String, Vec<IndexMeta>>, Error> {
let filter = config.compile_filter()?;
let mut n_excl = 0;
@ -362,29 +358,26 @@ async fn get_index_metas(
}
let crate_versions: Vec<Result<(String, Vec<IndexMeta>), Error>> =
futures::stream::iter(files.into_iter().map(|(crate_name, path)| {
async move {
let file = tokio::fs::File::open(&path).await.map_err(|e| {
error!(err = ?e, ?path, "failed to open file");
futures::stream::iter(files.into_iter().map(|(crate_name, path)| async move {
let file = tokio::fs::File::open(&path).await.map_err(|e| {
error!(err = ?e, ?path, "failed to open file");
e
})?;
let buf = tokio::io::BufReader::new(file);
let mut out = Vec::new();
let mut lines = buf.lines();
'lines: while let Some(line) = lines.next_line().await? {
let index_meta: IndexMeta = serde_json::from_str(&line).map_err(|e| {
error!(err = ?e, ?path, "failed to parse line");
e
})?;
let buf = tokio::io::BufReader::new(file);
let mut out = Vec::new();
let mut lines = buf.lines();
'lines: while let Some(line) = lines.next_line().await? {
let index_meta: IndexMeta = serde_json::from_str(&line)
.map_err(|e| {
error!(err = ?e, ?path, "failed to parse line");
e
})?;
out.push(index_meta);
}
debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"),
"parsed {} crate versions from metadata file", out.len()
);
Ok((crate_name, out))
out.push(index_meta);
}
debug!(crate_name = %out.first().map(|x| x.name.as_str()).unwrap_or("na"),
"parsed {} crate versions from metadata file", out.len()
);
Ok((crate_name, out))
}))
.buffer_unordered(num_cpus::get())
.collect()
@ -452,7 +445,10 @@ struct VersionMeta {
impl VersionMeta {
pub fn source_dir(&self) -> PathBuf {
self.tmp.path().join(&format!("{}-{}", self.index_meta.name, self.index_meta.vers))
self.tmp.path().join(&format!(
"{}-{}",
self.index_meta.name, self.index_meta.vers
))
}
}
@ -469,30 +465,32 @@ fn parse_one_manifest(
) -> Result<VersionMeta, Error> {
let version = index_meta.vers.clone();
trace!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir
let dot_crate_path = config
.src
.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path)?;
trace!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = std::fs::read(&dot_crate_path)
.with_context(|| {
format!("failed to read .crate file for \
let dot_crate_bytes = std::fs::read(&dot_crate_path).with_context(|| {
format!(
"failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers,
)
})?;
index_meta.vers,
)
})?;
trace!("extracting Cargo.toml from .crate targz archive");
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_files = extract_manifest_files_from_tar(decoder)
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
err
})?;
let manifest_files = extract_manifest_files_from_tar(decoder).map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
err
})?;
let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path())
tar::Archive::new(decoder)
.unpack(tmp.path())
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir");
err
@ -559,10 +557,7 @@ fn parse_manifests(
/// [target.'cfg(not(target_env = "msvc"))'.dependencies]
/// dep-one = { version = "0.1.0", registry = "old-registry" }
/// ```
fn edit_deps(
manifest: &mut toml_edit::Document,
config: &Config,
) {
fn edit_deps(manifest: &mut toml_edit::Document, config: &Config) {
use toml_edit::{visit_mut::VisitMut, TableLike};
struct DepsVisitor<'a>(&'a Config);
@ -609,21 +604,30 @@ fn edit_publish_registry(
src_registry_name: &str,
dst_registry_name: &str,
) -> Result<(), Error> {
let Some(package) = manifest.get_mut("package").and_then(|item| item.as_table_like_mut()) else {
let Some(package) = manifest
.get_mut("package")
.and_then(|item| item.as_table_like_mut())
else {
anyhow::bail!("package key not found in manifest toml");
};
let Some(publish_item) = package.get_mut("publish") else {
trace!("no 'publish' key in Cargo.toml package section");
return Ok(())
return Ok(());
};
let Some(publish_array) = publish_item.as_array_mut() else {
anyhow::bail!("failed to cast publish item as array");
};
let Some(i) = publish_array.iter().position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false)) else {
anyhow::bail!("publish key exists, but source registry name does not appear in it! (`{}`)", publish_array.to_string());
let Some(i) = publish_array
.iter()
.position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false))
else {
anyhow::bail!(
"publish key exists, but source registry name does not appear in it! (`{}`)",
publish_array.to_string()
);
};
let item_i = publish_array.get_mut(i).unwrap();
@ -634,10 +638,17 @@ fn edit_publish_registry(
fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Result<(), Error> {
let source_dir = meta.source_dir();
let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?;
let mut modified_manifest = meta
.manifest_files
.cargo_toml_orig
.parse::<toml_edit::Document>()?;
edit_deps(&mut modified_manifest, &config);
edit_publish_registry(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
edit_publish_registry(
&mut modified_manifest,
&config.src.registry_name,
&config.dst.registry_name,
)?;
// write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is)
let modified_manifest_toml = modified_manifest.to_string();
@ -675,7 +686,10 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
Ok(())
}
fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap<String, Vec<VersionMeta>>) -> Result<(), Error> {
fn prepare_source_dirs_for_publish(
config: &Config,
manifests: &mut HashMap<String, Vec<VersionMeta>>,
) -> Result<(), Error> {
let begin = Instant::now();
manifests.par_iter_mut()
.map(|(name, versions)| -> Result<(), Error> {
@ -695,8 +709,14 @@ fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap<Stri
fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Result<(), Error> {
let begin = Instant::now();
info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "publishing crate version");
let index_env_key = format!("CARGO_REGISTRIES_{}_INDEX", config.dst.registry_name.to_case(Case::ScreamingSnake));
let token_env_key = format!("CARGO_REGISTRIES_{}_TOKEN", config.dst.registry_name.to_case(Case::ScreamingSnake));
let index_env_key = format!(
"CARGO_REGISTRIES_{}_INDEX",
config.dst.registry_name.to_case(Case::ScreamingSnake)
);
let token_env_key = format!(
"CARGO_REGISTRIES_{}_TOKEN",
config.dst.registry_name.to_case(Case::ScreamingSnake)
);
let source_dir = meta.source_dir();
let manifest_path = source_dir.join("Cargo.toml");
@ -723,8 +743,14 @@ fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Res
let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err");
error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr);
if !stderr.contains("already exists") {
debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig);
debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap());
debug!(
"cargo publish error - original Cargo.toml:\n***\n{}\n***",
meta.manifest_files.cargo_toml_orig
);
debug!(
"cargo publish error - modified Cargo.toml:\n***\n{}\n***",
meta.modified_manifest_toml.as_ref().unwrap()
);
}
}
@ -762,15 +788,18 @@ fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> {
fn read_publish_log_csv(path: &Path) -> Result<Vec<PublishLogRow>, Error> {
let begin = Instant::now();
let CsvSetup { mut rdr, headers, mut row } = csv_setup(path)?;
let CsvSetup {
mut rdr,
headers,
mut row,
} = csv_setup(path)?;
let mut out = Vec::new();
while rdr.read_byte_record(&mut row)? {
// only partially deserialized after this
let parsed: PublishLogRow = row.deserialize(Some(&headers))
.map_err(|err| {
error!(?row, ?headers, ?err, "deserializing row failed");
err
})?;
let parsed: PublishLogRow = row.deserialize(Some(&headers)).map_err(|err| {
error!(?row, ?headers, ?err, "deserializing row failed");
err
})?;
out.push(parsed);
}
info!(?path, "parsed publish log csv in {:?}", begin.elapsed());
@ -796,7 +825,7 @@ fn main() -> Result<(), Error> {
if opt.validate {
println!("{:#?}", config);
return Ok(())
return Ok(());
}
let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?;
@ -805,9 +834,7 @@ fn main() -> Result<(), Error> {
info!(n_rows = publish_log.len(), "parsed publish log csv");
if let Some(filter) = config.compile_filter()? {
publish_log.retain(|x| {
filter.is_match(&x.crate_name)
});
publish_log.retain(|x| filter.is_match(&x.crate_name));
info!(n_filtered_rows = publish_log.len(), "filtered publish log");
}
@ -817,16 +844,18 @@ fn main() -> Result<(), Error> {
prepare_source_dirs_for_publish(&config, &mut manifests)?;
let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests.iter()
.flat_map(|(k, v)| {
v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m))
}).collect();
let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests
.iter()
.flat_map(|(k, v)| v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m)))
.collect();
for row in publish_log.iter() {
let Some(meta) = by_name_vers.remove(&(row.crate_name.as_str(), &row.version)) else {
warn!(?row, "crate version in publish log not found in index versions");
continue
warn!(
?row,
"crate version in publish log not found in index versions"
);
continue;
};
if let Err(err) = cargo_publish_modified_source_dir(&config, meta) {

Write Preview
Loading…
Cancel
Save