|
|
|
@ -2,7 +2,7 @@ |
|
|
|
|
|
|
|
|
|
|
|
use std::path::{Path, PathBuf}; |
|
|
|
use std::path::{Path, PathBuf}; |
|
|
|
use std::collections::{BTreeMap, HashMap}; |
|
|
|
use std::collections::{BTreeMap, HashMap}; |
|
|
|
use std::io::prelude::*; |
|
|
|
use std::io::{self, prelude::*}; |
|
|
|
use std::time::*; |
|
|
|
use std::time::*; |
|
|
|
|
|
|
|
|
|
|
|
use serde::{Serialize, Deserialize}; |
|
|
|
use serde::{Serialize, Deserialize}; |
|
|
|
@ -17,9 +17,8 @@ use tokio::io::AsyncBufReadExt; |
|
|
|
use reqwest::header::AUTHORIZATION; |
|
|
|
use reqwest::header::AUTHORIZATION; |
|
|
|
use tempfile::TempDir; |
|
|
|
use tempfile::TempDir; |
|
|
|
use rayon::prelude::*; |
|
|
|
use rayon::prelude::*; |
|
|
|
use petgraph::stable_graph::StableGraph; |
|
|
|
use chrono::prelude::*; |
|
|
|
use petgraph::visit::{Bfs, EdgeRef, Topo, Walker}; |
|
|
|
use convert_case::{Case, Casing}; |
|
|
|
use petgraph::graph::NodeIndex; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Parser, Debug)] |
|
|
|
#[derive(Parser, Debug)] |
|
|
|
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] |
|
|
|
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] |
|
|
|
@ -50,8 +49,16 @@ struct Opt { |
|
|
|
pub struct DestinationRegistryConfig { |
|
|
|
pub struct DestinationRegistryConfig { |
|
|
|
#[serde(alias = "api")] |
|
|
|
#[serde(alias = "api")] |
|
|
|
pub api_url: Url, |
|
|
|
pub api_url: Url, |
|
|
|
|
|
|
|
/// Registry index url, i.e. the url provided to Cargo via configuration
|
|
|
|
|
|
|
|
/// to identify where to pull the index metadata from.
|
|
|
|
|
|
|
|
#[serde(alias = "index")] |
|
|
|
|
|
|
|
pub index_url: String, |
|
|
|
#[serde(alias = "token")] |
|
|
|
#[serde(alias = "token")] |
|
|
|
pub auth_token: String, |
|
|
|
pub auth_token: String, |
|
|
|
|
|
|
|
/// The name the registry should have in the Cargo.toml files published to
|
|
|
|
|
|
|
|
/// the destination registry. This can be a rename (i.e. different than the
|
|
|
|
|
|
|
|
/// registry name provided in `SourceRegistryConfig`) or the same name.
|
|
|
|
|
|
|
|
pub registry_name: String, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, Deserialize)] |
|
|
|
#[derive(Debug, Clone, Deserialize)] |
|
|
|
@ -61,6 +68,10 @@ pub struct SourceRegistryConfig { |
|
|
|
pub index_dir: PathBuf, |
|
|
|
pub index_dir: PathBuf, |
|
|
|
#[serde(alias = "crate-files")] |
|
|
|
#[serde(alias = "crate-files")] |
|
|
|
pub crate_files_dir: PathBuf, |
|
|
|
pub crate_files_dir: PathBuf, |
|
|
|
|
|
|
|
/// Name used in Cargo.toml for dependencies from the registry.
|
|
|
|
|
|
|
|
pub registry_name: String, |
|
|
|
|
|
|
|
/// Path of CSV file with log of when each crate version was published.
|
|
|
|
|
|
|
|
pub publish_history_csv: PathBuf, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Deserialize, Debug, Clone)] |
|
|
|
#[derive(Deserialize, Debug, Clone)] |
|
|
|
@ -125,6 +136,38 @@ impl Config { |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
|
|
|
|
pub struct PublishLogRow { |
|
|
|
|
|
|
|
pub crate_name: String, |
|
|
|
|
|
|
|
pub version: Version, |
|
|
|
|
|
|
|
pub path: PathBuf, |
|
|
|
|
|
|
|
pub commit: String, |
|
|
|
|
|
|
|
pub author: String, |
|
|
|
|
|
|
|
pub time: DateTime<Utc>, |
|
|
|
|
|
|
|
pub unix_nanos: u64, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
struct CsvSetup { |
|
|
|
|
|
|
|
pub rdr: csv::Reader<io::BufReader<std::fs::File>>, |
|
|
|
|
|
|
|
pub headers: csv::ByteRecord, |
|
|
|
|
|
|
|
pub row: csv::ByteRecord, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn csv_setup(path: &Path) -> Result<CsvSetup, Error> { |
|
|
|
|
|
|
|
verify_file_exists(path)?; |
|
|
|
|
|
|
|
let file = std::fs::File::open(path)?; |
|
|
|
|
|
|
|
let buf = std::io::BufReader::new(file); |
|
|
|
|
|
|
|
let mut rdr = csv::Reader::from_reader(buf); |
|
|
|
|
|
|
|
let headers = |
|
|
|
|
|
|
|
rdr.byte_headers() |
|
|
|
|
|
|
|
.map_err(|e| anyhow!("failed to parse csv headers: {}", e))? |
|
|
|
|
|
|
|
.clone(); |
|
|
|
|
|
|
|
let row = csv::ByteRecord::new(); |
|
|
|
|
|
|
|
Ok(CsvSetup { rdr, headers, row }) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/// fields we need from Cargo.toml [package] section to combine with IndexMeta
|
|
|
|
/// fields we need from Cargo.toml [package] section to combine with IndexMeta
|
|
|
|
/// to form a PublishMeta.
|
|
|
|
/// to form a PublishMeta.
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
@ -590,7 +633,14 @@ struct VersionMeta { |
|
|
|
manifest: ManifestStub, |
|
|
|
manifest: ManifestStub, |
|
|
|
readme: Option<String>, |
|
|
|
readme: Option<String>, |
|
|
|
tmp: TempDir, |
|
|
|
tmp: TempDir, |
|
|
|
meta: cargo_metadata::Metadata, |
|
|
|
modified_manifest_toml: Option<String>, |
|
|
|
|
|
|
|
// meta: cargo_metadata::Metadata,
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
impl VersionMeta { |
|
|
|
|
|
|
|
pub fn source_dir(&self) -> PathBuf { |
|
|
|
|
|
|
|
self.tmp.path().join(&format!("{}-{}", self.index_meta.name, self.index_meta.vers)) |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)] |
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)] |
|
|
|
@ -679,8 +729,8 @@ fn parse_manifests( |
|
|
|
let begin = Instant::now(); |
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
|
|
|
|
let out: HashMap<String, Vec<VersionMeta>> = crate_versions |
|
|
|
let out: HashMap<String, Vec<VersionMeta>> = crate_versions |
|
|
|
// .into_par_iter()
|
|
|
|
.into_par_iter() |
|
|
|
.into_iter() |
|
|
|
// .into_iter()
|
|
|
|
.map(|(crate_name, versions)| -> Result<(String, Vec<VersionMeta>), Error> { |
|
|
|
.map(|(crate_name, versions)| -> Result<(String, Vec<VersionMeta>), Error> { |
|
|
|
let begin = Instant::now(); |
|
|
|
let begin = Instant::now(); |
|
|
|
let mut version_metas = Vec::new(); |
|
|
|
let mut version_metas = Vec::new(); |
|
|
|
@ -722,13 +772,13 @@ fn parse_manifests( |
|
|
|
let target_dir = tmp.path().join("target"); |
|
|
|
let target_dir = tmp.path().join("target"); |
|
|
|
std::fs::create_dir(&target_dir)?; |
|
|
|
std::fs::create_dir(&target_dir)?; |
|
|
|
|
|
|
|
|
|
|
|
let meta = cargo_metadata::MetadataCommand::new() |
|
|
|
// let meta = cargo_metadata::MetadataCommand::new()
|
|
|
|
.manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) |
|
|
|
// .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
|
|
|
|
//.env("CARGO_TARGET_DIR", &target_dir)
|
|
|
|
// //.env("CARGO_TARGET_DIR", &target_dir)
|
|
|
|
.other_options(vec!["-vv".to_string()]) |
|
|
|
// .other_options(vec!["-vv".to_string()])
|
|
|
|
.verbose(true) |
|
|
|
// .verbose(true)
|
|
|
|
// .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
|
|
|
|
// // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
|
|
|
|
.exec()?; |
|
|
|
// .exec()?;
|
|
|
|
|
|
|
|
|
|
|
|
version_metas.push(VersionMeta { |
|
|
|
version_metas.push(VersionMeta { |
|
|
|
index_meta, |
|
|
|
index_meta, |
|
|
|
@ -737,7 +787,8 @@ fn parse_manifests( |
|
|
|
manifest, |
|
|
|
manifest, |
|
|
|
readme, |
|
|
|
readme, |
|
|
|
tmp, |
|
|
|
tmp, |
|
|
|
meta, |
|
|
|
modified_manifest_toml: None, |
|
|
|
|
|
|
|
// meta,
|
|
|
|
}); |
|
|
|
}); |
|
|
|
} |
|
|
|
} |
|
|
|
debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); |
|
|
|
debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); |
|
|
|
@ -757,48 +808,160 @@ fn parse_manifests( |
|
|
|
// }
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
|
|
// fn build_dependency_graph<'a>(
|
|
|
|
fn edit_dep_registries( |
|
|
|
// crate_versions: &'a HashMap<String, Vec<VersionMeta>>,
|
|
|
|
dep_key: &str, |
|
|
|
// ) -> (StableGraph<Node<'a>, ()>, HashMap<Node<'a>, NodeIndex<u32>>) {
|
|
|
|
manifest: &mut toml_edit::Document, |
|
|
|
// let begin = Instant::now();
|
|
|
|
src_registry_name: &str, |
|
|
|
//
|
|
|
|
dst_registry_name: &str, |
|
|
|
// let mut graph = StableGraph::new();
|
|
|
|
) -> Result<(), Error> { |
|
|
|
// let mut index: HashMap<Node<'a>, NodeIndex<u32>> = Default::default();
|
|
|
|
let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else { |
|
|
|
//
|
|
|
|
trace!("missing key in manifest toml: {}", dep_key); |
|
|
|
// macro_rules! get_ix {
|
|
|
|
return Ok(()) |
|
|
|
// ($node:expr) => {{
|
|
|
|
}; |
|
|
|
// let key_exists = ix.contains_key($node);
|
|
|
|
|
|
|
|
// if !key_exists {
|
|
|
|
for (k, v) in deps.iter_mut() { |
|
|
|
// let ix = graph.add_node($node.clnoe());
|
|
|
|
let Some(t) = v.as_table_like_mut() else { continue }; |
|
|
|
// index.insert(node.clone(), ix);
|
|
|
|
if t.contains_key("registry-index") { |
|
|
|
// ix
|
|
|
|
warn!(dep_name = ?k, "dep table contains registry-index key!"); |
|
|
|
// } else {
|
|
|
|
} |
|
|
|
// *index[$node]
|
|
|
|
if let Some(registry_item) = t.get_mut("registry") { |
|
|
|
// }
|
|
|
|
if registry_item.as_str().unwrap_or("") == src_registry_name { |
|
|
|
// }}
|
|
|
|
trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml"); |
|
|
|
// }
|
|
|
|
*registry_item = toml_edit::value(dst_registry_name); |
|
|
|
//
|
|
|
|
} |
|
|
|
// for (name, versions) in crate_versions.iter() {
|
|
|
|
} |
|
|
|
// for version_meta in versions.iter() {
|
|
|
|
} |
|
|
|
// let v = &version_meta.index_meta;
|
|
|
|
Ok(()) |
|
|
|
// let node = Node { name: name.as_str(), vers: v.vers.clone() };
|
|
|
|
} |
|
|
|
// let key_exists = ix.contains_key(&node);
|
|
|
|
|
|
|
|
// let ix = get_ix!(&node);
|
|
|
|
fn edit_publish_registry_if_present( |
|
|
|
// for dep_node in v.deps.iter().filter_map(get_registry_dep) {
|
|
|
|
manifest: &mut toml_edit::Document, |
|
|
|
// let jx = get_ix!(dep_node);
|
|
|
|
src_registry_name: &str, |
|
|
|
// graph.add_edge(ix, js, ());
|
|
|
|
dst_registry_name: &str, |
|
|
|
// }
|
|
|
|
) -> Result<(), Error> { |
|
|
|
// }
|
|
|
|
let Some(package) = manifest.get_mut("package").and_then(|item| item.as_table_like_mut()) else { |
|
|
|
// }
|
|
|
|
anyhow::bail!("package key not found in manifest toml"); |
|
|
|
//
|
|
|
|
}; |
|
|
|
// info!(
|
|
|
|
|
|
|
|
// n_nodes = graph.node_count(),
|
|
|
|
let Some(publish_item) = package.get_mut("publish") else { |
|
|
|
// n_edges = graph.edge_count(),
|
|
|
|
trace!("no 'publish' key in Cargo.toml package section"); |
|
|
|
// "built dependency graph for entire registry in {:?}", begin.elapsed(),
|
|
|
|
return Ok(()) |
|
|
|
// );
|
|
|
|
}; |
|
|
|
//
|
|
|
|
|
|
|
|
// (graph, index)
|
|
|
|
let Some(publish_array) = publish_item.as_array_mut() else { |
|
|
|
// }
|
|
|
|
anyhow::bail!("failed to cast publish item as array"); |
|
|
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let Some(i) = publish_array.iter().position(|x| x.as_str().map(|s| s == src_registry_name).unwrap_or(false)) else { |
|
|
|
|
|
|
|
anyhow::bail!("publish key exists, but source registry name does not appear in it! (`{}`)", publish_array.to_string()); |
|
|
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let item_i = publish_array.get_mut(i).unwrap(); |
|
|
|
|
|
|
|
*item_i = toml_edit::Value::from(dst_registry_name); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Result<(), Error> { |
|
|
|
|
|
|
|
let source_dir = meta.source_dir(); |
|
|
|
|
|
|
|
let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
edit_dep_registries("dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; |
|
|
|
|
|
|
|
edit_dep_registries("dev-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; |
|
|
|
|
|
|
|
edit_dep_registries("build-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is)
|
|
|
|
|
|
|
|
let modified_manifest_toml = modified_manifest.to_string(); |
|
|
|
|
|
|
|
let cargo_toml_path = source_dir.join("Cargo.toml"); |
|
|
|
|
|
|
|
std::fs::write(&cargo_toml_path, modified_manifest_toml.as_bytes())?; |
|
|
|
|
|
|
|
debug!( |
|
|
|
|
|
|
|
crate_name = %meta.index_meta.name, |
|
|
|
|
|
|
|
vers = %meta.index_meta.vers, |
|
|
|
|
|
|
|
path = ?cargo_toml_path, |
|
|
|
|
|
|
|
"wrote modified manifest file", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
meta.modified_manifest_toml = Some(modified_manifest_toml); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let cargo_toml_orig_path = source_dir.join("Cargo.toml.orig"); |
|
|
|
|
|
|
|
if cargo_toml_orig_path.exists() { |
|
|
|
|
|
|
|
std::fs::remove_file(&cargo_toml_orig_path)?; |
|
|
|
|
|
|
|
trace!( |
|
|
|
|
|
|
|
crate_name = %meta.index_meta.name, |
|
|
|
|
|
|
|
vers = %meta.index_meta.vers, |
|
|
|
|
|
|
|
path = ?cargo_toml_orig_path, |
|
|
|
|
|
|
|
"removed Cargo.toml.orig file", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let cargo_lock_path = source_dir.join("Cargo.lock"); |
|
|
|
|
|
|
|
if cargo_lock_path.exists() { |
|
|
|
|
|
|
|
std::fs::remove_file(&cargo_lock_path)?; |
|
|
|
|
|
|
|
trace!( |
|
|
|
|
|
|
|
crate_name = %meta.index_meta.name, |
|
|
|
|
|
|
|
vers = %meta.index_meta.vers, |
|
|
|
|
|
|
|
path = ?cargo_lock_path, |
|
|
|
|
|
|
|
"removed Cargo.lock file", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn prepare_source_dirs_for_publish(config: &Config, manifests: &mut HashMap<String, Vec<VersionMeta>>) -> Result<(), Error> { |
|
|
|
|
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
manifests.par_iter_mut() |
|
|
|
|
|
|
|
.map(|(name, versions)| -> Result<(), Error> { |
|
|
|
|
|
|
|
for meta in versions.iter_mut() { |
|
|
|
|
|
|
|
prepare_source_dir_for_publish(&config, meta) |
|
|
|
|
|
|
|
.map_err(|err| { |
|
|
|
|
|
|
|
error!(%name, vers = %meta.index_meta.vers, ?err, "prepare_source_dir_for_publish failed"); |
|
|
|
|
|
|
|
err |
|
|
|
|
|
|
|
})?; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
}).collect::<Result<Vec<()>, Error>>()?; |
|
|
|
|
|
|
|
info!("modified Cargo.toml manifests in {:?}", begin.elapsed()); |
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Result<(), Error> { |
|
|
|
|
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "publishing crate version"); |
|
|
|
|
|
|
|
let index_env_key = format!("CARGO_REGISTRIES_{}_INDEX", config.dst.registry_name.to_case(Case::ScreamingSnake)); |
|
|
|
|
|
|
|
let token_env_key = format!("CARGO_REGISTRIES_{}_TOKEN", config.dst.registry_name.to_case(Case::ScreamingSnake)); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let source_dir = meta.source_dir(); |
|
|
|
|
|
|
|
let manifest_path = source_dir.join("Cargo.toml"); |
|
|
|
|
|
|
|
let manifest_path_str = manifest_path.display().to_string(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let mut args: Vec<&str> = vec!["publish"]; |
|
|
|
|
|
|
|
args.extend_from_slice(&["--registry", &config.dst.registry_name][..]); |
|
|
|
|
|
|
|
// args.extend_from_slice(&["--index", &config.dst.index_url][..]);
|
|
|
|
|
|
|
|
args.extend_from_slice(&["--token", &config.dst.auth_token][..]); |
|
|
|
|
|
|
|
args.extend_from_slice(&["--manifest-path", manifest_path_str.as_str()][..]); |
|
|
|
|
|
|
|
args.extend_from_slice(&["--no-verify", "--allow-dirty", "-vv"][..]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
debug!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "executing `cargo {}`", args.join(" ")); |
|
|
|
|
|
|
|
let output = std::process::Command::new("cargo") |
|
|
|
|
|
|
|
.env(&index_env_key, &config.dst.index_url) |
|
|
|
|
|
|
|
.env(&token_env_key, &config.dst.auth_token) |
|
|
|
|
|
|
|
.args(&args) |
|
|
|
|
|
|
|
.output()?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
debug!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, exit_status = ?output.status, "finished executing `cargo publish` command"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if !output.status.success() { |
|
|
|
|
|
|
|
let stdout = std::str::from_utf8(&output.stdout).unwrap_or("utf8err"); |
|
|
|
|
|
|
|
let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err"); |
|
|
|
|
|
|
|
error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr); |
|
|
|
|
|
|
|
debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig); |
|
|
|
|
|
|
|
debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "finished cargo publish in {:?}", begin.elapsed()); |
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
match tokio::fs::metadata(path.as_ref()).await { |
|
|
|
match tokio::fs::metadata(path.as_ref()).await { |
|
|
|
@ -828,6 +991,23 @@ fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn read_publish_log_csv(path: &Path) -> Result<Vec<PublishLogRow>, Error> { |
|
|
|
|
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
let CsvSetup { mut rdr, headers, mut row } = csv_setup(path)?; |
|
|
|
|
|
|
|
let mut out = Vec::new(); |
|
|
|
|
|
|
|
while rdr.read_byte_record(&mut row)? { |
|
|
|
|
|
|
|
// only partially deserialized after this
|
|
|
|
|
|
|
|
let parsed: PublishLogRow = row.deserialize(Some(&headers)) |
|
|
|
|
|
|
|
.map_err(|err| { |
|
|
|
|
|
|
|
error!(?row, ?headers, ?err, "deserializing row failed"); |
|
|
|
|
|
|
|
err |
|
|
|
|
|
|
|
})?; |
|
|
|
|
|
|
|
out.push(parsed); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
info!(?path, "parsed publish log csv in {:?}", begin.elapsed()); |
|
|
|
|
|
|
|
Ok(out) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fn main() -> Result<(), Error> { |
|
|
|
fn main() -> Result<(), Error> { |
|
|
|
let begin = Instant::now(); |
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
|
|
|
|
@ -843,14 +1023,41 @@ fn main() -> Result<(), Error> { |
|
|
|
rt.block_on(verify_dir_exists(&config.src.index_dir))?; |
|
|
|
rt.block_on(verify_dir_exists(&config.src.index_dir))?; |
|
|
|
rt.block_on(verify_dir_exists(&config.src.crate_files_dir))?; |
|
|
|
rt.block_on(verify_dir_exists(&config.src.crate_files_dir))?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
verify_file_exists(&config.src.publish_history_csv)?; |
|
|
|
|
|
|
|
|
|
|
|
if opt.validate { |
|
|
|
if opt.validate { |
|
|
|
println!("{:#?}", config); |
|
|
|
println!("{:#?}", config); |
|
|
|
return Ok(()) |
|
|
|
return Ok(()) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?; |
|
|
|
|
|
|
|
publish_log.sort_by_key(|x| x.unix_nanos); |
|
|
|
|
|
|
|
assert!(!publish_log.is_empty()); |
|
|
|
|
|
|
|
info!(n_rows = publish_log.len(), "parsed publish log csv"); |
|
|
|
|
|
|
|
|
|
|
|
let krates = rt.block_on(get_index_metas(&config))?; |
|
|
|
let krates = rt.block_on(get_index_metas(&config))?; |
|
|
|
|
|
|
|
|
|
|
|
let manifests = parse_manifests(&config, krates)?; |
|
|
|
let mut manifests = parse_manifests(&config, krates)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
prepare_source_dirs_for_publish(&config, &mut manifests)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let mut by_name_vers: HashMap<(&str, &Version), &VersionMeta> = manifests.iter() |
|
|
|
|
|
|
|
.flat_map(|(k, v)| { |
|
|
|
|
|
|
|
v.iter().map(|m| ((k.as_str(), &m.index_meta.vers), m)) |
|
|
|
|
|
|
|
}).collect(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for row in publish_log.iter() { |
|
|
|
|
|
|
|
let Some(meta) = by_name_vers.remove(&(row.crate_name.as_str(), &row.version)) else { |
|
|
|
|
|
|
|
warn!(?row, "crate version in publish log not found in index versions"); |
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if let Err(err) = cargo_publish_modified_source_dir(&config, meta) { |
|
|
|
|
|
|
|
error!(?err, name = %meta.index_meta.name, vers = %meta.index_meta.vers, "failed to publish crate version"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
info!("finished publishing crates to destination registry"); |
|
|
|
|
|
|
|
|
|
|
|
// let (graph, ix) = build_dependency_graph(&manifests);
|
|
|
|
// let (graph, ix) = build_dependency_graph(&manifests);
|
|
|
|
|
|
|
|
|
|
|
|
|
