diff --git a/Cargo.lock b/Cargo.lock index 906e4b0..e6aa5b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.4" @@ -192,6 +198,38 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.83" @@ -250,10 +288,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "termcolor", "textwrap", @@ -343,6 +381,30 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -427,6 +489,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -448,6 +516,28 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "filetime" version = "0.2.22" @@ -460,6 +550,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.28" @@ -632,7 +728,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "ignore", "walkdir", ] @@ -666,7 +762,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -879,6 +975,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.2", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -942,6 +1048,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" + [[package]] name = "lock_api" version = "0.4.11" @@ -1003,6 +1115,15 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1183,6 +1304,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "phf" version = "0.11.2" @@ -1400,7 +1531,27 @@ version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", ] [[package]] @@ -1418,7 +1569,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1427,7 +1578,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1479,6 +1630,7 @@ name = "registry-backup" version = "0.4.1" dependencies = [ "anyhow", + "cargo_metadata", "chrono", "clap", "dotenvy", @@ -1486,7 +1638,9 @@ dependencies = [ "futures", "governor", "num_cpus", + "petgraph", "pretty_toa", + "rayon", "regex", "reqwest", "semver", @@ -1494,6 +1648,7 @@ dependencies = [ "serde_json", "tar", "tempdir", + "tempfile", "tera", "tokio", "toml", @@ -1589,6 +1744,19 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustls" version = "0.21.8" @@ -1816,7 +1984,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "system-configuration-sys", ] @@ -1852,6 +2020,19 @@ dependencies = [ "remove_dir_all", ] +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.4.1", + "rustix", + "windows-sys", +] + [[package]] name = "tera" version = "1.19.1" diff --git a/Cargo.toml b/Cargo.toml index 19f46b4..895280d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,10 @@ tar = "0.4.38" anyhow = "1" dotenvy = "0.15" flate2 = "1" +tempfile = "3.8.1" +rayon = "1.8" +cargo_metadata = "0.18" +petgraph = "0.6.4" [features] default = [] diff --git a/src/publish.rs b/src/publish.rs index 334400a..86e9c0c 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -15,6 +15,11 @@ use semver::Version; use futures::stream::StreamExt; use tokio::io::AsyncBufReadExt; use reqwest::header::AUTHORIZATION; +use tempfile::TempDir; +use rayon::prelude::*; +use petgraph::stable_graph::StableGraph; +use petgraph::visit::{Bfs, EdgeRef, Topo, Walker}; +use petgraph::graph::NodeIndex; #[derive(Parser, Debug)] #[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] @@ -142,11 +147,34 @@ pub struct PackageStub { pub links: Option, } + +/// Example from post-cargo publish Cargo.toml +/// ```toml,ignore +/// [dependencies.docyard] +/// version = "0.31.0" +/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git" +/// features = [ +/// "auth", +/// "storage", +/// ] +/// default-features = false +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +pub struct GeneratedManifestDependency { + #[serde(rename = "registry-index")] + pub registry_index: Option, +} + /// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear /// in IndexMeta #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] pub struct ManifestStub { pub package: PackageStub, + + // TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original) + // to rewrite the `registry-index` fields. + // + // we will also need to recompute the cksum } /// full definition of cargo publish json @@ -333,19 +361,54 @@ fn serialize_publish_payload( out } -fn extract_manifest_from_tar(rdr: R) -> Result, Error> { +fn extract_manifest_files_from_tar(rdr: R) -> Result { let mut archive = tar::Archive::new(rdr); + + let mut cargo_toml = None; + let mut cargo_toml_orig = None; + let mut cargo_lock = None; + for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?; if path.ends_with("Cargo.toml.orig") { - let mut manifest_toml = String::new(); - entry.read_to_string(&mut manifest_toml)?; - return Ok(Some(manifest_toml)) + let mut data = String::new(); + entry.read_to_string(&mut data)?; + cargo_toml_orig = Some(data); + } else if path.ends_with("Cargo.toml") { + let mut data = String::new(); + entry.read_to_string(&mut data)?; + cargo_toml = Some(data); + } else if path.ends_with("Cargo.lock") { + let mut data = String::new(); + entry.read_to_string(&mut data)?; + cargo_lock = Some(data); + } + + if cargo_toml.is_some() + && cargo_toml_orig.is_some() + && cargo_lock.is_some() + { + break } } - Ok(None) + + if !(cargo_toml.is_some() && cargo_toml_orig.is_some()) + { + anyhow::bail!("some required manifest files missing in .crate archive \ + (cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})", + cargo_toml.is_some(), + cargo_toml_orig.is_some(), + cargo_lock.is_some(), + ); + } + + Ok(ManifestFiles { + cargo_toml: cargo_toml.unwrap(), + cargo_toml_orig: cargo_toml_orig.unwrap(), + cargo_lock, + }) } fn extract_readme_from_tar(rdr: R, readme_path: &Path) -> Result, Error> { @@ -498,121 +561,245 @@ async fn get_index_metas( Ok(crate_versions) } -#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] +#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] pub struct PublishWarnings { + #[serde(default)] pub invalid_categories: Vec, + #[serde(default)] pub invalid_badges: Vec, + #[serde(default)] pub other: Vec, } -async fn process_crates( - config: &Config, - crate_versions: HashMap>, -) -> Result<(), Error> { - let http_client = reqwest::Client::builder() - .user_agent(&config.http.user_agent) - .build()?; - - let publish_url = config.dst.api_url.join("/api/v1/crates/new")?; - - for (crate_name, versions) in crate_versions { - for index_meta in versions { - let version = index_meta.vers.clone(); - debug!(%crate_name, %version, "processing crate version"); - let dot_crate_path = config.src.crate_files_dir - .join(&format!("{}/{}/download", crate_name, index_meta.vers)); - verify_file_exists(&dot_crate_path).await?; - - debug!(path = ?dot_crate_path, "reading .crate file"); - let dot_crate_bytes = tokio::fs::read(&dot_crate_path) - .await - .with_context(|| { - format!("failed to read .crate file for \ - {crate_name} v{0} with path {dot_crate_path:?}", - index_meta.vers, - ) - })?; - - debug!("extracting Cargo.toml from .crate targz archive"); - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - let manifest_toml = extract_manifest_from_tar(decoder)? - .ok_or_else(|| anyhow!("Cargo.toml not found in .crate targz archive"))?; - - let manifest: ManifestStub = toml::from_str(&manifest_toml)?; - - let mut readme: Option = None; - if let Some(readme_path) = manifest.package.readme.as_ref() { - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? { - debug!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); - readme = Some(readme_content); - } - } - - let publish_meta = PublishMeta::new(index_meta, manifest, readme); - let publish_meta_json = serde_json::to_vec(&publish_meta)?; - let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes); - debug!( - n_bytes = payload.len(), - %crate_name, - %version, - "serialized publish payload", - ); - - if config.dry_run { - debug!( - %crate_name, - %version, - %publish_url, - "skipping publish (--dry-run mode)", - ); - continue; - } - - let resp = http_client.put(publish_url.clone()) - .header(AUTHORIZATION, &config.dst.auth_token) - .body(payload) - .send() - .await?; - - debug!(status = ?resp.status(), "rcvd server response to publish request"); +#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] +pub struct PublishResponse { + #[serde(default)] + pub warnings: PublishWarnings, +} - let warnings: PublishWarnings = resp - .error_for_status()? - .json() - .await?; +struct ManifestFiles { + cargo_toml: String, + cargo_toml_orig: String, + cargo_lock: Option, +} - let mut any_warnings = false; +struct VersionMeta { + index_meta: IndexMeta, + manifest_files: ManifestFiles, + dot_crate_path: PathBuf, + manifest: ManifestStub, + readme: Option, + tmp: TempDir, + meta: cargo_metadata::Metadata, +} - for warning in warnings.invalid_categories.iter() { - warn!(%crate_name, %version, "registry server invalid category warning: {}", warning); - any_warnings = true; - } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Node<'a> { + name: &'a str, + vers: Version, +} - for warning in warnings.invalid_badges.iter() { - warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning); - any_warnings = true; - } +// async fn process_crates( +// config: &Config, +// crate_versions: HashMap>, +// ) -> Result<(), Error> { +// let http_client = reqwest::Client::builder() +// .user_agent(&config.http.user_agent) +// .build()?; +// +// let publish_url = config.dst.api_url.join("/api/v1/crates/new")?; +// let publish_meta = PublishMeta::new(index_meta, manifest, readme); +// +// debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta); +// +// let publish_meta_json = serde_json::to_vec(&publish_meta)?; +// let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes); +// debug!( +// n_bytes = payload.len(), +// %crate_name, +// %version, +// "serialized publish payload", +// ); +// +// if config.dry_run { +// debug!( +// %crate_name, +// %version, +// %publish_url, +// "skipping publish (--dry-run mode)", +// ); +// continue; +// } +// +// let resp = http_client.put(publish_url.clone()) +// .header(AUTHORIZATION, &config.dst.auth_token) +// .body(payload) +// .send() +// .await?; +// +// debug!(status = ?resp.status(), "rcvd server response to publish request"); +// +// let resp: PublishResponse = resp +// .error_for_status()? +// .json() +// .await?; +// let PublishResponse { warnings } = resp; +// +// let mut any_warnings = false; +// +// for warning in warnings.invalid_categories.iter() { +// warn!(%crate_name, %version, "registry server invalid category warning: {}", warning); +// any_warnings = true; +// } +// +// for warning in warnings.invalid_badges.iter() { +// warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning); +// any_warnings = true; +// } +// +// for warning in warnings.other.iter() { +// warn!(%crate_name, %version, "registry server 'other' warning: {}", warning); +// any_warnings = true; +// } +// +// trace!("server response body:\n{warnings:#?}"); +// +// info!( +// %crate_name, +// %version, +// any_warnings, +// "published crate version in {:?}!", +// begin.elapsed(), +// ); + +fn parse_manifests( + config: &Config, + crate_versions: HashMap>, +) -> Result>, Error> { + let begin = Instant::now(); - for warning in warnings.other.iter() { - warn!(%crate_name, %version, "registry server 'other' warning: {}", warning); - any_warnings = true; + let out: HashMap> = crate_versions + // .into_par_iter() + .into_iter() + .map(|(crate_name, versions)| -> Result<(String, Vec), Error> { + let begin = Instant::now(); + let mut version_metas = Vec::new(); + for index_meta in versions { + let version = index_meta.vers.clone(); + trace!(%crate_name, %version, "processing crate version"); + let dot_crate_path = config.src.crate_files_dir + .join(&format!("{}/{}/download", crate_name, index_meta.vers)); + verify_file_exists(&dot_crate_path)?; + + trace!(path = ?dot_crate_path, "reading .crate file"); + let dot_crate_bytes = std::fs::read(&dot_crate_path) + .with_context(|| { + format!("failed to read .crate file for \ + {crate_name} v{0} with path {dot_crate_path:?}", + index_meta.vers, + ) + })?; + + trace!("extracting Cargo.toml from .crate targz archive"); + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + let manifest_files = extract_manifest_files_from_tar(decoder)?; + if manifest_files.cargo_lock.is_none() { + debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); + } + let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?; + let mut readme: Option = None; + if let Some(readme_path) = manifest.package.readme.as_ref() { + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? { + trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); + readme = Some(readme_content); + } + } + let tmp = TempDir::new()?; + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + tar::Archive::new(decoder).unpack(tmp.path())?; + trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir"); + let target_dir = tmp.path().join("target"); + std::fs::create_dir(&target_dir)?; + + let meta = cargo_metadata::MetadataCommand::new() + .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) + //.env("CARGO_TARGET_DIR", &target_dir) + .other_options(vec!["-vv".to_string()]) + .verbose(true) + // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::>()) + .exec()?; + + version_metas.push(VersionMeta { + index_meta, + manifest_files, + dot_crate_path, + manifest, + readme, + tmp, + meta, + }); } + debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); + Ok((crate_name, version_metas)) + }).collect::>()?; - trace!("server response body:\n{warnings:#?}"); + info!("parsed crate version manifests in {:?}", begin.elapsed()); - info!( - %crate_name, - %version, - any_warnings, - "published crate version!", - ); - } - } - - Ok(()) + Ok(out) } +// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option> { +// match index_dep.registry.as_ref() { +// None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }), +// Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None, +// Some(other) => panic!("unexpected registry value: {}", other), +// } +// } + +// fn build_dependency_graph<'a>( +// crate_versions: &'a HashMap>, +// ) -> (StableGraph, ()>, HashMap, NodeIndex>) { +// let begin = Instant::now(); +// +// let mut graph = StableGraph::new(); +// let mut index: HashMap, NodeIndex> = Default::default(); +// +// macro_rules! get_ix { +// ($node:expr) => {{ +// let key_exists = ix.contains_key($node); +// if !key_exists { +// let ix = graph.add_node($node.clnoe()); +// index.insert(node.clone(), ix); +// ix +// } else { +// *index[$node] +// } +// }} +// } +// +// for (name, versions) in crate_versions.iter() { +// for version_meta in versions.iter() { +// let v = &version_meta.index_meta; +// let node = Node { name: name.as_str(), vers: v.vers.clone() }; +// let key_exists = ix.contains_key(&node); +// let ix = get_ix!(&node); +// for dep_node in v.deps.iter().filter_map(get_registry_dep) { +// let jx = get_ix!(dep_node); +// graph.add_edge(ix, js, ()); +// } +// } +// } +// +// info!( +// n_nodes = graph.node_count(), +// n_edges = graph.edge_count(), +// "built dependency graph for entire registry in {:?}", begin.elapsed(), +// ); +// +// (graph, index) +// } + async fn verify_dir_exists>(path: P) -> Result<(), Error> { match tokio::fs::metadata(path.as_ref()).await { Ok(meta) if meta.is_dir() => Ok(()), @@ -627,8 +814,8 @@ async fn verify_dir_exists>(path: P) -> Result<(), Err } } -async fn verify_file_exists>(path: P) -> Result<(), Error> { - match tokio::fs::metadata(path.as_ref()).await { +fn verify_file_exists>(path: P) -> Result<(), Error> { + match std::fs::metadata(path.as_ref()) { Ok(meta) if meta.is_file() => Ok(()), Ok(meta) /* if ! meta.is_file() */ => { debug_assert!( ! meta.is_file()); @@ -663,7 +850,12 @@ fn main() -> Result<(), Error> { let krates = rt.block_on(get_index_metas(&config))?; - rt.block_on(process_crates(&config, krates))?; + let manifests = parse_manifests(&config, krates)?; + + // let (graph, ix) = build_dependency_graph(&manifests); + + drop(manifests); + drop(rt); info!("finished in {:?}", begin.elapsed()); Ok(())