|
|
|
@ -15,6 +15,11 @@ use semver::Version; |
|
|
|
use futures::stream::StreamExt; |
|
|
|
use futures::stream::StreamExt; |
|
|
|
use tokio::io::AsyncBufReadExt; |
|
|
|
use tokio::io::AsyncBufReadExt; |
|
|
|
use reqwest::header::AUTHORIZATION; |
|
|
|
use reqwest::header::AUTHORIZATION; |
|
|
|
|
|
|
|
use tempfile::TempDir; |
|
|
|
|
|
|
|
use rayon::prelude::*; |
|
|
|
|
|
|
|
use petgraph::stable_graph::StableGraph; |
|
|
|
|
|
|
|
use petgraph::visit::{Bfs, EdgeRef, Topo, Walker}; |
|
|
|
|
|
|
|
use petgraph::graph::NodeIndex; |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Parser, Debug)] |
|
|
|
#[derive(Parser, Debug)] |
|
|
|
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] |
|
|
|
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))] |
|
|
|
@ -142,11 +147,34 @@ pub struct PackageStub { |
|
|
|
pub links: Option<String>, |
|
|
|
pub links: Option<String>, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/// Example from post-cargo publish Cargo.toml
|
|
|
|
|
|
|
|
/// ```toml,ignore
|
|
|
|
|
|
|
|
/// [dependencies.docyard]
|
|
|
|
|
|
|
|
/// version = "0.31.0"
|
|
|
|
|
|
|
|
/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git"
|
|
|
|
|
|
|
|
/// features = [
|
|
|
|
|
|
|
|
/// "auth",
|
|
|
|
|
|
|
|
/// "storage",
|
|
|
|
|
|
|
|
/// ]
|
|
|
|
|
|
|
|
/// default-features = false
|
|
|
|
|
|
|
|
/// ```
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
|
|
|
|
pub struct GeneratedManifestDependency { |
|
|
|
|
|
|
|
#[serde(rename = "registry-index")] |
|
|
|
|
|
|
|
pub registry_index: Option<String>, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
|
|
|
|
/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
|
|
|
|
/// in IndexMeta
|
|
|
|
/// in IndexMeta
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] |
|
|
|
pub struct ManifestStub { |
|
|
|
pub struct ManifestStub { |
|
|
|
pub package: PackageStub, |
|
|
|
pub package: PackageStub, |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original)
|
|
|
|
|
|
|
|
// to rewrite the `registry-index` fields.
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// we will also need to recompute the cksum
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/// full definition of cargo publish json
|
|
|
|
/// full definition of cargo publish json
|
|
|
|
@ -333,19 +361,54 @@ fn serialize_publish_payload( |
|
|
|
out |
|
|
|
out |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fn extract_manifest_from_tar<R: Read>(rdr: R) -> Result<Option<String>, Error> { |
|
|
|
fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Error> { |
|
|
|
let mut archive = tar::Archive::new(rdr); |
|
|
|
let mut archive = tar::Archive::new(rdr); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let mut cargo_toml = None; |
|
|
|
|
|
|
|
let mut cargo_toml_orig = None; |
|
|
|
|
|
|
|
let mut cargo_lock = None; |
|
|
|
|
|
|
|
|
|
|
|
for entry in archive.entries()? { |
|
|
|
for entry in archive.entries()? { |
|
|
|
let mut entry = entry?; |
|
|
|
let mut entry = entry?; |
|
|
|
let path = entry.path()?; |
|
|
|
let path = entry.path()?; |
|
|
|
|
|
|
|
|
|
|
|
if path.ends_with("Cargo.toml.orig") { |
|
|
|
if path.ends_with("Cargo.toml.orig") { |
|
|
|
let mut manifest_toml = String::new(); |
|
|
|
let mut data = String::new(); |
|
|
|
entry.read_to_string(&mut manifest_toml)?; |
|
|
|
entry.read_to_string(&mut data)?; |
|
|
|
return Ok(Some(manifest_toml)) |
|
|
|
cargo_toml_orig = Some(data); |
|
|
|
|
|
|
|
} else if path.ends_with("Cargo.toml") { |
|
|
|
|
|
|
|
let mut data = String::new(); |
|
|
|
|
|
|
|
entry.read_to_string(&mut data)?; |
|
|
|
|
|
|
|
cargo_toml = Some(data); |
|
|
|
|
|
|
|
} else if path.ends_with("Cargo.lock") { |
|
|
|
|
|
|
|
let mut data = String::new(); |
|
|
|
|
|
|
|
entry.read_to_string(&mut data)?; |
|
|
|
|
|
|
|
cargo_lock = Some(data); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if cargo_toml.is_some() |
|
|
|
|
|
|
|
&& cargo_toml_orig.is_some() |
|
|
|
|
|
|
|
&& cargo_lock.is_some() |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
break
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
Ok(None) |
|
|
|
|
|
|
|
|
|
|
|
if !(cargo_toml.is_some() && cargo_toml_orig.is_some()) |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
anyhow::bail!("some required manifest files missing in .crate archive \ |
|
|
|
|
|
|
|
(cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})", |
|
|
|
|
|
|
|
cargo_toml.is_some(), |
|
|
|
|
|
|
|
cargo_toml_orig.is_some(), |
|
|
|
|
|
|
|
cargo_lock.is_some(), |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Ok(ManifestFiles { |
|
|
|
|
|
|
|
cargo_toml: cargo_toml.unwrap(), |
|
|
|
|
|
|
|
cargo_toml_orig: cargo_toml_orig.unwrap(), |
|
|
|
|
|
|
|
cargo_lock, |
|
|
|
|
|
|
|
}) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> { |
|
|
|
fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> { |
|
|
|
@ -498,121 +561,245 @@ async fn get_index_metas( |
|
|
|
Ok(crate_versions) |
|
|
|
Ok(crate_versions) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] |
|
|
|
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] |
|
|
|
pub struct PublishWarnings { |
|
|
|
pub struct PublishWarnings { |
|
|
|
|
|
|
|
#[serde(default)] |
|
|
|
pub invalid_categories: Vec<String>, |
|
|
|
pub invalid_categories: Vec<String>, |
|
|
|
|
|
|
|
#[serde(default)] |
|
|
|
pub invalid_badges: Vec<String>, |
|
|
|
pub invalid_badges: Vec<String>, |
|
|
|
|
|
|
|
#[serde(default)] |
|
|
|
pub other: Vec<String>, |
|
|
|
pub other: Vec<String>, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
async fn process_crates( |
|
|
|
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] |
|
|
|
config: &Config, |
|
|
|
pub struct PublishResponse { |
|
|
|
crate_versions: HashMap<String, Vec<IndexMeta>>, |
|
|
|
#[serde(default)] |
|
|
|
) -> Result<(), Error> { |
|
|
|
pub warnings: PublishWarnings, |
|
|
|
let http_client = reqwest::Client::builder() |
|
|
|
} |
|
|
|
.user_agent(&config.http.user_agent) |
|
|
|
|
|
|
|
.build()?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let publish_url = config.dst.api_url.join("/api/v1/crates/new")?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (crate_name, versions) in crate_versions { |
|
|
|
|
|
|
|
for index_meta in versions { |
|
|
|
|
|
|
|
let version = index_meta.vers.clone(); |
|
|
|
|
|
|
|
debug!(%crate_name, %version, "processing crate version"); |
|
|
|
|
|
|
|
let dot_crate_path = config.src.crate_files_dir |
|
|
|
|
|
|
|
.join(&format!("{}/{}/download", crate_name, index_meta.vers)); |
|
|
|
|
|
|
|
verify_file_exists(&dot_crate_path).await?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
debug!(path = ?dot_crate_path, "reading .crate file"); |
|
|
|
|
|
|
|
let dot_crate_bytes = tokio::fs::read(&dot_crate_path) |
|
|
|
|
|
|
|
.await |
|
|
|
|
|
|
|
.with_context(|| { |
|
|
|
|
|
|
|
format!("failed to read .crate file for \ |
|
|
|
|
|
|
|
{crate_name} v{0} with path {dot_crate_path:?}", |
|
|
|
|
|
|
|
index_meta.vers, |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
})?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
debug!("extracting Cargo.toml from .crate targz archive"); |
|
|
|
|
|
|
|
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); |
|
|
|
|
|
|
|
let manifest_toml = extract_manifest_from_tar(decoder)? |
|
|
|
|
|
|
|
.ok_or_else(|| anyhow!("Cargo.toml not found in .crate targz archive"))?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let manifest: ManifestStub = toml::from_str(&manifest_toml)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let mut readme: Option<String> = None; |
|
|
|
|
|
|
|
if let Some(readme_path) = manifest.package.readme.as_ref() { |
|
|
|
|
|
|
|
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); |
|
|
|
|
|
|
|
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? { |
|
|
|
|
|
|
|
debug!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); |
|
|
|
|
|
|
|
readme = Some(readme_content); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let publish_meta = PublishMeta::new(index_meta, manifest, readme); |
|
|
|
|
|
|
|
let publish_meta_json = serde_json::to_vec(&publish_meta)?; |
|
|
|
|
|
|
|
let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes); |
|
|
|
|
|
|
|
debug!( |
|
|
|
|
|
|
|
n_bytes = payload.len(), |
|
|
|
|
|
|
|
%crate_name, |
|
|
|
|
|
|
|
%version, |
|
|
|
|
|
|
|
"serialized publish payload", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if config.dry_run { |
|
|
|
|
|
|
|
debug!( |
|
|
|
|
|
|
|
%crate_name, |
|
|
|
|
|
|
|
%version, |
|
|
|
|
|
|
|
%publish_url, |
|
|
|
|
|
|
|
"skipping publish (--dry-run mode)", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
continue; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let resp = http_client.put(publish_url.clone()) |
|
|
|
|
|
|
|
.header(AUTHORIZATION, &config.dst.auth_token) |
|
|
|
|
|
|
|
.body(payload) |
|
|
|
|
|
|
|
.send() |
|
|
|
|
|
|
|
.await?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
debug!(status = ?resp.status(), "rcvd server response to publish request"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let warnings: PublishWarnings = resp |
|
|
|
struct ManifestFiles { |
|
|
|
.error_for_status()? |
|
|
|
cargo_toml: String, |
|
|
|
.json() |
|
|
|
cargo_toml_orig: String, |
|
|
|
.await?; |
|
|
|
cargo_lock: Option<String>, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
let mut any_warnings = false; |
|
|
|
struct VersionMeta { |
|
|
|
|
|
|
|
index_meta: IndexMeta, |
|
|
|
|
|
|
|
manifest_files: ManifestFiles, |
|
|
|
|
|
|
|
dot_crate_path: PathBuf, |
|
|
|
|
|
|
|
manifest: ManifestStub, |
|
|
|
|
|
|
|
readme: Option<String>, |
|
|
|
|
|
|
|
tmp: TempDir, |
|
|
|
|
|
|
|
meta: cargo_metadata::Metadata, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
for warning in warnings.invalid_categories.iter() { |
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)] |
|
|
|
warn!(%crate_name, %version, "registry server invalid category warning: {}", warning); |
|
|
|
struct Node<'a> { |
|
|
|
any_warnings = true; |
|
|
|
name: &'a str, |
|
|
|
} |
|
|
|
vers: Version, |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
for warning in warnings.invalid_badges.iter() { |
|
|
|
// async fn process_crates(
|
|
|
|
warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning); |
|
|
|
// config: &Config,
|
|
|
|
any_warnings = true; |
|
|
|
// crate_versions: HashMap<String, Vec<IndexMeta>>,
|
|
|
|
} |
|
|
|
// ) -> Result<(), Error> {
|
|
|
|
|
|
|
|
// let http_client = reqwest::Client::builder()
|
|
|
|
|
|
|
|
// .user_agent(&config.http.user_agent)
|
|
|
|
|
|
|
|
// .build()?;
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
|
|
|
|
|
|
|
|
// let publish_meta = PublishMeta::new(index_meta, manifest, readme);
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta);
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let publish_meta_json = serde_json::to_vec(&publish_meta)?;
|
|
|
|
|
|
|
|
// let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
|
|
|
|
|
|
|
|
// debug!(
|
|
|
|
|
|
|
|
// n_bytes = payload.len(),
|
|
|
|
|
|
|
|
// %crate_name,
|
|
|
|
|
|
|
|
// %version,
|
|
|
|
|
|
|
|
// "serialized publish payload",
|
|
|
|
|
|
|
|
// );
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// if config.dry_run {
|
|
|
|
|
|
|
|
// debug!(
|
|
|
|
|
|
|
|
// %crate_name,
|
|
|
|
|
|
|
|
// %version,
|
|
|
|
|
|
|
|
// %publish_url,
|
|
|
|
|
|
|
|
// "skipping publish (--dry-run mode)",
|
|
|
|
|
|
|
|
// );
|
|
|
|
|
|
|
|
// continue;
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let resp = http_client.put(publish_url.clone())
|
|
|
|
|
|
|
|
// .header(AUTHORIZATION, &config.dst.auth_token)
|
|
|
|
|
|
|
|
// .body(payload)
|
|
|
|
|
|
|
|
// .send()
|
|
|
|
|
|
|
|
// .await?;
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// debug!(status = ?resp.status(), "rcvd server response to publish request");
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let resp: PublishResponse = resp
|
|
|
|
|
|
|
|
// .error_for_status()?
|
|
|
|
|
|
|
|
// .json()
|
|
|
|
|
|
|
|
// .await?;
|
|
|
|
|
|
|
|
// let PublishResponse { warnings } = resp;
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let mut any_warnings = false;
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// for warning in warnings.invalid_categories.iter() {
|
|
|
|
|
|
|
|
// warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
|
|
|
|
|
|
|
|
// any_warnings = true;
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// for warning in warnings.invalid_badges.iter() {
|
|
|
|
|
|
|
|
// warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
|
|
|
|
|
|
|
|
// any_warnings = true;
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// for warning in warnings.other.iter() {
|
|
|
|
|
|
|
|
// warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
|
|
|
|
|
|
|
|
// any_warnings = true;
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// trace!("server response body:\n{warnings:#?}");
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// info!(
|
|
|
|
|
|
|
|
// %crate_name,
|
|
|
|
|
|
|
|
// %version,
|
|
|
|
|
|
|
|
// any_warnings,
|
|
|
|
|
|
|
|
// "published crate version in {:?}!",
|
|
|
|
|
|
|
|
// begin.elapsed(),
|
|
|
|
|
|
|
|
// );
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn parse_manifests( |
|
|
|
|
|
|
|
config: &Config, |
|
|
|
|
|
|
|
crate_versions: HashMap<String, Vec<IndexMeta>>, |
|
|
|
|
|
|
|
) -> Result<HashMap<String, Vec<VersionMeta>>, Error> { |
|
|
|
|
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
|
|
|
|
for warning in warnings.other.iter() { |
|
|
|
let out: HashMap<String, Vec<VersionMeta>> = crate_versions |
|
|
|
warn!(%crate_name, %version, "registry server 'other' warning: {}", warning); |
|
|
|
// .into_par_iter()
|
|
|
|
any_warnings = true; |
|
|
|
.into_iter() |
|
|
|
|
|
|
|
.map(|(crate_name, versions)| -> Result<(String, Vec<VersionMeta>), Error> { |
|
|
|
|
|
|
|
let begin = Instant::now(); |
|
|
|
|
|
|
|
let mut version_metas = Vec::new(); |
|
|
|
|
|
|
|
for index_meta in versions { |
|
|
|
|
|
|
|
let version = index_meta.vers.clone(); |
|
|
|
|
|
|
|
trace!(%crate_name, %version, "processing crate version"); |
|
|
|
|
|
|
|
let dot_crate_path = config.src.crate_files_dir |
|
|
|
|
|
|
|
.join(&format!("{}/{}/download", crate_name, index_meta.vers)); |
|
|
|
|
|
|
|
verify_file_exists(&dot_crate_path)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
trace!(path = ?dot_crate_path, "reading .crate file"); |
|
|
|
|
|
|
|
let dot_crate_bytes = std::fs::read(&dot_crate_path) |
|
|
|
|
|
|
|
.with_context(|| { |
|
|
|
|
|
|
|
format!("failed to read .crate file for \ |
|
|
|
|
|
|
|
{crate_name} v{0} with path {dot_crate_path:?}", |
|
|
|
|
|
|
|
index_meta.vers, |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
})?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
trace!("extracting Cargo.toml from .crate targz archive"); |
|
|
|
|
|
|
|
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); |
|
|
|
|
|
|
|
let manifest_files = extract_manifest_files_from_tar(decoder)?; |
|
|
|
|
|
|
|
if manifest_files.cargo_lock.is_none() { |
|
|
|
|
|
|
|
debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?; |
|
|
|
|
|
|
|
let mut readme: Option<String> = None; |
|
|
|
|
|
|
|
if let Some(readme_path) = manifest.package.readme.as_ref() { |
|
|
|
|
|
|
|
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); |
|
|
|
|
|
|
|
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? { |
|
|
|
|
|
|
|
trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); |
|
|
|
|
|
|
|
readme = Some(readme_content); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
let tmp = TempDir::new()?; |
|
|
|
|
|
|
|
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); |
|
|
|
|
|
|
|
tar::Archive::new(decoder).unpack(tmp.path())?; |
|
|
|
|
|
|
|
trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir"); |
|
|
|
|
|
|
|
let target_dir = tmp.path().join("target"); |
|
|
|
|
|
|
|
std::fs::create_dir(&target_dir)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let meta = cargo_metadata::MetadataCommand::new() |
|
|
|
|
|
|
|
.manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) |
|
|
|
|
|
|
|
//.env("CARGO_TARGET_DIR", &target_dir)
|
|
|
|
|
|
|
|
.other_options(vec!["-vv".to_string()]) |
|
|
|
|
|
|
|
.verbose(true) |
|
|
|
|
|
|
|
// .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
|
|
|
|
|
|
|
|
.exec()?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
version_metas.push(VersionMeta { |
|
|
|
|
|
|
|
index_meta, |
|
|
|
|
|
|
|
manifest_files, |
|
|
|
|
|
|
|
dot_crate_path, |
|
|
|
|
|
|
|
manifest, |
|
|
|
|
|
|
|
readme, |
|
|
|
|
|
|
|
tmp, |
|
|
|
|
|
|
|
meta, |
|
|
|
|
|
|
|
}); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); |
|
|
|
|
|
|
|
Ok((crate_name, version_metas)) |
|
|
|
|
|
|
|
}).collect::<Result<_, Error>>()?; |
|
|
|
|
|
|
|
|
|
|
|
trace!("server response body:\n{warnings:#?}"); |
|
|
|
info!("parsed crate version manifests in {:?}", begin.elapsed()); |
|
|
|
|
|
|
|
|
|
|
|
info!( |
|
|
|
Ok(out) |
|
|
|
%crate_name, |
|
|
|
|
|
|
|
%version, |
|
|
|
|
|
|
|
any_warnings, |
|
|
|
|
|
|
|
"published crate version!", |
|
|
|
|
|
|
|
); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Ok(()) |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option<Node<'a>> {
|
|
|
|
|
|
|
|
// match index_dep.registry.as_ref() {
|
|
|
|
|
|
|
|
// None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }),
|
|
|
|
|
|
|
|
// Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None,
|
|
|
|
|
|
|
|
// Some(other) => panic!("unexpected registry value: {}", other),
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// fn build_dependency_graph<'a>(
|
|
|
|
|
|
|
|
// crate_versions: &'a HashMap<String, Vec<VersionMeta>>,
|
|
|
|
|
|
|
|
// ) -> (StableGraph<Node<'a>, ()>, HashMap<Node<'a>, NodeIndex<u32>>) {
|
|
|
|
|
|
|
|
// let begin = Instant::now();
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// let mut graph = StableGraph::new();
|
|
|
|
|
|
|
|
// let mut index: HashMap<Node<'a>, NodeIndex<u32>> = Default::default();
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// macro_rules! get_ix {
|
|
|
|
|
|
|
|
// ($node:expr) => {{
|
|
|
|
|
|
|
|
// let key_exists = ix.contains_key($node);
|
|
|
|
|
|
|
|
// if !key_exists {
|
|
|
|
|
|
|
|
// let ix = graph.add_node($node.clnoe());
|
|
|
|
|
|
|
|
// index.insert(node.clone(), ix);
|
|
|
|
|
|
|
|
// ix
|
|
|
|
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
// *index[$node]
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }}
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// for (name, versions) in crate_versions.iter() {
|
|
|
|
|
|
|
|
// for version_meta in versions.iter() {
|
|
|
|
|
|
|
|
// let v = &version_meta.index_meta;
|
|
|
|
|
|
|
|
// let node = Node { name: name.as_str(), vers: v.vers.clone() };
|
|
|
|
|
|
|
|
// let key_exists = ix.contains_key(&node);
|
|
|
|
|
|
|
|
// let ix = get_ix!(&node);
|
|
|
|
|
|
|
|
// for dep_node in v.deps.iter().filter_map(get_registry_dep) {
|
|
|
|
|
|
|
|
// let jx = get_ix!(dep_node);
|
|
|
|
|
|
|
|
// graph.add_edge(ix, js, ());
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// info!(
|
|
|
|
|
|
|
|
// n_nodes = graph.node_count(),
|
|
|
|
|
|
|
|
// n_edges = graph.edge_count(),
|
|
|
|
|
|
|
|
// "built dependency graph for entire registry in {:?}", begin.elapsed(),
|
|
|
|
|
|
|
|
// );
|
|
|
|
|
|
|
|
//
|
|
|
|
|
|
|
|
// (graph, index)
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
|
|
async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
match tokio::fs::metadata(path.as_ref()).await { |
|
|
|
match tokio::fs::metadata(path.as_ref()).await { |
|
|
|
Ok(meta) if meta.is_dir() => Ok(()), |
|
|
|
Ok(meta) if meta.is_dir() => Ok(()), |
|
|
|
@ -627,8 +814,8 @@ async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Err |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
async fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> { |
|
|
|
match tokio::fs::metadata(path.as_ref()).await { |
|
|
|
match std::fs::metadata(path.as_ref()) { |
|
|
|
Ok(meta) if meta.is_file() => Ok(()), |
|
|
|
Ok(meta) if meta.is_file() => Ok(()), |
|
|
|
Ok(meta) /* if ! meta.is_file() */ => { |
|
|
|
Ok(meta) /* if ! meta.is_file() */ => { |
|
|
|
debug_assert!( ! meta.is_file()); |
|
|
|
debug_assert!( ! meta.is_file()); |
|
|
|
@ -663,7 +850,12 @@ fn main() -> Result<(), Error> { |
|
|
|
|
|
|
|
|
|
|
|
let krates = rt.block_on(get_index_metas(&config))?; |
|
|
|
let krates = rt.block_on(get_index_metas(&config))?; |
|
|
|
|
|
|
|
|
|
|
|
rt.block_on(process_crates(&config, krates))?; |
|
|
|
let manifests = parse_manifests(&config, krates)?; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// let (graph, ix) = build_dependency_graph(&manifests);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
drop(manifests); |
|
|
|
|
|
|
|
drop(rt); |
|
|
|
|
|
|
|
|
|
|
|
info!("finished in {:?}", begin.elapsed()); |
|
|
|
info!("finished in {:?}", begin.elapsed()); |
|
|
|
Ok(()) |
|
|
|
Ok(()) |
|
|
|
|
