diff --git a/src/publish.rs b/src/publish.rs index be9d9b6..ab869dc 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -723,6 +723,80 @@ struct Node<'a> { // begin.elapsed(), // ); +fn parse_one_manifest( + config: &Config, + crate_name: &str, + index_meta: IndexMeta, +) -> Result { + let version = index_meta.vers.clone(); + trace!(%crate_name, %version, "processing crate version"); + let dot_crate_path = config.src.crate_files_dir + .join(&format!("{}/{}/download", crate_name, index_meta.vers)); + verify_file_exists(&dot_crate_path)?; + + trace!(path = ?dot_crate_path, "reading .crate file"); + let dot_crate_bytes = std::fs::read(&dot_crate_path) + .with_context(|| { + format!("failed to read .crate file for \ + {crate_name} v{0} with path {dot_crate_path:?}", + index_meta.vers, + ) + })?; + + trace!("extracting Cargo.toml from .crate targz archive"); + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + let manifest_files = extract_manifest_files_from_tar(decoder) + .map_err(|err| { + error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); + err + })?; + if manifest_files.cargo_lock.is_none() { + debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); + } + let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?; + let mut readme: Option = None; + if let Some(readme_path) = manifest.package.readme.as_ref() { + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| { + error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme"); + err + })? { + trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); + readme = Some(readme_content); + } + } + let tmp = TempDir::new()?; + let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); + tar::Archive::new(decoder).unpack(tmp.path()) + .map_err(|err| { + error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir"); + err + })?; + + trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir"); + let target_dir = tmp.path().join("target"); + std::fs::create_dir(&target_dir)?; + + // let meta = cargo_metadata::MetadataCommand::new() + // .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) + // //.env("CARGO_TARGET_DIR", &target_dir) + // .other_options(vec!["-vv".to_string()]) + // .verbose(true) + // // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::>()) + // .exec()?; + + Ok(VersionMeta { + index_meta, + manifest_files, + dot_crate_path, + manifest, + readme, + tmp, + modified_manifest_toml: None, + // meta, + }) +} + fn parse_manifests( config: &Config, crate_versions: HashMap>, @@ -732,82 +806,32 @@ fn parse_manifests( let out: HashMap> = crate_versions .into_par_iter() // .into_iter() - .map(|(crate_name, versions)| -> Result<(String, Vec), Error> { + .filter_map(|(crate_name, versions)| -> Option<(String, Vec)> { let begin = Instant::now(); debug!(%crate_name, "parsing manifests"); let mut version_metas = Vec::new(); + let mut n_err = 0; for index_meta in versions { let version = index_meta.vers.clone(); - trace!(%crate_name, %version, "processing crate version"); - let dot_crate_path = config.src.crate_files_dir - .join(&format!("{}/{}/download", crate_name, index_meta.vers)); - verify_file_exists(&dot_crate_path)?; - - trace!(path = ?dot_crate_path, "reading .crate file"); - let dot_crate_bytes = std::fs::read(&dot_crate_path) - .with_context(|| { - format!("failed to read .crate file for \ - {crate_name} v{0} with path {dot_crate_path:?}", - index_meta.vers, - ) - })?; + match parse_one_manifest(config, &crate_name, index_meta) { + Ok(meta) => { + version_metas.push(meta); + } - trace!("extracting Cargo.toml from .crate targz archive"); - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - let manifest_files = extract_manifest_files_from_tar(decoder) - .map_err(|err| { - error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); - err - })?; - if manifest_files.cargo_lock.is_none() { - debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); - } - let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?; - let mut readme: Option = None; - if let Some(readme_path) = manifest.package.readme.as_ref() { - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| { - error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme"); - err - })? { - trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); - readme = Some(readme_content); + Err(err) => { + error!(?err, %crate_name, %version, "failed to parse manifest; skipping"); + n_err += 1; } } - let tmp = TempDir::new()?; - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - tar::Archive::new(decoder).unpack(tmp.path()) - .map_err(|err| { - error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir"); - err - })?; - - trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir"); - let target_dir = tmp.path().join("target"); - std::fs::create_dir(&target_dir)?; - - // let meta = cargo_metadata::MetadataCommand::new() - // .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) - // //.env("CARGO_TARGET_DIR", &target_dir) - // .other_options(vec!["-vv".to_string()]) - // .verbose(true) - // // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::>()) - // .exec()?; - - version_metas.push(VersionMeta { - index_meta, - manifest_files, - dot_crate_path, - manifest, - readme, - tmp, - modified_manifest_toml: None, - // meta, - }); } - debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); - Ok((crate_name, version_metas)) - }).collect::>()?; + debug!(%crate_name, n_err, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed()); + if version_metas.is_empty() { + warn!(%crate_name, n_err, "parsed zero manifests successfully for crate!"); + None + } else { + Some((crate_name, version_metas)) + } + }).collect(); info!("parsed crate version manifests in {:?}", begin.elapsed()); @@ -822,6 +846,11 @@ fn parse_manifests( // } // } +// conditional dep tables aren't handled right: +// +// [target.'cfg(not(target_env = "msvc"))'.dependencies] +// dep-one = { version = "0.1.0", registry = "old-registry" } +// fn edit_dep_registries( dep_key: &str, manifest: &mut toml_edit::Document, @@ -832,25 +861,32 @@ fn edit_dep_registries( let src_index_url = config.src.index_url.as_str(); let dst_index_url = config.dst.index_url.as_str(); - let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else { - trace!("missing key in manifest toml: {}", dep_key); - return Ok(()) - }; + let dep_key_ends_with = format!(".{dep_key}"); - for (k, v) in deps.iter_mut() { - let Some(t) = v.as_table_like_mut() else { continue }; + let it = manifest.iter_mut() + .filter(|(k, v)| { + k == dep_key || k.ends_with(&dep_key_ends_with) + }); + + for (outer_k, outer_v) in it { + let Some(deps) = outer_v.as_table_like_mut() else { + anyhow::bail!("failed to cast deps item as table"); + }; + for (k, v) in deps.iter_mut() { + let Some(t) = v.as_table_like_mut() else { continue }; - if let Some(registry_item) = t.get_mut("registry") { - if registry_item.as_str().unwrap_or("") == src_registry_name { - trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml"); - *registry_item = toml_edit::value(dst_registry_name); + if let Some(registry_item) = t.get_mut("registry") { + if registry_item.as_str().unwrap_or("") == src_registry_name { + trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml"); + *registry_item = toml_edit::value(dst_registry_name); + } } - } - if let Some(registry_index_item) = t.get_mut("registry-index") { - if registry_index_item.as_str().unwrap_or("") == src_index_url { - trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml"); - *registry_index_item = toml_edit::value(dst_index_url); + if let Some(registry_index_item) = t.get_mut("registry-index") { + if registry_index_item.as_str().unwrap_or("") == src_index_url { + trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml"); + *registry_index_item = toml_edit::value(dst_index_url); + } } } } @@ -978,8 +1014,10 @@ fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Res let stdout = std::str::from_utf8(&output.stdout).unwrap_or("utf8err"); let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err"); error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr); - debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig); - debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap()); + if !stderr.contains("already exists") { + debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig); + debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap()); + } } info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "finished cargo publish in {:?}", begin.elapsed());