jstrong
/
registry-backup
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity
Browse Source

first attempt at dealing with conditional keys - correct solution will use toml_edit::visit_mut::VisitMut

feat/mass-publish-tool
Jonathan Strong 1 year ago
parent
9545b350a4
commit
5d3ea67dfc
1 changed files with 122 additions and 84 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 206
      src/publish.rs

206
src/publish.rs
Unescape View File

@ -723,6 +723,80 @@ struct Node<'a> {
// begin.elapsed(),
// );
fn parse_one_manifest(
config: &Config,
crate_name: &str,
index_meta: IndexMeta,
) -> Result<VersionMeta, Error> {
let version = index_meta.vers.clone();
trace!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path)?;
trace!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = std::fs::read(&dot_crate_path)
.with_context(|| {
format!("failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers,
)
})?;
trace!("extracting Cargo.toml from .crate targz archive");
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_files = extract_manifest_files_from_tar(decoder)
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
err
})?;
if manifest_files.cargo_lock.is_none() {
debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
}
let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
let mut readme: Option<String> = None;
if let Some(readme_path) = manifest.package.readme.as_ref() {
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme");
err
})? {
trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
readme = Some(readme_content);
}
}
let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path())
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir");
err
})?;
trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir");
let target_dir = tmp.path().join("target");
std::fs::create_dir(&target_dir)?;
// let meta = cargo_metadata::MetadataCommand::new()
// .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
// //.env("CARGO_TARGET_DIR", &target_dir)
// .other_options(vec!["-vv".to_string()])
// .verbose(true)
// // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
// .exec()?;
Ok(VersionMeta {
index_meta,
manifest_files,
dot_crate_path,
manifest,
readme,
tmp,
modified_manifest_toml: None,
// meta,
})
}
fn parse_manifests(
config: &Config,
crate_versions: HashMap<String, Vec<IndexMeta>>,
@ -732,82 +806,32 @@ fn parse_manifests(
let out: HashMap<String, Vec<VersionMeta>> = crate_versions
.into_par_iter()
// .into_iter()
.map(|(crate_name, versions)| -> Result<(String, Vec<VersionMeta>), Error> {
.filter_map(|(crate_name, versions)| -> Option<(String, Vec<VersionMeta>)> {
let begin = Instant::now();
debug!(%crate_name, "parsing manifests");
let mut version_metas = Vec::new();
let mut n_err = 0;
for index_meta in versions {
let version = index_meta.vers.clone();
trace!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path)?;
trace!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = std::fs::read(&dot_crate_path)
.with_context(|| {
format!("failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers,
)
})?;
match parse_one_manifest(config, &crate_name, index_meta) {
Ok(meta) => {
version_metas.push(meta);
}
trace!("extracting Cargo.toml from .crate targz archive");
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_files = extract_manifest_files_from_tar(decoder)
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
err
})?;
if manifest_files.cargo_lock.is_none() {
debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
}
let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
let mut readme: Option<String> = None;
if let Some(readme_path) = manifest.package.readme.as_ref() {
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme");
err
})? {
trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
readme = Some(readme_content);
Err(err) => {
error!(?err, %crate_name, %version, "failed to parse manifest; skipping");
n_err += 1;
}
}
let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path())
.map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to unpack to temp dir");
err
})?;
trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir");
let target_dir = tmp.path().join("target");
std::fs::create_dir(&target_dir)?;
// let meta = cargo_metadata::MetadataCommand::new()
// .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
// //.env("CARGO_TARGET_DIR", &target_dir)
// .other_options(vec!["-vv".to_string()])
// .verbose(true)
// // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
// .exec()?;
version_metas.push(VersionMeta {
index_meta,
manifest_files,
dot_crate_path,
manifest,
readme,
tmp,
modified_manifest_toml: None,
// meta,
});
}
debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed());
Ok((crate_name, version_metas))
}).collect::<Result<_, Error>>()?;
debug!(%crate_name, n_err, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed());
if version_metas.is_empty() {
warn!(%crate_name, n_err, "parsed zero manifests successfully for crate!");
None
} else {
Some((crate_name, version_metas))
}
}).collect();
info!("parsed crate version manifests in {:?}", begin.elapsed());
@ -822,6 +846,11 @@ fn parse_manifests(
// }
// }
// conditional dep tables aren't handled right:
//
// [target.'cfg(not(target_env = "msvc"))'.dependencies]
// dep-one = { version = "0.1.0", registry = "old-registry" }
//
fn edit_dep_registries(
dep_key: &str,
manifest: &mut toml_edit::Document,
@ -832,25 +861,32 @@ fn edit_dep_registries(
let src_index_url = config.src.index_url.as_str();
let dst_index_url = config.dst.index_url.as_str();
let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else {
trace!("missing key in manifest toml: {}", dep_key);
return Ok(())
};
let dep_key_ends_with = format!(".{dep_key}");
for (k, v) in deps.iter_mut() {
let Some(t) = v.as_table_like_mut() else { continue };
let it = manifest.iter_mut()
.filter(|(k, v)| {
k == dep_key || k.ends_with(&dep_key_ends_with)
});
for (outer_k, outer_v) in it {
let Some(deps) = outer_v.as_table_like_mut() else {
anyhow::bail!("failed to cast deps item as table");
};
for (k, v) in deps.iter_mut() {
let Some(t) = v.as_table_like_mut() else { continue };
if let Some(registry_item) = t.get_mut("registry") {
if registry_item.as_str().unwrap_or("") == src_registry_name {
trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
*registry_item = toml_edit::value(dst_registry_name);
if let Some(registry_item) = t.get_mut("registry") {
if registry_item.as_str().unwrap_or("") == src_registry_name {
trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
*registry_item = toml_edit::value(dst_registry_name);
}
}
}
if let Some(registry_index_item) = t.get_mut("registry-index") {
if registry_index_item.as_str().unwrap_or("") == src_index_url {
trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
*registry_index_item = toml_edit::value(dst_index_url);
if let Some(registry_index_item) = t.get_mut("registry-index") {
if registry_index_item.as_str().unwrap_or("") == src_index_url {
trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
*registry_index_item = toml_edit::value(dst_index_url);
}
}
}
}
@ -978,8 +1014,10 @@ fn cargo_publish_modified_source_dir(config: &Config, meta: &VersionMeta) -> Res
let stdout = std::str::from_utf8(&output.stdout).unwrap_or("utf8err");
let stderr = std::str::from_utf8(&output.stderr).unwrap_or("utf8err");
error!(exit_status = ?output.status, "cargo publish error!\nstdout:\n{}\nstderr:\n:{}\n\n", stdout, stderr);
debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig);
debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap());
if !stderr.contains("already exists") {
debug!("cargo publish error - original Cargo.toml:\n***\n{}\n***", meta.manifest_files.cargo_toml_orig);
debug!("cargo publish error - modified Cargo.toml:\n***\n{}\n***", meta.modified_manifest_toml.as_ref().unwrap());
}
}
info!(name = %meta.index_meta.name, vers = %meta.index_meta.vers, "finished cargo publish in {:?}", begin.elapsed());

Write Preview
Loading…
Cancel
Save