clean up code, removed unused stuff, etc.

Cargo.lock

@@ -192,38 +192,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
-[[package]]
-name = "camino"
-version = "1.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "cargo-platform"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "cargo_metadata"
-version = "0.18.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
-dependencies = [
- "camino",
- "cargo-platform",
- "semver",
- "serde",
- "serde_json",
- "thiserror",
-]
-
 [[package]]
 name = "cc"
 version = "1.0.83"
@@ -1642,7 +1610,6 @@ name = "registry-backup"
 version = "0.4.1"
 dependencies = [
  "anyhow",
- "cargo_metadata",
  "chrono",
  "clap",
  "convert_case",

Cargo.toml

@@ -50,8 +50,6 @@ dotenvy = "0.15"
 flate2 = "1"
 tempfile = { version = "3.8.1", optional = true }
 rayon = { version = "1.8", optional = true }
-cargo_metadata = { version = "0.18", optional = true }
-#petgraph = { version = "0.6.4", optional = true }
 csv = { version = "1", optional = true }
 convert_case = { version = "0.6", optional = true }
 toml_edit = { version = "0.21", optional = true }
@@ -64,8 +62,6 @@ publish = [
     "chrono",
     "tempfile",
     "rayon",
-    "cargo_metadata",
-#    "petgraph",
     "convert_case",
     "toml_edit",
 ]

src/publish.rs

@@ -9,12 +9,10 @@ use serde::{Serialize, Deserialize};
 use clap::Parser;
 use tracing::{debug, error, info, trace, warn};
 use tracing_subscriber::filter::EnvFilter;
-use url::Url;
 use anyhow::{anyhow, bail, Error, Context};
 use semver::Version;
 use futures::stream::StreamExt;
 use tokio::io::AsyncBufReadExt;
-use reqwest::header::AUTHORIZATION;
 use tempfile::TempDir;
 use rayon::prelude::*;
 use chrono::prelude::*;
@@ -46,9 +44,7 @@ struct Opt {
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-pub struct DestinationRegistryConfig {
+struct DestinationRegistryConfig {
-    #[serde(alias = "api")]
-    pub api_url: Url,
     /// Registry index url, i.e. the url provided to Cargo via configuration
     /// to identify where to pull the index metadata from.
     #[serde(alias = "index")]
@@ -63,7 +59,7 @@ pub struct DestinationRegistryConfig {
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-pub struct SourceRegistryConfig {
+struct SourceRegistryConfig {
     #[serde(alias = "index")]
     pub index_dir: PathBuf,
     #[serde(alias = "crate-files")]
@@ -75,31 +71,9 @@ pub struct SourceRegistryConfig {
     pub index_url: String,
 }
 
-#[derive(Deserialize, Debug, Clone)]
-#[serde(rename_all = "kebab-case")]
-pub struct HttpConfig {
-    /// Value of user-agent HTTP header
-    #[serde(default = "default_user_agent")]
-    pub user_agent: String,
-}
-
-const DEFAULT_USER_AGENT: &str = concat!("shipyard.rs-publish-tool/v", env!("CARGO_PKG_VERSION"));
-
-fn default_user_agent() -> String {
-    DEFAULT_USER_AGENT.to_string()
-}
-
-impl Default for HttpConfig {
-    fn default() -> Self {
-        Self {
-            user_agent: default_user_agent(),
-        }
-    }
-}
-
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-pub struct Config {
+struct Config {
     /// Do everything except actually publish to the destination registry. Can also be
     /// toggled using the --dry-run command line flag.
     #[serde(default)]
@@ -111,9 +85,6 @@ pub struct Config {
     /// destination registry
     #[serde(alias = "destination")]
     pub dst: DestinationRegistryConfig,
-    /// Settings controlling the HTTP publish requests to the destination registry
-    #[serde(default)]
-    pub http: HttpConfig,
     /// Use to limit which crates from the source registry are published to the
     /// destination registry. Expects a regular expression which will be matched
     /// against the names of crates. Only crates with names that match the regex
@@ -138,7 +109,7 @@ impl Config {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct PublishLogRow {
+struct PublishLogRow {
     pub crate_name: String,
     pub version: Version,
     pub path: PathBuf,
@@ -167,99 +138,8 @@ fn csv_setup(path: &Path) -> Result<CsvSetup, Error> {
     Ok(CsvSetup { rdr, headers, row })
 }
 
-
-
-/// fields we need from Cargo.toml [package] section to combine with IndexMeta
-/// to form a PublishMeta.
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct PackageStub {
-    pub name: String,
-    pub version: Version,
-    #[serde(default)]
-    pub authors: Vec<String>,
-    pub description: Option<String>,
-    pub license: Option<String>,
-    pub license_file: Option<PathBuf>,
-    #[serde(default)]
-    pub categories: Vec<String>,
-    #[serde(default)]
-    pub keywords: Vec<String>,
-    pub readme: Option<PathBuf>,
-    pub repository: Option<String>,
-    pub homepage: Option<String>,
-    pub documentation: Option<String>,
-    pub links: Option<String>,
-}
-
-
-/// Example from post-cargo publish Cargo.toml
-/// ```toml,ignore
-/// [dependencies.docyard]
-/// version = "0.31.0"
-/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git"
-/// features = [
-///     "auth",
-///     "storage",
-/// ]
-/// default-features = false
-/// ```
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct GeneratedManifestDependency {
-    #[serde(rename = "registry-index")]
-    pub registry_index: Option<String>,
-}
-
-/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
-/// in IndexMeta
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct ManifestStub {
-    pub package: PackageStub,
-
-    // TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original)
-    // to rewrite the `registry-index` fields.
-    //
-    // we will also need to recompute the cksum
-}
-
-/// full definition of cargo publish json
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct PublishMeta {
-    pub name: String,
-    #[serde(alias = "version")]
-    pub vers: semver::Version,
-    #[serde(alias = "dependencies")]
-    #[serde(default)]
-    pub deps: Vec<PublishDependency>,
-    #[serde(default)]
-    pub features: BTreeMap<String, Vec<String>>,
-    #[serde(default)]
-    pub authors: Vec<String>,
-    pub description: Option<String>,
-    pub documentation: Option<String>,
-    pub homepage: Option<String>,
-    pub readme: Option<String>,
-    pub readme_file: Option<PathBuf>,
-    #[serde(default)]
-    pub keywords: Vec<String>,
-    #[serde(default)]
-    pub categories: Vec<String>,
-    pub license: Option<String>,
-    pub license_file: Option<PathBuf>,
-    pub repository: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub links: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub badges: Option<BTreeMap<String, String>>,
-    /// from ancient cargo versions
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub features2: Option<BTreeMap<String, Vec<String>>>,
-    /// from ancient cargo versions
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub v: Option<u8>,
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct PublishDependency {
+struct PublishDependency {
     pub optional: bool,
     pub default_features: bool,
     pub name: String,
@@ -293,7 +173,7 @@ impl From<IndexDependency> for PublishDependency {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct IndexMeta {
+struct IndexMeta {
     // same everything as publish metadata
     pub name: String,
     #[serde(alias = "version")]
@@ -321,7 +201,7 @@ pub struct IndexMeta {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct IndexDependency {
+struct IndexDependency {
     /// corresponds to `explicit_name_in_toml` field in `publish::Dependency`
     /// when a dep is renamed in Cargo.toml, otherwise same as `package`.
     pub name: String,
@@ -353,58 +233,6 @@ pub enum DependencyKind {
     Dev,
 }
 
-impl PublishMeta {
-    pub fn new(
-        index_meta: IndexMeta,
-        manifest: ManifestStub,
-        readme: Option<String>,
-    ) -> Self {
-        let ManifestStub { package } = manifest;
-        PublishMeta {
-            name: package.name,
-            vers: package.version,
-            deps: index_meta.deps.into_iter().map(From::from).collect(),
-            features: index_meta.features,
-            authors: package.authors,
-            description: package.description,
-            documentation: package.documentation,
-            homepage: package.homepage,
-            readme,
-            readme_file: package.readme,
-            keywords: package.keywords,
-            categories: package.categories,
-            license: package.license,
-            license_file: package.license_file,
-            repository: package.repository,
-            links: package.links,
-            badges: index_meta.badges,
-            features2: index_meta.features2,
-            v: index_meta.v,
-        }
-    }
-}
-
-fn serialize_publish_payload(
-    publish_meta_json: &[u8],
-    dot_crate_bytes: &[u8],
-) -> Vec<u8> {
-    assert!(publish_meta_json.len() <= u32::MAX as usize);
-    assert!(dot_crate_bytes.len() <= u32::MAX as usize);
-
-    let mut out: Vec<u8> = Vec::with_capacity(
-        publish_meta_json.len()
-        + dot_crate_bytes.len()
-        + 8 // 2x u32 lengths
-    );
-
-    out.extend_from_slice(&(publish_meta_json.len() as u32).to_le_bytes()[..]);
-    out.extend_from_slice(publish_meta_json);
-    out.extend_from_slice(&(dot_crate_bytes.len() as u32).to_le_bytes()[..]);
-    out.extend_from_slice(dot_crate_bytes);
-
-    out
-}
-
 fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Error> {
     let mut archive = tar::Archive::new(rdr);
 
@@ -455,20 +283,6 @@ fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Err
     })
 }
 
-fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> {
-    let mut archive = tar::Archive::new(rdr);
-    for entry in archive.entries()? {
-        let mut entry = entry?;
-        let path = entry.path()?;
-        if path == readme_path || path.ends_with(readme_path) {
-            let mut out = String::new();
-            entry.read_to_string(&mut out)?;
-            return Ok(Some(out))
-        }
-    }
-    Ok(None)
-}
-
 fn setup_logger() {
     let env_filter = EnvFilter::from_default_env();
     let builder = tracing_subscriber::fmt()
@@ -606,7 +420,7 @@ async fn get_index_metas(
 }
 
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
-pub struct PublishWarnings {
+struct PublishWarnings {
     #[serde(default)]
     pub invalid_categories: Vec<String>,
     #[serde(default)]
@@ -616,26 +430,24 @@ pub struct PublishWarnings {
 }
 
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
-pub struct PublishResponse {
+struct PublishResponse {
     #[serde(default)]
     pub warnings: PublishWarnings,
 }
 
 struct ManifestFiles {
+    #[allow(dead_code)]
     cargo_toml: String,
     cargo_toml_orig: String,
+    #[allow(dead_code)]
     cargo_lock: Option<String>,
 }
 
 struct VersionMeta {
     index_meta: IndexMeta,
     manifest_files: ManifestFiles,
-    dot_crate_path: PathBuf,
-    manifest: ManifestStub,
-    readme: Option<String>,
     tmp: TempDir,
     modified_manifest_toml: Option<String>,
-    // meta: cargo_metadata::Metadata,
 }
 
 impl VersionMeta {
@@ -650,79 +462,6 @@ struct Node<'a> {
     vers: Version,
 }
 
-// async fn process_crates(
-//     config: &Config,
-//     crate_versions: HashMap<String, Vec<IndexMeta>>,
-// ) -> Result<(), Error> {
-//     let http_client = reqwest::Client::builder()
-//         .user_agent(&config.http.user_agent)
-//         .build()?;
-// 
-//     let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
-//             let publish_meta = PublishMeta::new(index_meta, manifest, readme);
-// 
-//             debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta);
-// 
-//             let publish_meta_json = serde_json::to_vec(&publish_meta)?;
-//             let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
-//             debug!(
-//                 n_bytes = payload.len(),
-//                 %crate_name,
-//                 %version,
-//                 "serialized publish payload",
-//             );
-// 
-//             if config.dry_run {
-//                 debug!(
-//                     %crate_name,
-//                     %version,
-//                     %publish_url,
-//                     "skipping publish (--dry-run mode)",
-//                 );
-//                 continue;
-//             }
-// 
-//             let resp = http_client.put(publish_url.clone())
-//                 .header(AUTHORIZATION, &config.dst.auth_token)
-//                 .body(payload)
-//                 .send()
-//                 .await?;
-// 
-//             debug!(status = ?resp.status(), "rcvd server response to publish request");
-// 
-//             let resp: PublishResponse = resp
-//                 .error_for_status()?
-//                 .json()
-//                 .await?;
-//             let PublishResponse { warnings } = resp;
-// 
-//             let mut any_warnings = false;
-// 
-//             for warning in warnings.invalid_categories.iter() {
-//                 warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
-//                 any_warnings = true;
-//             }
-// 
-//             for warning in warnings.invalid_badges.iter() {
-//                 warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
-//                 any_warnings = true;
-//             }
-// 
-//             for warning in warnings.other.iter() {
-//                 warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
-//                 any_warnings = true;
-//             }
-// 
-//             trace!("server response body:\n{warnings:#?}");
-// 
-//             info!(
-//                 %crate_name,
-//                 %version,
-//                 any_warnings,
-//                 "published crate version in {:?}!",
-//                 begin.elapsed(),
-//             );
-
 fn parse_one_manifest(
     config: &Config,
     crate_name: &str,
@@ -750,21 +489,7 @@ fn parse_one_manifest(
             error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
             err
         })?;
-    if manifest_files.cargo_lock.is_none() {
+
-        debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
-    }
-    let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
-    let mut readme: Option<String> = None;
-    if let Some(readme_path) = manifest.package.readme.as_ref() {
-        let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
-        if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| {
-            error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme");
-            err
-        })? {
-            trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
-            readme = Some(readme_content);
-        }
-    }
     let tmp = TempDir::new()?;
     let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
     tar::Archive::new(decoder).unpack(tmp.path())
@@ -777,23 +502,11 @@ fn parse_one_manifest(
     let target_dir = tmp.path().join("target");
     std::fs::create_dir(&target_dir)?;
 
-    // let meta = cargo_metadata::MetadataCommand::new()
-    //     .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
-    //     //.env("CARGO_TARGET_DIR", &target_dir)
-    //     .other_options(vec!["-vv".to_string()])
-    //     .verbose(true)
-    //     // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
-    //     .exec()?;
-
     Ok(VersionMeta {
         index_meta,
         manifest_files,
-        dot_crate_path,
-        manifest,
-        readme,
         tmp,
         modified_manifest_toml: None,
-        // meta,
     })
 }
 
@@ -805,7 +518,6 @@ fn parse_manifests(
 
     let out: HashMap<String, Vec<VersionMeta>> = crate_versions
         .into_par_iter()
-        // .into_iter()
         .filter_map(|(crate_name, versions)| -> Option<(String, Vec<VersionMeta>)> {
             let begin = Instant::now();
             debug!(%crate_name, "parsing manifests");
@@ -838,62 +550,61 @@ fn parse_manifests(
     Ok(out)
 }
 
-// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option<Node<'a>> {
+/// edit registry deps to point to the destination registry.
-//     match index_dep.registry.as_ref() {
+///
-//         None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }),
+/// NOTE: recursive traversing of the toml is needed to handle things
-//         Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None,
+/// like conditional deps blocks like:
-//         Some(other) => panic!("unexpected registry value: {}", other),
+///
-//     }
+/// ```toml,ignore
-// }
+/// [target.'cfg(not(target_env = "msvc"))'.dependencies]
-
+/// dep-one = { version = "0.1.0", registry = "old-registry" }
-// conditional dep tables aren't handled right:
+/// ```
-//
+fn edit_deps(
-// [target.'cfg(not(target_env = "msvc"))'.dependencies]
-// dep-one = { version = "0.1.0", registry = "old-registry" }
-//
-fn edit_dep_registries(
-    dep_key: &str,
     manifest: &mut toml_edit::Document,
     config: &Config,
-) -> Result<(), Error> {
+) {
-    let src_registry_name = config.src.registry_name.as_str();
+    use toml_edit::{visit_mut::VisitMut, TableLike};
-    let dst_registry_name = config.dst.registry_name.as_str();
-    let src_index_url = config.src.index_url.as_str();
-    let dst_index_url = config.dst.index_url.as_str();
 
-    let dep_key_ends_with = format!(".{dep_key}");
+    struct DepsVisitor<'a>(&'a Config);
 
-    let it = manifest.iter_mut()
+    impl<'a> VisitMut for DepsVisitor<'a> {
-        .filter(|(k, v)| {
+        fn visit_table_like_mut(&mut self, dep: &mut dyn TableLike) {
-            k == dep_key || k.ends_with(&dep_key_ends_with)
+            let config = self.0;
-        });
+            let src_registry_name = config.src.registry_name.as_str();
-
+            let dst_registry_name = config.dst.registry_name.as_str();
-    for (outer_k, outer_v) in it {
+            let src_index_url = config.src.index_url.as_str();
-        let Some(deps) = outer_v.as_table_like_mut() else {
+            let dst_index_url = config.dst.index_url.as_str();
-            anyhow::bail!("failed to cast deps item as table");
+            let mut edited = false;
-        };
-        for (k, v) in deps.iter_mut() {
-            let Some(t) = v.as_table_like_mut() else { continue };
 
-            if let Some(registry_item) = t.get_mut("registry") {
+            if let Some(registry_item) = dep.get_mut("registry") {
                 if registry_item.as_str().unwrap_or("") == src_registry_name {
-                    trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
                     *registry_item = toml_edit::value(dst_registry_name);
+                    edited = true;
                 }
             }
 
-            if let Some(registry_index_item) = t.get_mut("registry-index") {
+            if let Some(registry_index_item) = dep.get_mut("registry-index") {
                 if registry_index_item.as_str().unwrap_or("") == src_index_url {
-                    trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
                     *registry_index_item = toml_edit::value(dst_index_url);
+                    edited = true;
+                }
+            }
+
+            if !edited {
+                for (_, v) in dep.iter_mut() {
+                    if let Some(t) = v.as_table_like_mut() {
+                        toml_edit::visit_mut::visit_table_like_mut(self, t);
+                    }
                 }
             }
         }
     }
-    Ok(())
+
+    let mut visitor = DepsVisitor(config);
+    visitor.visit_document_mut(&mut *manifest);
 }
 
-fn edit_publish_registry_if_present(
+fn edit_publish_registry(
     manifest: &mut toml_edit::Document,
     src_registry_name: &str,
     dst_registry_name: &str,
@@ -925,11 +636,8 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
     let source_dir = meta.source_dir();
     let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?;
 
-    edit_dep_registries("dependencies", &mut modified_manifest, &config)?;
+    edit_deps(&mut modified_manifest, &config);
-    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config)?;
+    edit_publish_registry(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
-    edit_dep_registries("build-dependencies", &mut modified_manifest, &config)?;
-
-    edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
 
     // write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is)
     let modified_manifest_toml = modified_manifest.to_string();
@@ -1127,8 +835,6 @@ fn main() -> Result<(), Error> {
     }
     info!("finished publishing crates to destination registry");
 
-    // let (graph, ix) = build_dependency_graph(&manifests);
-
     drop(manifests);
     drop(rt);