Cargo.lock

@@ -192,6 +192,38 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
+[[package]]
+name = "camino"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.83"
@@ -1610,6 +1642,7 @@ name = "registry-backup"
 version = "0.4.1"
 dependencies = [
  "anyhow",
+ "cargo_metadata",
  "chrono",
  "clap",
  "convert_case",

Cargo.toml

@@ -50,6 +50,8 @@ dotenvy = "0.15"
 flate2 = "1"
 tempfile = { version = "3.8.1", optional = true }
 rayon = { version = "1.8", optional = true }
+cargo_metadata = { version = "0.18", optional = true }
+#petgraph = { version = "0.6.4", optional = true }
 csv = { version = "1", optional = true }
 convert_case = { version = "0.6", optional = true }
 toml_edit = { version = "0.21", optional = true }
@@ -62,6 +64,8 @@ publish = [
     "chrono",
     "tempfile",
     "rayon",
+    "cargo_metadata",
+#    "petgraph",
     "convert_case",
     "toml_edit",
 ]

src/publish.rs

@@ -9,10 +9,12 @@ use serde::{Serialize, Deserialize};
 use clap::Parser;
 use tracing::{debug, error, info, trace, warn};
 use tracing_subscriber::filter::EnvFilter;
+use url::Url;
 use anyhow::{anyhow, bail, Error, Context};
 use semver::Version;
 use futures::stream::StreamExt;
 use tokio::io::AsyncBufReadExt;
+use reqwest::header::AUTHORIZATION;
 use tempfile::TempDir;
 use rayon::prelude::*;
 use chrono::prelude::*;
@@ -44,7 +46,9 @@ struct Opt {
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-struct DestinationRegistryConfig {
+pub struct DestinationRegistryConfig {
+    #[serde(alias = "api")]
+    pub api_url: Url,
     /// Registry index url, i.e. the url provided to Cargo via configuration
     /// to identify where to pull the index metadata from.
     #[serde(alias = "index")]
@@ -59,7 +63,7 @@ struct DestinationRegistryConfig {
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-struct SourceRegistryConfig {
+pub struct SourceRegistryConfig {
     #[serde(alias = "index")]
     pub index_dir: PathBuf,
     #[serde(alias = "crate-files")]
@@ -71,9 +75,31 @@ struct SourceRegistryConfig {
     pub index_url: String,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub struct HttpConfig {
+    /// Value of user-agent HTTP header
+    #[serde(default = "default_user_agent")]
+    pub user_agent: String,
+}
+
+const DEFAULT_USER_AGENT: &str = concat!("shipyard.rs-publish-tool/v", env!("CARGO_PKG_VERSION"));
+
+fn default_user_agent() -> String {
+    DEFAULT_USER_AGENT.to_string()
+}
+
+impl Default for HttpConfig {
+    fn default() -> Self {
+        Self {
+            user_agent: default_user_agent(),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-struct Config {
+pub struct Config {
     /// Do everything except actually publish to the destination registry. Can also be
     /// toggled using the --dry-run command line flag.
     #[serde(default)]
@@ -85,6 +111,9 @@ struct Config {
     /// destination registry
     #[serde(alias = "destination")]
     pub dst: DestinationRegistryConfig,
+    /// Settings controlling the HTTP publish requests to the destination registry
+    #[serde(default)]
+    pub http: HttpConfig,
     /// Use to limit which crates from the source registry are published to the
     /// destination registry. Expects a regular expression which will be matched
     /// against the names of crates. Only crates with names that match the regex
@@ -109,7 +138,7 @@ impl Config {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-struct PublishLogRow {
+pub struct PublishLogRow {
     pub crate_name: String,
     pub version: Version,
     pub path: PathBuf,
@@ -138,8 +167,99 @@ fn csv_setup(path: &Path) -> Result<CsvSetup, Error> {
     Ok(CsvSetup { rdr, headers, row })
 }
 
+
+
+/// fields we need from Cargo.toml [package] section to combine with IndexMeta
+/// to form a PublishMeta.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct PackageStub {
+    pub name: String,
+    pub version: Version,
+    #[serde(default)]
+    pub authors: Vec<String>,
+    pub description: Option<String>,
+    pub license: Option<String>,
+    pub license_file: Option<PathBuf>,
+    #[serde(default)]
+    pub categories: Vec<String>,
+    #[serde(default)]
+    pub keywords: Vec<String>,
+    pub readme: Option<PathBuf>,
+    pub repository: Option<String>,
+    pub homepage: Option<String>,
+    pub documentation: Option<String>,
+    pub links: Option<String>,
+}
+
+
+/// Example from post-cargo publish Cargo.toml
+/// ```toml,ignore
+/// [dependencies.docyard]
+/// version = "0.31.0"
+/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git"
+/// features = [
+///     "auth",
+///     "storage",
+/// ]
+/// default-features = false
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct GeneratedManifestDependency {
+    #[serde(rename = "registry-index")]
+    pub registry_index: Option<String>,
+}
+
+/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
+/// in IndexMeta
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct ManifestStub {
+    pub package: PackageStub,
+
+    // TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original)
+    // to rewrite the `registry-index` fields.
+    //
+    // we will also need to recompute the cksum
+}
+
+/// full definition of cargo publish json
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct PublishMeta {
+    pub name: String,
+    #[serde(alias = "version")]
+    pub vers: semver::Version,
+    #[serde(alias = "dependencies")]
+    #[serde(default)]
+    pub deps: Vec<PublishDependency>,
+    #[serde(default)]
+    pub features: BTreeMap<String, Vec<String>>,
+    #[serde(default)]
+    pub authors: Vec<String>,
+    pub description: Option<String>,
+    pub documentation: Option<String>,
+    pub homepage: Option<String>,
+    pub readme: Option<String>,
+    pub readme_file: Option<PathBuf>,
+    #[serde(default)]
+    pub keywords: Vec<String>,
+    #[serde(default)]
+    pub categories: Vec<String>,
+    pub license: Option<String>,
+    pub license_file: Option<PathBuf>,
+    pub repository: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub links: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub badges: Option<BTreeMap<String, String>>,
+    /// from ancient cargo versions
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub features2: Option<BTreeMap<String, Vec<String>>>,
+    /// from ancient cargo versions
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub v: Option<u8>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-struct PublishDependency {
+pub struct PublishDependency {
     pub optional: bool,
     pub default_features: bool,
     pub name: String,
@@ -173,7 +293,7 @@ impl From<IndexDependency> for PublishDependency {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-struct IndexMeta {
+pub struct IndexMeta {
     // same everything as publish metadata
     pub name: String,
     #[serde(alias = "version")]
@@ -201,7 +321,7 @@ struct IndexMeta {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-struct IndexDependency {
+pub struct IndexDependency {
     /// corresponds to `explicit_name_in_toml` field in `publish::Dependency`
     /// when a dep is renamed in Cargo.toml, otherwise same as `package`.
     pub name: String,
@@ -233,6 +353,58 @@ pub enum DependencyKind {
     Dev,
 }
 
+impl PublishMeta {
+    pub fn new(
+        index_meta: IndexMeta,
+        manifest: ManifestStub,
+        readme: Option<String>,
+    ) -> Self {
+        let ManifestStub { package } = manifest;
+        PublishMeta {
+            name: package.name,
+            vers: package.version,
+            deps: index_meta.deps.into_iter().map(From::from).collect(),
+            features: index_meta.features,
+            authors: package.authors,
+            description: package.description,
+            documentation: package.documentation,
+            homepage: package.homepage,
+            readme,
+            readme_file: package.readme,
+            keywords: package.keywords,
+            categories: package.categories,
+            license: package.license,
+            license_file: package.license_file,
+            repository: package.repository,
+            links: package.links,
+            badges: index_meta.badges,
+            features2: index_meta.features2,
+            v: index_meta.v,
+        }
+    }
+}
+
+fn serialize_publish_payload(
+    publish_meta_json: &[u8],
+    dot_crate_bytes: &[u8],
+) -> Vec<u8> {
+    assert!(publish_meta_json.len() <= u32::MAX as usize);
+    assert!(dot_crate_bytes.len() <= u32::MAX as usize);
+
+    let mut out: Vec<u8> = Vec::with_capacity(
+        publish_meta_json.len()
+        + dot_crate_bytes.len()
+        + 8 // 2x u32 lengths
+    );
+
+    out.extend_from_slice(&(publish_meta_json.len() as u32).to_le_bytes()[..]);
+    out.extend_from_slice(publish_meta_json);
+    out.extend_from_slice(&(dot_crate_bytes.len() as u32).to_le_bytes()[..]);
+    out.extend_from_slice(dot_crate_bytes);
+
+    out
+}
+
 fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Error> {
     let mut archive = tar::Archive::new(rdr);
 
@@ -283,6 +455,20 @@ fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Err
     })
 }
 
+fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> {
+    let mut archive = tar::Archive::new(rdr);
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let path = entry.path()?;
+        if path == readme_path || path.ends_with(readme_path) {
+            let mut out = String::new();
+            entry.read_to_string(&mut out)?;
+            return Ok(Some(out))
+        }
+    }
+    Ok(None)
+}
+
 fn setup_logger() {
     let env_filter = EnvFilter::from_default_env();
     let builder = tracing_subscriber::fmt()
@@ -420,7 +606,7 @@ async fn get_index_metas(
 }
 
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
-struct PublishWarnings {
+pub struct PublishWarnings {
     #[serde(default)]
     pub invalid_categories: Vec<String>,
     #[serde(default)]
@@ -430,24 +616,26 @@ struct PublishWarnings {
 }
 
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
-struct PublishResponse {
+pub struct PublishResponse {
     #[serde(default)]
     pub warnings: PublishWarnings,
 }
 
 struct ManifestFiles {
-    #[allow(dead_code)]
     cargo_toml: String,
     cargo_toml_orig: String,
-    #[allow(dead_code)]
     cargo_lock: Option<String>,
 }
 
 struct VersionMeta {
     index_meta: IndexMeta,
     manifest_files: ManifestFiles,
+    dot_crate_path: PathBuf,
+    manifest: ManifestStub,
+    readme: Option<String>,
     tmp: TempDir,
     modified_manifest_toml: Option<String>,
+    // meta: cargo_metadata::Metadata,
 }
 
 impl VersionMeta {
@@ -462,6 +650,79 @@ struct Node<'a> {
     vers: Version,
 }
 
+// async fn process_crates(
+//     config: &Config,
+//     crate_versions: HashMap<String, Vec<IndexMeta>>,
+// ) -> Result<(), Error> {
+//     let http_client = reqwest::Client::builder()
+//         .user_agent(&config.http.user_agent)
+//         .build()?;
+// 
+//     let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
+//             let publish_meta = PublishMeta::new(index_meta, manifest, readme);
+// 
+//             debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta);
+// 
+//             let publish_meta_json = serde_json::to_vec(&publish_meta)?;
+//             let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
+//             debug!(
+//                 n_bytes = payload.len(),
+//                 %crate_name,
+//                 %version,
+//                 "serialized publish payload",
+//             );
+// 
+//             if config.dry_run {
+//                 debug!(
+//                     %crate_name,
+//                     %version,
+//                     %publish_url,
+//                     "skipping publish (--dry-run mode)",
+//                 );
+//                 continue;
+//             }
+// 
+//             let resp = http_client.put(publish_url.clone())
+//                 .header(AUTHORIZATION, &config.dst.auth_token)
+//                 .body(payload)
+//                 .send()
+//                 .await?;
+// 
+//             debug!(status = ?resp.status(), "rcvd server response to publish request");
+// 
+//             let resp: PublishResponse = resp
+//                 .error_for_status()?
+//                 .json()
+//                 .await?;
+//             let PublishResponse { warnings } = resp;
+// 
+//             let mut any_warnings = false;
+// 
+//             for warning in warnings.invalid_categories.iter() {
+//                 warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
+//                 any_warnings = true;
+//             }
+// 
+//             for warning in warnings.invalid_badges.iter() {
+//                 warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
+//                 any_warnings = true;
+//             }
+// 
+//             for warning in warnings.other.iter() {
+//                 warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
+//                 any_warnings = true;
+//             }
+// 
+//             trace!("server response body:\n{warnings:#?}");
+// 
+//             info!(
+//                 %crate_name,
+//                 %version,
+//                 any_warnings,
+//                 "published crate version in {:?}!",
+//                 begin.elapsed(),
+//             );
+
 fn parse_one_manifest(
     config: &Config,
     crate_name: &str,
@@ -489,7 +750,21 @@ fn parse_one_manifest(
             error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
             err
         })?;
-
+    if manifest_files.cargo_lock.is_none() {
+        debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
+    }
+    let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
+    let mut readme: Option<String> = None;
+    if let Some(readme_path) = manifest.package.readme.as_ref() {
+        let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
+        if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| {
+            error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme");
+            err
+        })? {
+            trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
+            readme = Some(readme_content);
+        }
+    }
     let tmp = TempDir::new()?;
     let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
     tar::Archive::new(decoder).unpack(tmp.path())
@@ -502,11 +777,23 @@ fn parse_one_manifest(
     let target_dir = tmp.path().join("target");
     std::fs::create_dir(&target_dir)?;
 
+    // let meta = cargo_metadata::MetadataCommand::new()
+    //     .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
+    //     //.env("CARGO_TARGET_DIR", &target_dir)
+    //     .other_options(vec!["-vv".to_string()])
+    //     .verbose(true)
+    //     // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
+    //     .exec()?;
+
     Ok(VersionMeta {
         index_meta,
         manifest_files,
+        dot_crate_path,
+        manifest,
+        readme,
         tmp,
         modified_manifest_toml: None,
+        // meta,
     })
 }
 
@@ -518,6 +805,7 @@ fn parse_manifests(
 
     let out: HashMap<String, Vec<VersionMeta>> = crate_versions
         .into_par_iter()
+        // .into_iter()
         .filter_map(|(crate_name, versions)| -> Option<(String, Vec<VersionMeta>)> {
             let begin = Instant::now();
             debug!(%crate_name, "parsing manifests");
@@ -550,61 +838,62 @@ fn parse_manifests(
     Ok(out)
 }
 
-/// edit registry deps to point to the destination registry.
+// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option<Node<'a>> {
-///
+//     match index_dep.registry.as_ref() {
-/// NOTE: recursive traversing of the toml is needed to handle things
+//         None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }),
-/// like conditional deps blocks like:
+//         Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None,
-///
+//         Some(other) => panic!("unexpected registry value: {}", other),
-/// ```toml,ignore
+//     }
-/// [target.'cfg(not(target_env = "msvc"))'.dependencies]
+// }
-/// dep-one = { version = "0.1.0", registry = "old-registry" }
+
-/// ```
+// conditional dep tables aren't handled right:
-fn edit_deps(
+//
+// [target.'cfg(not(target_env = "msvc"))'.dependencies]
+// dep-one = { version = "0.1.0", registry = "old-registry" }
+//
+fn edit_dep_registries(
+    dep_key: &str,
     manifest: &mut toml_edit::Document,
     config: &Config,
-) {
+) -> Result<(), Error> {
-    use toml_edit::{visit_mut::VisitMut, TableLike};
-
-    struct DepsVisitor<'a>(&'a Config);
-
-    impl<'a> VisitMut for DepsVisitor<'a> {
-        fn visit_table_like_mut(&mut self, dep: &mut dyn TableLike) {
-            let config = self.0;
     let src_registry_name = config.src.registry_name.as_str();
     let dst_registry_name = config.dst.registry_name.as_str();
     let src_index_url = config.src.index_url.as_str();
     let dst_index_url = config.dst.index_url.as_str();
-            let mut edited = false;
 
-            if let Some(registry_item) = dep.get_mut("registry") {
+    let dep_key_ends_with = format!(".{dep_key}");
+
+    let it = manifest.iter_mut()
+        .filter(|(k, v)| {
+            k == dep_key || k.ends_with(&dep_key_ends_with)
+        });
+
+    for (outer_k, outer_v) in it {
+        let Some(deps) = outer_v.as_table_like_mut() else {
+            anyhow::bail!("failed to cast deps item as table");
+        };
+        for (k, v) in deps.iter_mut() {
+            let Some(t) = v.as_table_like_mut() else { continue };
+
+            if let Some(registry_item) = t.get_mut("registry") {
                 if registry_item.as_str().unwrap_or("") == src_registry_name {
+                    trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
                     *registry_item = toml_edit::value(dst_registry_name);
-                    edited = true;
                 }
             }
 
-            if let Some(registry_index_item) = dep.get_mut("registry-index") {
+            if let Some(registry_index_item) = t.get_mut("registry-index") {
                 if registry_index_item.as_str().unwrap_or("") == src_index_url {
+                    trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
                     *registry_index_item = toml_edit::value(dst_index_url);
-                    edited = true;
-                }
-            }
-
-            if !edited {
-                for (_, v) in dep.iter_mut() {
-                    if let Some(t) = v.as_table_like_mut() {
-                        toml_edit::visit_mut::visit_table_like_mut(self, t);
                 }
             }
         }
     }
+    Ok(())
 }
 
-    let mut visitor = DepsVisitor(config);
+fn edit_publish_registry_if_present(
-    visitor.visit_document_mut(&mut *manifest);
-}
-
-fn edit_publish_registry(
     manifest: &mut toml_edit::Document,
     src_registry_name: &str,
     dst_registry_name: &str,
@@ -636,8 +925,11 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
     let source_dir = meta.source_dir();
     let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?;
 
-    edit_deps(&mut modified_manifest, &config);
+    edit_dep_registries("dependencies", &mut modified_manifest, &config)?;
-    edit_publish_registry(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
+    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config)?;
+    edit_dep_registries("build-dependencies", &mut modified_manifest, &config)?;
+
+    edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
 
     // write modified manifest over Cargo.toml (leaves Cargo.toml.orig as is)
     let modified_manifest_toml = modified_manifest.to_string();
@@ -835,6 +1127,8 @@ fn main() -> Result<(), Error> {
     }
     info!("finished publishing crates to destination registry");
 
+    // let (graph, ix) = build_dependency_graph(&manifests);
+
     drop(manifests);
     drop(rt);