begin work on mass publish tool - scoping out how to build publish meta from index meta + Cargo.toml

Cargo.toml

@@ -3,7 +3,7 @@ name = "registry-backup"
 authors = ["Jonathan Strong <jstrong@shipyard.rs>"]
 version = "0.4.1"
 edition = "2021"
-publish = ["shipyard-rs-public"]
+#publish = ["shipyard-rs-public"]
 readme = "README.md"
 repository = "https://git.shipyard.rs/jstrong/registry-backup"
 homepage = "https://git.shipyard.rs/jstrong/registry-backup"

src/publish.rs

@@ -0,0 +1,224 @@
+use std::path::{Path, PathBuf};
+use std::collections::BTreeMap;
+use std::borrow::Cow;
+
+use serde::Deserialize;
+use clap::Parser;
+use tracing::{debug, error, info, warn};
+use tracing_subscriber::filter::EnvFilter;
+use url::Url;
+use anyhow::{anyhow, bail, Error};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DestinationRegistryConfig {
+    pub api_url: Url,
+    pub token: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SourceRegistryConfig {
+    pub index_dir: PathBuf,
+    pub crate_files_dir: PathBuf,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Config {
+    #[serde(alias = "source")]
+    pub src: SourceRegistryConfig,
+    #[serde(alias = "destination")]
+    pub dst: DestinationRegistryConfig,
+}
+
+/// fields we need from Cargo.toml [package] section to combine with IndexMeta
+/// to form a PublishMeta.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct PackageStub {
+    pub name: String,
+    pub version: Version,
+    #[serde(default)]
+    pub authors: Vec<String>,
+    pub description: Option<String>,
+    pub license: Option<String>,
+    pub license_file: Option<PathBuf>,
+    #[serde(default)]
+    pub categories: Vec<String>,
+    #[serde(default)]
+    pub keywords: Vec<String>,
+    pub readme: Option<PathBuf>,
+    pub repository: Option<String>,
+    pub homepage: Option<String>,
+    pub documentation: Option<String>,
+}
+
+/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
+/// in IndexMeta
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct ManifestStub {
+    pub package: PackageStub,
+}
+
+/// full definition of cargo publish json
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct PublishMeta {
+    #[serde(borrow)]
+    pub name: String,
+    #[serde(alias = "version")]
+    pub vers: semver::Version,
+    #[serde(alias = "dependencies")]
+    #[serde(default)]
+    pub deps: Vec<PublishDependency>,
+    #[serde(default, borrow)]
+    pub features: BTreeMap<String, Vec<String>>,
+    #[serde(default, borrow)]
+    pub authors: Vec<String>,
+    #[serde(borrow)]
+    pub description: Option<String>,
+    #[serde(borrow)]
+    pub documentation: Option<String>,
+    #[serde(borrow)]
+    pub homepage: Option<String>,
+    #[serde(borrow)]
+    pub readme: Option<String>,
+    #[serde(borrow)]
+    pub readme_file: Option<String>,
+    #[serde(default, borrow)]
+    pub keywords: Vec<String>,
+    #[serde(default, borrow)]
+    pub categories: Vec<String>,
+    #[serde(borrow)]
+    pub license: Option<String>,
+    #[serde(borrow)]
+    pub license_file: Option<String>,
+    #[serde(borrow)]
+    pub repository: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub links: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub badges: Option<BTreeMap<String, String>>,
+    /// from ancient cargo versions
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub features2: Option<BTreeMap<String, Vec<String>>>,
+    /// from ancient cargo versions
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub v: Option<u8>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct PublishDependency {
+    pub optional: bool,
+    pub default_features: bool,
+    #[serde(borrow)]
+    pub name: String,
+    #[serde(borrow)]
+    pub features: Vec<String>,
+    // cargo and crates-io have this as string
+    #[serde(alias = "req")]
+    pub version_req: semver::VersionReq,
+    #[serde(borrow)]
+    pub target: Option<String>,
+    // crates-io has this as option
+    pub kind: PublishDependencyKind,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub registry: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub explicit_name_in_toml: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct IndexMeta {
+    // same everything as publish metadata
+    #[serde(borrow)]
+    pub name: String,
+    #[serde(alias = "version")]
+    pub vers: semver::Version,
+    #[serde(alias = "dependencies", borrow)]
+    pub features: BTreeMap<String, Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub links: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub badges: Option<BTreeMap<String, String>>,
+
+    // modified format/field names
+    pub deps: Vec<IndexDependency>,
+
+    // fields that don't appear in publish metadata
+    pub cksum: String,
+    pub yanked: bool,
+
+    // ancient fields, these were actually written
+    // on sanskrit on stone tablets
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub features2: Option<BTreeMap<String, Vec<String>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub v: Option<u8>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+pub struct IndexDependency {
+    /// corresponds to `explicit_name_in_toml` field in `publish::Dependency`
+    /// when a dep is renamed in Cargo.toml, otherwise same as `package`.
+    #[serde(borrow)]
+    pub name: String,
+    /// corresponds to `name` in `publish::Dependency`
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub package: Option<String>,
+    /// in publish meta, this field is called `version_req`, and the index
+    /// format requires it to be renamed to `req`
+    #[serde(alias = "version_req")]
+    pub req: semver::VersionReq,
+    #[serde(borrow)]
+    pub features: Vec<String>,
+    pub optional: bool,
+    pub default_features: bool,
+    #[serde(borrow)]
+    pub target: Option<String>,
+    pub kind: DependencyKind,
+    #[serde(skip_serializing_if = "Option::is_none", borrow)]
+    pub registry: Option<String>,
+}
+
+/// Section in which this dependency was defined
+#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]
+#[serde(rename_all = "lowercase")]
+pub enum DependencyKind {
+    /// Used at run time
+    Normal,
+    /// Used at build time, not available at run time
+    Build,
+    /// Not fetched and not used, except for when used direclty in a workspace
+    Dev,
+}
+
+fn extract_manifest_from_tar<R: Read>(rdr: R) -> Result<Option<String>, Error> {
+    let mut archive = tar::Archive::new(rdr);
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let path = entry.path()?;
+
+        if path.ends_with("Cargo.toml.orig") {
+            let mut manifest_toml = String::new();
+            entry.read_to_string(&mut manifest_toml)?;
+            return Ok(Some(manifest_toml))
+        }
+    }
+    Ok(None)
+}
+
+fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> {
+    let mut archive = tar::Archive::new(rdr);
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let path = entry.path()?;
+        if path == readme_path || path.ends_with(readme_path) {
+            let mut out = String::new();
+            entry.read_to_string(&mut out)?;
+            return Ok(Some(out))
+        }
+    }
+    Ok(None)
+}
+
+