diff --git a/Cargo.lock b/Cargo.lock index b27f715..b2009fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,38 +192,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" -[[package]] -name = "camino" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo-platform" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo_metadata" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "cc" version = "1.0.83" @@ -1642,7 +1610,6 @@ name = "registry-backup" version = "0.4.1" dependencies = [ "anyhow", - "cargo_metadata", "chrono", "clap", "convert_case", diff --git a/Cargo.toml b/Cargo.toml index 564f10a..adaa273 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,8 +50,6 @@ dotenvy = "0.15" flate2 = "1" tempfile = { version = "3.8.1", optional = true } rayon = { version = "1.8", optional = true } -cargo_metadata = { version = "0.18", optional = true } -#petgraph = { version = "0.6.4", optional = true } csv = { version = "1", optional = true } convert_case = { version = "0.6", optional = true } toml_edit = { version = "0.21", optional = true } @@ -64,8 +62,6 @@ publish = [ "chrono", "tempfile", "rayon", - "cargo_metadata", -# "petgraph", "convert_case", "toml_edit", ] diff --git a/src/publish.rs b/src/publish.rs index f7fe0a4..a1b7d3f 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -9,12 +9,10 @@ use serde::{Serialize, Deserialize}; use clap::Parser; use tracing::{debug, error, info, trace, warn}; use tracing_subscriber::filter::EnvFilter; -use url::Url; use anyhow::{anyhow, bail, Error, Context}; use semver::Version; use futures::stream::StreamExt; use tokio::io::AsyncBufReadExt; -use reqwest::header::AUTHORIZATION; use tempfile::TempDir; use rayon::prelude::*; use chrono::prelude::*; @@ -46,9 +44,7 @@ struct Opt { #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "kebab-case")] -pub struct DestinationRegistryConfig { - #[serde(alias = "api")] - pub api_url: Url, +struct DestinationRegistryConfig { /// Registry index url, i.e. the url provided to Cargo via configuration /// to identify where to pull the index metadata from. #[serde(alias = "index")] @@ -63,7 +59,7 @@ pub struct DestinationRegistryConfig { #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "kebab-case")] -pub struct SourceRegistryConfig { +struct SourceRegistryConfig { #[serde(alias = "index")] pub index_dir: PathBuf, #[serde(alias = "crate-files")] @@ -75,31 +71,9 @@ pub struct SourceRegistryConfig { pub index_url: String, } -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct HttpConfig { - /// Value of user-agent HTTP header - #[serde(default = "default_user_agent")] - pub user_agent: String, -} - -const DEFAULT_USER_AGENT: &str = concat!("shipyard.rs-publish-tool/v", env!("CARGO_PKG_VERSION")); - -fn default_user_agent() -> String { - DEFAULT_USER_AGENT.to_string() -} - -impl Default for HttpConfig { - fn default() -> Self { - Self { - user_agent: default_user_agent(), - } - } -} - #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "kebab-case")] -pub struct Config { +struct Config { /// Do everything except actually publish to the destination registry. Can also be /// toggled using the --dry-run command line flag. #[serde(default)] @@ -111,9 +85,6 @@ pub struct Config { /// destination registry #[serde(alias = "destination")] pub dst: DestinationRegistryConfig, - /// Settings controlling the HTTP publish requests to the destination registry - #[serde(default)] - pub http: HttpConfig, /// Use to limit which crates from the source registry are published to the /// destination registry. Expects a regular expression which will be matched /// against the names of crates. Only crates with names that match the regex @@ -138,7 +109,7 @@ impl Config { } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct PublishLogRow { +struct PublishLogRow { pub crate_name: String, pub version: Version, pub path: PathBuf, @@ -167,99 +138,8 @@ fn csv_setup(path: &Path) -> Result { Ok(CsvSetup { rdr, headers, row }) } - - -/// fields we need from Cargo.toml [package] section to combine with IndexMeta -/// to form a PublishMeta. -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct PackageStub { - pub name: String, - pub version: Version, - #[serde(default)] - pub authors: Vec, - pub description: Option, - pub license: Option, - pub license_file: Option, - #[serde(default)] - pub categories: Vec, - #[serde(default)] - pub keywords: Vec, - pub readme: Option, - pub repository: Option, - pub homepage: Option, - pub documentation: Option, - pub links: Option, -} - - -/// Example from post-cargo publish Cargo.toml -/// ```toml,ignore -/// [dependencies.docyard] -/// version = "0.31.0" -/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git" -/// features = [ -/// "auth", -/// "storage", -/// ] -/// default-features = false -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct GeneratedManifestDependency { - #[serde(rename = "registry-index")] - pub registry_index: Option, -} - -/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear -/// in IndexMeta -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct ManifestStub { - pub package: PackageStub, - - // TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original) - // to rewrite the `registry-index` fields. - // - // we will also need to recompute the cksum -} - -/// full definition of cargo publish json -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct PublishMeta { - pub name: String, - #[serde(alias = "version")] - pub vers: semver::Version, - #[serde(alias = "dependencies")] - #[serde(default)] - pub deps: Vec, - #[serde(default)] - pub features: BTreeMap>, - #[serde(default)] - pub authors: Vec, - pub description: Option, - pub documentation: Option, - pub homepage: Option, - pub readme: Option, - pub readme_file: Option, - #[serde(default)] - pub keywords: Vec, - #[serde(default)] - pub categories: Vec, - pub license: Option, - pub license_file: Option, - pub repository: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub links: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub badges: Option>, - /// from ancient cargo versions - #[serde(skip_serializing_if = "Option::is_none")] - pub features2: Option>>, - /// from ancient cargo versions - #[serde(skip_serializing_if = "Option::is_none")] - pub v: Option, -} - #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct PublishDependency { +struct PublishDependency { pub optional: bool, pub default_features: bool, pub name: String, @@ -293,7 +173,7 @@ impl From for PublishDependency { } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct IndexMeta { +struct IndexMeta { // same everything as publish metadata pub name: String, #[serde(alias = "version")] @@ -321,7 +201,7 @@ pub struct IndexMeta { } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct IndexDependency { +struct IndexDependency { /// corresponds to `explicit_name_in_toml` field in `publish::Dependency` /// when a dep is renamed in Cargo.toml, otherwise same as `package`. pub name: String, @@ -353,58 +233,6 @@ pub enum DependencyKind { Dev, } -impl PublishMeta { - pub fn new( - index_meta: IndexMeta, - manifest: ManifestStub, - readme: Option, - ) -> Self { - let ManifestStub { package } = manifest; - PublishMeta { - name: package.name, - vers: package.version, - deps: index_meta.deps.into_iter().map(From::from).collect(), - features: index_meta.features, - authors: package.authors, - description: package.description, - documentation: package.documentation, - homepage: package.homepage, - readme, - readme_file: package.readme, - keywords: package.keywords, - categories: package.categories, - license: package.license, - license_file: package.license_file, - repository: package.repository, - links: package.links, - badges: index_meta.badges, - features2: index_meta.features2, - v: index_meta.v, - } - } -} - -fn serialize_publish_payload( - publish_meta_json: &[u8], - dot_crate_bytes: &[u8], -) -> Vec { - assert!(publish_meta_json.len() <= u32::MAX as usize); - assert!(dot_crate_bytes.len() <= u32::MAX as usize); - - let mut out: Vec = Vec::with_capacity( - publish_meta_json.len() - + dot_crate_bytes.len() - + 8 // 2x u32 lengths - ); - - out.extend_from_slice(&(publish_meta_json.len() as u32).to_le_bytes()[..]); - out.extend_from_slice(publish_meta_json); - out.extend_from_slice(&(dot_crate_bytes.len() as u32).to_le_bytes()[..]); - out.extend_from_slice(dot_crate_bytes); - - out -} - fn extract_manifest_files_from_tar(rdr: R) -> Result { let mut archive = tar::Archive::new(rdr); @@ -455,20 +283,6 @@ fn extract_manifest_files_from_tar(rdr: R) -> Result(rdr: R, readme_path: &Path) -> Result, Error> { - let mut archive = tar::Archive::new(rdr); - for entry in archive.entries()? { - let mut entry = entry?; - let path = entry.path()?; - if path == readme_path || path.ends_with(readme_path) { - let mut out = String::new(); - entry.read_to_string(&mut out)?; - return Ok(Some(out)) - } - } - Ok(None) -} - fn setup_logger() { let env_filter = EnvFilter::from_default_env(); let builder = tracing_subscriber::fmt() @@ -606,7 +420,7 @@ async fn get_index_metas( } #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] -pub struct PublishWarnings { +struct PublishWarnings { #[serde(default)] pub invalid_categories: Vec, #[serde(default)] @@ -616,26 +430,24 @@ pub struct PublishWarnings { } #[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)] -pub struct PublishResponse { +struct PublishResponse { #[serde(default)] pub warnings: PublishWarnings, } struct ManifestFiles { + #[allow(dead_code)] cargo_toml: String, cargo_toml_orig: String, + #[allow(dead_code)] cargo_lock: Option, } struct VersionMeta { index_meta: IndexMeta, manifest_files: ManifestFiles, - dot_crate_path: PathBuf, - manifest: ManifestStub, - readme: Option, tmp: TempDir, modified_manifest_toml: Option, - // meta: cargo_metadata::Metadata, } impl VersionMeta { @@ -650,79 +462,6 @@ struct Node<'a> { vers: Version, } -// async fn process_crates( -// config: &Config, -// crate_versions: HashMap>, -// ) -> Result<(), Error> { -// let http_client = reqwest::Client::builder() -// .user_agent(&config.http.user_agent) -// .build()?; -// -// let publish_url = config.dst.api_url.join("/api/v1/crates/new")?; -// let publish_meta = PublishMeta::new(index_meta, manifest, readme); -// -// debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta); -// -// let publish_meta_json = serde_json::to_vec(&publish_meta)?; -// let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes); -// debug!( -// n_bytes = payload.len(), -// %crate_name, -// %version, -// "serialized publish payload", -// ); -// -// if config.dry_run { -// debug!( -// %crate_name, -// %version, -// %publish_url, -// "skipping publish (--dry-run mode)", -// ); -// continue; -// } -// -// let resp = http_client.put(publish_url.clone()) -// .header(AUTHORIZATION, &config.dst.auth_token) -// .body(payload) -// .send() -// .await?; -// -// debug!(status = ?resp.status(), "rcvd server response to publish request"); -// -// let resp: PublishResponse = resp -// .error_for_status()? -// .json() -// .await?; -// let PublishResponse { warnings } = resp; -// -// let mut any_warnings = false; -// -// for warning in warnings.invalid_categories.iter() { -// warn!(%crate_name, %version, "registry server invalid category warning: {}", warning); -// any_warnings = true; -// } -// -// for warning in warnings.invalid_badges.iter() { -// warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning); -// any_warnings = true; -// } -// -// for warning in warnings.other.iter() { -// warn!(%crate_name, %version, "registry server 'other' warning: {}", warning); -// any_warnings = true; -// } -// -// trace!("server response body:\n{warnings:#?}"); -// -// info!( -// %crate_name, -// %version, -// any_warnings, -// "published crate version in {:?}!", -// begin.elapsed(), -// ); - fn parse_one_manifest( config: &Config, crate_name: &str, @@ -750,21 +489,7 @@ fn parse_one_manifest( error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); err })?; - if manifest_files.cargo_lock.is_none() { - debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); - } - let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?; - let mut readme: Option = None; - if let Some(readme_path) = manifest.package.readme.as_ref() { - let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| { - error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme"); - err - })? { - trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive"); - readme = Some(readme_content); - } - } + let tmp = TempDir::new()?; let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); tar::Archive::new(decoder).unpack(tmp.path()) @@ -777,23 +502,11 @@ fn parse_one_manifest( let target_dir = tmp.path().join("target"); std::fs::create_dir(&target_dir)?; - // let meta = cargo_metadata::MetadataCommand::new() - // .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml"))) - // //.env("CARGO_TARGET_DIR", &target_dir) - // .other_options(vec!["-vv".to_string()]) - // .verbose(true) - // // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::>()) - // .exec()?; - Ok(VersionMeta { index_meta, manifest_files, - dot_crate_path, - manifest, - readme, tmp, modified_manifest_toml: None, - // meta, }) } @@ -805,7 +518,6 @@ fn parse_manifests( let out: HashMap> = crate_versions .into_par_iter() - // .into_iter() .filter_map(|(crate_name, versions)| -> Option<(String, Vec)> { let begin = Instant::now(); debug!(%crate_name, "parsing manifests"); @@ -838,14 +550,6 @@ fn parse_manifests( Ok(out) } -// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option> { -// match index_dep.registry.as_ref() { -// None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }), -// Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None, -// Some(other) => panic!("unexpected registry value: {}", other), -// } -// } - /// edit registry deps to point to the destination registry. /// /// NOTE: recursive traversing of the toml is needed to handle things @@ -859,7 +563,7 @@ fn edit_deps( manifest: &mut toml_edit::Document, config: &Config, ) { - use toml_edit::{visit_mut::VisitMut, KeyMut, Item, TableLike}; + use toml_edit::{visit_mut::VisitMut, TableLike}; struct DepsVisitor<'a>(&'a Config); @@ -1131,8 +835,6 @@ fn main() -> Result<(), Error> { } info!("finished publishing crates to destination registry"); - // let (graph, ix) = build_dependency_graph(&manifests); - drop(manifests); drop(rt);