Browse Source

clean up code, removed unused stuff, etc.

feat/mass-publish-tool
Jonathan Strong 1 year ago
parent
commit
934559fd41
  1. 33
      Cargo.lock
  2. 4
      Cargo.toml
  3. 324
      src/publish.rs

33
Cargo.lock generated

@ -192,38 +192,6 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "camino"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
dependencies = [
"serde",
]
[[package]]
name = "cargo-platform"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36"
dependencies = [
"serde",
]
[[package]]
name = "cargo_metadata"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
dependencies = [
"camino",
"cargo-platform",
"semver",
"serde",
"serde_json",
"thiserror",
]
[[package]]
name = "cc"
version = "1.0.83"
@ -1642,7 +1610,6 @@ name = "registry-backup"
version = "0.4.1"
dependencies = [
"anyhow",
"cargo_metadata",
"chrono",
"clap",
"convert_case",

4
Cargo.toml

@ -50,8 +50,6 @@ dotenvy = "0.15"
flate2 = "1"
tempfile = { version = "3.8.1", optional = true }
rayon = { version = "1.8", optional = true }
cargo_metadata = { version = "0.18", optional = true }
#petgraph = { version = "0.6.4", optional = true }
csv = { version = "1", optional = true }
convert_case = { version = "0.6", optional = true }
toml_edit = { version = "0.21", optional = true }
@ -64,8 +62,6 @@ publish = [
"chrono",
"tempfile",
"rayon",
"cargo_metadata",
# "petgraph",
"convert_case",
"toml_edit",
]

324
src/publish.rs

@ -9,12 +9,10 @@ use serde::{Serialize, Deserialize};
use clap::Parser;
use tracing::{debug, error, info, trace, warn};
use tracing_subscriber::filter::EnvFilter;
use url::Url;
use anyhow::{anyhow, bail, Error, Context};
use semver::Version;
use futures::stream::StreamExt;
use tokio::io::AsyncBufReadExt;
use reqwest::header::AUTHORIZATION;
use tempfile::TempDir;
use rayon::prelude::*;
use chrono::prelude::*;
@ -46,9 +44,7 @@ struct Opt {
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct DestinationRegistryConfig {
#[serde(alias = "api")]
pub api_url: Url,
struct DestinationRegistryConfig {
/// Registry index url, i.e. the url provided to Cargo via configuration
/// to identify where to pull the index metadata from.
#[serde(alias = "index")]
@ -63,7 +59,7 @@ pub struct DestinationRegistryConfig {
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct SourceRegistryConfig {
struct SourceRegistryConfig {
#[serde(alias = "index")]
pub index_dir: PathBuf,
#[serde(alias = "crate-files")]
@ -75,31 +71,9 @@ pub struct SourceRegistryConfig {
pub index_url: String,
}
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct HttpConfig {
/// Value of user-agent HTTP header
#[serde(default = "default_user_agent")]
pub user_agent: String,
}
const DEFAULT_USER_AGENT: &str = concat!("shipyard.rs-publish-tool/v", env!("CARGO_PKG_VERSION"));
fn default_user_agent() -> String {
DEFAULT_USER_AGENT.to_string()
}
impl Default for HttpConfig {
fn default() -> Self {
Self {
user_agent: default_user_agent(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Config {
struct Config {
/// Do everything except actually publish to the destination registry. Can also be
/// toggled using the --dry-run command line flag.
#[serde(default)]
@ -111,9 +85,6 @@ pub struct Config {
/// destination registry
#[serde(alias = "destination")]
pub dst: DestinationRegistryConfig,
/// Settings controlling the HTTP publish requests to the destination registry
#[serde(default)]
pub http: HttpConfig,
/// Use to limit which crates from the source registry are published to the
/// destination registry. Expects a regular expression which will be matched
/// against the names of crates. Only crates with names that match the regex
@ -138,7 +109,7 @@ impl Config {
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct PublishLogRow {
struct PublishLogRow {
pub crate_name: String,
pub version: Version,
pub path: PathBuf,
@ -167,99 +138,8 @@ fn csv_setup(path: &Path) -> Result<CsvSetup, Error> {
Ok(CsvSetup { rdr, headers, row })
}
/// fields we need from Cargo.toml [package] section to combine with IndexMeta
/// to form a PublishMeta.
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct PackageStub {
pub name: String,
pub version: Version,
#[serde(default)]
pub authors: Vec<String>,
pub description: Option<String>,
pub license: Option<String>,
pub license_file: Option<PathBuf>,
#[serde(default)]
pub categories: Vec<String>,
#[serde(default)]
pub keywords: Vec<String>,
pub readme: Option<PathBuf>,
pub repository: Option<String>,
pub homepage: Option<String>,
pub documentation: Option<String>,
pub links: Option<String>,
}
/// Example from post-cargo publish Cargo.toml
/// ```toml,ignore
/// [dependencies.docyard]
/// version = "0.31.0"
/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git"
/// features = [
/// "auth",
/// "storage",
/// ]
/// default-features = false
/// ```
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct GeneratedManifestDependency {
#[serde(rename = "registry-index")]
pub registry_index: Option<String>,
}
/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
/// in IndexMeta
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct ManifestStub {
pub package: PackageStub,
// TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original)
// to rewrite the `registry-index` fields.
//
// we will also need to recompute the cksum
}
/// full definition of cargo publish json
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct PublishMeta {
pub name: String,
#[serde(alias = "version")]
pub vers: semver::Version,
#[serde(alias = "dependencies")]
#[serde(default)]
pub deps: Vec<PublishDependency>,
#[serde(default)]
pub features: BTreeMap<String, Vec<String>>,
#[serde(default)]
pub authors: Vec<String>,
pub description: Option<String>,
pub documentation: Option<String>,
pub homepage: Option<String>,
pub readme: Option<String>,
pub readme_file: Option<PathBuf>,
#[serde(default)]
pub keywords: Vec<String>,
#[serde(default)]
pub categories: Vec<String>,
pub license: Option<String>,
pub license_file: Option<PathBuf>,
pub repository: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub links: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub badges: Option<BTreeMap<String, String>>,
/// from ancient cargo versions
#[serde(skip_serializing_if = "Option::is_none")]
pub features2: Option<BTreeMap<String, Vec<String>>>,
/// from ancient cargo versions
#[serde(skip_serializing_if = "Option::is_none")]
pub v: Option<u8>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct PublishDependency {
struct PublishDependency {
pub optional: bool,
pub default_features: bool,
pub name: String,
@ -293,7 +173,7 @@ impl From<IndexDependency> for PublishDependency {
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct IndexMeta {
struct IndexMeta {
// same everything as publish metadata
pub name: String,
#[serde(alias = "version")]
@ -321,7 +201,7 @@ pub struct IndexMeta {
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct IndexDependency {
struct IndexDependency {
/// corresponds to `explicit_name_in_toml` field in `publish::Dependency`
/// when a dep is renamed in Cargo.toml, otherwise same as `package`.
pub name: String,
@ -353,58 +233,6 @@ pub enum DependencyKind {
Dev,
}
impl PublishMeta {
pub fn new(
index_meta: IndexMeta,
manifest: ManifestStub,
readme: Option<String>,
) -> Self {
let ManifestStub { package } = manifest;
PublishMeta {
name: package.name,
vers: package.version,
deps: index_meta.deps.into_iter().map(From::from).collect(),
features: index_meta.features,
authors: package.authors,
description: package.description,
documentation: package.documentation,
homepage: package.homepage,
readme,
readme_file: package.readme,
keywords: package.keywords,
categories: package.categories,
license: package.license,
license_file: package.license_file,
repository: package.repository,
links: package.links,
badges: index_meta.badges,
features2: index_meta.features2,
v: index_meta.v,
}
}
}
fn serialize_publish_payload(
publish_meta_json: &[u8],
dot_crate_bytes: &[u8],
) -> Vec<u8> {
assert!(publish_meta_json.len() <= u32::MAX as usize);
assert!(dot_crate_bytes.len() <= u32::MAX as usize);
let mut out: Vec<u8> = Vec::with_capacity(
publish_meta_json.len()
+ dot_crate_bytes.len()
+ 8 // 2x u32 lengths
);
out.extend_from_slice(&(publish_meta_json.len() as u32).to_le_bytes()[..]);
out.extend_from_slice(publish_meta_json);
out.extend_from_slice(&(dot_crate_bytes.len() as u32).to_le_bytes()[..]);
out.extend_from_slice(dot_crate_bytes);
out
}
fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Error> {
let mut archive = tar::Archive::new(rdr);
@ -455,20 +283,6 @@ fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Err
})
}
fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> {
let mut archive = tar::Archive::new(rdr);
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
if path == readme_path || path.ends_with(readme_path) {
let mut out = String::new();
entry.read_to_string(&mut out)?;
return Ok(Some(out))
}
}
Ok(None)
}
fn setup_logger() {
let env_filter = EnvFilter::from_default_env();
let builder = tracing_subscriber::fmt()
@ -606,7 +420,7 @@ async fn get_index_metas(
}
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
pub struct PublishWarnings {
struct PublishWarnings {
#[serde(default)]
pub invalid_categories: Vec<String>,
#[serde(default)]
@ -616,26 +430,24 @@ pub struct PublishWarnings {
}
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
pub struct PublishResponse {
struct PublishResponse {
#[serde(default)]
pub warnings: PublishWarnings,
}
struct ManifestFiles {
#[allow(dead_code)]
cargo_toml: String,
cargo_toml_orig: String,
#[allow(dead_code)]
cargo_lock: Option<String>,
}
struct VersionMeta {
index_meta: IndexMeta,
manifest_files: ManifestFiles,
dot_crate_path: PathBuf,
manifest: ManifestStub,
readme: Option<String>,
tmp: TempDir,
modified_manifest_toml: Option<String>,
// meta: cargo_metadata::Metadata,
}
impl VersionMeta {
@ -650,79 +462,6 @@ struct Node<'a> {
vers: Version,
}
// async fn process_crates(
// config: &Config,
// crate_versions: HashMap<String, Vec<IndexMeta>>,
// ) -> Result<(), Error> {
// let http_client = reqwest::Client::builder()
// .user_agent(&config.http.user_agent)
// .build()?;
//
// let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
// let publish_meta = PublishMeta::new(index_meta, manifest, readme);
//
// debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta);
//
// let publish_meta_json = serde_json::to_vec(&publish_meta)?;
// let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
// debug!(
// n_bytes = payload.len(),
// %crate_name,
// %version,
// "serialized publish payload",
// );
//
// if config.dry_run {
// debug!(
// %crate_name,
// %version,
// %publish_url,
// "skipping publish (--dry-run mode)",
// );
// continue;
// }
//
// let resp = http_client.put(publish_url.clone())
// .header(AUTHORIZATION, &config.dst.auth_token)
// .body(payload)
// .send()
// .await?;
//
// debug!(status = ?resp.status(), "rcvd server response to publish request");
//
// let resp: PublishResponse = resp
// .error_for_status()?
// .json()
// .await?;
// let PublishResponse { warnings } = resp;
//
// let mut any_warnings = false;
//
// for warning in warnings.invalid_categories.iter() {
// warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
// any_warnings = true;
// }
//
// for warning in warnings.invalid_badges.iter() {
// warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
// any_warnings = true;
// }
//
// for warning in warnings.other.iter() {
// warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
// any_warnings = true;
// }
//
// trace!("server response body:\n{warnings:#?}");
//
// info!(
// %crate_name,
// %version,
// any_warnings,
// "published crate version in {:?}!",
// begin.elapsed(),
// );
fn parse_one_manifest(
config: &Config,
crate_name: &str,
@ -750,21 +489,7 @@ fn parse_one_manifest(
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
err
})?;
if manifest_files.cargo_lock.is_none() {
debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
}
let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
let mut readme: Option<String> = None;
if let Some(readme_path) = manifest.package.readme.as_ref() {
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path).map_err(|err| {
error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract readme");
err
})? {
trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
readme = Some(readme_content);
}
}
let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path())
@ -777,23 +502,11 @@ fn parse_one_manifest(
let target_dir = tmp.path().join("target");
std::fs::create_dir(&target_dir)?;
// let meta = cargo_metadata::MetadataCommand::new()
// .manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
// //.env("CARGO_TARGET_DIR", &target_dir)
// .other_options(vec!["-vv".to_string()])
// .verbose(true)
// // .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
// .exec()?;
Ok(VersionMeta {
index_meta,
manifest_files,
dot_crate_path,
manifest,
readme,
tmp,
modified_manifest_toml: None,
// meta,
})
}
@ -805,7 +518,6 @@ fn parse_manifests(
let out: HashMap<String, Vec<VersionMeta>> = crate_versions
.into_par_iter()
// .into_iter()
.filter_map(|(crate_name, versions)| -> Option<(String, Vec<VersionMeta>)> {
let begin = Instant::now();
debug!(%crate_name, "parsing manifests");
@ -838,14 +550,6 @@ fn parse_manifests(
Ok(out)
}
// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option<Node<'a>> {
// match index_dep.registry.as_ref() {
// None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }),
// Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None,
// Some(other) => panic!("unexpected registry value: {}", other),
// }
// }
/// edit registry deps to point to the destination registry.
///
/// NOTE: recursive traversing of the toml is needed to handle things
@ -859,7 +563,7 @@ fn edit_deps(
manifest: &mut toml_edit::Document,
config: &Config,
) {
use toml_edit::{visit_mut::VisitMut, KeyMut, Item, TableLike};
use toml_edit::{visit_mut::VisitMut, TableLike};
struct DepsVisitor<'a>(&'a Config);
@ -1131,8 +835,6 @@ fn main() -> Result<(), Error> {
}
info!("finished publishing crates to destination registry");
// let (graph, ix) = build_dependency_graph(&manifests);
drop(manifests);
drop(rt);

Loading…
Cancel
Save