Compare commits

..

No commits in common. 'cc275bacb7add62696efd2559660038a9a87fc9f' and '7ae5ab55d59978ca5c8e9d119ccbd12952f04d8e' have entirely different histories.

  1. 197
      Cargo.lock
  2. 4
      Cargo.toml
  3. 118
      script/get-publish-history.py
  4. 406
      src/publish.rs

197
Cargo.lock generated

@ -140,12 +140,6 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -198,38 +192,6 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "camino"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
dependencies = [
"serde",
]
[[package]]
name = "cargo-platform"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36"
dependencies = [
"serde",
]
[[package]]
name = "cargo_metadata"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
dependencies = [
"camino",
"cargo-platform",
"semver",
"serde",
"serde_json",
"thiserror",
]
[[package]]
name = "cc"
version = "1.0.83"
@ -288,10 +250,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"atty",
"bitflags 1.3.2",
"bitflags",
"clap_derive",
"clap_lex",
"indexmap 1.9.3",
"indexmap",
"once_cell",
"termcolor",
"textwrap",
@ -381,30 +343,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.16"
@ -489,12 +427,6 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
name = "either"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "encoding_rs"
version = "0.8.33"
@ -516,28 +448,6 @@ dependencies = [
"syn 2.0.38",
]
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
[[package]]
name = "filetime"
version = "0.2.22"
@ -550,12 +460,6 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "fixedbitset"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flate2"
version = "1.0.28"
@ -728,7 +632,7 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"ignore",
"walkdir",
]
@ -762,7 +666,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http",
"indexmap 1.9.3",
"indexmap",
"slab",
"tokio",
"tokio-util",
@ -975,16 +879,6 @@ dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [
"equivalent",
"hashbrown 0.14.2",
]
[[package]]
name = "ipconfig"
version = "0.3.2"
@ -1048,12 +942,6 @@ version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
[[package]]
name = "lock_api"
version = "0.4.11"
@ -1115,15 +1003,6 @@ version = "2.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
[[package]]
name = "memoffset"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -1304,16 +1183,6 @@ dependencies = [
"sha2",
]
[[package]]
name = "petgraph"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
dependencies = [
"fixedbitset",
"indexmap 2.1.0",
]
[[package]]
name = "phf"
version = "0.11.2"
@ -1531,27 +1400,7 @@ version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "rayon"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
"bitflags",
]
[[package]]
@ -1569,7 +1418,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags 1.3.2",
"bitflags",
]
[[package]]
@ -1578,7 +1427,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
dependencies = [
"bitflags 1.3.2",
"bitflags",
]
[[package]]
@ -1630,7 +1479,6 @@ name = "registry-backup"
version = "0.4.1"
dependencies = [
"anyhow",
"cargo_metadata",
"chrono",
"clap",
"dotenvy",
@ -1638,9 +1486,7 @@ dependencies = [
"futures",
"governor",
"num_cpus",
"petgraph",
"pretty_toa",
"rayon",
"regex",
"reqwest",
"semver",
@ -1648,7 +1494,6 @@ dependencies = [
"serde_json",
"tar",
"tempdir",
"tempfile",
"tera",
"tokio",
"toml",
@ -1744,19 +1589,6 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustix"
version = "0.38.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3"
dependencies = [
"bitflags 2.4.1",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustls"
version = "0.21.8"
@ -1984,7 +1816,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"core-foundation",
"system-configuration-sys",
]
@ -2020,19 +1852,6 @@ dependencies = [
"remove_dir_all",
]
[[package]]
name = "tempfile"
version = "3.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
dependencies = [
"cfg-if",
"fastrand",
"redox_syscall 0.4.1",
"rustix",
"windows-sys",
]
[[package]]
name = "tera"
version = "1.19.1"

4
Cargo.toml

@ -48,10 +48,6 @@ tar = "0.4.38"
anyhow = "1"
dotenvy = "0.15"
flate2 = "1"
tempfile = "3.8.1"
rayon = "1.8"
cargo_metadata = "0.18"
petgraph = "0.6.4"
[features]
default = []

118
script/get-publish-history.py

@ -1,118 +0,0 @@
## Imports
import os
import sys
import git
import io
from pathlib import Path
import json
import pandas as pd
## Module Constants
DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
def versions(path, branch='master'):
"""
This function returns a generator which iterates through all commits of
the repository located in the given path for the given branch. It yields
file diff information to show a timeseries of file changes.
"""
# Create the repository, raises an error if it isn't one.
repo = git.Repo(path)
# Iterate through every commit for the given branch in the repository
for commit in repo.iter_commits(branch):
# Determine the parent of the commit to diff against.
# If no parent, this is the first commit, so use empty tree.
# Then create a mapping of path to diff for each file changed.
parent = commit.parents[0] if commit.parents else EMPTY_TREE_SHA
diffs = {
diff.a_path: diff for diff in commit.diff(parent)
}
# The stats on the commit is a summary of all the changes for this
# commit, we'll iterate through it to get the information we need.
for objpath, stats in commit.stats.files.items():
# Select the diff for the path in the stats
diff = diffs.get(objpath)
# If the path is not in the dictionary, it's because it was
# renamed, so search through the b_paths for the current name.
if not diff:
for diff in diffs.values():
if diff.b_path == path and diff.renamed:
break
p = Path(objpath)
if len(p.parts) != 3:
print(f'skipping path: wrong depth ({p.parts})', file=sys.stderr)
continue
try:
obj = commit.tree / objpath
with io.BytesIO(obj.data_stream.read()) as f:
lastline = list(f.readlines())[-1].decode('utf-8')
except Exception as e:
print(f'failed to load file at commit {commit}', file=sys.stderr)
continue
lastline = lastline.strip()
try:
d = json.loads(lastline)
except Exception as e:
print(f'failed to parse json at commit {commit}: {e}', file=sys.stderr)
continue
row = {
'path': os.path.join(path, objpath),
'commit': commit.hexsha,
'author': commit.author.email,
'time': commit.authored_datetime.strftime(DATE_TIME_FORMAT),
#'size': diff_size(diff),
#'type': diff_type(diff),
'crate': d['name'],
'vers': d['vers'],
#'json': lastline,
}
# Update the stats with the additional information
# stats.update(row)
# yield stats
yield row
def diff_size(diff):
"""
Computes the size of the diff by comparing the size of the blobs.
"""
if diff.b_blob is None and diff.deleted_file:
# This is a deletion, so return negative the size of the original.
return diff.a_blob.size * -1
if diff.a_blob is None and diff.new_file:
# This is a new file, so return the size of the new value.
return diff.b_blob.size
# Otherwise just return the size a-b
return diff.a_blob.size - diff.b_blob.size
def diff_type(diff):
"""
Determines the type of the diff by looking at the diff flags.
"""
if diff.renamed: return 'R'
if diff.deleted_file: return 'D'
if diff.new_file: return 'A'
return 'M'
df = pd.DataFrame(versions('crate-index'))
df['time'] = pd.to_datetime(df['time'])
df['unix_nanos'] = df['time'].astype('int')
df = df.sort_values(by='time').groupby(['crate', 'vers']).last().reset_index()
buf = io.StringIO()
df.to_csv(buf, index=False)
print(buf.getvalue())

406
src/publish.rs

@ -15,11 +15,6 @@ use semver::Version;
use futures::stream::StreamExt;
use tokio::io::AsyncBufReadExt;
use reqwest::header::AUTHORIZATION;
use tempfile::TempDir;
use rayon::prelude::*;
use petgraph::stable_graph::StableGraph;
use petgraph::visit::{Bfs, EdgeRef, Topo, Walker};
use petgraph::graph::NodeIndex;
#[derive(Parser, Debug)]
#[clap(author, version, global_setting(clap::AppSettings::DeriveDisplayOrder))]
@ -147,34 +142,11 @@ pub struct PackageStub {
pub links: Option<String>,
}
/// Example from post-cargo publish Cargo.toml
/// ```toml,ignore
/// [dependencies.docyard]
/// version = "0.31.0"
/// registry-index = "ssh://git@ssh.shipyard.rs/shipyard-rs/crate-index.git"
/// features = [
/// "auth",
/// "storage",
/// ]
/// default-features = false
/// ```
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct GeneratedManifestDependency {
#[serde(rename = "registry-index")]
pub registry_index: Option<String>,
}
/// for parsing Cargo.toml to extract missing PublishMeta fields that do not appear
/// in IndexMeta
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct ManifestStub {
pub package: PackageStub,
// TODO: parse + modify the generated Cargo.toml (not Cargo.toml.original)
// to rewrite the `registry-index` fields.
//
// we will also need to recompute the cksum
}
/// full definition of cargo publish json
@ -361,54 +333,19 @@ fn serialize_publish_payload(
out
}
fn extract_manifest_files_from_tar<R: Read>(rdr: R) -> Result<ManifestFiles, Error> {
fn extract_manifest_from_tar<R: Read>(rdr: R) -> Result<Option<String>, Error> {
let mut archive = tar::Archive::new(rdr);
let mut cargo_toml = None;
let mut cargo_toml_orig = None;
let mut cargo_lock = None;
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
if path.ends_with("Cargo.toml.orig") {
let mut data = String::new();
entry.read_to_string(&mut data)?;
cargo_toml_orig = Some(data);
} else if path.ends_with("Cargo.toml") {
let mut data = String::new();
entry.read_to_string(&mut data)?;
cargo_toml = Some(data);
} else if path.ends_with("Cargo.lock") {
let mut data = String::new();
entry.read_to_string(&mut data)?;
cargo_lock = Some(data);
}
if cargo_toml.is_some()
&& cargo_toml_orig.is_some()
&& cargo_lock.is_some()
{
break
let mut manifest_toml = String::new();
entry.read_to_string(&mut manifest_toml)?;
return Ok(Some(manifest_toml))
}
}
if !(cargo_toml.is_some() && cargo_toml_orig.is_some())
{
anyhow::bail!("some required manifest files missing in .crate archive \
(cargo_toml={:?} cargo_toml_orig={:?} cargo_lock={:?})",
cargo_toml.is_some(),
cargo_toml_orig.is_some(),
cargo_lock.is_some(),
);
}
Ok(ManifestFiles {
cargo_toml: cargo_toml.unwrap(),
cargo_toml_orig: cargo_toml_orig.unwrap(),
cargo_lock,
})
Ok(None)
}
fn extract_readme_from_tar<R: Read>(rdr: R, readme_path: &Path) -> Result<Option<String>, Error> {
@ -561,244 +498,120 @@ async fn get_index_metas(
Ok(crate_versions)
}
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
#[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
pub struct PublishWarnings {
#[serde(default)]
pub invalid_categories: Vec<String>,
#[serde(default)]
pub invalid_badges: Vec<String>,
#[serde(default)]
pub other: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Default)]
pub struct PublishResponse {
#[serde(default)]
pub warnings: PublishWarnings,
}
struct ManifestFiles {
cargo_toml: String,
cargo_toml_orig: String,
cargo_lock: Option<String>,
}
struct VersionMeta {
index_meta: IndexMeta,
manifest_files: ManifestFiles,
dot_crate_path: PathBuf,
manifest: ManifestStub,
readme: Option<String>,
tmp: TempDir,
meta: cargo_metadata::Metadata,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct Node<'a> {
name: &'a str,
vers: Version,
}
// async fn process_crates(
// config: &Config,
// crate_versions: HashMap<String, Vec<IndexMeta>>,
// ) -> Result<(), Error> {
// let http_client = reqwest::Client::builder()
// .user_agent(&config.http.user_agent)
// .build()?;
//
// let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
// let publish_meta = PublishMeta::new(index_meta, manifest, readme);
//
// debug!("built publish meta using crate index json and the Cargo.toml manifest in the .crate targz archive:\n{:#?}\n", publish_meta);
//
// let publish_meta_json = serde_json::to_vec(&publish_meta)?;
// let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
// debug!(
// n_bytes = payload.len(),
// %crate_name,
// %version,
// "serialized publish payload",
// );
//
// if config.dry_run {
// debug!(
// %crate_name,
// %version,
// %publish_url,
// "skipping publish (--dry-run mode)",
// );
// continue;
// }
//
// let resp = http_client.put(publish_url.clone())
// .header(AUTHORIZATION, &config.dst.auth_token)
// .body(payload)
// .send()
// .await?;
//
// debug!(status = ?resp.status(), "rcvd server response to publish request");
//
// let resp: PublishResponse = resp
// .error_for_status()?
// .json()
// .await?;
// let PublishResponse { warnings } = resp;
//
// let mut any_warnings = false;
//
// for warning in warnings.invalid_categories.iter() {
// warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
// any_warnings = true;
// }
//
// for warning in warnings.invalid_badges.iter() {
// warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
// any_warnings = true;
// }
//
// for warning in warnings.other.iter() {
// warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
// any_warnings = true;
// }
//
// trace!("server response body:\n{warnings:#?}");
//
// info!(
// %crate_name,
// %version,
// any_warnings,
// "published crate version in {:?}!",
// begin.elapsed(),
// );
fn parse_manifests(
async fn process_crates(
config: &Config,
crate_versions: HashMap<String, Vec<IndexMeta>>,
) -> Result<HashMap<String, Vec<VersionMeta>>, Error> {
let begin = Instant::now();
) -> Result<(), Error> {
let http_client = reqwest::Client::builder()
.user_agent(&config.http.user_agent)
.build()?;
let publish_url = config.dst.api_url.join("/api/v1/crates/new")?;
for (crate_name, versions) in crate_versions {
for index_meta in versions {
let version = index_meta.vers.clone();
debug!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path).await?;
debug!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = tokio::fs::read(&dot_crate_path)
.await
.with_context(|| {
format!("failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers,
)
})?;
let out: HashMap<String, Vec<VersionMeta>> = crate_versions
// .into_par_iter()
.into_iter()
.map(|(crate_name, versions)| -> Result<(String, Vec<VersionMeta>), Error> {
let begin = Instant::now();
let mut version_metas = Vec::new();
for index_meta in versions {
let version = index_meta.vers.clone();
trace!(%crate_name, %version, "processing crate version");
let dot_crate_path = config.src.crate_files_dir
.join(&format!("{}/{}/download", crate_name, index_meta.vers));
verify_file_exists(&dot_crate_path)?;
trace!(path = ?dot_crate_path, "reading .crate file");
let dot_crate_bytes = std::fs::read(&dot_crate_path)
.with_context(|| {
format!("failed to read .crate file for \
{crate_name} v{0} with path {dot_crate_path:?}",
index_meta.vers,
)
})?;
trace!("extracting Cargo.toml from .crate targz archive");
debug!("extracting Cargo.toml from .crate targz archive");
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_toml = extract_manifest_from_tar(decoder)?
.ok_or_else(|| anyhow!("Cargo.toml not found in .crate targz archive"))?;
let manifest: ManifestStub = toml::from_str(&manifest_toml)?;
let mut readme: Option<String> = None;
if let Some(readme_path) = manifest.package.readme.as_ref() {
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
let manifest_files = extract_manifest_files_from_tar(decoder)?;
if manifest_files.cargo_lock.is_none() {
debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? {
debug!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
readme = Some(readme_content);
}
let manifest: ManifestStub = toml::from_str(&manifest_files.cargo_toml)?;
let mut readme: Option<String> = None;
if let Some(readme_path) = manifest.package.readme.as_ref() {
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
if let Some(readme_content) = extract_readme_from_tar(decoder, readme_path)? {
trace!(length = readme_content.len(), "extracted readme file content from .crate targz archive");
readme = Some(readme_content);
}
}
let tmp = TempDir::new()?;
let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
tar::Archive::new(decoder).unpack(tmp.path())?;
trace!(tmpdir = ?tmp.path(), "unpacked .crate archive to temp dir");
let target_dir = tmp.path().join("target");
std::fs::create_dir(&target_dir)?;
let meta = cargo_metadata::MetadataCommand::new()
.manifest_path(tmp.path().join(&format!("{crate_name}-{version}/Cargo.toml")))
//.env("CARGO_TARGET_DIR", &target_dir)
.other_options(vec!["-vv".to_string()])
.verbose(true)
// .other_options(["--frozen"].into_iter().map(|x| x.to_owned()).collect::<Vec<_>>())
.exec()?;
version_metas.push(VersionMeta {
index_meta,
manifest_files,
dot_crate_path,
manifest,
readme,
tmp,
meta,
});
}
debug!(%crate_name, "parsed {} manifests in {:?}", version_metas.len(), begin.elapsed());
Ok((crate_name, version_metas))
}).collect::<Result<_, Error>>()?;
info!("parsed crate version manifests in {:?}", begin.elapsed());
let publish_meta = PublishMeta::new(index_meta, manifest, readme);
let publish_meta_json = serde_json::to_vec(&publish_meta)?;
let payload = serialize_publish_payload(&publish_meta_json, &dot_crate_bytes);
debug!(
n_bytes = payload.len(),
%crate_name,
%version,
"serialized publish payload",
);
if config.dry_run {
debug!(
%crate_name,
%version,
%publish_url,
"skipping publish (--dry-run mode)",
);
continue;
}
let resp = http_client.put(publish_url.clone())
.header(AUTHORIZATION, &config.dst.auth_token)
.body(payload)
.send()
.await?;
Ok(out)
}
debug!(status = ?resp.status(), "rcvd server response to publish request");
let warnings: PublishWarnings = resp
.error_for_status()?
.json()
.await?;
// fn get_registry_dep<'a>(index_dep: &'a IndexDependency) -> Option<Node<'a>> {
// match index_dep.registry.as_ref() {
// None => Some(Node { name: index_dep.name.as_str(), vers: index_dep.vers.clone() }),
// Some(index) if index.contains("github.com/rust-lang/crates.io-index") => None,
// Some(other) => panic!("unexpected registry value: {}", other),
// }
// }
// fn build_dependency_graph<'a>(
// crate_versions: &'a HashMap<String, Vec<VersionMeta>>,
// ) -> (StableGraph<Node<'a>, ()>, HashMap<Node<'a>, NodeIndex<u32>>) {
// let begin = Instant::now();
//
// let mut graph = StableGraph::new();
// let mut index: HashMap<Node<'a>, NodeIndex<u32>> = Default::default();
//
// macro_rules! get_ix {
// ($node:expr) => {{
// let key_exists = ix.contains_key($node);
// if !key_exists {
// let ix = graph.add_node($node.clnoe());
// index.insert(node.clone(), ix);
// ix
// } else {
// *index[$node]
// }
// }}
// }
//
// for (name, versions) in crate_versions.iter() {
// for version_meta in versions.iter() {
// let v = &version_meta.index_meta;
// let node = Node { name: name.as_str(), vers: v.vers.clone() };
// let key_exists = ix.contains_key(&node);
// let ix = get_ix!(&node);
// for dep_node in v.deps.iter().filter_map(get_registry_dep) {
// let jx = get_ix!(dep_node);
// graph.add_edge(ix, js, ());
// }
// }
// }
//
// info!(
// n_nodes = graph.node_count(),
// n_edges = graph.edge_count(),
// "built dependency graph for entire registry in {:?}", begin.elapsed(),
// );
//
// (graph, index)
// }
let mut any_warnings = false;
for warning in warnings.invalid_categories.iter() {
warn!(%crate_name, %version, "registry server invalid category warning: {}", warning);
any_warnings = true;
}
for warning in warnings.invalid_badges.iter() {
warn!(%crate_name, %version, "registry server invalid badge warning: {}", warning);
any_warnings = true;
}
for warning in warnings.other.iter() {
warn!(%crate_name, %version, "registry server 'other' warning: {}", warning);
any_warnings = true;
}
trace!("server response body:\n{warnings:#?}");
info!(
%crate_name,
%version,
any_warnings,
"published crate version!",
);
}
}
Ok(())
}
async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> {
match tokio::fs::metadata(path.as_ref()).await {
@ -814,8 +627,8 @@ async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Err
}
}
fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> {
match std::fs::metadata(path.as_ref()) {
async fn verify_file_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Error> {
match tokio::fs::metadata(path.as_ref()).await {
Ok(meta) if meta.is_file() => Ok(()),
Ok(meta) /* if ! meta.is_file() */ => {
debug_assert!( ! meta.is_file());
@ -850,12 +663,7 @@ fn main() -> Result<(), Error> {
let krates = rt.block_on(get_index_metas(&config))?;
let manifests = parse_manifests(&config, krates)?;
// let (graph, ix) = build_dependency_graph(&manifests);
drop(manifests);
drop(rt);
rt.block_on(process_crates(&config, krates))?;
info!("finished in {:?}", begin.elapsed());
Ok(())

Loading…
Cancel
Save