diff --git a/script/get-publish-history.py b/script/get-publish-history.py index a76fe97..4bc3797 100644 --- a/script/get-publish-history.py +++ b/script/get-publish-history.py @@ -111,8 +111,8 @@ def versions(path, branch='master'): def main(path): df = pd.DataFrame(versions(path)) - df['time'] = pd.to_datetime(df['time']) - df['unix_nanos'] = df['time'].astype('int') + df['time'] = pd.to_datetime(df['time'], utc=True) + # df['unix_nanos'] = df['time'].astype('int') df = df.sort_values(by='time').groupby(['crate_name', 'version']).last().reset_index() buf = io.StringIO() df.to_csv(buf, index=False) diff --git a/src/publish.rs b/src/publish.rs index 6a845c1..9a536fd 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -72,6 +72,7 @@ pub struct SourceRegistryConfig { pub registry_name: String, /// Path of CSV file with log of when each crate version was published. pub publish_history_csv: PathBuf, + pub index_url: String, } #[derive(Deserialize, Debug, Clone)] @@ -144,7 +145,7 @@ pub struct PublishLogRow { pub commit: String, pub author: String, pub time: DateTime, - pub unix_nanos: u64, + // pub unix_nanos: u64, } struct CsvSetup { @@ -752,7 +753,11 @@ fn parse_manifests( trace!("extracting Cargo.toml from .crate targz archive"); let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]); - let manifest_files = extract_manifest_files_from_tar(decoder)?; + let manifest_files = extract_manifest_files_from_tar(decoder) + .map_err(|err| { + error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files"); + err + })?; if manifest_files.cargo_lock.is_none() { debug!(%crate_name, %version, "Cargo.lock not present in .crate archive"); } @@ -811,9 +816,13 @@ fn parse_manifests( fn edit_dep_registries( dep_key: &str, manifest: &mut toml_edit::Document, - src_registry_name: &str, - dst_registry_name: &str, + config: &Config, ) -> Result<(), Error> { + let src_registry_name = config.src.registry_name.as_str(); + let dst_registry_name = config.dst.registry_name.as_str(); + let src_index_url = config.src.index_url.as_str(); + let dst_index_url = config.dst.index_url.as_str(); + let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else { trace!("missing key in manifest toml: {}", dep_key); return Ok(()) @@ -821,15 +830,20 @@ fn edit_dep_registries( for (k, v) in deps.iter_mut() { let Some(t) = v.as_table_like_mut() else { continue }; - if t.contains_key("registry-index") { - warn!(dep_name = ?k, "dep table contains registry-index key!"); - } + if let Some(registry_item) = t.get_mut("registry") { if registry_item.as_str().unwrap_or("") == src_registry_name { trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml"); *registry_item = toml_edit::value(dst_registry_name); } } + + if let Some(registry_index_item) = t.get_mut("registry-index") { + if registry_index_item.as_str().unwrap_or("") == src_index_url { + trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml"); + *registry_index_item = toml_edit::value(dst_index_url); + } + } } Ok(()) } @@ -866,9 +880,9 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re let source_dir = meta.source_dir(); let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::()?; - edit_dep_registries("dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; - edit_dep_registries("dev-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; - edit_dep_registries("build-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; + edit_dep_registries("dependencies", &mut modified_manifest, &config)?; + edit_dep_registries("dev-dependencies", &mut modified_manifest, &config)?; + edit_dep_registries("build-dependencies", &mut modified_manifest, &config)?; edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?; @@ -1031,10 +1045,17 @@ fn main() -> Result<(), Error> { } let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?; - publish_log.sort_by_key(|x| x.unix_nanos); + publish_log.sort_by_key(|x| x.time); assert!(!publish_log.is_empty()); info!(n_rows = publish_log.len(), "parsed publish log csv"); + if let Some(filter) = config.compile_filter()? { + publish_log.retain(|x| { + filter.is_match(&x.crate_name) + }); + info!(n_filtered_rows = publish_log.len(), "filtered publish log"); + } + let krates = rt.block_on(get_index_metas(&config))?; let mut manifests = parse_manifests(&config, krates)?;