From 24bbd17e82751770a004cb866a1f532b7c530a7b Mon Sep 17 00:00:00 2001
From: Jonathan Strong <jstrong@shipyard.rs>
Date: Tue, 7 Nov 2023 16:23:05 -0500
Subject: [PATCH] bugfixes, etc.

---
 script/get-publish-history.py |  4 ++--
 src/publish.rs                | 43 ++++++++++++++++++++++++++---------
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/script/get-publish-history.py b/script/get-publish-history.py
index a76fe97..4bc3797 100644
--- a/script/get-publish-history.py
+++ b/script/get-publish-history.py
@@ -111,8 +111,8 @@ def versions(path, branch='master'):
 
 def main(path):
     df = pd.DataFrame(versions(path))
-    df['time'] = pd.to_datetime(df['time'])
-    df['unix_nanos'] = df['time'].astype('int')
+    df['time'] = pd.to_datetime(df['time'], utc=True)
+    # df['unix_nanos'] = df['time'].astype('int')
     df = df.sort_values(by='time').groupby(['crate_name', 'version']).last().reset_index()
     buf = io.StringIO()
     df.to_csv(buf, index=False)
diff --git a/src/publish.rs b/src/publish.rs
index 6a845c1..9a536fd 100644
--- a/src/publish.rs
+++ b/src/publish.rs
@@ -72,6 +72,7 @@ pub struct SourceRegistryConfig {
     pub registry_name: String,
     /// Path of CSV file with log of when each crate version was published.
     pub publish_history_csv: PathBuf,
+    pub index_url: String,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -144,7 +145,7 @@ pub struct PublishLogRow {
     pub commit: String,
     pub author: String,
     pub time: DateTime<Utc>,
-    pub unix_nanos: u64,
+    // pub unix_nanos: u64,
 }
 
 struct CsvSetup {
@@ -752,7 +753,11 @@ fn parse_manifests(
 
                 trace!("extracting Cargo.toml from .crate targz archive");
                 let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
-                let manifest_files = extract_manifest_files_from_tar(decoder)?;
+                let manifest_files = extract_manifest_files_from_tar(decoder)
+                    .map_err(|err| {
+                        error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
+                        err
+                    })?;
                 if manifest_files.cargo_lock.is_none() {
                     debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
                 }
@@ -811,9 +816,13 @@ fn parse_manifests(
 fn edit_dep_registries(
     dep_key: &str,
     manifest: &mut toml_edit::Document,
-    src_registry_name: &str,
-    dst_registry_name: &str,
+    config: &Config,
 ) -> Result<(), Error> {
+    let src_registry_name = config.src.registry_name.as_str();
+    let dst_registry_name = config.dst.registry_name.as_str();
+    let src_index_url = config.src.index_url.as_str();
+    let dst_index_url = config.dst.index_url.as_str();
+
     let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else {
         trace!("missing key in manifest toml: {}", dep_key);
         return Ok(())
@@ -821,15 +830,20 @@ fn edit_dep_registries(
 
     for (k, v) in deps.iter_mut() {
         let Some(t) = v.as_table_like_mut() else { continue };
-        if t.contains_key("registry-index") {
-            warn!(dep_name = ?k, "dep table contains registry-index key!");
-        }
+
         if let Some(registry_item) = t.get_mut("registry") {
             if registry_item.as_str().unwrap_or("") == src_registry_name {
                 trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
                 *registry_item = toml_edit::value(dst_registry_name);
             }
         }
+
+        if let Some(registry_index_item) = t.get_mut("registry-index") {
+            if registry_index_item.as_str().unwrap_or("") == src_index_url {
+                trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
+                *registry_index_item = toml_edit::value(dst_index_url);
+            }
+        }
     }
     Ok(())
 }
@@ -866,9 +880,9 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
     let source_dir = meta.source_dir();
     let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?;
 
-    edit_dep_registries("dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
-    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
-    edit_dep_registries("build-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
+    edit_dep_registries("dependencies", &mut modified_manifest, &config)?;
+    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config)?;
+    edit_dep_registries("build-dependencies", &mut modified_manifest, &config)?;
 
     edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
 
@@ -1031,10 +1045,17 @@ fn main() -> Result<(), Error> {
     }
 
     let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?;
-    publish_log.sort_by_key(|x| x.unix_nanos);
+    publish_log.sort_by_key(|x| x.time);
     assert!(!publish_log.is_empty());
     info!(n_rows = publish_log.len(), "parsed publish log csv");
 
+    if let Some(filter) = config.compile_filter()? {
+        publish_log.retain(|x| {
+            filter.is_match(&x.crate_name)
+        });
+        info!(n_filtered_rows = publish_log.len(), "filtered publish log");
+    }
+
     let krates = rt.block_on(get_index_metas(&config))?;
 
     let mut manifests = parse_manifests(&config, krates)?;