bugfixes, etc.

script/get-publish-history.py

@@ -111,8 +111,8 @@ def versions(path, branch='master'):
 
 def main(path):
     df = pd.DataFrame(versions(path))
-    df['time'] = pd.to_datetime(df['time'])
-    df['unix_nanos'] = df['time'].astype('int')
+    df['time'] = pd.to_datetime(df['time'], utc=True)
+    # df['unix_nanos'] = df['time'].astype('int')
     df = df.sort_values(by='time').groupby(['crate_name', 'version']).last().reset_index()
     buf = io.StringIO()
     df.to_csv(buf, index=False)

src/main.rs

@@ -512,7 +512,7 @@ async fn get_crate_versions(
     let crate_versions: Vec<CrateVersion> = crate_versions
         .into_iter()
         .flat_map(|result| match result {
-            Ok(xs) => xs.into_iter().filter(|x| x.name != "vst").collect(),
+            Ok(xs) => xs.into_iter().collect(),
             Err(e) => {
                 error!(err = ?e, "parsing metadata failed, skipping file");
                 vec![]
@@ -550,6 +550,23 @@ async fn ensure_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), Any
     }
 }
 
+async fn verify_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), AnyError> {
+    match tokio::fs::metadata(path.as_ref()).await {
+        Ok(meta) if meta.is_dir() => Ok(()),
+
+        Ok(meta) /* if ! meta.is_dir() */ => {
+            debug_assert!( ! meta.is_dir());
+            Err(format!("path exists, but is not a directory: {:?}", path.as_ref()).into())
+        }
+
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+            Err(format!("path does not exist: {}", path.as_ref().display()).into())
+        }
+
+        Err(e) => Err(e.into()),
+    }
+}
+
 async fn ensure_file_parent_dir_exists<P: AsRef<std::path::Path>>(path: P) -> Result<(), AnyError> {
     if let Some(parent_dir) = path.as_ref().parent() {
         ensure_dir_exists(parent_dir).await
@@ -757,7 +774,10 @@ async fn run(config: Config) -> Result<(), AnyError> {
             tmp
         }
 
-        (_, Some(path)) => path,
+        (_, Some(path)) => {
+            verify_dir_exists(&path).await?;
+            path
+        }
 
         _ => unreachable!(),
     };

src/publish.rs

@@ -72,6 +72,7 @@ pub struct SourceRegistryConfig {
     pub registry_name: String,
     /// Path of CSV file with log of when each crate version was published.
     pub publish_history_csv: PathBuf,
+    pub index_url: String,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -144,7 +145,7 @@ pub struct PublishLogRow {
     pub commit: String,
     pub author: String,
     pub time: DateTime<Utc>,
-    pub unix_nanos: u64,
+    // pub unix_nanos: u64,
 }
 
 struct CsvSetup {
@@ -752,7 +753,11 @@ fn parse_manifests(
 
                 trace!("extracting Cargo.toml from .crate targz archive");
                 let decoder = flate2::read::GzDecoder::new(&dot_crate_bytes[..]);
-                let manifest_files = extract_manifest_files_from_tar(decoder)?;
+                let manifest_files = extract_manifest_files_from_tar(decoder)
+                    .map_err(|err| {
+                        error!(%crate_name, vers = %index_meta.vers, ?err, "failed to extract manifest files");
+                        err
+                    })?;
                 if manifest_files.cargo_lock.is_none() {
                     debug!(%crate_name, %version, "Cargo.lock not present in .crate archive");
                 }
@@ -811,9 +816,13 @@ fn parse_manifests(
 fn edit_dep_registries(
     dep_key: &str,
     manifest: &mut toml_edit::Document,
-    src_registry_name: &str,
-    dst_registry_name: &str,
+    config: &Config,
 ) -> Result<(), Error> {
+    let src_registry_name = config.src.registry_name.as_str();
+    let dst_registry_name = config.dst.registry_name.as_str();
+    let src_index_url = config.src.index_url.as_str();
+    let dst_index_url = config.dst.index_url.as_str();
+
     let Some(deps) = manifest.get_mut(dep_key).and_then(|item| item.as_table_like_mut()) else {
         trace!("missing key in manifest toml: {}", dep_key);
         return Ok(())
@@ -821,15 +830,20 @@ fn edit_dep_registries(
 
     for (k, v) in deps.iter_mut() {
         let Some(t) = v.as_table_like_mut() else { continue };
-        if t.contains_key("registry-index") {
-            warn!(dep_name = ?k, "dep table contains registry-index key!");
-        }
+
         if let Some(registry_item) = t.get_mut("registry") {
             if registry_item.as_str().unwrap_or("") == src_registry_name {
                 trace!(dep_name = ?k, %dep_key, ?src_registry_name, ?dst_registry_name, "modifying registry in Cargo.toml");
                 *registry_item = toml_edit::value(dst_registry_name);
             }
         }
+
+        if let Some(registry_index_item) = t.get_mut("registry-index") {
+            if registry_index_item.as_str().unwrap_or("") == src_index_url {
+                trace!(dep_name = ?k, %dep_key, ?src_index_url, ?dst_index_url, "modifying registry-index in Cargo.toml");
+                *registry_index_item = toml_edit::value(dst_index_url);
+            }
+        }
     }
     Ok(())
 }
@@ -866,9 +880,9 @@ fn prepare_source_dir_for_publish(config: &Config, meta: &mut VersionMeta) -> Re
     let source_dir = meta.source_dir();
     let mut modified_manifest = meta.manifest_files.cargo_toml_orig.parse::<toml_edit::Document>()?;
 
-    edit_dep_registries("dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
-    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
-    edit_dep_registries("build-dependencies", &mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
+    edit_dep_registries("dependencies", &mut modified_manifest, &config)?;
+    edit_dep_registries("dev-dependencies", &mut modified_manifest, &config)?;
+    edit_dep_registries("build-dependencies", &mut modified_manifest, &config)?;
 
     edit_publish_registry_if_present(&mut modified_manifest, &config.src.registry_name, &config.dst.registry_name)?;
 
@@ -1031,10 +1045,17 @@ fn main() -> Result<(), Error> {
     }
 
     let mut publish_log = read_publish_log_csv(&config.src.publish_history_csv)?;
-    publish_log.sort_by_key(|x| x.unix_nanos);
+    publish_log.sort_by_key(|x| x.time);
     assert!(!publish_log.is_empty());
     info!(n_rows = publish_log.len(), "parsed publish log csv");
 
+    if let Some(filter) = config.compile_filter()? {
+        publish_log.retain(|x| {
+            filter.is_match(&x.crate_name)
+        });
+        info!(n_filtered_rows = publish_log.len(), "filtered publish log");
+    }
+
     let krates = rt.block_on(get_index_metas(&config))?;
 
     let mut manifests = parse_manifests(&config, krates)?;