From 1892048654cc0299d2ddeaab94c0b20bb0da1129 Mon Sep 17 00:00:00 2001 From: Jonathan Strong Date: Thu, 15 Sep 2022 16:19:37 -0400 Subject: [PATCH] implement template-based output paths (--output-format) --- Cargo.lock | 142 +++++++++++++++++++++++++++------------------------- Cargo.toml | 2 +- src/main.rs | 64 +++++++++++++++-------- 3 files changed, 119 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4dc7b6..b9c4351 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,15 +19,15 @@ dependencies = [ [[package]] name = "alloc-no-stdlib" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ "alloc-no-stdlib", ] @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.20" +version = "3.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b71c3ce99b7611011217b366d923f1d0a7e07a92bb2dbf1e84508c673ca3bd" +checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7" dependencies = [ "atty", "bitflags", @@ -260,7 +260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" dependencies = [ "cookie", - "idna", + "idna 0.2.3", "log", "publicsuffix", "serde", @@ -393,11 +393,10 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", "percent-encoding", ] @@ -694,9 +693,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.47" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" +checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -717,6 +716,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "ignore" version = "0.4.18" @@ -777,9 +786,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "js-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] @@ -884,9 +893,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" dependencies = [ "adler", ] @@ -1008,15 +1017,15 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4" +checksum = "cb779fcf4bb850fbbb0edc96ff6cf34fd90c4b1a112ce042653280d9a7364048" dependencies = [ "thiserror", "ucd-trie", @@ -1024,9 +1033,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905708f7f674518498c1f8d644481440f476d39ca6ecae83319bba7c6c12da91" +checksum = "502b62a6d0245378b04ffe0a7fb4f4419a4815fce813bd8a0ec89a56e07d67b1" dependencies = [ "pest", "pest_generator", @@ -1034,9 +1043,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5803d8284a629cc999094ecd630f55e91b561a1d1ba75e233b00ae13b91a69ad" +checksum = "451e629bf49b750254da26132f1a5a9d11fd8a95a3df51d15c4abd1ba154cb6c" dependencies = [ "pest", "pest_meta", @@ -1047,13 +1056,13 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1538eb784f07615c6d9a8ab061089c6c54a344c5b4301db51990ca1c241e8c04" +checksum = "bcec162c71c45e269dfc3fc2916eaeb97feab22993a21bcce4721d08cd7801a6" dependencies = [ "once_cell", "pest", - "sha-1", + "sha1", ] [[package]] @@ -1175,7 +1184,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aeeedb0b429dc462f30ad27ef3de97058b060016f47790c066757be38ef792b4" dependencies = [ - "idna", + "idna 0.2.3", "psl-types", ] @@ -1231,7 +1240,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -1241,7 +1250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -1261,18 +1270,18 @@ checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] [[package]] name = "raw-cpuid" -version = "10.5.0" +version = "10.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6" +checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb" dependencies = [ "bitflags", ] @@ -1323,7 +1332,7 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "registry-backup" -version = "0.4.0-rc.2" +version = "0.4.0-rc.3" dependencies = [ "chrono", "clap", @@ -1521,10 +1530,10 @@ dependencies = [ ] [[package]] -name = "sha-1" -version = "0.10.0" +name = "sha1" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" dependencies = [ "cfg-if", "cpufeatures", @@ -1655,18 +1664,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0a539a918745651435ac7db7a18761589a94cd7e94cd56999f828bf73c8a57" +checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c251e90f708e16c49a16f4917dc2131e75222b72edfa9cb7f7c58ae56aae0c09" +checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" dependencies = [ "proc-macro2", "quote", @@ -1727,9 +1736,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.0" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89797afd69d206ccd11fb0ea560a44bbb87731d020670e79416d442919257d42" +checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" dependencies = [ "autocfg", "bytes", @@ -1770,9 +1779,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes", "futures-core", @@ -1886,7 +1895,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.2.3", "ipnet", "lazy_static", "log", @@ -2012,9 +2021,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-normalization" @@ -2033,13 +2042,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", - "matches", + "idna 0.3.0", "percent-encoding", ] @@ -2090,9 +2098,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2100,9 +2108,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", @@ -2115,9 +2123,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ "cfg-if", "js-sys", @@ -2127,9 +2135,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2137,9 +2145,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -2150,15 +2158,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 74c087e..e670426 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "registry-backup" authors = ["Jonathan Strong "] -version = "0.4.0-rc.2" +version = "0.4.0-rc.3" edition = "2021" publish = ["shipyard-rs-public"] readme = "README.md" diff --git a/src/main.rs b/src/main.rs index 2eae4c9..ed5857a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -113,26 +113,24 @@ pub struct OutputConfig { #[clap(short = 'o', long = "output-path", default_value = DEFAULT_OUTPUT_PATH)] #[serde(default = "default_output_path")] pub path: PathBuf, - /// Download files when if .crate file already exists in output dir for a /// given crate version, and overwrite the existing file with the new one. /// Default behavior is to skip downloading if .crate file already exists. #[serde(default)] #[clap(long)] pub overwrite_existing: bool, - // /// What format to use for the output filenames. Works the same as - // /// Cargo's registry syntax for the "dl" key in the `config.json` - // /// file in a reigstry index. See [Cargo - // /// docs](https://doc.rust-lang.org/cargo/reference/registries.html#index-format) - // /// for additional details. Not specifying this field is equivalent - // /// to specifying "/{crate}/{version}/download", the default. - // #[clap(long = "output-format")] - // pub format: Option, - // /// Use whatever output filename syntax is specified in the target - // /// registry's `config.json` file. Conflicts with --output-format. - // #[serde(default)] - // #[clap(long)] - // pub mirror_registry_format: bool, + /// What format to use for the output filenames. Works the same as + /// Cargo's registry syntax for the "dl" key in the `config.json` + /// file in a reigstry index. See [Cargo + /// docs](https://doc.rust-lang.org/cargo/reference/registries.html#index-format) + /// for additional details. Not specifying this field is equivalent + /// to specifying "{crate}/{version}/download", the default. + /// + /// The resulting path specified by the format should be relative; + /// it will be joined with the --output-path. (i.e. it should not start + /// with "/".) + #[clap(long = "output-format")] + pub format: Option, } #[derive(Deserialize, Parser)] @@ -598,6 +596,15 @@ async fn download_versions( let rate_limit = RateLimiter::direct(Quota::per_second(config.http.requests_per_second)); + // piggy back on Registry Config's url template functionality + // to render the relative output path + let output_config = RegistryConfig { + dl: config.output.format.as_deref().unwrap_or("").to_string(), + api: String::new(), + allowed_registries: vec![], + auth_required: None, + }; + let http_client = reqwest::Client::builder() .user_agent(&config.http.user_agent) .build()?; @@ -612,17 +619,17 @@ async fn download_versions( let inner_stream = futures::stream::iter(versions.into_iter().map(|vers| { let req_begin = Instant::now(); let http_client = http_client.clone(); + let output_config = &output_config; async move { - // TODO actually parse and use the format let url = url::Url::parse(®istry_config.get_dl_url(&vers.name, &vers.vers, &vers.cksum))?; - //let output_path = config.output.path.join(format!("{}/{}/download", vers.name, vers.vers)); - assert!( - url.path().starts_with('/'), - "sanity check in case there are unforseen exceptions to this" - ); - let output_path = config.output.path.join(&url.path()[1..]); + + let output_path = config.output.path.join(output_config.get_dl_url( + &vers.name, + &vers.vers, + &vers.cksum, + )); if config.dry_run { debug!(%url, "skipping download (--dry-run mode)"); @@ -786,4 +793,19 @@ mod tests { "iM/-1/iM-14yo-LoL__im/-1/im-14yo-lol__c5b6fc73__iM-14yo-LoL__0.69.42-rc.123.tar.gz", ); } + + #[test] + fn sanity_check_url_template_used_for_output_path() { + let config = RegistryConfig { + dl: "{crate}/{sha256-checksum}".to_string(), + api: String::new(), + allowed_registries: vec![], + auth_required: Some(true), + }; + + let output_path = + Path::new("output").join(config.get_dl_url("lazy-static", "1.0.0", "x7b2z899")); + + assert_eq!(output_path, Path::new("output/lazy-static/x7b2z899"),); + } }