jstrong
/
registry-backup
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity
Browse Source

implement template-based output paths (--output-format)

master
Jonathan Strong 2 years ago
parent
50017aa456
commit
1892048654
3 changed files with 119 additions and 89 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 142
      Cargo.lock
  2. 2
      Cargo.toml
  3. 64
      src/main.rs

142
Cargo.lock generated
Unescape View File

@ -19,15 +19,15 @@ dependencies = [
[[package]]
name = "alloc-no-stdlib"
version = "2.0.3"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
@ -206,9 +206,9 @@ dependencies = [
[[package]]
name = "clap"
version = "3.2.20"
version = "3.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b71c3ce99b7611011217b366d923f1d0a7e07a92bb2dbf1e84508c673ca3bd"
checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7"
dependencies = [
"atty",
"bitflags",
@ -260,7 +260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd"
dependencies = [
"cookie",
"idna",
"idna 0.2.3",
"log",
"publicsuffix",
"serde",
@ -393,11 +393,10 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.0.1"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
dependencies = [
"matches",
"percent-encoding",
]
@ -694,9 +693,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
version = "0.1.47"
version = "0.1.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7"
checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@ -717,6 +716,16 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "idna"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
dependencies = [
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "ignore"
version = "0.4.18"
@ -777,9 +786,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
[[package]]
name = "js-sys"
version = "0.3.59"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
dependencies = [
"wasm-bindgen",
]
@ -884,9 +893,9 @@ dependencies = [
[[package]]
name = "miniz_oxide"
version = "0.5.3"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
dependencies = [
"adler",
]
@ -1008,15 +1017,15 @@ dependencies = [
[[package]]
name = "percent-encoding"
version = "2.1.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "pest"
version = "2.3.0"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4"
checksum = "cb779fcf4bb850fbbb0edc96ff6cf34fd90c4b1a112ce042653280d9a7364048"
dependencies = [
"thiserror",
"ucd-trie",
@ -1024,9 +1033,9 @@ dependencies = [
[[package]]
name = "pest_derive"
version = "2.3.0"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "905708f7f674518498c1f8d644481440f476d39ca6ecae83319bba7c6c12da91"
checksum = "502b62a6d0245378b04ffe0a7fb4f4419a4815fce813bd8a0ec89a56e07d67b1"
dependencies = [
"pest",
"pest_generator",
@ -1034,9 +1043,9 @@ dependencies = [
[[package]]
name = "pest_generator"
version = "2.3.0"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5803d8284a629cc999094ecd630f55e91b561a1d1ba75e233b00ae13b91a69ad"
checksum = "451e629bf49b750254da26132f1a5a9d11fd8a95a3df51d15c4abd1ba154cb6c"
dependencies = [
"pest",
"pest_meta",
@ -1047,13 +1056,13 @@ dependencies = [
[[package]]
name = "pest_meta"
version = "2.3.0"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1538eb784f07615c6d9a8ab061089c6c54a344c5b4301db51990ca1c241e8c04"
checksum = "bcec162c71c45e269dfc3fc2916eaeb97feab22993a21bcce4721d08cd7801a6"
dependencies = [
"once_cell",
"pest",
"sha-1",
"sha1",
]
[[package]]
@ -1175,7 +1184,7 @@ version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aeeedb0b429dc462f30ad27ef3de97058b060016f47790c066757be38ef792b4"
dependencies = [
"idna",
"idna 0.2.3",
"psl-types",
]
@ -1231,7 +1240,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core 0.6.3",
"rand_core 0.6.4",
]
[[package]]
@ -1241,7 +1250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.3",
"rand_core 0.6.4",
]
[[package]]
@ -1261,18 +1270,18 @@ checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_core"
version = "0.6.3"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "raw-cpuid"
version = "10.5.0"
version = "10.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6"
checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb"
dependencies = [
"bitflags",
]
@ -1323,7 +1332,7 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
[[package]]
name = "registry-backup"
version = "0.4.0-rc.2"
version = "0.4.0-rc.3"
dependencies = [
"chrono",
"clap",
@ -1521,10 +1530,10 @@ dependencies = [
]
[[package]]
name = "sha-1"
version = "0.10.0"
name = "sha1"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549"
dependencies = [
"cfg-if",
"cpufeatures",
@ -1655,18 +1664,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "thiserror"
version = "1.0.33"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d0a539a918745651435ac7db7a18761589a94cd7e94cd56999f828bf73c8a57"
checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.33"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c251e90f708e16c49a16f4917dc2131e75222b72edfa9cb7f7c58ae56aae0c09"
checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783"
dependencies = [
"proc-macro2",
"quote",
@ -1727,9 +1736,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.21.0"
version = "1.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89797afd69d206ccd11fb0ea560a44bbb87731d020670e79416d442919257d42"
checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95"
dependencies = [
"autocfg",
"bytes",
@ -1770,9 +1779,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.7.3"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45"
checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
dependencies = [
"bytes",
"futures-core",
@ -1886,7 +1895,7 @@ dependencies = [
"futures-channel",
"futures-io",
"futures-util",
"idna",
"idna 0.2.3",
"ipnet",
"lazy_static",
"log",
@ -2012,9 +2021,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
[[package]]
name = "unicode-ident"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
[[package]]
name = "unicode-normalization"
@ -2033,13 +2042,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.2.2"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
dependencies = [
"form_urlencoded",
"idna",
"matches",
"idna 0.3.0",
"percent-encoding",
]
@ -2090,9 +2098,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.82"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -2100,9 +2108,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.82"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
dependencies = [
"bumpalo",
"log",
@ -2115,9 +2123,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.32"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad"
checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
dependencies = [
"cfg-if",
"js-sys",
@ -2127,9 +2135,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.82"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2137,9 +2145,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.82"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [
"proc-macro2",
"quote",
@ -2150,15 +2158,15 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.82"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]]
name = "web-sys"
version = "0.3.59"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1"
checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
dependencies = [
"js-sys",
"wasm-bindgen",

2
Cargo.toml
Unescape View File

@ -1,7 +1,7 @@
[package]
name = "registry-backup"
authors = ["Jonathan Strong <jstrong@shipyard.rs>"]
version = "0.4.0-rc.2"
version = "0.4.0-rc.3"
edition = "2021"
publish = ["shipyard-rs-public"]
readme = "README.md"

64
src/main.rs
Unescape View File

@ -113,26 +113,24 @@ pub struct OutputConfig {
#[clap(short = 'o', long = "output-path", default_value = DEFAULT_OUTPUT_PATH)]
#[serde(default = "default_output_path")]
pub path: PathBuf,
/// Download files when if .crate file already exists in output dir for a
/// given crate version, and overwrite the existing file with the new one.
/// Default behavior is to skip downloading if .crate file already exists.
#[serde(default)]
#[clap(long)]
pub overwrite_existing: bool,
// /// What format to use for the output filenames. Works the same as
// /// Cargo's registry syntax for the "dl" key in the `config.json`
// /// file in a reigstry index. See [Cargo
// /// docs](https://doc.rust-lang.org/cargo/reference/registries.html#index-format)
// /// for additional details. Not specifying this field is equivalent
// /// to specifying "/{crate}/{version}/download", the default.
// #[clap(long = "output-format")]
// pub format: Option<String>,
// /// Use whatever output filename syntax is specified in the target
// /// registry's `config.json` file. Conflicts with --output-format.
// #[serde(default)]
// #[clap(long)]
// pub mirror_registry_format: bool,
/// What format to use for the output filenames. Works the same as
/// Cargo's registry syntax for the "dl" key in the `config.json`
/// file in a reigstry index. See [Cargo
/// docs](https://doc.rust-lang.org/cargo/reference/registries.html#index-format)
/// for additional details. Not specifying this field is equivalent
/// to specifying "{crate}/{version}/download", the default.
///
/// The resulting path specified by the format should be relative;
/// it will be joined with the --output-path. (i.e. it should not start
/// with "/".)
#[clap(long = "output-format")]
pub format: Option<String>,
}
#[derive(Deserialize, Parser)]
@ -598,6 +596,15 @@ async fn download_versions(
let rate_limit = RateLimiter::direct(Quota::per_second(config.http.requests_per_second));
// piggy back on Registry Config's url template functionality
// to render the relative output path
let output_config = RegistryConfig {
dl: config.output.format.as_deref().unwrap_or("").to_string(),
api: String::new(),
allowed_registries: vec![],
auth_required: None,
};
let http_client = reqwest::Client::builder()
.user_agent(&config.http.user_agent)
.build()?;
@ -612,17 +619,17 @@ async fn download_versions(
let inner_stream = futures::stream::iter(versions.into_iter().map(|vers| {
let req_begin = Instant::now();
let http_client = http_client.clone();
let output_config = &output_config;
async move {
// TODO actually parse and use the format
let url =
url::Url::parse(&registry_config.get_dl_url(&vers.name, &vers.vers, &vers.cksum))?;
//let output_path = config.output.path.join(format!("{}/{}/download", vers.name, vers.vers));
assert!(
url.path().starts_with('/'),
"sanity check in case there are unforseen exceptions to this"
);
let output_path = config.output.path.join(&url.path()[1..]);
let output_path = config.output.path.join(output_config.get_dl_url(
&vers.name,
&vers.vers,
&vers.cksum,
));
if config.dry_run {
debug!(%url, "skipping download (--dry-run mode)");
@ -786,4 +793,19 @@ mod tests {
"iM/-1/iM-14yo-LoL__im/-1/im-14yo-lol__c5b6fc73__iM-14yo-LoL__0.69.42-rc.123.tar.gz",
);
}
#[test]
fn sanity_check_url_template_used_for_output_path() {
let config = RegistryConfig {
dl: "{crate}/{sha256-checksum}".to_string(),
api: String::new(),
allowed_registries: vec![],
auth_required: Some(true),
};
let output_path =
Path::new("output").join(config.get_dl_url("lazy-static", "1.0.0", "x7b2z899"));
assert_eq!(output_path, Path::new("output/lazy-static/x7b2z899"),);
}
}

Write Preview
Loading…
Cancel
Save