jstrong
/
registry-backup
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity
Browse Source

adds template-based README.md with script to re-generate

master
Jonathan Strong 3 years ago
parent
a6d6f5a0c3
commit
3a8426f172
9 changed files with 700 additions and 6 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 376
      Cargo.lock
  2. 17
      Cargo.toml
  3. 128
      README.md
  4. 56
      doc/README.tera.md
  5. 43
      doc/cli-menu.txt
  6. 12
      doc/just-commands.txt
  7. 40
      justfile
  8. 31
      src/generate-readme.rs
  9. 3
      src/main.rs

376
Cargo.lock generated
Unescape View File

@ -8,6 +8,15 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "0.7.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "alloc-no-stdlib" name = "alloc-no-stdlib"
version = "2.0.3" version = "2.0.3"
@ -84,6 +93,15 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "brotli" name = "brotli"
version = "3.3.4" version = "3.3.4"
@ -105,6 +123,15 @@ dependencies = [
"alloc-stdlib", "alloc-stdlib",
] ]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.11.0" version = "3.11.0"
@ -136,11 +163,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
dependencies = [ dependencies = [
"iana-time-zone", "iana-time-zone",
"js-sys",
"num-integer", "num-integer",
"num-traits 0.2.15", "num-traits 0.2.15",
"time 0.1.43",
"wasm-bindgen",
"winapi", "winapi",
] ]
[[package]]
name = "chrono-tz"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c39203181991a7dd4343b8005bd804e7a9a37afb8ac070e43771e8c820bbde"
dependencies = [
"chrono",
"chrono-tz-build",
"phf",
]
[[package]]
name = "chrono-tz-build"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f509c3a87b33437b05e2458750a0700e5bdd6956176773e6c7d6dd15a283a0c"
dependencies = [
"parse-zoneinfo",
"phf",
"phf_codegen",
]
[[package]] [[package]]
name = "clap" name = "clap"
version = "3.2.20" version = "3.2.20"
@ -184,7 +236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05"
dependencies = [ dependencies = [
"percent-encoding", "percent-encoding",
"time", "time 0.3.14",
"version_check", "version_check",
] ]
@ -200,7 +252,7 @@ dependencies = [
"publicsuffix", "publicsuffix",
"serde", "serde",
"serde_json", "serde_json",
"time", "time 0.3.14",
"url", "url",
] ]
@ -210,6 +262,15 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cpufeatures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.3.2" version = "1.3.2"
@ -229,6 +290,16 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "5.4.0" version = "5.4.0"
@ -248,6 +319,22 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
[[package]]
name = "deunicode"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690"
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]] [[package]]
name = "dtoa" name = "dtoa"
version = "0.4.8" version = "0.4.8"
@ -402,6 +489,16 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "generic-array"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
dependencies = [
"typenum",
"version_check",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.7" version = "0.2.7"
@ -413,6 +510,30 @@ dependencies = [
"wasi 0.11.0+wasi-snapshot-preview1", "wasi 0.11.0+wasi-snapshot-preview1",
] ]
[[package]]
name = "globset"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a"
dependencies = [
"aho-corasick",
"bstr",
"fnv",
"log",
"regex",
]
[[package]]
name = "globwalk"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
dependencies = [
"bitflags",
"ignore",
"walkdir",
]
[[package]] [[package]]
name = "governor" name = "governor"
version = "0.4.2" version = "0.4.2"
@ -515,6 +636,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "humansize"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"
[[package]] [[package]]
name = "hyper" name = "hyper"
version = "0.14.20" version = "0.14.20"
@ -577,6 +704,24 @@ dependencies = [
"unicode-normalization", "unicode-normalization",
] ]
[[package]]
name = "ignore"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d"
dependencies = [
"crossbeam-utils",
"globset",
"lazy_static",
"log",
"memchr",
"regex",
"same-file",
"thread_local",
"walkdir",
"winapi-util",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "1.9.1" version = "1.9.1"
@ -839,12 +984,104 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "parse-zoneinfo"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41"
dependencies = [
"regex",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.1.0" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pest"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4"
dependencies = [
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "905708f7f674518498c1f8d644481440f476d39ca6ecae83319bba7c6c12da91"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5803d8284a629cc999094ecd630f55e91b561a1d1ba75e233b00ae13b91a69ad"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pest_meta"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1538eb784f07615c6d9a8ab061089c6c54a344c5b4301db51990ca1c241e8c04"
dependencies = [
"once_cell",
"pest",
"sha-1",
]
[[package]]
name = "phf"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_codegen"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf"
dependencies = [
"phf_shared",
"rand 0.8.5",
]
[[package]]
name = "phf_shared"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676"
dependencies = [
"siphasher",
"uncased",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.9" version = "0.2.9"
@ -1051,6 +1288,8 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
dependencies = [ dependencies = [
"aho-corasick",
"memchr",
"regex-syntax", "regex-syntax",
] ]
@ -1071,8 +1310,9 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
[[package]] [[package]]
name = "registry-backup" name = "registry-backup"
version = "0.1.0" version = "0.2.0"
dependencies = [ dependencies = [
"chrono",
"clap", "clap",
"futures", "futures",
"governor", "governor",
@ -1082,6 +1322,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"tempdir", "tempdir",
"tera",
"tokio", "tokio",
"toml", "toml",
"tracing", "tracing",
@ -1264,6 +1505,17 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "sha-1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]] [[package]]
name = "sharded-slab" name = "sharded-slab"
version = "0.1.4" version = "0.1.4"
@ -1282,6 +1534,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "siphasher"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.7" version = "0.4.7"
@ -1291,6 +1549,15 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "slug"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373"
dependencies = [
"deunicode",
]
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.9.0" version = "1.9.0"
@ -1334,6 +1601,28 @@ dependencies = [
"remove_dir_all", "remove_dir_all",
] ]
[[package]]
name = "tera"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d4685e72cb35f0eb74319c8fe2d3b61e93da5609841cde2cb87fcc3bea56d20"
dependencies = [
"chrono",
"chrono-tz",
"globwalk",
"humansize",
"lazy_static",
"percent-encoding",
"pest",
"pest_derive",
"rand 0.8.5",
"regex",
"serde",
"serde_json",
"slug",
"unic-segment",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.15.0" version = "0.15.0"
@ -1369,6 +1658,16 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "time"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.14" version = "0.3.14"
@ -1601,6 +1900,77 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "typenum"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "ucd-trie"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
[[package]]
name = "uncased"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622"
dependencies = [
"version_check",
]
[[package]]
name = "unic-char-property"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
dependencies = [
"unic-char-range",
]
[[package]]
name = "unic-char-range"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
[[package]]
name = "unic-common"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
[[package]]
name = "unic-segment"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23"
dependencies = [
"unic-ucd-segment",
]
[[package]]
name = "unic-ucd-segment"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-version"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
dependencies = [
"unic-common",
]
[[package]] [[package]]
name = "unicase" name = "unicase"
version = "2.6.0" version = "2.6.0"

17
Cargo.toml
Unescape View File

@ -1,10 +1,19 @@
[package] [package]
name = "registry-backup" name = "registry-backup"
authors = ["Jonathan Strong <jstrong@shipyard.rs>"] authors = ["Jonathan Strong <jstrong@shipyard.rs>"]
version = "0.1.0" version = "0.2.0"
edition = "2021" edition = "2021"
publish = ["shipyard-rs-public"] publish = ["shipyard-rs-public"]
[[bin]]
name = "registry-backup"
path = "src/main.rs"
[[bin]]
name = "generate-readme"
path = "src/generate-readme.rs"
required-features = ["docs"]
[dependencies] [dependencies]
clap = { version = "3.1", default-features = false, features = ["std", "derive", "cargo"] } clap = { version = "3.1", default-features = false, features = ["std", "derive", "cargo"] }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
@ -20,3 +29,9 @@ walkdir = "2.3"
num_cpus = "1.3" num_cpus = "1.3"
governor = "0.4.2" governor = "0.4.2"
pretty_toa = "1" pretty_toa = "1"
tera = { version = "1", optional = true }
chrono = { version = "0.4", optional = true }
[features]
default = []
docs = ["tera", "chrono"]

128
README.md
Unescape View File

@ -0,0 +1,128 @@
# registry-backup
A command line utility for downloading all .crate files hosted by a Cargo registry server.
Use cases:
- **Backup:** retrieve a registry server's files for backup storage
- **Export:** pull the files so you can host them at another registry server
## Runtime Options
```console
$ ./target/release/registry-backup --help
registry-backup 0.2.0
Jonathan Strong <jstrong@shipyard.rs>
Download all .crate files from a registry server
USAGE:
registry-backup [OPTIONS]
OPTIONS:
--index-url <INDEX_URL>
URL of the registry index we are downloading .crate files from. The program expects that
it will be able to clone the index to a local temporary directory; the user must handle
authentication if needed
--index-path <INDEX_PATH>
instead of an index url, just point to a local path where the index is already cloned
-p, --output-path <PATH>
Directory where downloaded .crate files will be saved to [default: output]
-u, --user-agent <USER_AGENT>
Value of user-agent HTTP header [default: registry-backup/v0.2.0]
--requests-per-second <REQUESTS_PER_SECOND>
Requests to registry server will not exceed this rate [default: 25]
--max-concurrent-requests <MAX_CONCURRENT_REQUESTS>
Independent of the requests per second rate limit, no more than
`max_concurrent_requests` will be in flight at any given moment [default: 10]
-a, --auth-token <AUTH_TOKEN>
If registry requires authorization (i.e. "auth-required" key is set to `true` in the
`config.json` file), the token to include using the Authorization HTTP header
-c, --config-file <CONFIG_FILE>
Specify configuration values using the provided TOML file, instead of via command line
flags. The values in the config file will override any values passed as command line
flags. See config.toml.sample for syntax of the config file
-h, --help
Print help information
-V, --version
Print version information
```
## Configuration File
A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file:
```toml
# index repo url
index-url = "ssh://git@ssh.shipyard.rs/shipyard-rs-public/crate-index.git"
# alternatively, specify a local dir
# index-path = "/path/to/cloned/index"
[http]
user-agent = "registry-backup/v0.1.0"
requests-per-second = 40
max-concurrent-requests = 20
[output]
path = "output"
format = "/{crate}/{version}/download"
# auth-token = "xxx"
```
## Justfile
The repository includes a `justfile` with functionality for building, testing, etc.
Included commands:
```console
$ just --list
Available recipes:
cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.)
check +args='' # cargo check wrapper
debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode
generate-readme # generate updated README.md
get-crate-version
pre-release # check, run tests, check non-error output for clippy, run rustfmt
release
release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode
release-prep
test +args='' # cargo test wrapper
verify-clean-git # verify no uncommitted changes
```
The commands that mirror cargo commands (e.g. `just test`) are included for the purpose of convenience, so that various options (e.g. `RUSTFLAGS='-C target-cpu=native`) can be included without typing them out each time.
There are several variables defined in the `justfile` which pertain to rust build settings:
```
rustc-version
## Generating README.md
This file is generated using a template (`doc/README.tera.md`) rendered using updated outputs of the CLI menu, config sample, and other values.
This version of README.md was generated at `Thu, 08 Sep 2022 21:32:44 +0000` based on git commit `a6d6f5a0`.
To (re-)generate the README.md file, use the justfile command:
```console
$ just generate-readme
```

56
doc/README.tera.md
Unescape View File

@ -0,0 +1,56 @@
# registry-backup
A command line utility for downloading all .crate files hosted by a Cargo registry server.
Use cases:
- **Backup:** retrieve a registry server's files for backup storage
- **Export:** pull the files so you can host them at another registry server
## Runtime Options
```console
$ ./target/release/registry-backup --help
{{ cli_menu }}
```
## Configuration File
A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file:
```toml
{{ config_sample }}
```
## Justfile
The repository includes a `justfile` with functionality for building, testing, etc.
Included commands:
```console
$ just --list
{{ just_commands }}
```
The commands that mirror cargo commands (e.g. `just test`) are included for the purpose of convenience, so that various options (e.g. `RUSTFLAGS='-C target-cpu=native`) can be included without typing them out each time.
There are several variables defined in the `justfile` which pertain to rust build settings:
```
rustc-version
## Generating README.md
This file is generated using a template (`doc/README.tera.md`) rendered using updated outputs of the CLI menu, config sample, and other values.
This version of README.md was generated at `{{ generation_time }}` based on git commit `{{ git_commit }}`.
To (re-)generate the README.md file, use the justfile command:
```console
$ just generate-readme
```

43
doc/cli-menu.txt
Unescape View File

@ -0,0 +1,43 @@
registry-backup 0.2.0
Jonathan Strong <jstrong@shipyard.rs>
Download all .crate files from a registry server
USAGE:
registry-backup [OPTIONS]
OPTIONS:
--index-url <INDEX_URL>
URL of the registry index we are downloading .crate files from. The program expects that
it will be able to clone the index to a local temporary directory; the user must handle
authentication if needed
--index-path <INDEX_PATH>
instead of an index url, just point to a local path where the index is already cloned
-p, --output-path <PATH>
Directory where downloaded .crate files will be saved to [default: output]
-u, --user-agent <USER_AGENT>
Value of user-agent HTTP header [default: registry-backup/v0.2.0]
--requests-per-second <REQUESTS_PER_SECOND>
Requests to registry server will not exceed this rate [default: 25]
--max-concurrent-requests <MAX_CONCURRENT_REQUESTS>
Independent of the requests per second rate limit, no more than
`max_concurrent_requests` will be in flight at any given moment [default: 10]
-a, --auth-token <AUTH_TOKEN>
If registry requires authorization (i.e. "auth-required" key is set to `true` in the
`config.json` file), the token to include using the Authorization HTTP header
-c, --config-file <CONFIG_FILE>
Specify configuration values using the provided TOML file, instead of via command line
flags. The values in the config file will override any values passed as command line
flags. See config.toml.sample for syntax of the config file
-h, --help
Print help information
-V, --version
Print version information

12
doc/just-commands.txt
Unescape View File

@ -0,0 +1,12 @@
Available recipes:
cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.)
check +args='' # cargo check wrapper
debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode
generate-readme # generate updated README.md
get-crate-version
pre-release # check, run tests, check non-error output for clippy, run rustfmt
release
release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode
release-prep
test +args='' # cargo test wrapper
verify-clean-git # verify no uncommitted changes

40
justfile
Unescape View File

@ -1,25 +1,65 @@
# load any variables in .env
set dotenv-load := true set dotenv-load := true
# version of rustc used for cargo commands
rustc-version := "nightly" rustc-version := "nightly"
# name of crate registry to publish versions of this tool to
publish-registry := "shipyard-rs-public" publish-registry := "shipyard-rs-public"
# env settings exported for cargo settings
export RUSTFLAGS := "-C link-arg=-fuse-ld=lld -C target-cpu=native" export RUSTFLAGS := "-C link-arg=-fuse-ld=lld -C target-cpu=native"
# cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.)
cargo +args='': cargo +args='':
cargo +{{rustc-version}} {{args}} cargo +{{rustc-version}} {{args}}
# cargo check wrapper
check +args='': check +args='':
@just cargo check {{args}} @just cargo check {{args}}
# cargo test wrapper
test +args='': test +args='':
just cargo test {{args}} just cargo test {{args}}
# cargo build wrapper - builds registry-backup in debug mode
debug-build +args='': debug-build +args='':
@just cargo build --bin registry-backup {{args}} @just cargo build --bin registry-backup {{args}}
# cargo build --release wrapper - builds registry-backup in release mode
release-build +args='': release-build +args='':
@just cargo build --bin registry-backup --release {{args}} @just cargo build --bin registry-backup --release {{args}}
# generate updated README.md
generate-readme:
just debug-build
./target/debug/registry-backup --help > doc/cli-menu.txt
just --list > doc/just-commands.txt
just cargo run --bin generate-readme --features docs > README.md
# check, run tests, check non-error output for clippy, run rustfmt
pre-release:
just cargo check \
&& just cargo test \
&& just cargo clippy \
&& just cargo fmt
# verify no uncommitted changes
verify-clean-git:
test "$(echo `git status --porcelain` | wc -c)" -eq "1"
get-crate-version:
cat Cargo.toml | rg '^version =' | tr -d "version = " | tr -d '"'
release-prep:
just verify-clean-git
just pre-release
VERSION=$(just get-crate-version) \
&& test -z "$(git tag | rg \"v${VERSION}\")" # Error: tag appears to exist already
just generate-readme
release: release-prep
just verify-clean-git || ( git add README.md doc/*.txt && git commit -m 'update docs' )
git tag "v$(just get-crate-version)"
git push && git push --tags

31
src/generate-readme.rs
Unescape View File

@ -0,0 +1,31 @@
use chrono::prelude::*;
const README_TEMPLATE: &str = include_str!("../doc/README.tera.md");
const CLI_MENU: &str = include_str!("../doc/cli-menu.txt");
const JUST_COMMANDS: &str = include_str!("../doc/just-commands.txt");
const CONFIG_SAMPLE: &str = include_str!("../config.toml.sample");
fn get_commit_ref() -> Result<String, Box<dyn std::error::Error>> {
let output = std::process::Command::new("git")
.args(&["rev-parse", "HEAD"][..])
.output()?
.stdout;
let commit = std::str::from_utf8(&output[..8]).map(|x| x.to_string())?;
Ok(commit)
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut tera = tera::Tera::default();
tera.add_raw_template("README.md", README_TEMPLATE).unwrap();
let mut ctx = tera::Context::new();
ctx.insert("cli_menu", CLI_MENU);
ctx.insert("config_sample", CONFIG_SAMPLE);
ctx.insert("just_commands", JUST_COMMANDS);
ctx.insert("git_commit", &get_commit_ref()?);
ctx.insert("generation_time", &Utc::now().to_rfc2822());
let readme = tera.render("README.md", &ctx)?;
println!("{}", readme);
Ok(())
}

3
src/main.rs
Unescape View File

@ -55,7 +55,6 @@ pub struct OutputConfig {
#[clap(short, long = "output-path", default_value = DEFAULT_OUTPUT_PATH)] #[clap(short, long = "output-path", default_value = DEFAULT_OUTPUT_PATH)]
#[serde(default = "default_output_path")] #[serde(default = "default_output_path")]
pub path: PathBuf, pub path: PathBuf,
// /// What format to use for the output filenames. Works the same as // /// What format to use for the output filenames. Works the same as
// /// Cargo's registry syntax for the "dl" key in the `config.json` // /// Cargo's registry syntax for the "dl" key in the `config.json`
// /// file in a reigstry index. See [Cargo // /// file in a reigstry index. See [Cargo
@ -109,7 +108,7 @@ pub struct Config {
/// instead of an index url, just point to a local path where the index /// instead of an index url, just point to a local path where the index
/// is already cloned. /// is already cloned.
#[serde(default)] #[serde(default)]
#[clap(long, conflicts_with = "index_url")] #[clap(long, conflicts_with = "index-url")]
pub index_path: Option<PathBuf>, pub index_path: Option<PathBuf>,
/// Where to save the downloaded files /// Where to save the downloaded files
#[clap(flatten)] #[clap(flatten)]

Write Preview
Loading…
Cancel
Save