diff --git a/Cargo.lock b/Cargo.lock index b2009fb..fe4cde4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,7 +1607,7 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "registry-backup" -version = "0.4.1" +version = "0.5.0-beta.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index adaa273..def2e9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "registry-backup" authors = ["Jonathan Strong "] -version = "0.4.1" +version = "0.5.0-beta.1" edition = "2021" #publish = ["shipyard-rs-public"] readme = "README.md" diff --git a/README.md b/README.md index 1e3dc60..2045505 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,25 @@ # registry-backup -A command line utility for downloading all .crate files hosted by a Cargo registry server. +Command line utilities for backup, export, and migration of a Rust private crate registry. Use cases: - **Backup:** retrieve a registry server's files for backup storage - **Export:** pull the files so you can host them at another registry server +- **Migration:** publish downloaded .crate files to a new private registry, including modifying the `Cargo.toml` manifests of each published crate version to make it compatible with the destination registry -## Example Usage: +## Tools + +There are two binaries in the repo: + +- `registry-backup`: for downloading all .crate files hosted by a Cargo registry server +- `publish`: for publishing the .crate files downloaded by `registry-backup` to a different registry + +## `registry-backup` + +`registry-backup` is a tool to download all of the .crate files hosted by a Cargo registry server. + +### Example Usage Specify the registry index either as a local path (`--index-path`)... @@ -28,18 +40,18 @@ $ RUST_LOG=info registry-backup \ --auth-token ${AUTH_TOKEN} # for private registry, need auth ``` -## Install +### Install ```console $ cargo install registry-backup --git https://git.shipyard.rs/jstrong/registry-backup.git ``` -## Runtime Options +### Runtime Options ```console $ ./target/release/registry-backup --help -registry-backup 0.4.1 +registry-backup 0.5.0-beta.1 Jonathan Strong Download all .crate files from a registry server @@ -82,7 +94,7 @@ OPTIONS: -U, --user-agent Value of user-agent HTTP header - [default: registry-backup/v0.4.1] + [default: registry-backup/v0.5.0-beta.1] -R, --requests-per-second Requests to registry server will not exceed this rate @@ -116,7 +128,7 @@ OPTIONS: ``` -## Configuration File +### Configuration File A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file: @@ -152,6 +164,251 @@ $ just release-build # alternatively, cargo build --bin registry-backup --releas # cp target/release/registry-backup ~/.cargo/bin/ ``` +## `publish` + +`publish` is a tool to publish all of the crate versions from a *source registry* to second *destination registry*. + +### Usage Overview + +`publish` is different from `registry-backup` in that in requires several steps, including the use of a Python script. + +In general, migrating all of the crate versions to another registry is relatively complex, compared to just downloading the .crate files. Migrating to a new registry involves the following (big picture) steps: + +1) extracting the order that crate versions were published to the source registry from the git history of the crate index repository +2) extracting the source files, including `Cargo.toml` manifests, from the downloaded `.crate` files +3) modifying the `Cargo.toml` manifests for each crate version so the crate will be compatible with the destination registry +4) publishing the crate versions, in the right order and using the modified `Cargo.toml` manifests, to the destination registry + +### Background Context: `cargo publish`, `.crate` Files, and `Cargo.toml.orig` + +When you run the `cargo publish` command to publish a crate version to a registry server, it generates an alternate `Cargo.toml` manifest based on the contents of the original `Cargo.toml` in combination with the configured settings with which the command was invoked. + +For example, if you had configured a private registry in `~/.cargo/config.toml`: + +```toml +# ~/.cargo/config.toml + +[registries.my-private-registry] +index = "ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git" +``` + +And then added a dependency from that registry in a `Cargo.toml` for a crate: + +```toml +# Cargo.toml +[package] +name = "foo" +publish = ["my-private-registry"] + +[dependencies] +bar = { version = "1.0", registry = "my-private-registry" } +``` + +...`cargo publish` would convert the dependency into one with a hard-coded `registry-index` field that points to the specific index URL that was configured at the time it was invoked: + +``` +# cargo publish-generated Cargo.toml +[package] +name = "foo" +publish = ["my-private-registry"] + +[dependencies] +bar = { version = "1.0", registry-index = "ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git" } +``` + +`cargo publish` includes the original `Cargo.toml` file at the path `Cargo.toml.orig` in the `.crate` file (actually a `.tar.gz` archive). + +Since the `registry-index` entries generated by `cargo publish` point to the specific URL of the source registry, just publishing the `.crate` file as is to the destination registry will not suffice. To resolve this problem, `publish` uses the `Cargo.toml.orig` file contained in the `.crate` file, modifies the dependency entries according to the settings of the destination registry, and publishes them to the destination registry using `cargo publish` (i.e. discard the `cargo publish`-generated `Cargo.toml`, relying instead on the modified `Cargo.toml.orig` in combination with runtime settings provided as env vars to `cargo`). + +### The Global Dependency Graph of a Registry and `publish-log.csv` + +Once we have solved how to take a `.crate` file from the source registry and publish it to the destination registry, there is still the issue of which order the crate versions should be published. If crate `a` version 1.2.3 depends on crate `b` version 2.3.4, then crate `b` version 2.3.4 needs to have already been published to the registry at the time crate `a` version 1.2.3 is published, otherwise it will depend on a crate that does not (yet) exist (in the destination registry, at least). If you try to publish crates without respecting this global dependency graph using `cargo publish`, it will exit with an error, and it's not a good idea otherwise, either. + +Building a dependency graph for the entire registry is certainly possible, theoretically. However, in practice it is tedious to do, mainly because it requires mirroring `cargo`'s dependency resolution process, just to be able to identify the full set of dependencies that would end up in the `Cargo.lock` file. That, in turn, requires using `cargo` (i.e. via the `cargo metadata` command), which is slow for large registries (only a single `cargo metadata` command can run at a time due to the use of lock files), and quite involved in terms of parsing the programmatically-generated outputs (wow it is amazing how many different forms crate metadata is represented in various `cargo`/registry contexts!). + +To shortcut these complexities, `publish` relies on the use of a Python script to extract the order in which crate versions were published to a registry using the git history of the crate index repository. + +The tool (`script/get-publish-history.py`) was based on an open source script that utilizes the `GitPython` library to traverse the commit history of a repo. In a few minutes work, we were able to modify the script to extract the publish order of all the crate versions appearing in the crate index repository. And, as much as we love Rust (and do not share the same passion for Python), porting the code to Rust using the `git2` crate appeared like quite a tedious project itself. + +To generate a `.csv` file with the order in which crates were published, first clone the crate index repository, e.g.: + +``` +$ git clone ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git +``` + +Then run the script (it has two dependencies `GitPython` and `pandas`, both of which can be `pip install`ed or otherwise acquired using whatever terrible Python package manager you want): + +``` +$ python script/get-publish-history.py path/to/crate-index > publish-log.csv +``` + +You will need a `publish-log.csv` generated from the source registry to use `publish`. + +(You might be wondering why we are relying on git history to reconstruct the publishing order. The primary reason is the crate index metadata (or any other metadata universally available from a crate registry) does not include any information about when each crate version was published.) + +### Detailed Usage Example + +##### 1) Clone the source registry crate index repository: + +``` +$ mkdir source-registry +$ git clone source-registry/crate-index +``` + +##### 2) Use `registry-backup` to download all the `.crate` files from the source registry: + +``` +$ cargo install registry-backup --git https://git.shipyard.rs/jstrong/registry-backup.git # or build from source +$ RUST_LOG=info registry-backup \ + --index-path source-registry/crate-index \ + --output-path source-registry/crate-files +``` + +##### 3) Use the `get-publish-history.py` script to extract the crate version publish history: + +``` +$ . ../virtualenvs/my-env/activate # or whatever you use +$ pip install GitPython +$ pip install pandas +$ python3 script/get-publish-history.py source-registry/crate-index > source-registry/publish-log.csv +``` + +##### 4) Create a configuration file: + +```toml +# publish-config.toml + +# source registry config +[src] +index-dir = "source-registry/crate-index" # <- see step 1 +crate-files-dir = "source-registry/crate-files" # <- see step 2 +publish-history-csv = "source-registry/publish-log.csv" # <- see step 3 +registry-name = "my-old-registry" # <- whatever label the source registry was given in Cargo.toml files +index-url = "https://github.com/my-org/crate-index.git" # <- index url, i.e. same as one provided in ~/.cargo/config.toml + +# destination registry config +[dst] +index-url = "ssh://git@ssh.shipyard.rs/my-new-registry/crate-index.git" +registry-name = "my-new-registry" # can be same as old name or a different name +auth-token = "xxx" # auth token for publishing to the destination registry +``` + +##### 5) Build `publish`: + +``` +$ cargo bulid --bin publish --features publish --release +``` + +##### 6) Validate your config file (optional): + +``` +$ ./target/release/publish --config publish-config.toml --validate +``` + +##### 7) Publish to the destination registry using `publish`: + +``` +$ RUST_LOG=info ./target/release/publish --config publish-config.toml +``` + +### Expected Runtime + +As an example, using `publish`, it took us about 50 minutes to migrate a registry with 77 crates and 937 versions. Results may vary based on the machine used to run `publish` as well as the performance of the destination registry server. + +### Building `publish` (Full Example) + +``` +$ git clone https://git.shipyard.rs/jstrong/registry-backup.git +$ cd registry-backup +$ just release-build-publish # alternately, cargo build --bin publish --features publish --release +``` + +Note: `--release` really is quite a bit faster, at least for larger registries. + +### Configuration File + +Annotated example configuration file: + +```toml +# optional field for providing a regex-based filter +# to limit which crates are published to the destination +# registry. only crates with names matching the regex will +# be published. +# +filter-crates = "^." + +# do everything except actually publish to the destination registry +dry-run = false + +# source registry config +[src] +index-dir = "path/to/crate-index/repo" # git clone of crate index repository +crate-files-dir = "path/to/crate/files" # i.e. files downloaded by registry-backup tool +publish-history-csv = "path/to/publish-log.csv" # see docs above +registry-name = "my-old-registry" # whatever label the source registry was given in Cargo.toml files +index-url = "https://github.com/my-org/crate-index.git" # index url, i.e. same as one provided in ~/.cargo/config.toml + +# destination registry config +[dst] +index-url = "ssh://git@ssh.shipyard.rs/my-new-registry/crate-index.git" # index url of new registry +registry-name = "my-new-registry" # can be same as old name or a different name +auth-token = "xxx" # auth token for publishing to the destination registry + +``` + +### Runtime Options + +```console +$ ./target/release/publish --help + +registry-backup 0.5.0-beta.1 +Jonathan Strong + +USAGE: + publish [OPTIONS] --config-file + +OPTIONS: + -c, --config-file Config file with source directories and destination registry info + --dry-run Perform all the work of generating `cargo publish` payloads, but + don't send them to the destination registry server + --validate Load config file, validate the settings, and display the final + loaded content to stdout, then exit + --filter-crates Use to limit which crates from the source registry are published + to the destination registry. Expects a regular expression which + will be matched against the names of crates. Only crates with + names that match the regex will be published. This field may also + be specified at the top level of the config file + -h, --help Print help information + -V, --version Print version information + +``` + +### Configuration File + +A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file: + +```toml +dry-run = false +filter-crates = "^." + +[registry] +index-url = "ssh://git@ssh.shipyard.rs/shipyard-rs-public/crate-index.git" +# alternatively, specify a local dir +# index-path = "/path/to/cloned/index" +auth-token = "xxx" + +[http] +user-agent = "registry-backup/v0.1.0" +requests-per-second = 100 +max-concurrent-requests = 50 + +[output] +path = "output" +overwrite-existing = false +format = "{crate}/{version}/download" + +``` + ## Running Tests ```console @@ -168,22 +425,24 @@ Included commands: $ just --list Available recipes: - cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.) - check +args='' # cargo check wrapper - debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode - generate-readme # generate updated README.md + cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.) + check +args='' # cargo check wrapper + debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode + debug-build-publish +args='' # cargo build wrapper - builds publish tool in debug mode + generate-readme # generate updated README.md get-crate-version - install # cargo install registry-backup via git dep - pre-release # check, run tests, check non-error output for clippy, run rustfmt - release # release version (regenerate docs, git tag v0.0.0) - release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode - release-prep # get everything all ready for release - show-build-env # diagnostic command for viewing value of build variables at runtime - test +args='' # cargo test wrapper - update-readme # re-generate README.md and overwrite existing file with output - update-readme-and-commit # re-generate, overwrite, stage, and commit - update-readme-and-stage # re-generate, overwrite, and stage changes - verify-clean-git # verify no uncommitted changes + install # cargo install registry-backup via git dep + pre-release # check, run tests, check non-error output for clippy, run rustfmt + release # release version (regenerate docs, git tag v0.0.0) + release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode + release-build-publish +args='' # cargo build --release wrapper - builds publish tool in release mode + release-prep # get everything all ready for release + show-build-env # diagnostic command for viewing value of build variables at runtime + test +args='' # cargo test wrapper + update-readme # re-generate README.md and overwrite existing file with output + update-readme-and-commit # re-generate, overwrite, stage, and commit + update-readme-and-stage # re-generate, overwrite, and stage changes + verify-clean-git # verify no uncommitted changes ``` @@ -193,7 +452,7 @@ The commands that mirror cargo commands (e.g. `just test`) are included for the This file is generated using a template (`doc/README.tera.md`) rendered using updated outputs of the CLI menu, config sample, and other values. -This version of `README.md` was generated at `Thu, 08 Dec 2022 02:23:17 +0000` based on git commit `3241e207`. +This version of `README.md` was generated at `Fri, 10 Nov 2023 01:27:13 +0000` based on git commit `b717fc95`. To (re-)generate the `README.md` file, use the justfile command: diff --git a/doc/README.tera.md b/doc/README.tera.md index f855c73..8ffe19f 100644 --- a/doc/README.tera.md +++ b/doc/README.tera.md @@ -1,13 +1,25 @@ # registry-backup -A command line utility for downloading all .crate files hosted by a Cargo registry server. +Command line utilities for backup, export, and migration of a Rust private crate registry. Use cases: - **Backup:** retrieve a registry server's files for backup storage - **Export:** pull the files so you can host them at another registry server +- **Migration:** publish downloaded .crate files to a new private registry, including modifying the `Cargo.toml` manifests of each published crate version to make it compatible with the destination registry -## Example Usage: +## Tools + +There are two binaries in the repo: + +- `registry-backup`: for downloading all .crate files hosted by a Cargo registry server +- `publish`: for publishing the .crate files downloaded by `registry-backup` to a different registry + +## `registry-backup` + +`registry-backup` is a tool to download all of the .crate files hosted by a Cargo registry server. + +### Example Usage Specify the registry index either as a local path (`--index-path`)... @@ -28,13 +40,13 @@ $ RUST_LOG=info registry-backup \ --auth-token ${AUTH_TOKEN} # for private registry, need auth ``` -## Install +### Install ```console $ cargo install registry-backup --git https://git.shipyard.rs/jstrong/registry-backup.git ``` -## Runtime Options +### Runtime Options ```console $ ./target/release/registry-backup --help @@ -42,7 +54,7 @@ $ ./target/release/registry-backup --help {{ cli_menu }} ``` -## Configuration File +### Configuration File A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file: @@ -60,6 +72,191 @@ $ just release-build # alternatively, cargo build --bin registry-backup --releas # cp target/release/registry-backup ~/.cargo/bin/ ``` +## `publish` + +`publish` is a tool to publish all of the crate versions from a *source registry* to second *destination registry*. + +### Usage Overview + +`publish` is different from `registry-backup` in that in requires several steps, including the use of a Python script. + +In general, migrating all of the crate versions to another registry is relatively complex, compared to just downloading the .crate files. Migrating to a new registry involves the following (big picture) steps: + +1) extracting the order that crate versions were published to the source registry from the git history of the crate index repository +2) extracting the source files, including `Cargo.toml` manifests, from the downloaded `.crate` files +3) modifying the `Cargo.toml` manifests for each crate version so the crate will be compatible with the destination registry +4) publishing the crate versions, in the right order and using the modified `Cargo.toml` manifests, to the destination registry + +### Background Context: `cargo publish`, `.crate` Files, and `Cargo.toml.orig` + +When you run the `cargo publish` command to publish a crate version to a registry server, it generates an alternate `Cargo.toml` manifest based on the contents of the original `Cargo.toml` in combination with the configured settings with which the command was invoked. + +For example, if you had configured a private registry in `~/.cargo/config.toml`: + +```toml +# ~/.cargo/config.toml + +[registries.my-private-registry] +index = "ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git" +``` + +And then added a dependency from that registry in a `Cargo.toml` for a crate: + +```toml +# Cargo.toml +[package] +name = "foo" +publish = ["my-private-registry"] + +[dependencies] +bar = { version = "1.0", registry = "my-private-registry" } +``` + +...`cargo publish` would convert the dependency into one with a hard-coded `registry-index` field that points to the specific index URL that was configured at the time it was invoked: + +``` +# cargo publish-generated Cargo.toml +[package] +name = "foo" +publish = ["my-private-registry"] + +[dependencies] +bar = { version = "1.0", registry-index = "ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git" } +``` + +`cargo publish` includes the original `Cargo.toml` file at the path `Cargo.toml.orig` in the `.crate` file (actually a `.tar.gz` archive). + +Since the `registry-index` entries generated by `cargo publish` point to the specific URL of the source registry, just publishing the `.crate` file as is to the destination registry will not suffice. To resolve this problem, `publish` uses the `Cargo.toml.orig` file contained in the `.crate` file, modifies the dependency entries according to the settings of the destination registry, and publishes them to the destination registry using `cargo publish` (i.e. discard the `cargo publish`-generated `Cargo.toml`, relying instead on the modified `Cargo.toml.orig` in combination with runtime settings provided as env vars to `cargo`). + +### The Global Dependency Graph of a Registry and `publish-log.csv` + +Once we have solved how to take a `.crate` file from the source registry and publish it to the destination registry, there is still the issue of which order the crate versions should be published. If crate `a` version 1.2.3 depends on crate `b` version 2.3.4, then crate `b` version 2.3.4 needs to have already been published to the registry at the time crate `a` version 1.2.3 is published, otherwise it will depend on a crate that does not (yet) exist (in the destination registry, at least). If you try to publish crates without respecting this global dependency graph using `cargo publish`, it will exit with an error, and it's not a good idea otherwise, either. + +Building a dependency graph for the entire registry is certainly possible, theoretically. However, in practice it is tedious to do, mainly because it requires mirroring `cargo`'s dependency resolution process, just to be able to identify the full set of dependencies that would end up in the `Cargo.lock` file. That, in turn, requires using `cargo` (i.e. via the `cargo metadata` command), which is slow for large registries (only a single `cargo metadata` command can run at a time due to the use of lock files), and quite involved in terms of parsing the programmatically-generated outputs (wow it is amazing how many different forms crate metadata is represented in various `cargo`/registry contexts!). + +To shortcut these complexities, `publish` relies on the use of a Python script to extract the order in which crate versions were published to a registry using the git history of the crate index repository. + +The tool (`script/get-publish-history.py`) was based on an open source script that utilizes the `GitPython` library to traverse the commit history of a repo. In a few minutes work, we were able to modify the script to extract the publish order of all the crate versions appearing in the crate index repository. And, as much as we love Rust (and do not share the same passion for Python), porting the code to Rust using the `git2` crate appeared like quite a tedious project itself. + +To generate a `.csv` file with the order in which crates were published, first clone the crate index repository, e.g.: + +``` +$ git clone ssh://git@ssh.shipyard.rs/my-private-registry/crate-index.git +``` + +Then run the script (it has two dependencies `GitPython` and `pandas`, both of which can be `pip install`ed or otherwise acquired using whatever terrible Python package manager you want): + +``` +$ python script/get-publish-history.py path/to/crate-index > publish-log.csv +``` + +You will need a `publish-log.csv` generated from the source registry to use `publish`. + +(You might be wondering why we are relying on git history to reconstruct the publishing order. The primary reason is the crate index metadata (or any other metadata universally available from a crate registry) does not include any information about when each crate version was published.) + +### Detailed Usage Example + +##### 1) Clone the source registry crate index repository: + +``` +$ mkdir source-registry +$ git clone source-registry/crate-index +``` + +##### 2) Use `registry-backup` to download all the `.crate` files from the source registry: + +``` +$ cargo install registry-backup --git https://git.shipyard.rs/jstrong/registry-backup.git # or build from source +$ RUST_LOG=info registry-backup \ + --index-path source-registry/crate-index \ + --output-path source-registry/crate-files +``` + +##### 3) Use the `get-publish-history.py` script to extract the crate version publish history: + +``` +$ . ../virtualenvs/my-env/activate # or whatever you use +$ pip install GitPython +$ pip install pandas +$ python3 script/get-publish-history.py source-registry/crate-index > source-registry/publish-log.csv +``` + +##### 4) Create a configuration file: + +```toml +# publish-config.toml + +# source registry config +[src] +index-dir = "source-registry/crate-index" # <- see step 1 +crate-files-dir = "source-registry/crate-files" # <- see step 2 +publish-history-csv = "source-registry/publish-log.csv" # <- see step 3 +registry-name = "my-old-registry" # <- whatever label the source registry was given in Cargo.toml files +index-url = "https://github.com/my-org/crate-index.git" # <- index url, i.e. same as one provided in ~/.cargo/config.toml + +# destination registry config +[dst] +index-url = "ssh://git@ssh.shipyard.rs/my-new-registry/crate-index.git" +registry-name = "my-new-registry" # can be same as old name or a different name +auth-token = "xxx" # auth token for publishing to the destination registry +``` + +##### 5) Build `publish`: + +``` +$ cargo bulid --bin publish --features publish --release +``` + +##### 6) Validate your config file (optional): + +``` +$ ./target/release/publish --config publish-config.toml --validate +``` + +##### 7) Publish to the destination registry using `publish`: + +``` +$ RUST_LOG=info ./target/release/publish --config publish-config.toml +``` + +### Expected Runtime + +As an example, using `publish`, it took us about 50 minutes to migrate a registry with 77 crates and 937 versions. Results may vary based on the machine used to run `publish` as well as the performance of the destination registry server. + +### Building `publish` (Full Example) + +``` +$ git clone https://git.shipyard.rs/jstrong/registry-backup.git +$ cd registry-backup +$ just release-build-publish # alternately, cargo build --bin publish --features publish --release +``` + +Note: `--release` really is quite a bit faster, at least for larger registries. + +### Configuration File + +Annotated example configuration file: + +```toml +{{ publish_config_sample }} +``` + +### Runtime Options + +```console +$ ./target/release/publish --help + +{{ publish_cli_menu }} +``` + +### Configuration File + +A toml configuration file may be used instead of command line flags. A sample file (`config.toml.sample`) is included. From the example file: + +```toml +{{ config_sample }} +``` + ## Running Tests ```console diff --git a/doc/cli-menu.txt b/doc/cli-menu.txt index 58212c2..b12be30 100644 --- a/doc/cli-menu.txt +++ b/doc/cli-menu.txt @@ -1,4 +1,4 @@ -registry-backup 0.4.1 +registry-backup 0.5.0-beta.1 Jonathan Strong Download all .crate files from a registry server @@ -41,7 +41,7 @@ OPTIONS: -U, --user-agent Value of user-agent HTTP header - [default: registry-backup/v0.4.1] + [default: registry-backup/v0.5.0-beta.1] -R, --requests-per-second Requests to registry server will not exceed this rate diff --git a/doc/just-commands.txt b/doc/just-commands.txt index a097138..7458d8b 100644 --- a/doc/just-commands.txt +++ b/doc/just-commands.txt @@ -1,17 +1,19 @@ Available recipes: - cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.) - check +args='' # cargo check wrapper - debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode - generate-readme # generate updated README.md + cargo +args='' # cargo wrapper; executes a cargo command using the settings in justfile (RUSTFLAGS, etc.) + check +args='' # cargo check wrapper + debug-build +args='' # cargo build wrapper - builds registry-backup in debug mode + debug-build-publish +args='' # cargo build wrapper - builds publish tool in debug mode + generate-readme # generate updated README.md get-crate-version - install # cargo install registry-backup via git dep - pre-release # check, run tests, check non-error output for clippy, run rustfmt - release # release version (regenerate docs, git tag v0.0.0) - release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode - release-prep # get everything all ready for release - show-build-env # diagnostic command for viewing value of build variables at runtime - test +args='' # cargo test wrapper - update-readme # re-generate README.md and overwrite existing file with output - update-readme-and-commit # re-generate, overwrite, stage, and commit - update-readme-and-stage # re-generate, overwrite, and stage changes - verify-clean-git # verify no uncommitted changes + install # cargo install registry-backup via git dep + pre-release # check, run tests, check non-error output for clippy, run rustfmt + release # release version (regenerate docs, git tag v0.0.0) + release-build +args='' # cargo build --release wrapper - builds registry-backup in release mode + release-build-publish +args='' # cargo build --release wrapper - builds publish tool in release mode + release-prep # get everything all ready for release + show-build-env # diagnostic command for viewing value of build variables at runtime + test +args='' # cargo test wrapper + update-readme # re-generate README.md and overwrite existing file with output + update-readme-and-commit # re-generate, overwrite, stage, and commit + update-readme-and-stage # re-generate, overwrite, and stage changes + verify-clean-git # verify no uncommitted changes diff --git a/doc/publish-cli-menu.txt b/doc/publish-cli-menu.txt new file mode 100644 index 0000000..5ca595f --- /dev/null +++ b/doc/publish-cli-menu.txt @@ -0,0 +1,19 @@ +registry-backup 0.5.0-beta.1 +Jonathan Strong + +USAGE: + publish [OPTIONS] --config-file + +OPTIONS: + -c, --config-file Config file with source directories and destination registry info + --dry-run Perform all the work of generating `cargo publish` payloads, but + don't send them to the destination registry server + --validate Load config file, validate the settings, and display the final + loaded content to stdout, then exit + --filter-crates Use to limit which crates from the source registry are published + to the destination registry. Expects a regular expression which + will be matched against the names of crates. Only crates with + names that match the regex will be published. This field may also + be specified at the top level of the config file + -h, --help Print help information + -V, --version Print version information diff --git a/justfile b/justfile index 197567e..db716c0 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,8 @@ release-build-publish +args='': generate-readme: just debug-build ./target/debug/registry-backup --help > doc/cli-menu.txt + just debug-build-publish + ./target/debug/publish --help > doc/publish-cli-menu.txt just --list > doc/just-commands.txt just cargo run --bin generate-readme --features docs diff --git a/publish-config.toml.sample b/publish-config.toml.sample index e4287fb..0ff3f71 100644 --- a/publish-config.toml.sample +++ b/publish-config.toml.sample @@ -3,22 +3,21 @@ # registry. only crates with names matching the regex will # be published. # -# filter-crates = "" +filter-crates = "^." # do everything except actually publish to the destination registry dry-run = false +# source registry config [src] -# path of local dir where crate index repository has been cloned -index-dir = "path/to/cloned/rrate-index/repo" -# path of dir where .crate files were downloaded to. use the -# registry-backup tool to quickly and easily download all of -# a registry's files -crate-files-dir = "path/to/crate/files" +index-dir = "path/to/crate-index/repo" # git clone of crate index repository +crate-files-dir = "path/to/crate/files" # i.e. files downloaded by registry-backup tool +publish-history-csv = "path/to/publish-log.csv" # see docs above +registry-name = "my-old-registry" # whatever label the source registry was given in Cargo.toml files +index-url = "https://github.com/my-org/crate-index.git" # index url, i.e. same as one provided in ~/.cargo/config.toml +# destination registry config [dst] -# the value of the `api` field in the destination registry's -# config.json file (part of the crate index -api-url = "https://crates.shipyard.rs" -# auth token to use when publishing crate versions -auth-token = "xxx" +index-url = "ssh://git@ssh.shipyard.rs/my-new-registry/crate-index.git" # index url of new registry +registry-name = "my-new-registry" # can be same as old name or a different name +auth-token = "xxx" # auth token for publishing to the destination registry diff --git a/src/generate-readme.rs b/src/generate-readme.rs index 8b4584d..6c1a3d2 100644 --- a/src/generate-readme.rs +++ b/src/generate-readme.rs @@ -2,8 +2,10 @@ use chrono::prelude::*; const README_TEMPLATE: &str = include_str!("../doc/README.tera.md"); const CLI_MENU: &str = include_str!("../doc/cli-menu.txt"); +const PUBLISH_CLI_MENU: &str = include_str!("../doc/publish-cli-menu.txt"); const JUST_COMMANDS: &str = include_str!("../doc/just-commands.txt"); const CONFIG_SAMPLE: &str = include_str!("../config.toml.sample"); +const PUBLISH_CONFIG_SAMPLE: &str = include_str!("../publish-config.toml.sample"); fn get_commit_ref() -> Result> { let output = std::process::Command::new("git") @@ -21,7 +23,9 @@ fn main() -> Result<(), Box> { tera.add_raw_template("README.md", README_TEMPLATE).unwrap(); let mut ctx = tera::Context::new(); ctx.insert("cli_menu", CLI_MENU); + ctx.insert("publish_cli_menu", PUBLISH_CLI_MENU); ctx.insert("config_sample", CONFIG_SAMPLE); + ctx.insert("publish_config_sample", PUBLISH_CONFIG_SAMPLE); ctx.insert("just_commands", JUST_COMMANDS); ctx.insert("git_commit", &get_commit_ref()?); ctx.insert("generation_time", &Utc::now().to_rfc2822()); diff --git a/src/main.rs b/src/main.rs index a24c91d..5437ad5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -797,10 +797,10 @@ fn main() -> Result<(), anyhow::Error> { setup_logger(); - info!("initializing..."); - let config = Config::parse(); + info!("initializing..."); + let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()