diff --git a/CHANGELOG.md b/CHANGELOG.md index 345cbedb16..2ddf8022dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## mdBook 0.4.43+ (tba) + +### Fixed + +- Allow doctests to use external crates by referencing a `Cargo.toml` + ## mdBook 0.4.43 [v0.4.42...v0.4.43](https://github.com/rust-lang/mdBook/compare/v0.4.42...v0.4.43) diff --git a/Cargo.lock b/Cargo.lock index 3fb221bcef..08e58b1dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,6 +217,17 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "cargo-manifest" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2ce2075c35e4b492b93e3d5dd1dd3670de553f15045595daef8164ed9a3751" +dependencies = [ + "serde", + "thiserror", + "toml 0.8.19", +] + [[package]] name = "cc" version = "1.1.36" @@ -1144,6 +1155,7 @@ dependencies = [ "ammonia", "anyhow", "assert_cmd", + "cargo-manifest", "chrono", "clap", "clap_complete", @@ -1170,12 +1182,26 @@ dependencies = [ "shlex", "tempfile", "tokio", - "toml", + "toml 0.5.11", "topological-sort", "walkdir", "warp", ] +[[package]] +name = "mdbook-book-code-samples" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "mdbook", + "pulldown-cmark 0.10.3", + "pulldown-cmark-to-cmark", + "semver", + "serde", + "serde_json", +] + [[package]] name = "mdbook-remove-emphasis" version = "0.1.0" @@ -1811,6 +1837,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2084,6 +2119,41 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "topological-sort" version = "0.2.2" @@ -2511,6 +2581,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 04063a45a6..f20e0008e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,17 @@ [workspace] -members = [".", "examples/remove-emphasis/mdbook-remove-emphasis"] +members = [".", "examples/remove-emphasis/mdbook-remove-emphasis", "guide"] + +[workspace.dependencies] +anyhow = "1.0.71" +clap = { version = "4.3.12", features = ["cargo", "wrap_help"] } +mdbook = { path = "." } +pulldown-cmark = { version = "0.10.0", default-features = false, features = [ + "html", +] } # Do not update, part of the public api. +serde = { version = "1.0.163", features = ["derive"] } +serde_json = "1.0.96" +semver = "1.0.17" + [package] name = "mdbook" @@ -7,7 +19,7 @@ version = "0.4.43" authors = [ "Mathieu David ", "Michael-F-Bryan ", - "Matt Ickstadt " + "Matt Ickstadt ", ] documentation = "https://rust-lang.github.io/mdBook/index.html" edition = "2021" @@ -20,23 +32,24 @@ description = "Creates a book from markdown files" rust-version = "1.74" [dependencies] -anyhow = "1.0.71" +anyhow.workspace = true chrono = { version = "0.4.24", default-features = false, features = ["clock"] } -clap = { version = "4.3.12", features = ["cargo", "wrap_help"] } +clap.workspace = true clap_complete = "4.3.2" +cargo-manifest = "0.17.0" once_cell = "1.17.1" env_logger = "0.11.1" handlebars = "6.0" log = "0.4.17" memchr = "2.5.0" opener = "0.7.0" -pulldown-cmark = { version = "0.10.0", default-features = false, features = ["html"] } # Do not update, part of the public api. +pulldown-cmark.workspace = true regex = "1.8.1" -serde = { version = "1.0.163", features = ["derive"] } -serde_json = "1.0.96" +serde.workspace = true +serde_json.workspace = true shlex = "1.3.0" tempfile = "3.4.0" -toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 +toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 topological-sort = "0.2.2" # Watch feature @@ -59,7 +72,7 @@ ammonia = { version = "4.0.0", optional = true } assert_cmd = "2.0.11" predicates = "3.0.3" select = "0.6.0" -semver = "1.0.17" +semver.workspace = true pretty_assertions = "1.3.0" walkdir = "2.3.3" diff --git a/guide/Cargo.toml b/guide/Cargo.toml new file mode 100644 index 0000000000..48ded4467e --- /dev/null +++ b/guide/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mdbook-book-code-samples" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +mdbook.workspace = true +pulldown-cmark.workspace = true +pulldown-cmark-to-cmark = "18.0.0" +serde.workspace = true +serde_json.workspace = true +semver.workspace = true diff --git a/guide/book.toml b/guide/book.toml index 817f8b07b7..32acd675d6 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -5,7 +5,8 @@ authors = ["Mathieu David", "Michael-F-Bryan"] language = "en" [rust] -edition = "2018" +## not needed, and will cause an error, if using Cargo.toml: edition = "2021" +manifest = "Cargo.toml" [output.html] smart-punctuation = true diff --git a/guide/src/cli/test.md b/guide/src/cli/test.md index ba06bd7082..e8fd9210e6 100644 --- a/guide/src/cli/test.md +++ b/guide/src/cli/test.md @@ -1,32 +1,16 @@ # The test command -When writing a book, you sometimes need to automate some tests. For example, +When writing a book, you may want to provide some code samples, +and it's important that these be kept accurate as your software API evolves. +For example, [The Rust Programming Book](https://doc.rust-lang.org/stable/book/) uses a lot -of code examples that could get outdated. Therefore it is very important for -them to be able to automatically test these code examples. +of code samples that could become outdated as the language evolves. -mdBook supports a `test` command that will run all available tests in a book. At -the moment, only Rust tests are supported. +MdBook supports a `test` command which runs code samples in your book as doc tests to verify they +will compile, and, optionally, run correctly. +For details on how to specify the test to be done and outcome to be expected, see [Code Blocks](/format/mdbook.md#code-blocks). -#### Disable tests on a code block - -rustdoc doesn't test code blocks which contain the `ignore` attribute: - - ```rust,ignore - fn main() {} - ``` - -rustdoc also doesn't test code blocks which specify a language other than Rust: - - ```markdown - **Foo**: _bar_ - ``` - -rustdoc *does* test code blocks which have no language specified: - - ``` - This is going to cause an error! - ``` +At the moment, mdBook only supports doc *tests* written in Rust, although code samples can be written and *displayed* in many programming languages. #### Specify a directory @@ -37,7 +21,22 @@ instead of the current working directory. mdbook test path/to/book ``` -#### `--library-path` +#### `--dest-dir` + +The `--dest-dir` (`-d`) option allows you to change the output directory for the +book. Relative paths are interpreted relative to the book's root directory. If +not specified it will default to the value of the `build.build-dir` key in +`book.toml`, or to `./book`. + +#### `--chapter` + +The `--chapter` (`-c`) option allows you to test a specific chapter of the +book using the chapter name or the relative path to the chapter. + +#### `--library-path` `[`deprecated`]` + +***Note*** This argument is deprecated. Since Rust edition 2018, the compiler needs an explicit `--extern` argument for each external crate used in a doc test, it no longer simply scans the library path for likely-looking crates. +New projects should list external crates as dependencies in a **Cargo.toml** file and reference that file in your ***book.toml***, as described in [rust configuration](/format/configuration/general.html#rust-options). The `--library-path` (`-L`) option allows you to add directories to the library search path used by `rustdoc` when it builds and tests the examples. Multiple @@ -53,15 +52,3 @@ mdbook test my-book -L target/debug/deps/ See the `rustdoc` command-line [documentation](https://doc.rust-lang.org/rustdoc/command-line-arguments.html#-l--library-path-where-to-look-for-dependencies) for more information. - -#### `--dest-dir` - -The `--dest-dir` (`-d`) option allows you to change the output directory for the -book. Relative paths are interpreted relative to the book's root directory. If -not specified it will default to the value of the `build.build-dir` key in -`book.toml`, or to `./book`. - -#### `--chapter` - -The `--chapter` (`-c`) option allows you to test a specific chapter of the -book using the chapter name or the relative path to the chapter. diff --git a/guide/src/for_developers/backends.md b/guide/src/for_developers/backends.md index 72f8263eb5..6f3062075d 100644 --- a/guide/src/for_developers/backends.md +++ b/guide/src/for_developers/backends.md @@ -31,9 +31,9 @@ a [`RenderContext::from_json()`] constructor which will load a `RenderContext`. This is all the boilerplate necessary for our backend to load the book. -```rust +```rust,should_panic +# // this sample panics because it can't open stdin // src/main.rs -extern crate mdbook; use std::io; use mdbook::renderer::RenderContext; @@ -55,14 +55,18 @@ fn main() { ## Inspecting the Book -Now our backend has a copy of the book, lets count how many words are in each +Now our backend has a copy of the book, let's count how many words are in each chapter! Because the `RenderContext` contains a [`Book`] field (`book`), and a `Book` has the [`Book::iter()`] method for iterating over all items in a `Book`, this step turns out to be just as easy as the first. -```rust +```rust,should_panic +# // this sample panics because it can't open stdin +use std::io; +use mdbook::renderer::RenderContext; +use mdbook::book::{BookItem, Chapter}; fn main() { let mut stdin = io::stdin(); @@ -174,26 +178,25 @@ deserializing to some arbitrary type `T`. To implement this, we'll create our own serializable `WordcountConfig` struct which will encapsulate all configuration for this backend. -First add `serde` and `serde_derive` to your `Cargo.toml`, +First add `serde` to your `Cargo.toml`, -``` -$ cargo add serde serde_derive +```shell +$ cargo add serde ``` And then you can create the config struct, ```rust -extern crate serde; -#[macro_use] -extern crate serde_derive; +use serde::{Serialize, Deserialize}; -... +fn main() { #[derive(Debug, Default, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct WordcountConfig { pub ignores: Vec, } +} ``` Now we just need to deserialize the `WordcountConfig` from our `RenderContext` diff --git a/guide/src/for_developers/preprocessors.md b/guide/src/for_developers/preprocessors.md index 1455aceb7a..7a52fce25d 100644 --- a/guide/src/for_developers/preprocessors.md +++ b/guide/src/for_developers/preprocessors.md @@ -36,9 +36,8 @@ be adapted for other preprocessors.
Example no-op preprocessor -```rust +```rust,no_run // nop-preprocessors.rs - {{#include ../../../examples/nop-preprocessor.rs}} ```
@@ -67,7 +66,12 @@ translate events back into markdown text. The following code block shows how to remove all emphasis from markdown, without accidentally breaking the document. -```rust +```rust,compile_fail +# // tagged compile_fail because +# // sample fails to compile here: +# // "trait Borrow not implemented for pulldown_cmark_to_cmark::..." +# // Probably due to version skew on pulldown-cmark +# // between examples/remove-emphasis/Cargo.toml and /Cargo.toml {{#rustdoc_include ../../../examples/remove-emphasis/mdbook-remove-emphasis/src/main.rs:remove_emphasis}} ``` diff --git a/guide/src/format/configuration/general.md b/guide/src/format/configuration/general.md index 40a4570132..0963eb0887 100644 --- a/guide/src/format/configuration/general.md +++ b/guide/src/format/configuration/general.md @@ -11,7 +11,7 @@ authors = ["John Doe"] description = "The example book covers examples." [rust] -edition = "2018" +manifest = "./Cargo.toml" [build] build-dir = "my-example-book" @@ -30,7 +30,7 @@ limit-results = 15 ## Supported configuration options -It is important to note that **any** relative path specified in the +> Note: **any** relative path specified in the configuration will always be taken relative from the root of the book where the configuration file is located. @@ -38,6 +38,16 @@ configuration file is located. This is general information about your book. +```toml +[book] +title = "Example book" +authors = ["John Doe", "Jane Doe"] +description = "The example book covers examples." +src = "my-src" # source files in `root/my-src` instead of `root/src` +language = "en" +text-direction = "ltr" +``` + - **title:** The title of the book - **authors:** The author(s) of the book - **description:** A description for the book, which is added as meta @@ -50,17 +60,6 @@ This is general information about your book. - **text-direction**: The direction of text in the book: Left-to-right (LTR) or Right-to-left (RTL). Possible values: `ltr`, `rtl`. When not specified, the text direction is derived from the book's `language` attribute. -**book.toml** -```toml -[book] -title = "Example book" -authors = ["John Doe", "Jane Doe"] -description = "The example book covers examples." -src = "my-src" # the source files will be found in `root/my-src` instead of `root/src` -language = "en" -text-direction = "ltr" -``` - ### Rust options Options for the Rust language, relevant to running tests and playground @@ -68,19 +67,19 @@ integration. ```toml [rust] -edition = "2015" # the default edition for code blocks +manifest = "path/for/Cargo.toml" +edition = "2015" # [deprecated] the default edition for code blocks ``` -- **edition**: Rust edition to use by default for the code snippets. Default - is `"2015"`. Individual code blocks can be controlled with the `edition2015`, - `edition2018` or `edition2021` annotations, such as: +- **manifest**: Path to a ***Cargo.toml*** file which is used to resolve dependencies of your sample code. mdBook also uses the `package.edition` configured in the cargo project as the default for code snippets in your book. +See [Using External Crates and Dependencies](/format/mdbook.html#using-external-crates-and-dependencies) for details. + +- **edition** `[`deprecated`]`: Rust edition to use by default for the code snippets. +Default is `"2015"`. Individual code blocks can be controlled with the `edition2015`, + `edition2018` or `edition2021` annotations, as described in [Rust code block attributes](/format/mdbook.html#rust-code-block-attributes). + This option is deprecated because it's only useful if your code samples don't depend on external crates or you're not doctest'ing them. In any case, this option cannot be specified if **manifest** is configured. + - ~~~text - ```rust,edition2015 - // This only works in 2015. - let try = true; - ``` - ~~~ ### Build options diff --git a/guide/src/format/mathjax.md b/guide/src/format/mathjax.md index 3dd792159d..642a3aaa19 100644 --- a/guide/src/format/mathjax.md +++ b/guide/src/format/mathjax.md @@ -20,24 +20,25 @@ extra backslash to work. Hopefully this limitation will be lifted soon. > to add _two extra_ backslashes (e.g., `\begin{cases} \frac 1 2 \\\\ \frac 3 4 > \end{cases}`). - ### Inline equations + Inline equations are delimited by `\\(` and `\\)`. So for example, to render the following inline equation \\( \int x dx = \frac{x^2}{2} + C \\) you would write the following: -``` + +```text \\( \int x dx = \frac{x^2}{2} + C \\) ``` ### Block equations + Block equations are delimited by `\\[` and `\\]`. To render the following equation \\[ \mu = \frac{1}{N} \sum_{i=0} x_i \\] - you would write: -```bash +```text \\[ \mu = \frac{1}{N} \sum_{i=0} x_i \\] ``` diff --git a/guide/src/format/mdbook.md b/guide/src/format/mdbook.md index 9bb94615ce..f3bada29c3 100644 --- a/guide/src/format/mdbook.md +++ b/guide/src/format/mdbook.md @@ -1,5 +1,9 @@ # mdBook-specific features +# Code blocks + +These capabilities primarily affect how the user sees or interacts with code samples in your book and are supported directly by mdBook. Some also affect running the sample as a documentation test. (for which mdBook invokes `rustdoc --test`), so : this is detailed in the sections below. + ## Hiding code lines There is a feature in mdBook that lets you hide code lines by prepending them with a specific prefix. @@ -8,7 +12,7 @@ For the Rust language, you can use the `#` character as a prefix which will hide [rustdoc-hide]: https://doc.rust-lang.org/stable/rustdoc/write-documentation/documentation-tests.html#hiding-portions-of-the-example -```bash +```text # fn main() { let x = 5; let y = 6; @@ -40,7 +44,7 @@ python = "~" The prefix will hide any lines that begin with the given prefix. With the python prefix shown above, this: -```bash +```text ~hidden() nothidden(): ~ hidden() @@ -151,6 +155,7 @@ interpreting them. ```` ## Including portions of a file + Often you only need a specific part of the file, e.g. relevant lines for an example. We support four different modes of partial includes: @@ -175,6 +180,7 @@ the regex `ANCHOR_END:\s*[\w_-]+`. This allows you to put anchors in any kind of commented line. Consider the following file to include: + ```rs /* ANCHOR: all */ @@ -192,6 +198,7 @@ impl System for MySystem { ... } ``` Then in the book, all you have to do is: + ````hbs Here is a component: ```rust,no_run,noplayground @@ -223,7 +230,7 @@ Rustdoc will use the complete example when you run `mdbook test`. For example, consider a file named `file.rs` that contains this Rust program: -```rust +```rust,editable fn main() { let x = add_one(2); assert_eq!(x, 3); @@ -306,6 +313,21 @@ And the `editable` attribute will enable the [editor] as described at [Rust code [Rust Playground]: https://play.rust-lang.org/ +## Using external crates and dependencies + +If your code samples depend on external crates, you will probably want to include `use ` statements in the code and want them to resolve and allow documentation tests to run. +To configure this: + +1. Create a ***Cargo.toml*** file with a `[package.dependencies]` section that defines a dependency for each `` you want to use in any sample. If your book is already embedded in an existing Cargo project, you may be able to use the existing project `Cargo.toml`. +2. In your ***book.toml***: + * configure the path to ***Cargo.toml*** in `rust.manifest`, as described in [rust configuration](/format/configuration/general.html#rust-options). + * remove `rust.edition` if it is configured. The default rust edition will be as specified in the ***Cargo.toml*** (though this can be overridden for a specific code block). + * Refrain from invoking `mdbook test` with `-L` or `--library-path` argument. This, too, will be inferred from cargo project configuration + +# Features for general content + +These can be used in markdown text (outside code blocks). + ## Controlling page \ A chapter can set a \ that is different from its entry in the table of diff --git a/guide/src/lib.rs b/guide/src/lib.rs new file mode 100644 index 0000000000..cefd0a0e41 --- /dev/null +++ b/guide/src/lib.rs @@ -0,0 +1,8 @@ +// no code yet? +#![allow(unused)] +use mdbook; +use pulldown_cmark; +use pulldown_cmark_to_cmark; +use serde_json; + +pub fn marco() {} diff --git a/src/book/mod.rs b/src/book/mod.rs index b33ec6f00c..ff99a016fb 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -31,6 +31,7 @@ use crate::renderer::{CmdRenderer, HtmlHandlebars, MarkdownRenderer, RenderConte use crate::utils; use crate::config::{Config, RustEdition}; +use crate::utils::extern_args::ExternArgs; /// The object used to manage and build a book. pub struct MDBook { @@ -304,6 +305,14 @@ impl MDBook { let (book, _) = self.preprocess_book(&TestRenderer)?; let color_output = std::io::stderr().is_terminal(); + + // get extra args we'll need for rustdoc, if config points to a cargo project. + + let mut extern_args = ExternArgs::new(); + if let Some(manifest) = &self.config.rust.manifest { + extern_args.load(&self.root.join(manifest))?; + } + let mut failed = false; for item in book.iter() { if let BookItem::Chapter(ref ch) = *item { @@ -332,9 +341,14 @@ impl MDBook { cmd.current_dir(temp_dir.path()) .arg(chapter_path) .arg("--test") - .args(&library_args); - - if let Some(edition) = self.config.rust.edition { + .args(&library_args) // also need --extern for doctest to actually work + .args(extern_args.get_args()); + + // rustdoc edition from cargo manifest takes precedence over book.toml + // bugbug but also takes precedence over command line flag -- that seems rude. + if !extern_args.edition.is_empty() { + cmd.args(["--edition", &extern_args.edition]); + } else if let Some(edition) = self.config.rust.edition { match edition { RustEdition::E2015 => { cmd.args(["--edition", "2015"]); @@ -351,6 +365,7 @@ impl MDBook { } } + // bugbug Why show color in hidden invocation of rustdoc? if color_output { cmd.args(["--color", "always"]); } diff --git a/src/cmd/test.rs b/src/cmd/test.rs index d41e9ef9eb..4fcbf527f4 100644 --- a/src/cmd/test.rs +++ b/src/cmd/test.rs @@ -28,7 +28,7 @@ pub fn make_subcommand() -> Command { .value_parser(NonEmptyStringValueParser::new()) .action(ArgAction::Append) .help( - "A comma-separated list of directories to add to the crate \ + "[deprecated] A comma-separated list of directories to add to the crate \ search path when building tests", ), ) diff --git a/src/config.rs b/src/config.rs index b87ad27644..dd5657e811 100644 --- a/src/config.rs +++ b/src/config.rs @@ -497,6 +497,8 @@ impl Default for BuildConfig { #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct RustConfig { + /// Path to a Cargo.toml + pub manifest: Option, /// Rust edition used in playground pub edition: Option, } @@ -798,6 +800,9 @@ mod tests { create-missing = false use-default-preprocessors = true + [rust] + manifest = "./Cargo.toml" + [output.html] theme = "./themedir" default-theme = "rust" @@ -839,7 +844,10 @@ mod tests { use_default_preprocessors: true, extra_watch_dirs: Vec::new(), }; - let rust_should_be = RustConfig { edition: None }; + let rust_should_be = RustConfig { + manifest: Some(PathBuf::from("./Cargo.toml")), + edition: None, + }; let playground_should_be = Playground { editable: true, copyable: true, @@ -918,6 +926,7 @@ mod tests { assert_eq!(got.book, book_should_be); let rust_should_be = RustConfig { + manifest: None, edition: Some(RustEdition::E2015), }; let got = Config::from_str(src).unwrap(); @@ -937,6 +946,7 @@ mod tests { "#; let rust_should_be = RustConfig { + manifest: None, edition: Some(RustEdition::E2018), }; @@ -957,6 +967,7 @@ mod tests { "#; let rust_should_be = RustConfig { + manifest: None, edition: Some(RustEdition::E2021), }; @@ -1356,4 +1367,18 @@ mod tests { false ); } + + /* todo -- make this test fail, as it should + #[test] + #[should_panic(expected = "Invalid configuration file")] + // invalid key in config file should really generate an error... + fn invalid_rust_setting() { + let src = r#" + [rust] + foo = "bar" + "#; + + Config::from_str(src).unwrap(); + } + */ } diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs new file mode 100644 index 0000000000..9520eb931f --- /dev/null +++ b/src/utils/extern_args.rs @@ -0,0 +1,342 @@ +//! Get "compiler" args from cargo + +use crate::errors::*; +use anyhow::anyhow; +use cargo_manifest::{Edition, Manifest, MaybeInherited::Local}; +use log::{debug, info}; +use std::fs; +use std::fs::File; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Get the arguments needed to invoke rustc so it can find external crates +/// when invoked by rustdoc to compile doctests. +/// +/// It seems the `-L ` and `--extern =` args are sufficient. +/// +/// Cargo doesn't expose a stable API to get this information. +/// `cargo metadata` does not include the hash suffix in ``. +/// But it does leak when doing a build in verbose mode. +/// So we force a cargo build, capture the console output and parse the args therefrom. +/// +/// Example: +/// ```rust +/// +/// use mdbook::utils::extern_args::ExternArgs; +/// # use mdbook::errors::*; +/// +/// # fn main() -> Result<()> { +/// // Get cargo to say what the compiler args need to be... +/// let manifest_file = std::env::current_dir()?.join("Cargo.toml"); // or other path to `Cargo.toml` +/// let mut extern_args = ExternArgs::new(); +/// extern_args.load(&manifest_file)?; +/// +/// // then, when actually invoking rustdoc or some other compiler-like tool... +/// +/// assert!(extern_args.get_args().iter().any(|e| e == "-L")); // args contains "-L".to_string() +/// assert!(extern_args.get_args().iter().any(|e| e == "--extern")); +/// # Ok(()) +/// # } +/// ``` + +#[derive(Debug)] +pub struct ExternArgs { + /// rust edition as specified in manifest + pub edition: String, // where default value of "" means arg wasn't specified + /// crate name as specified in manifest + pub crate_name: String, + // accumulated library path(s), as observed from live cargo run + lib_list: Vec, + // explicit extern crates, as observed from live cargo run + extern_list: Vec, +} + +impl ExternArgs { + /// simple constructor + pub fn new() -> Self { + ExternArgs { + edition: String::default(), + crate_name: String::default(), + lib_list: vec![], + extern_list: vec![], + } + } + + /// Run a `cargo build` to see what args Cargo is using for library paths and extern crates. + /// Touch a source file in the crate to ensure something is compiled and the args will be visible. + pub fn load(&mut self, cargo_path: &Path) -> Result<&Self> { + // find Cargo.toml and determine the package name and lib or bin source file. + let proj_root = cargo_path + .canonicalize() + .context(format!( + "can't find cargo manifest {}", + &cargo_path.to_string_lossy() + ))? + .parent() + .ok_or(anyhow!("can't find parent of {:?}", cargo_path))? + .to_owned(); + let mut manifest = Manifest::from_path(cargo_path).context(format!( + "can't open cargo manifest {}", + &cargo_path.to_string_lossy() + ))?; + manifest.complete_from_path(&proj_root)?; // try real hard to determine bin or lib + let package = manifest + .package + .expect("doctest Cargo.toml must include a [package] section"); + + self.crate_name = package.name.replace('-', "_"); // maybe cargo shouldn't allow packages to include non-identifier characters? + // in any case, this won't work when default crate doesn't have package name (which I'm sure cargo allows somehow or another) + self.edition = if let Some(Local(edition)) = package.edition { + my_display_edition(edition) + } else { + "".to_owned() // + }; + + debug!( + "parsed from manifest: name: {}, edition: {}", + self.crate_name, + format!("{:?}", self.edition) + ); + + // touch (change) a file in the project to force check to do something + // I haven't figured out how to determine bin or lib source file from cargo, fall back on heuristics here. + + for fname in ["main.rs", "lib.rs"] { + let try_path: PathBuf = proj_root.join("src").join(fname); + if try_path.exists() { + touch(&try_path)?; + self.run_cargo(&proj_root, cargo_path)?; + return Ok(self); + // file should be closed when f goes out of scope at bottom of this loop + } + } + bail!("Couldn't find lib or bin source in project {:?}", proj_root) + } + + fn run_cargo(&mut self, proj_root: &Path, manifest_path: &Path) -> Result<&Self> { + let mut cmd = Command::new("cargo"); + cmd.current_dir(proj_root) + .arg("build") + .arg("--verbose") + .arg("--manifest-path") + .arg(manifest_path); + info!("running {:?}", cmd); + + let output = cmd.output()?; + + if !output.status.success() { + bail!( + "Exit status {} from {:?}\nMessage:\n{:?}", + output.status, + cmd, + std::string::String::from_utf8_lossy(&output.stderr) + ); + } + + //ultimatedebug std::fs::write(proj_root.join("mdbook_cargo_out.txt"), &output.stderr)?; + + let cmd_resp: &str = std::str::from_utf8(&output.stderr)?; + self.parse_response(self.crate_name.clone().as_str(), cmd_resp)?; + + Ok(self) + } + + /// Parse response stdout+stderr response from `cargo build` + /// into arguments we can use to invoke rustdoc (--edition --extern and -L). + /// The response may contain multiple builds, scan for the one that corresponds to the doctest crate. + /// + /// > This parser is broken, doesn't handle arg values with embedded spaces (single quoted). + /// > Fortunately, the args we care about (so far) don't have those kinds of values. + pub fn parse_response(&mut self, my_crate: &str, buf: &str) -> Result<()> { + let mut builds_ignored = 0; + + let my_cn_arg = format!(" --crate-name {}", my_crate); + for l in buf.lines() { + if let Some(_i) = l.find(" Running ") { + if let Some(_cn_pos) = l.find(&my_cn_arg) { + let args_seg: &str = l.split('`').skip(1).take(1).collect::>()[0]; // sadly, cargo decorates string with backticks + let mut arg_iter = args_seg.split_whitespace(); + + while let Some(arg) = arg_iter.next() { + match arg { + "-L" | "--library-path" => { + self.lib_list + .push(arg_iter.next().unwrap_or_default().to_owned()); + } + + "--extern" => { + let mut dep_arg = arg_iter.next().unwrap_or_default().to_owned(); + + // sometimes, build references the.rmeta even though our doctests will require .rlib + // so convert the argument and hope for the best. + // if .rlib is not there when the doctest runs, it will complain. + if dep_arg.ends_with(".rmeta") { + debug!( + "Build referenced {}, converted to .rlib hoping that actual file will be there in time.", + dep_arg); + dep_arg = dep_arg.replace(".rmeta", ".rlib"); + } + self.extern_list.push(dep_arg); + } + + "--crate-name" => { + self.crate_name = arg_iter.next().unwrap_or_default().to_owned(); + } + + _ => { + if let Some((kw, val)) = arg.split_once('=') { + if kw == "--edition" { + self.edition = val.to_owned(); + } + } + } + } + } + } else { + builds_ignored += 1; + } + }; + } + + if self.extern_list.is_empty() || self.lib_list.is_empty() { + bail!("Couldn't extract -L or --extern args from Cargo, is current directory == cargo project root?"); + } + + debug!( + "Ignored {} other builds performed in this run", + builds_ignored + ); + + Ok(()) + } + + /// provide the parsed external args used to invoke rustdoc (--edition, -L and --extern). + pub fn get_args(&self) -> Vec { + let mut ret_val: Vec = vec![]; + for i in &self.lib_list { + ret_val.push("-L".to_owned()); + ret_val.push(i.clone()); + } + for j in &self.extern_list { + ret_val.push("--extern".to_owned()); + ret_val.push(j.clone()); + } + ret_val + } +} + +impl Default for ExternArgs { + fn default() -> Self { + Self::new() + } +} + +fn my_display_edition(edition: Edition) -> String { + match edition { + Edition::E2015 => "2015", + Edition::E2018 => "2018", + Edition::E2021 => "2021", + Edition::E2024 => "2024", + } + .to_owned() +} +// Private "touch" function to update file modification time without changing content. +// needed because [std::fs::set_modified] is unstable in rust 1.74, +// which is currently the MSRV for mdBook. It is available in rust 1.76 onward. + +fn touch(victim: &Path) -> Result<()> { + let curr_content = fs::read(victim).with_context(|| "reading existing file")?; + let mut touchfs = File::options() + .append(true) + .open(victim) + .with_context(|| "opening for touch")?; + + let _len_written = touchfs.write(b"z")?; // write a byte + touchfs.flush().expect("closing"); // close the file + drop(touchfs); // close modified file, hopefully updating modification time + + fs::write(victim, curr_content).with_context(|| "trying to restore old content") +} + +#[cfg(test)] +mod test { + use super::*; + use std::fs; + use std::thread; + use std::time::Duration; + use tempfile; + + #[test] + fn parse_response_parses_string() -> Result<()> { + let test_str = r###" + Fresh unicode-ident v1.0.14 + Fresh cfg-if v1.0.0 + Fresh memchr v2.7.4 + Fresh autocfg v1.4.0 + Fresh version_check v0.9.5 + --- clip --- + Fresh bytecount v0.6.8 + Fresh leptos_router v0.7.0 + Fresh leptos_meta v0.7.0 + Fresh console_error_panic_hook v0.1.7 + Fresh mdbook-keeper v0.5.0 + Dirty leptos-book v0.1.0 (/home/bobhy/src/localdep/book): the file `src/lib.rs` has changed (1733758773.052514835s, 10h 32m 29s after last build at 1733720824.458358565s) + Compiling leptos-book v0.1.0 (/home/bobhy/src/localdep/book) + Running `/home/bobhy/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc --crate-name leptos_book --edition=2021 src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type cdylib --crate-type rlib --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("hydrate", "ssr"))' -C metadata=2eec49d479de095c --out-dir /home/bobhy/src/localdep/book/target/debug/deps -C incremental=/home/bobhy/src/localdep/book/target/debug/incremental -L dependency=/home/bobhy/src/localdep/book/target/debug/deps --extern console_error_panic_hook=/home/bobhy/src/localdep/book/target/debug/deps/libconsole_error_panic_hook-d34cf0116774f283.rlib --extern http=/home/bobhy/src/localdep/book/target/debug/deps/libhttp-d4d503240b7a6b18.rlib --extern leptos=/home/bobhy/src/localdep/book/target/debug/deps/libleptos-1dabf2e09ca58f3d.rlib --extern leptos_meta=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_meta-df8ce1704acca063.rlib --extern leptos_router=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_router-df109cd2ee44b2a0.rlib --extern mdbook_keeper_lib=/home/bobhy/src/localdep/book/target/debug/deps/libmdbook_keeper_lib-f4016aaf2c5da5f2.rlib --extern thiserror=/home/bobhy/src/localdep/book/target/debug/deps/libthiserror-acc5435cdf9551fe.rlib --extern wasm_bindgen=/home/bobhy/src/localdep/book/target/debug/deps/libwasm_bindgen-89a7b1dccd9668ae.rlib` + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.60s + + "###; + + let mut ea = ExternArgs::new(); + ea.parse_response("leptos_book", &test_str)?; + + let args = ea.get_args(); + + assert_eq!(ea.edition, "2021"); + assert_eq!(ea.crate_name, "leptos_book"); + + assert_eq!(18, args.len()); + + assert_eq!(1, args.iter().filter(|i| *i == "-L").count()); + assert_eq!(8, args.iter().filter(|i| *i == "--extern").count()); + + Ok(()) + } + + #[test] + fn verify_touch() -> Result<()> { + const FILE_CONTENT: &[u8] = + b"I am some random text with crlfs \r\n but also nls \n and terminated with a nl \n"; + const DELAY: Duration = Duration::from_millis(20); // don't hang up tests for too long, but maybe 10ms is too short? + + let temp_dir = tempfile::TempDir::new()?; + let mut victim_path = temp_dir.path().to_owned(); + victim_path.push("workfile.dir"); + fs::write(&victim_path, FILE_CONTENT)?; + let old_md = fs::metadata(&victim_path)?; + thread::sleep(DELAY); + + touch(&victim_path)?; + let new_md = fs::metadata(&victim_path)?; + + let act_content = fs::read(&victim_path)?; + + assert_eq!(FILE_CONTENT, act_content); + let tdif = new_md + .modified() + .expect("getting modified time new") + .duration_since(old_md.modified().expect("getting modified time old")) + .expect("system time botch"); + // can't expect sleep 20ms to actually delay exactly that -- + // but the test is to verify that `touch` made the file look any newer. + // Give ourselves 50% slop under what we were aiming for and call it good enough. + assert!( + tdif >= (DELAY / 2), + "verify_touch: expected {:?}, actual {:?}", + DELAY, + tdif + ); + Ok(()) + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index a53f79c0e9..4650339723 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,5 +1,6 @@ #![allow(missing_docs)] // FIXME: Document this +pub mod extern_args; pub mod fs; mod string; pub(crate) mod toml_ext;