From b1c7c54108e4eaeeb2a549debdef53db795f8d4e Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 25 Sep 2018 19:41:38 +0800 Subject: [PATCH] Rewrote a large proportion of the Preprocessor docs to be up-to-date --- .../src/for_developers/preprocessors.md | 104 +++++++------- examples/de-emphasize.rs | 24 +--- examples/nop-preprocessor.rs | 130 +++++++++++------- src/preprocess/cmd.rs | 19 +-- src/preprocess/links.rs | 114 ++++++++++++--- 5 files changed, 244 insertions(+), 147 deletions(-) diff --git a/book-example/src/for_developers/preprocessors.md b/book-example/src/for_developers/preprocessors.md index 03c915bb..e8bcaf0a 100644 --- a/book-example/src/for_developers/preprocessors.md +++ b/book-example/src/for_developers/preprocessors.md @@ -11,68 +11,71 @@ the book. Possible use cases are: mathjax equivalents -## Implementing a Preprocessor +## Hooking Into MDBook -A preprocessor is represented by the `Preprocessor` trait. +MDBook uses a fairly simple mechanism for discovering third party plugins. +A new table is added to `book.toml` (e.g. `preprocessor.foo` for the `foo` +preprocessor) and then `mdbook` will try to invoke the `mdbook-foo` program as +part of the build process. -```rust -pub trait Preprocessor { - fn name(&self) -> &str; - fn run(&self, ctx: &PreprocessorContext, book: Book) -> Result; - fn supports_renderer(&self, _renderer: &str) -> bool { - true - } -} +While preprocessors can be hard-coded to specify which backend it should be run +for (e.g. it doesn't make sense for MathJax to be used for non-HTML renderers) +with the `preprocessor.foo.renderer` key. + +```toml +[book] +title = "My Book" +authors = ["Michael-F-Bryan"] + +[preprocessor.foo] +# The command can also be specified manually +command = "python3 /path/to/foo.py" +# Only run the `foo` preprocessor for the HTML and EPUB renderer +renderer = ["html", "epub"] ``` -Where the `PreprocessorContext` is defined as +In typical unix style, all inputs to the plugin will be written to `stdin` as +JSON and `mdbook` will read from `stdout` if it is expecting output. + +The easiest way to get started is by creating your own implementation of the +`Preprocessor` trait (e.g. in `lib.rs`) and then creating a shell binary which +translates inputs to the correct `Preprocessor` method. For convenience, there +is [an example no-op preprocessor] in the `examples/` directory which can easily +be adapted for other preprocessors. + +
+Example no-op preprocessor ```rust -pub struct PreprocessorContext { - pub root: PathBuf, - pub config: Config, - /// The `Renderer` this preprocessor is being used with. - pub renderer: String, -} +// nop-preprocessors.rs + +{{#include ../../../examples/nop-preprocessor.rs}} ``` +
-The `renderer` value allows you react accordingly, for example, PDF or HTML. +## Hints For Implementing A Preprocessor -## A complete Example +By pulling in `mdbook` as a library, preprocessors can have access to the +existing infrastructure for dealing with books. -The magic happens within the `run(...)` method of the -[`Preprocessor`][preprocessor-docs] trait implementation. +For example, a custom preprocessor could use the +[`CmdPreprocessor::parse_input()`] function to deserialize the JSON written to +`stdin`. Then each chapter of the `Book` can be mutated in-place via +[`Book::for_each_mut()`], and then written to `stdout` with the `serde_json` +crate. -As direct access to the chapters is not possible, you will probably end up -iterating them using `for_each_mut(...)`: +Chapters can be accessed either directly (by recursively iterating over +chapters) or via the `Book::for_each_mut()` convenience method. -```rust -book.for_each_mut(|item: &mut BookItem| { - if let BookItem::Chapter(ref mut chapter) = *item { - eprintln!("{}: processing chapter '{}'", self.name(), chapter.name); - res = Some( - match Deemphasize::remove_emphasis(&mut num_removed_items, chapter) { - Ok(md) => { - chapter.content = md; - Ok(()) - } - Err(err) => Err(err), - }, - ); - } -}); -``` +The `chapter.content` is just a string which happens to be markdown. While it's +entirely possible to use regular expressions or do a manual find & replace, +you'll probably want to process the input into something more computer-friendly. +The [`pulldown-cmark`][pc] crate implements a production-quality event-based +Markdown parser, with the [`pulldown-cmark-to-cmark`][pctc] allowing you to +translate events back into markdown text. -The `chapter.content` is just a markdown formatted string, and you will have to -process it in some way. Even though it's entirely possible to implement some -sort of manual find & replace operation, if that feels too unsafe you can use -[`pulldown-cmark`][pc] to parse the string into events and work on them instead. - -Finally you can use [`pulldown-cmark-to-cmark`][pctc] to transform these events -back to a string. - -The following code block shows how to remove all emphasis from markdown, and do -so safely. +The following code block shows how to remove all emphasis from markdown, +without accidentally breaking the document. ```rust fn remove_emphasis( @@ -107,3 +110,6 @@ For everything else, have a look [at the complete example][example]. [pc]: https://crates.io/crates/pulldown-cmark [pctc]: https://crates.io/crates/pulldown-cmark-to-cmark [example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/de-emphasize.rs +[an example no-op preprocessor]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs +[`CmdPreprocessor::parse_input()`]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html#method.parse_input +[`Book::for_each_mut()`]: https://docs.rs/mdbook/latest/mdbook/book/struct.Book.html#method.for_each_mut diff --git a/examples/de-emphasize.rs b/examples/de-emphasize.rs index 88c1b3a4..933e5c45 100644 --- a/examples/de-emphasize.rs +++ b/examples/de-emphasize.rs @@ -1,4 +1,6 @@ -//! This program removes all forms of emphasis from the markdown of the book. +//! An example preprocessor for removing all forms of emphasis from a markdown +//! book. + extern crate mdbook; extern crate pulldown_cmark; extern crate pulldown_cmark_to_cmark; @@ -6,31 +8,13 @@ extern crate pulldown_cmark_to_cmark; use mdbook::book::{Book, BookItem, Chapter}; use mdbook::errors::{Error, Result}; use mdbook::preprocess::{Preprocessor, PreprocessorContext}; -use mdbook::MDBook; use pulldown_cmark::{Event, Parser, Tag}; use pulldown_cmark_to_cmark::fmt::cmark; -use std::env::{args, args_os}; -use std::ffi::OsString; -use std::process; - const NAME: &str = "md-links-to-html-links"; -fn do_it(book: OsString) -> Result<()> { - let mut book = MDBook::load(book)?; - book.with_preprecessor(Deemphasize); - book.build() -} - fn main() { - if args_os().count() != 2 { - eprintln!("USAGE: {} ", args().next().expect("executable")); - return; - } - if let Err(e) = do_it(args_os().skip(1).next().expect("one argument")) { - eprintln!("{}", e); - process::exit(1); - } + panic!("This example is intended to be part of a library"); } struct Deemphasize; diff --git a/examples/nop-preprocessor.rs b/examples/nop-preprocessor.rs index 4cd8693d..d4615ef6 100644 --- a/examples/nop-preprocessor.rs +++ b/examples/nop-preprocessor.rs @@ -1,56 +1,64 @@ +extern crate clap; extern crate mdbook; extern crate serde_json; -#[macro_use] -extern crate clap; use clap::{App, Arg, ArgMatches, SubCommand}; -use mdbook::preprocess::CmdPreprocessor; use mdbook::book::Book; -use std::process; +use mdbook::errors::Error; +use mdbook::preprocess::{CmdPreprocessor, Preprocessor, PreprocessorContext}; use std::io; +use std::process; +use nop_lib::Nop; + +pub fn make_app() -> App<'static, 'static> { + App::new("nop-preprocessor") + .about("A mdbook preprocessor which does precisely nothing") + .subcommand( + SubCommand::with_name("supports") + .arg(Arg::with_name("renderer").required(true)) + .about("Check whether a renderer is supported by this preprocessor")) +} fn main() { - let matches = app().get_matches(); + let matches = make_app().get_matches(); + + // Users will want to construct their own preprocessor here + let preprocessor = Nop::new(); if let Some(sub_args) = matches.subcommand_matches("supports") { - handle_supports(sub_args); + handle_supports(&preprocessor, sub_args); } else { - handle_preprocessing(); - } -} - -fn handle_preprocessing() { - let (ctx, book) = CmdPreprocessor::parse_input(io::stdin()) - .expect("Couldn't parse the input"); - - // You can inspect the calling mdbook's version to check for compatibility - if ctx.mdbook_version != mdbook::MDBOOK_VERSION { - panic!("The version check failed!"); - } - - // In testing we want to tell the preprocessor to blow up by setting a - // particular config value - if let Some(table) = ctx.config.get_preprocessor("nop-preprocessor") { - let should_blow_up = table.get("blow-up").is_some(); - - if should_blow_up { - panic!("Boom!!!1!"); + if let Err(e) = handle_preprocessing(&preprocessor) { + eprintln!("{}", e); + process::exit(1); } } - - let processed_book = do_processing(book); - - serde_json::to_writer(io::stdout(), &processed_book).unwrap(); } -fn do_processing(book: Book) -> Book { - // We *are* a nop preprocessor after all... - book +fn handle_preprocessing(pre: &dyn Preprocessor) -> Result<(), Error> { + let (ctx, book) = CmdPreprocessor::parse_input(io::stdin())?; + + if ctx.mdbook_version != mdbook::MDBOOK_VERSION { + // We should probably use the `semver` crate to check compatibility + // here... + eprintln!( + "Warning: The {} plugin was built against version {} of mdbook, \ + but we're being called from version {}", + pre.name(), + mdbook::MDBOOK_VERSION, + ctx.mdbook_version + ); + } + + let processed_book = pre.run(&ctx, book)?; + serde_json::to_writer(io::stdout(), &processed_book)?; + + Ok(()) } -fn handle_supports(sub_args: &ArgMatches) { +fn handle_supports(pre: &dyn Preprocessor, sub_args: &ArgMatches) -> ! { let renderer = sub_args.value_of("renderer").expect("Required argument"); - let supported = renderer_is_supported(&renderer); + let supported = pre.supports_renderer(&renderer); // Signal whether the renderer is supported by exiting with 1 or 0. if supported { @@ -60,17 +68,45 @@ fn handle_supports(sub_args: &ArgMatches) { } } -fn renderer_is_supported(renderer: &str) -> bool { - // We support everything except the `not-supported` renderer - renderer != "not-supported" +/// The actual implementation of the `Nop` preprocessor. This would usually go +/// in your main `lib.rs` file. +mod nop_lib { + use super::*; + + /// A no-op preprocessor. + pub struct Nop; + + impl Nop { + pub fn new() -> Nop { + Nop + } + } + + impl Preprocessor for Nop { + fn name(&self) -> &str { + "nop-preprocessor" + } + + fn run( + &self, + ctx: &PreprocessorContext, + book: Book, + ) -> Result { + // In testing we want to tell the preprocessor to blow up by setting a + // particular config value + if let Some(nop_cfg) = ctx.config.get_preprocessor(self.name()) { + if nop_cfg.contains_key("blow-up") { + return Err("Boom!!1!".into()); + } + } + + // we *are* a no-op preprocessor after all + Ok(book) + } + + fn supports_renderer(&self, renderer: &str) -> bool { + renderer != "not-supported" + } + } } -fn app() -> App<'static, 'static> { - app_from_crate!().subcommand( - SubCommand::with_name("supports") - .arg(Arg::with_name("renderer").required(true)) - .about( - "Check whether a renderer is supported by this preprocessor", - ), - ) -} diff --git a/src/preprocess/cmd.rs b/src/preprocess/cmd.rs index 870b1015..4398eba6 100644 --- a/src/preprocess/cmd.rs +++ b/src/preprocess/cmd.rs @@ -3,7 +3,7 @@ use book::Book; use errors::*; use serde_json; use shlex::Shlex; -use std::io::{self, Read}; +use std::io::{self, Read, Write}; use std::process::{Child, Command, Stdio}; /// A custom preprocessor which will shell out to a 3rd-party program. @@ -50,23 +50,24 @@ impl CmdPreprocessor { .chain_err(|| "Unable to parse the input") } - fn write_input( + fn write_input_to_child( &self, child: &mut Child, - book: Book, + book: &Book, ctx: &PreprocessorContext, ) { - let mut stdin = child.stdin.take().expect("Child has stdin"); - let input = (ctx, book); + let stdin = child.stdin.take().expect("Child has stdin"); - if let Err(e) = serde_json::to_writer(&mut stdin, &input) { + if let Err(e) = self.write_input(stdin, &book, &ctx) { // Looks like the backend hung up before we could finish // sending it the render context. Log the error and keep going warn!("Error writing the RenderContext to the backend, {}", e); } + } - // explicitly close the `stdin` file handle - drop(stdin); + fn write_input(&self, writer: W, book: &Book, ctx: &PreprocessorContext) -> Result<()> { + serde_json::to_writer(writer, &(ctx, book)) + .map_err(Into::into) } /// The command this `Preprocessor` will invoke. @@ -106,7 +107,7 @@ impl Preprocessor for CmdPreprocessor { .spawn() .chain_err(|| format!("Unable to start the \"{}\" preprocessor. Is it installed?", self.name()))?; - self.write_input(&mut child, book, ctx); + self.write_input_to_child(&mut child, &book, ctx); let output = child .wait_with_output() diff --git a/src/preprocess/links.rs b/src/preprocess/links.rs index 870f96a8..3403891a 100644 --- a/src/preprocess/links.rs +++ b/src/preprocess/links.rs @@ -69,7 +69,12 @@ where Ok(new_content) => { if depth < MAX_LINK_NESTED_DEPTH { if let Some(rel_path) = playpen.link.relative_path(path) { - replaced.push_str(&replace_all(&new_content, rel_path, source, depth + 1)); + replaced.push_str(&replace_all( + &new_content, + rel_path, + source, + depth + 1, + )); } else { replaced.push_str(&new_content); } @@ -83,6 +88,10 @@ where } Err(e) => { error!("Error updating \"{}\", {}", playpen.link_text, e); + for cause in e.iter().skip(1) { + warn!("Caused By: {}", cause); + } + // This should make sure we include the raw `{{# ... }}` snippet // in the page content if there are any errors. previous_end_index = playpen.start_index; @@ -109,10 +118,18 @@ impl<'a> LinkType<'a> { let base = base.as_ref(); match self { LinkType::Escaped => None, - LinkType::IncludeRange(p, _) => Some(return_relative_path(base, &p)), - LinkType::IncludeRangeFrom(p, _) => Some(return_relative_path(base, &p)), - LinkType::IncludeRangeTo(p, _) => Some(return_relative_path(base, &p)), - LinkType::IncludeRangeFull(p, _) => Some(return_relative_path(base, &p)), + LinkType::IncludeRange(p, _) => { + Some(return_relative_path(base, &p)) + } + LinkType::IncludeRangeFrom(p, _) => { + Some(return_relative_path(base, &p)) + } + LinkType::IncludeRangeTo(p, _) => { + Some(return_relative_path(base, &p)) + } + LinkType::IncludeRangeFull(p, _) => { + Some(return_relative_path(base, &p)) + } LinkType::Playpen(p, _) => Some(return_relative_path(base, &p)), } } @@ -182,11 +199,15 @@ impl<'a> Link<'a> { match (typ.as_str(), file_arg) { ("include", Some(pth)) => Some(parse_include_path(pth)), - ("playpen", Some(pth)) => Some(LinkType::Playpen(pth.into(), props)), + ("playpen", Some(pth)) => { + Some(LinkType::Playpen(pth.into(), props)) + } _ => None, } } - (Some(mat), None, None) if mat.as_str().starts_with(ESCAPE_CHAR) => { + (Some(mat), None, None) + if mat.as_str().starts_with(ESCAPE_CHAR) => + { Some(LinkType::Escaped) } _ => None, @@ -207,20 +228,65 @@ impl<'a> Link<'a> { match self.link { // omit the escape char LinkType::Escaped => Ok((&self.link_text[1..]).to_owned()), - LinkType::IncludeRange(ref pat, ref range) => file_to_string(base.join(pat)) - .map(|s| take_lines(&s, range.clone())) - .chain_err(|| format!("Could not read file for link {}", self.link_text)), - LinkType::IncludeRangeFrom(ref pat, ref range) => file_to_string(base.join(pat)) - .map(|s| take_lines(&s, range.clone())) - .chain_err(|| format!("Could not read file for link {}", self.link_text)), - LinkType::IncludeRangeTo(ref pat, ref range) => file_to_string(base.join(pat)) - .map(|s| take_lines(&s, range.clone())) - .chain_err(|| format!("Could not read file for link {}", self.link_text)), - LinkType::IncludeRangeFull(ref pat, _) => file_to_string(base.join(pat)) - .chain_err(|| format!("Could not read file for link {}", self.link_text)), + LinkType::IncludeRange(ref pat, ref range) => { + let target = base.join(pat); + + file_to_string(&target) + .map(|s| take_lines(&s, range.clone())) + .chain_err(|| { + format!( + "Could not read file for link {} ({})", + self.link_text, + target.display(), + ) + }) + } + LinkType::IncludeRangeFrom(ref pat, ref range) => { + let target = base.join(pat); + + file_to_string(&target) + .map(|s| take_lines(&s, range.clone())) + .chain_err(|| { + format!( + "Could not read file for link {} ({})", + self.link_text, + target.display(), + ) + }) + } + LinkType::IncludeRangeTo(ref pat, ref range) => { + let target = base.join(pat); + + file_to_string(&target) + .map(|s| take_lines(&s, range.clone())) + .chain_err(|| { + format!( + "Could not read file for link {} ({})", + self.link_text, + target.display(), + ) + }) + } + LinkType::IncludeRangeFull(ref pat, _) => { + let target = base.join(pat); + + file_to_string(&target).chain_err(|| { + format!("Could not read file for link {} ({})", + self.link_text, + target.display()) + }) + } LinkType::Playpen(ref pat, ref attrs) => { - let contents = file_to_string(base.join(pat)) - .chain_err(|| format!("Could not read file for link {}", self.link_text))?; + let target = base.join(pat); + + let contents = + file_to_string(&target).chain_err(|| { + format!( + "Could not read file for link {} ({})", + self.link_text, + target.display() + ) + })?; let ftype = if !attrs.is_empty() { "rust," } else { "rust" }; Ok(format!( "```{}{}\n{}\n```\n", @@ -465,7 +531,10 @@ mod tests { Link { start_index: 38, end_index: 68, - link: LinkType::Playpen(PathBuf::from("file.rs"), vec!["editable"]), + link: LinkType::Playpen( + PathBuf::from("file.rs"), + vec!["editable"] + ), link_text: "{{#playpen file.rs editable }}", }, Link { @@ -475,7 +544,8 @@ mod tests { PathBuf::from("my.rs"), vec!["editable", "no_run", "should_panic"], ), - link_text: "{{#playpen my.rs editable no_run should_panic}}", + link_text: + "{{#playpen my.rs editable no_run should_panic}}", }, ] );