From bb043ef66027f1fb5cf9b8c8c8f420b575a07ed7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Feb 2018 11:14:52 +0100 Subject: [PATCH] Add complete preprocessor example (#629) * First version of preprocessor example, with quicli It seems it's not worth it right now. * Remove quicli, just to simplify everything * Finish de-emphasise example * Finish preprocessor example in book * Rename preprocessor type * Apply changes requested in review * Update preprocessor docs with latest code [skip CI] --- Cargo.lock | 10 ++ Cargo.toml | 1 + .../src/for_developers/preprocessors.md | 72 +++++++++++++- examples/de-emphasize.rs | 94 +++++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 examples/de-emphasize.rs diff --git a/Cargo.lock b/Cargo.lock index 45a16704..5a2bdce7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -465,6 +465,7 @@ dependencies = [ "open 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "select 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", @@ -732,6 +733,14 @@ dependencies = [ "getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pulldown-cmark-to-cmark" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pulldown-cmark 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quick-error" version = "1.2.1" @@ -1312,6 +1321,7 @@ dependencies = [ "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" "checksum pretty_assertions 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28ea5118e2f41bfbc974b28d88c07621befd1fa5d6ec23549be96302a1a59dd2" "checksum pulldown-cmark 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a656fdb8b6848f896df5e478a0eb9083681663e37dcb77dd16981ff65329fe8b" +"checksum pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57efca5f52f63336ee3a49bceee1a1169f18ef01c75aa7e71949441b49bbe7e4" "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" diff --git a/Cargo.toml b/Cargo.toml index 89228011..fb5a8734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ error-chain = "0.11" select = "0.4" pretty_assertions = "0.4" walkdir = "2.0" +pulldown-cmark-to-cmark = "1.1.0" [features] default = ["output", "watch", "serve"] diff --git a/book-example/src/for_developers/preprocessors.md b/book-example/src/for_developers/preprocessors.md index c935f2f4..5581efc1 100644 --- a/book-example/src/for_developers/preprocessors.md +++ b/book-example/src/for_developers/preprocessors.md @@ -5,13 +5,13 @@ book is loaded and before it gets rendered, allowing you to update and mutate the book. Possible use cases are: - Creating custom helpers like `\{{#include /path/to/file.md}}` -- Updating links so `[some chapter](some_chapter.md)` is automatically changed +- Updating links so `[some chapter](some_chapter.md)` is automatically changed to `[some chapter](some_chapter.html)` for the HTML renderer -- Substituting in latex-style expressions (`$$ \frac{1}{3} $$`) with their +- Substituting in latex-style expressions (`$$ \frac{1}{3} $$`) with their mathjax equivalents -## Implementing a Preprocessor +## Implementing a Preprocessor A preprocessor is represented by the `Preprocessor` trait. @@ -29,4 +29,68 @@ pub struct PreprocessorContext { pub root: PathBuf, pub config: Config, } -``` \ No newline at end of file +``` + +## A complete Example + +The magic happens within the `run(...)` method of the [`Preprocessor`][preprocessor-docs] trait implementation. + +As direct access to the chapters is not possible, you will probably end up iterating +them using `for_each_mut(...)`: + +```rust +book.for_each_mut(|item: &mut BookItem| { + if let BookItem::Chapter(ref mut chapter) = *item { + eprintln!("{}: processing chapter '{}'", self.name(), chapter.name); + res = Some( + match Deemphasize::remove_emphasis(&mut num_removed_items, chapter) { + Ok(md) => { + chapter.content = md; + Ok(()) + } + Err(err) => Err(err), + }, + ); + } +}); +``` + +The `chapter.content` is just a markdown formatted string, and you will have to +process it in some way. Even though it's entirely possible to implement some sort of +manual find & replace operation, if that feels too unsafe you can use [`pulldown-cmark`][pc] +to parse the string into events and work on them instead. + +Finally you can use [`pulldown-cmark-to-cmark`][pctc] to transform these events back to +a string. + +The following code block shows how to remove all emphasis from markdown, and do so +safely. + +```rust +fn remove_emphasis(num_removed_items: &mut i32, chapter: &mut Chapter) -> Result { + let mut buf = String::with_capacity(chapter.content.len()); + let events = Parser::new(&chapter.content).filter(|e| { + let should_keep = match *e { + Event::Start(Tag::Emphasis) + | Event::Start(Tag::Strong) + | Event::End(Tag::Emphasis) + | Event::End(Tag::Strong) => false, + _ => true, + }; + if !should_keep { + *num_removed_items += 1; + } + should_keep + }); + cmark(events, &mut buf, None) + .map(|_| buf) + .map_err(|err| Error::from(format!("Markdown serialization failed: {}", err))) +} +``` + +For everything else, have a look [at the complete example][example]. + +[preprocessor-docs]: https://docs.rs/mdbook/0.1.3/mdbook/preprocess/trait.Preprocessor.html +[pc]: https://crates.io/crates/pulldown-cmark +[pctc]: https://crates.io/crates/pulldown-cmark-to-cmark +[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/de-emphasize.rs diff --git a/examples/de-emphasize.rs b/examples/de-emphasize.rs new file mode 100644 index 00000000..2e5c7968 --- /dev/null +++ b/examples/de-emphasize.rs @@ -0,0 +1,94 @@ +//! This program removes all forms of emphasis from the markdown of the book. +extern crate mdbook; +extern crate pulldown_cmark; +extern crate pulldown_cmark_to_cmark; + +use mdbook::errors::{Error, Result}; +use mdbook::MDBook; +use mdbook::book::{Book, BookItem, Chapter}; +use mdbook::preprocess::{Preprocessor, PreprocessorContext}; +use pulldown_cmark::{Event, Parser, Tag}; +use pulldown_cmark_to_cmark::fmt::cmark; + +use std::ffi::OsString; +use std::env::{args, args_os}; +use std::process; + +struct Deemphasize; + +impl Preprocessor for Deemphasize { + fn name(&self) -> &str { + "md-links-to-html-links" + } + + fn run(&self, _ctx: &PreprocessorContext, book: &mut Book) -> Result<()> { + eprintln!("Running '{}' preprocessor", self.name()); + let mut res: Option<_> = None; + let mut num_removed_items = 0; + book.for_each_mut(|item: &mut BookItem| { + if let Some(Err(_)) = res { + return; + } + if let BookItem::Chapter(ref mut chapter) = *item { + eprintln!("{}: processing chapter '{}'", self.name(), chapter.name); + res = Some( + match Deemphasize::remove_emphasis(&mut num_removed_items, chapter) { + Ok(md) => { + chapter.content = md; + Ok(()) + } + Err(err) => Err(err), + }, + ); + } + }); + eprintln!( + "{}: removed {} events from markdown stream.", + self.name(), + num_removed_items + ); + match res { + Some(res) => res, + None => Ok(()), + } + } +} + +fn do_it(book: OsString) -> Result<()> { + let mut book = MDBook::load(book)?; + book.with_preprecessor(Deemphasize); + book.build() +} + +fn main() { + if args_os().count() != 2 { + eprintln!("USAGE: {} ", args().next().expect("executable")); + return; + } + if let Err(e) = do_it(args_os().skip(1).next().expect("one argument")) { + eprintln!("{}", e); + process::exit(1); + } +} + +impl Deemphasize { + fn remove_emphasis(num_removed_items: &mut i32, chapter: &mut Chapter) -> Result { + let mut buf = String::with_capacity(chapter.content.len()); + let events = Parser::new(&chapter.content).filter(|e| { + let should_keep = match *e { + Event::Start(Tag::Emphasis) + | Event::Start(Tag::Strong) + | Event::End(Tag::Emphasis) + | Event::End(Tag::Strong) => false, + _ => true, + }; + if !should_keep { + *num_removed_items += 1; + } + should_keep + }); + cmark(events, &mut buf, None) + .map(|_| buf) + .map_err(|err| Error::from(format!("Markdown serialization failed: {}", err))) + } +}