From 3cfed10098a7736505a6af8616ddf66ca9787b9b Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Jun 2019 09:26:24 -0700 Subject: [PATCH] Update to pulldown-cmark 0.5. (#898) * Update to pulldown-cmark 0.4.1. * Update to pulldown-cmark 0.5.2. * Remove pulldown-cmark-to-cmark dependency. Since it is not compatible with the new pulldown-cmark. This example isn't directly usable, anyways, and I think the no-op example sufficiently shows how to make a preprocessor. * cargo fmt * Fix example link. --- Cargo.lock | 35 ++++----- Cargo.toml | 4 +- .../src/for_developers/preprocessors.md | 2 +- examples/de-emphasize.rs | 75 ------------------- src/book/summary.rs | 8 +- src/renderer/html_handlebars/helpers/toc.rs | 7 +- src/renderer/html_handlebars/search.rs | 17 ++++- src/utils/mod.rs | 28 ++++--- 8 files changed, 49 insertions(+), 127 deletions(-) delete mode 100644 examples/de-emphasize.rs diff --git a/Cargo.lock b/Cargo.lock index 0ae397e3..e26997e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,11 +87,6 @@ name = "bit-vec" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "bitflags" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "bitflags" version = "1.0.4" @@ -646,8 +641,7 @@ dependencies = [ "notify 4.0.12 (registry+https://github.com/rust-lang/crates.io-index)", "open 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pulldown-cmark 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "select 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", @@ -936,19 +930,13 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.1.2" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "getopts 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "pulldown-cmark-to-cmark" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1479,6 +1467,14 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "unicase" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicode-bidi" version = "0.3.4" @@ -1659,7 +1655,6 @@ dependencies = [ "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" "checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" "checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" -"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" "checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09" @@ -1755,8 +1750,7 @@ dependencies = [ "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -"checksum pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d6fdf85cda6cadfae5428a54661d431330b312bc767ddbc57adbedc24da66e32" -"checksum pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57efca5f52f63336ee3a49bceee1a1169f18ef01c75aa7e71949441b49bbe7e4" +"checksum pulldown-cmark 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "051e60ace841b3bfecd402fe5051c06cb3bec4a6e6fdd060a37aa8eb829a1db3" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" @@ -1821,6 +1815,7 @@ dependencies = [ "checksum ucd-trie 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71a9c5b1fe77426cf144cc30e49e955270f5086e31a6441dfa8b32efc09b9d77" "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +"checksum unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a84e5511b2a947f3ae965dcb29b13b7b1691b6e7332cf5dbc1744138d5acb7f6" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" diff --git a/Cargo.toml b/Cargo.toml index 94488d40..9750d70b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ lazy_static = "1.0" log = "0.4" memchr = "2.0" open = "1.1" -pulldown-cmark = "0.1.2" +pulldown-cmark = "0.5" regex = "1.0.0" serde = "1.0" serde_derive = "1.0" @@ -52,8 +52,6 @@ ammonia = { version = "2.1", optional = true } select = "0.4" pretty_assertions = "0.6" walkdir = "2.0" -# FIXME(issue #898) -pulldown-cmark-to-cmark = "=1.1.0" [features] default = ["output", "watch", "serve", "search"] diff --git a/book-example/src/for_developers/preprocessors.md b/book-example/src/for_developers/preprocessors.md index e8bcaf0a..9a607401 100644 --- a/book-example/src/for_developers/preprocessors.md +++ b/book-example/src/for_developers/preprocessors.md @@ -109,7 +109,7 @@ For everything else, have a look [at the complete example][example]. [preprocessor-docs]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html [pc]: https://crates.io/crates/pulldown-cmark [pctc]: https://crates.io/crates/pulldown-cmark-to-cmark -[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/de-emphasize.rs +[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs [an example no-op preprocessor]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs [`CmdPreprocessor::parse_input()`]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html#method.parse_input [`Book::for_each_mut()`]: https://docs.rs/mdbook/latest/mdbook/book/struct.Book.html#method.for_each_mut diff --git a/examples/de-emphasize.rs b/examples/de-emphasize.rs deleted file mode 100644 index eefab73d..00000000 --- a/examples/de-emphasize.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! An example preprocessor for removing all forms of emphasis from a markdown -//! book. -use mdbook::book::{Book, BookItem, Chapter}; -use mdbook::errors::{Error, Result}; -use mdbook::preprocess::{Preprocessor, PreprocessorContext}; -use pulldown_cmark::{Event, Parser, Tag}; -use pulldown_cmark_to_cmark::fmt::cmark; - -const NAME: &str = "md-links-to-html-links"; - -fn main() { - panic!("This example is intended to be part of a library"); -} - -#[allow(dead_code)] -struct Deemphasize; - -impl Preprocessor for Deemphasize { - fn name(&self) -> &str { - NAME - } - - fn run(&self, _ctx: &PreprocessorContext, mut book: Book) -> Result { - eprintln!("Running '{}' preprocessor", self.name()); - let mut num_removed_items = 0; - - process(&mut book.sections, &mut num_removed_items)?; - - eprintln!( - "{}: removed {} events from markdown stream.", - self.name(), - num_removed_items - ); - - Ok(book) - } -} - -fn process<'a, I>(items: I, num_removed_items: &mut usize) -> Result<()> -where - I: IntoIterator + 'a, -{ - for item in items { - if let BookItem::Chapter(ref mut chapter) = *item { - eprintln!("{}: processing chapter '{}'", NAME, chapter.name); - - let md = remove_emphasis(num_removed_items, chapter)?; - chapter.content = md; - } - } - - Ok(()) -} - -fn remove_emphasis(num_removed_items: &mut usize, chapter: &mut Chapter) -> Result { - let mut buf = String::with_capacity(chapter.content.len()); - - let events = Parser::new(&chapter.content).filter(|e| { - let should_keep = match *e { - Event::Start(Tag::Emphasis) - | Event::Start(Tag::Strong) - | Event::End(Tag::Emphasis) - | Event::End(Tag::Strong) => false, - _ => true, - }; - if !should_keep { - *num_removed_items += 1; - } - should_keep - }); - - cmark(events, &mut buf, None) - .map(|_| buf) - .map_err(|err| Error::from(format!("Markdown serialization failed: {}", err))) -} diff --git a/src/book/summary.rs b/src/book/summary.rs index 2c32cbd8..817179c9 100644 --- a/src/book/summary.rs +++ b/src/book/summary.rs @@ -259,7 +259,7 @@ impl<'a> SummaryParser<'a> { bail!(self.parse_error("Suffix chapters cannot be followed by a list")); } } - Some(Event::Start(Tag::Link(href, _))) => { + Some(Event::Start(Tag::Link(_type, href, _title))) => { let link = self.parse_link(href.to_string())?; items.push(SummaryItem::Link(link)); } @@ -397,7 +397,7 @@ impl<'a> SummaryParser<'a> { loop { match self.next_event() { Some(Event::Start(Tag::Paragraph)) => continue, - Some(Event::Start(Tag::Link(href, _))) => { + Some(Event::Start(Tag::Link(_type, href, _title))) => { let mut link = self.parse_link(href.to_string())?; let mut number = parent.clone(); @@ -475,7 +475,7 @@ fn stringify_events(events: Vec>) -> String { events .into_iter() .filter_map(|t| match t { - Event::Text(text) => Some(text.into_owned()), + Event::Text(text) | Event::Code(text) => Some(text.into_string()), _ => None, }) .collect() @@ -629,7 +629,7 @@ mod tests { let _ = parser.stream.next(); // skip past start of paragraph let href = match parser.stream.next() { - Some(Event::Start(Tag::Link(href, _))) => href.to_string(), + Some(Event::Start(Tag::Link(_type, href, _title))) => href.to_string(), other => panic!("Unreachable, {:?}", other), }; diff --git a/src/renderer/html_handlebars/helpers/toc.rs b/src/renderer/html_handlebars/helpers/toc.rs index 27986c8f..3fdec599 100644 --- a/src/renderer/html_handlebars/helpers/toc.rs +++ b/src/renderer/html_handlebars/helpers/toc.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::utils; use handlebars::{Context, Handlebars, Helper, HelperDef, Output, RenderContext, RenderError}; -use pulldown_cmark::{html, Event, Parser, Tag}; +use pulldown_cmark::{html, Event, Parser}; // Handlebars helper to construct TOC #[derive(Clone, Copy)] @@ -117,10 +117,7 @@ impl HelperDef for RenderToc { // filter all events that are not inline code blocks let parser = Parser::new(name).filter(|event| match *event { - Event::Start(Tag::Code) - | Event::End(Tag::Code) - | Event::InlineHtml(_) - | Event::Text(_) => true, + Event::Code(_) | Event::InlineHtml(_) | Event::Text(_) => true, _ => false, }); diff --git a/src/renderer/html_handlebars/search.rs b/src/renderer/html_handlebars/search.rs index c5422b39..bce0844a 100644 --- a/src/renderer/html_handlebars/search.rs +++ b/src/renderer/html_handlebars/search.rs @@ -82,8 +82,8 @@ fn render_item( let anchor_base = utils::fs::normalize_path(filepath); let mut opts = Options::empty(); - opts.insert(OPTION_ENABLE_TABLES); - opts.insert(OPTION_ENABLE_FOOTNOTES); + opts.insert(Options::ENABLE_TABLES); + opts.insert(Options::ENABLE_FOOTNOTES); let p = Parser::new_ext(&chapter.content, opts); let mut in_header = false; @@ -91,6 +91,7 @@ fn render_item( let mut section_id = None; let mut heading = String::new(); let mut body = String::new(); + let mut html_block = String::new(); let mut breadcrumbs = chapter.parent_names.clone(); let mut footnote_numbers = HashMap::new(); @@ -124,6 +125,13 @@ fn render_item( let number = footnote_numbers.len() + 1; footnote_numbers.entry(name).or_insert(number); } + Event::Html(html) => { + html_block.push_str(&html); + } + Event::End(Tag::HtmlBlock) => { + body.push_str(&clean_html(&html_block)); + html_block.clear(); + } Event::Start(_) | Event::End(_) | Event::SoftBreak | Event::HardBreak => { // Insert spaces where HTML output would usually seperate text // to ensure words don't get merged together @@ -133,14 +141,14 @@ fn render_item( body.push(' '); } } - Event::Text(text) => { + Event::Text(text) | Event::Code(text) => { if in_header { heading.push_str(&text); } else { body.push_str(&text); } } - Event::Html(html) | Event::InlineHtml(html) => { + Event::InlineHtml(html) => { body.push_str(&clean_html(&html)); } Event::FootnoteReference(name) => { @@ -148,6 +156,7 @@ fn render_item( let number = footnote_numbers.entry(name).or_insert(len); body.push_str(&format!(" [{}] ", number)); } + Event::TaskListMarker(_checked) => {} } } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 3f542cf8..ace28589 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,9 +5,7 @@ mod string; use crate::errors::Error; use regex::Regex; -use pulldown_cmark::{ - html, Event, Options, Parser, Tag, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES, -}; +use pulldown_cmark::{html, CowStr, Event, Options, Parser, Tag}; use std::borrow::Cow; @@ -73,7 +71,7 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> { static ref MD_LINK: Regex = Regex::new(r"(?P.*)\.md(?P#.*)?").unwrap(); } - fn fix<'a>(dest: Cow<'a, str>, base: &str) -> Cow<'a, str> { + fn fix<'a>(dest: CowStr<'a>, base: &str) -> CowStr<'a> { // Don't modify links with schemes like `https`. if !SCHEME_LINK.is_match(&dest) { // This is a relative link, adjust it as necessary. @@ -92,17 +90,17 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> { } else { fixed_link.push_str(&dest); }; - return Cow::from(fixed_link); + return CowStr::from(fixed_link); } dest } match event { - Event::Start(Tag::Link(dest, title)) => { - Event::Start(Tag::Link(fix(dest, with_base), title)) + Event::Start(Tag::Link(link_type, dest, title)) => { + Event::Start(Tag::Link(link_type, fix(dest, with_base), title)) } - Event::Start(Tag::Image(dest, title)) => { - Event::Start(Tag::Image(fix(dest, with_base), title)) + Event::Start(Tag::Image(link_type, dest, title)) => { + Event::Start(Tag::Image(link_type, fix(dest, with_base), title)) } _ => event, } @@ -117,8 +115,8 @@ pub fn render_markdown_with_base(text: &str, curly_quotes: bool, base: &str) -> let mut s = String::with_capacity(text.len() * 3 / 2); let mut opts = Options::empty(); - opts.insert(OPTION_ENABLE_TABLES); - opts.insert(OPTION_ENABLE_FOOTNOTES); + opts.insert(Options::ENABLE_TABLES); + opts.insert(Options::ENABLE_FOOTNOTES); let p = Parser::new_ext(text, opts); let mut converter = EventQuoteConverter::new(curly_quotes); @@ -150,16 +148,16 @@ impl EventQuoteConverter { } match event { - Event::Start(Tag::CodeBlock(_)) | Event::Start(Tag::Code) => { + Event::Start(Tag::CodeBlock(_)) => { self.convert_text = false; event } - Event::End(Tag::CodeBlock(_)) | Event::End(Tag::Code) => { + Event::End(Tag::CodeBlock(_)) => { self.convert_text = true; event } Event::Text(ref text) if self.convert_text => { - Event::Text(Cow::from(convert_quotes_to_curly(text))) + Event::Text(CowStr::from(convert_quotes_to_curly(text))) } _ => event, } @@ -171,7 +169,7 @@ fn clean_codeblock_headers(event: Event<'_>) -> Event<'_> { Event::Start(Tag::CodeBlock(ref info)) => { let info: String = info.chars().filter(|ch| !ch.is_whitespace()).collect(); - Event::Start(Tag::CodeBlock(Cow::from(info))) + Event::Start(Tag::CodeBlock(CowStr::from(info))) } _ => event, }