From 1fe86a5a0c2ffaec4101b4749d2c2b5728fc01d0 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Fri, 17 Sep 2021 15:25:44 -0700 Subject: [PATCH] Move Playground and comma/space class normalization into a pulldown-cmark filter This avoids scanning over all the HTML with a regex, so it should be faster. --- src/renderer/html_handlebars/hbs_renderer.rs | 30 +- src/utils/highlight.rs | 6 +- src/utils/mod.rs | 442 +++++++++++++++---- 3 files changed, 383 insertions(+), 95 deletions(-) diff --git a/src/renderer/html_handlebars/hbs_renderer.rs b/src/renderer/html_handlebars/hbs_renderer.rs index 79368584..0b8dea0e 100644 --- a/src/renderer/html_handlebars/hbs_renderer.rs +++ b/src/renderer/html_handlebars/hbs_renderer.rs @@ -62,6 +62,8 @@ impl HtmlHandlebars { &content, ctx.html_config.curly_quotes, self.syntaxes.borrow().as_ref().unwrap(), + &ctx.html_config.playground, + ctx.edition, ); let fixed_content = utils::render_markdown_with_path( @@ -69,6 +71,8 @@ impl HtmlHandlebars { ctx.html_config.curly_quotes, Some(&path), self.syntaxes.borrow().as_ref().unwrap(), + &ctx.html_config.playground, + ctx.edition, ); if !ctx.is_index && ctx.html_config.print.page_break { // Add page break between chapters @@ -121,7 +125,7 @@ impl HtmlHandlebars { debug!("Render template"); let rendered = ctx.handlebars.render("index", &ctx.data)?; - let rendered = self.post_process(rendered, &ctx.html_config.playground, ctx.edition); + let rendered = self.post_process(rendered); // Write to file debug!("Creating {}", filepath.display()); @@ -132,8 +136,7 @@ impl HtmlHandlebars { ctx.data.insert("path_to_root".to_owned(), json!("")); ctx.data.insert("is_index".to_owned(), json!(true)); let rendered_index = ctx.handlebars.render("index", &ctx.data)?; - let rendered_index = - self.post_process(rendered_index, &ctx.html_config.playground, ctx.edition); + let rendered_index = self.post_process(rendered_index); debug!("Creating index.html from {}", ctx_path); utils::fs::write_file(&ctx.destination, "index.html", rendered_index.as_bytes())?; } @@ -171,6 +174,8 @@ impl HtmlHandlebars { &content_404, html_config.curly_quotes, self.syntaxes.borrow().as_ref().unwrap(), + &html_config.playground, + ctx.config.rust.edition, ); let mut data_404 = data.clone(); @@ -197,8 +202,7 @@ impl HtmlHandlebars { data_404.insert("title".to_owned(), json!(title)); let rendered = handlebars.render("index", &data_404)?; - let rendered = - self.post_process(rendered, &html_config.playground, ctx.config.rust.edition); + let rendered = self.post_process(rendered); let output_file = get_404_output_file(&html_config.input_404); utils::fs::write_file(destination, output_file, rendered.as_bytes())?; debug!("Creating 404.html ✓"); @@ -206,15 +210,8 @@ impl HtmlHandlebars { } #[cfg_attr(feature = "cargo-clippy", allow(clippy::let_and_return))] - fn post_process( - &self, - rendered: String, - playground_config: &Playground, - edition: Option, - ) -> String { + fn post_process(&self, rendered: String) -> String { let rendered = build_header_links(&rendered); - let rendered = fix_code_blocks(&rendered); - let rendered = add_playground_pre(&rendered, playground_config, edition); rendered } @@ -595,8 +592,7 @@ impl Renderer for HtmlHandlebars { debug!("Render template"); let rendered = handlebars.render("index", &data)?; - let rendered = - self.post_process(rendered, &html_config.playground, ctx.config.rust.edition); + let rendered = self.post_process(rendered); utils::fs::write_file(destination, "print.html", rendered.as_bytes())?; debug!("Creating print.html ✓"); @@ -1000,6 +996,10 @@ fn partition_source(s: &str) -> (String, String) { (before, after) } +lazy_static! { + static ref BORING_LINES_REGEX: Regex = Regex::new(r"^(\s*)#(.?)(.*)$").unwrap(); +} + struct RenderItemContext<'a> { handlebars: &'a Handlebars<'a>, destination: PathBuf, diff --git a/src/utils/highlight.rs b/src/utils/highlight.rs index 954ab59d..8325a5eb 100644 --- a/src/utils/highlight.rs +++ b/src/utils/highlight.rs @@ -86,10 +86,8 @@ impl<'a> HtmlGenerator<'a> { if did_boringify { // Since the boring scope is preceded only by a Pop operation, // it must be the first match on the line for - formatted_line = formatted_line.replace( - r#""#, - r#""#, - ); + formatted_line = + formatted_line.replace(r#""#, r#""#); } self.open_spans += delta; self.html.push_str(&formatted_line); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index c590c87a..45413f36 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod toml_ext; use crate::errors::Error; use regex::Regex; +use crate::config::{Playground, RustEdition}; use pulldown_cmark::{html, CodeBlockKind, CowStr, Event, Options, Parser, Tag}; use syntect::html::ClassStyle; use syntect::parsing::SyntaxReference; @@ -184,8 +185,21 @@ fn adjust_links<'a>(event: Event<'a>, path: Option<&Path>) -> Event<'a> { } /// Wrapper around the pulldown-cmark parser for rendering markdown to HTML. -pub fn render_markdown(text: &str, curly_quotes: bool, syntaxes: &SyntaxSet) -> String { - render_markdown_with_path(text, curly_quotes, None, syntaxes) +pub fn render_markdown( + text: &str, + curly_quotes: bool, + syntaxes: &SyntaxSet, + playground_config: &Playground, + default_edition: Option, +) -> String { + render_markdown_with_path( + text, + curly_quotes, + None, + syntaxes, + playground_config, + default_edition, + ) } pub fn new_cmark_parser(text: &str, curly_quotes: bool) -> Parser<'_, '_> { @@ -205,18 +219,20 @@ pub fn render_markdown_with_path( curly_quotes: bool, path: Option<&Path>, syntaxes: &SyntaxSet, + playground_config: &Playground, + default_edition: Option, ) -> String { let mut s = String::with_capacity(text.len() * 3 / 2); let p = new_cmark_parser(text, curly_quotes); - let mut highlighter = SyntaxHighlighter::default(); + let mut highlighter = SyntaxHighlighter::new(playground_config, default_edition); let events = p .map(clean_codeblock_headers) - .map(|event| highlighter.highlight(syntaxes, event)) .map(|event| adjust_links(event, path)) .flat_map(|event| { let (a, b) = wrap_tables(event); a.into_iter().chain(b) - }); + }) + .map(|event| highlighter.highlight(syntaxes, event)); html::push_html(&mut s, events); s @@ -234,20 +250,40 @@ fn wrap_tables(event: Event<'_>) -> (Option>, Option>) { } } -#[derive(Default)] struct SyntaxHighlighter<'a> { highlight: bool, is_rust: bool, + is_playground: bool, + is_editable: bool, syntax: Option<&'a SyntaxReference>, + playground_config: &'a Playground, + default_edition: Option, } impl<'a> SyntaxHighlighter<'a> { + fn new(playground_config: &'a Playground, default_edition: Option) -> Self { + SyntaxHighlighter { + highlight: false, + is_rust: false, + is_playground: false, + is_editable: false, + syntax: None, + playground_config, + default_edition, + } + } + fn highlight<'b>(&mut self, syntaxes: &'a SyntaxSet, event: Event<'b>) -> Event<'b> { match event { Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref info))) => { self.highlight = true; - let lang_name = info.split(',').next(); - if let Some(name) = lang_name { + let mut classes: Vec<_> = info + .replace(",", " ") + .split(' ') + .map(String::from) + .filter(|x| !x.is_empty()) + .collect(); + if let Some(name) = classes.first() { // If we're given an empty name, or it is marked as // plaintext, we shouldn't highlight it. if name.is_empty() @@ -270,19 +306,64 @@ impl<'a> SyntaxHighlighter<'a> { self.is_rust = true; } } - event + if self.is_rust { + let ignore = classes.iter().find(|&x| x == "ignore").is_some(); + let noplayground = classes.iter().find(|&x| x == "noplayground").is_some(); + let noplaypen = classes.iter().find(|&x| x == "noplaypen").is_some(); + let mdbook_runnable = + classes.iter().find(|&x| x == "mdbook-runnable").is_some(); + // Enable playground + if (!ignore && !noplayground && !noplaypen) || mdbook_runnable { + self.is_editable = classes.iter().find(|&x| x == "editable").is_some(); + let contains_e2015 = + classes.iter().find(|&x| x == "edition2015").is_some(); + let contains_e2018 = + classes.iter().find(|&x| x == "edition2018").is_some(); + let contains_e2021 = + classes.iter().find(|&x| x == "edition2021").is_some(); + // if the user forced edition, we should not overwrite it + if !contains_e2015 && !contains_e2018 && !contains_e2021 { + match self.default_edition { + Some(RustEdition::E2015) => { + classes.push("edition2015".to_owned()) + } + Some(RustEdition::E2018) => { + classes.push("edition2018".to_owned()) + } + Some(RustEdition::E2021) => { + classes.push("edition2021".to_owned()) + } + None => {} + } + } + self.is_playground = true; + return Event::Html(CowStr::from(format!( + r#"
"#,
+                                classes.join(" ")
+                            )));
+                        }
+                    }
                 } else {
                     // We also don't perform auto-detection of languages, so we
                     // shouldn't highlight code blocks without lang tags.
                     self.highlight = false;
-                    event
+                }
+                if classes.is_empty() {
+                    Event::Html(CowStr::from("
"))
+                } else {
+                    Event::Html(CowStr::from(format!(
+                        r#"
"#,
+                        classes.join(" ")
+                    )))
                 }
             }
             Event::End(Tag::CodeBlock(CodeBlockKind::Fenced(_))) => {
                 self.highlight = false;
                 self.is_rust = false;
                 self.syntax = None;
-                event
+                self.is_playground = false;
+                self.is_editable = false;
+                Event::Html(CowStr::from("
"))
             }
             Event::Text(ref code) if self.highlight => {
                 let mut gen = highlight::HtmlGenerator::new(
@@ -290,9 +371,19 @@ impl<'a> SyntaxHighlighter<'a> {
                     syntaxes,
                     ClassStyle::SpacedPrefixed { prefix: "syn-" },
                 );
+                let needs_wrapped = self.is_rust
+                    && !(self.playground_config.editable && self.is_editable)
+                    && !code.contains("fn main")
+                    && !code.contains("quick_main!");
+                if needs_wrapped {
+                    gen.parse_line("# fn main() {\n", self.is_rust);
+                }
                 for line in LinesWithEndings::from(code) {
                     gen.parse_line(line, self.is_rust);
                 }
+                if needs_wrapped {
+                    gen.parse_line("# }\n", self.is_rust);
+                }
                 Event::Html(CowStr::from(gen.finalize()))
             }
             _ => event,
@@ -300,41 +391,6 @@ impl<'a> SyntaxHighlighter<'a> {
     }
 }
 
-struct EventQuoteConverter {
-    enabled: bool,
-    convert_text: bool,
-}
-
-impl EventQuoteConverter {
-    fn new(enabled: bool) -> Self {
-        EventQuoteConverter {
-            enabled,
-            convert_text: true,
-        }
-    }
-
-    fn convert<'a>(&mut self, event: Event<'a>) -> Event<'a> {
-        if !self.enabled {
-            return event;
-        }
-
-        match event {
-            Event::Start(Tag::CodeBlock(_)) => {
-                self.convert_text = false;
-                event
-            }
-            Event::End(Tag::CodeBlock(_)) => {
-                self.convert_text = true;
-                event
-            }
-            Event::Text(ref text) if self.convert_text => {
-                Event::Text(CowStr::from(convert_quotes_to_curly(text)))
-            }
-            _ => event,
-        }
-    }
-}
-
 fn clean_codeblock_headers(event: Event<'_>) -> Event<'_> {
     match event {
         Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref info))) => {
@@ -389,6 +445,7 @@ mod tests {
         }
 
         use super::super::render_markdown;
+        use crate::config::{Playground, RustEdition};
 
         #[test]
         fn preserves_external_links() {
@@ -396,7 +453,9 @@ mod tests {
                 render_markdown(
                     "[example](https://www.rust-lang.org/)",
                     false,
-                    &default_syntaxes()
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
                 ),
                 "
example
\n"
             );
@@ -405,14 +464,22 @@ mod tests {
         #[test]
         fn it_can_adjust_markdown_links() {
             assert_eq!(
-                render_markdown("[example](example.md)", false, &default_syntaxes()),
+                render_markdown(
+                    "[example](example.md)",
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
                 "
example
\n"
             );
             assert_eq!(
                 render_markdown(
                     "[example_anchor](example.md#anchor)",
                     false,
-                    &default_syntaxes()
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
                 ),
                 "
example_anchor
\n"
             );
@@ -422,7 +489,9 @@ mod tests {
                 render_markdown(
                     "[phantom data](foo.html#phantomdata)",
                     false,
-                    &default_syntaxes()
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
                 ),
                 "
phantom data
\n"
             );
@@ -441,13 +510,28 @@ mod tests {
 
 
 "#.trim();
-            assert_eq!(render_markdown(src, false), out);
+            assert_eq!(
+                render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                out
+            );
         }
 
         #[test]
         fn it_can_keep_quotes_straight() {
             assert_eq!(
-                render_markdown("'one'", false, &default_syntaxes()),
+                render_markdown(
+                    "'one'",
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
                 "
'one'
\n"
             );
         }
@@ -465,7 +549,16 @@ mod tests {
 

 
'three' ‘four’

 "#;
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+            assert_eq!(
+                render_markdown(
+                    input,
+                    true,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
         }
 
         #[test]
@@ -481,7 +574,7 @@ more text with spaces
 "#;
 
             let expected = r#"
some text with spaces

-
fn main() {
+
fn main() {
 
 // code inside is unchanged
 
@@ -490,8 +583,26 @@ more text with spaces
 

 
more text with spaces

 "#;
-            assert_eq!(render_markdown(input, false, &default_syntaxes()), expected);
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+            assert_eq!(
+                render_markdown(
+                    input,
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
+            assert_eq!(
+                render_markdown(
+                    input,
+                    true,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
         }
 
         #[test]
@@ -501,10 +612,27 @@ more text with spaces
 ```
 "#;
 
-            let expected = r#"

-"#;
-            assert_eq!(render_markdown(input, false, &default_syntaxes()), expected);
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+            let expected = r#"
"#;
+            assert_eq!(
+                render_markdown(
+                    input,
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
+            assert_eq!(
+                render_markdown(
+                    input,
+                    true,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
         }
 
         #[test]
@@ -514,10 +642,27 @@ more text with spaces
 ```
 "#;
 
-            let expected = r#"

-"#;
-            assert_eq!(render_markdown(input, false, &default_syntaxes()), expected);
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+            let expected = r#"
"#;
+            assert_eq!(
+                render_markdown(
+                    input,
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
+            assert_eq!(
+                render_markdown(
+                    input,
+                    true,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
         }
 
         #[test]
@@ -527,19 +672,164 @@ more text with spaces
 ```
 "#;
 
-            let expected = r#"

-"#;
-            assert_eq!(render_markdown(input, false, &default_syntaxes()), expected);
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+            let expected = r#"
"#;
+            assert_eq!(
+                render_markdown(
+                    input,
+                    false,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
+            assert_eq!(
+                render_markdown(
+                    input,
+                    true,
+                    &default_syntaxes(),
+                    &Playground::default(),
+                    None,
+                ),
+                expected
+            );
+        }
 
-            // FIXME: Why are we doing this twice? It seems to be the same
-            // input and assertions.
-            let input = r#"
-```rust
-```
-"#;
-            assert_eq!(render_markdown(input, false, &default_syntaxes()), expected);
-            assert_eq!(render_markdown(input, true, &default_syntaxes()), expected);
+        // These HTML strings get very, very long.
+        // What I do is copy them out of here,
+        // paste them into a document.write() call in the JavaScript console,
+        // and then I can read the HTML in the DOM inspector to see if it looks right.
+        #[test]
+        fn add_playground() {
+            let inputs = [
+              ("```rust\nx()\n```",
+               "
fn main() {\nx()\n\n}\n
"),
+              ("```rust\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust editable\nlet s = \"foo\n # bar\n\";\n```",
+               "
let s = "foo\n\n bar\n";\n\n
"),
+              ("```rust editable\nlet s = \"foo\n ## bar\n\";\n```",
+                "
let s = "foo\n\n # bar\n";\n\n
"),
+              ("```rust editable\nlet s = \"foo\n # bar\n#\n\";\n```",
+                "
let s = "foo\n\n bar\n\n";\n\n
"),
+              ("```rust ignore\nlet s = \"foo\n # bar\n\";\n```",
+                "
fn main() {\nlet s = "foo\n\n bar\n";\n\n}\n
"),
+              ("```rust editable\n#![no_std]\nlet s = \"foo\";\n #[some_attr]\n```",
+                "
#![no_std]\n\nlet s = "foo";\n\n #[some_attr]\n\n
"),
+            ];
+            for (src, should_be) in &inputs {
+                let got = render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground {
+                        editable: true,
+                        ..Playground::default()
+                    },
+                    None,
+                );
+                assert_eq!(&*got, *should_be);
+            }
+        }
+        #[test]
+        fn add_playground_edition2015() {
+            let inputs = [
+              ("```rust\nx()\n```",
+               "
fn main() {\nx()\n\n}\n
"),
+              ("```rust\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2015\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2018\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+            ];
+            for (src, should_be) in &inputs {
+                let got = render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground {
+                        editable: true,
+                        ..Playground::default()
+                    },
+                    Some(RustEdition::E2015),
+                );
+                assert_eq!(&*got, *should_be);
+            }
+        }
+        #[test]
+        fn add_playground_edition2018() {
+            let inputs = [
+              ("```rust\nx()\n```",
+               "
fn main() {\nx()\n\n}\n
"),
+              ("```rust\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2015\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2018\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+            ];
+            for (src, should_be) in &inputs {
+                let got = render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground {
+                        editable: true,
+                        ..Playground::default()
+                    },
+                    Some(RustEdition::E2018),
+                );
+                assert_eq!(&*got, *should_be);
+            }
+        }
+        #[test]
+        fn add_playground_edition2021() {
+            let inputs = [
+              ("```rust\nx()\n```",
+               "
fn main() {\nx()\n\n}\n
"),
+              ("```rust\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2015\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+              ("```rust edition2018\nfn main() {}\n```",
+               "
fn main() {}\n\n
"),
+            ];
+            for (src, should_be) in &inputs {
+                let got = render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground {
+                        editable: true,
+                        ..Playground::default()
+                    },
+                    Some(RustEdition::E2021),
+                );
+                assert_eq!(&*got, *should_be);
+            }
+        }
+        #[test]
+        fn no_add_playground_to_other_languages() {
+            let inputs = [
+              ("```html,testhtml\n
\n```",
+               "
<p>\n
"),
+              ("```js es7\nf()\n```",
+               "
f()\n
"),
+            ];
+            for (src, should_be) in &inputs {
+                let got = render_markdown(
+                    src,
+                    false,
+                    &default_syntaxes(),
+                    &Playground {
+                        editable: true,
+                        ..Playground::default()
+                    },
+                    Some(RustEdition::E2021),
+                );
+                assert_eq!(&*got, *should_be);
+            }
         }
     }