Rewrite of {{#}} links handling in preprocess module

- Replaced link parser with a Regex
- Implemented {{#include}} links
- Will display relatively nice error when cannot open {{#}} linked file
- Escaped links no longer render with escape char
- utils::fs::file_to_path no takes AsRef<Path>
- sorted export/mod in lib.rs

Cargo.toml

@@ -22,6 +22,7 @@ serde_derive = "1.0"
 error-chain = "0.10.0"
 serde_json = "1.0"
 pulldown-cmark = "0.0.14"
+lazy_static = "0.2"
 log = "0.3"
 env_logger = "0.4.0"
 toml = { version = "0.4", features = ["serde"] }

src/lib.rs

@@ -71,22 +71,23 @@
 
 #[macro_use]
 extern crate error_chain;
+extern crate handlebars;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate log;
+extern crate pulldown_cmark;
+extern crate regex;
 #[macro_use]
 extern crate serde_derive;
 extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
-extern crate handlebars;
-extern crate pulldown_cmark;
-extern crate regex;
-
-#[macro_use]
-extern crate log;
-pub mod book;
-pub mod config;
 mod parse;
 mod preprocess;
+pub mod book;
+pub mod config;
 pub mod renderer;
 pub mod theme;
 pub mod utils;

src/preprocess/links.rs

@@ -1,202 +1,240 @@
 use std::path::{Path, PathBuf};
-use std::fs::File;
+use regex::{CaptureMatches, Captures, Regex};
-use std::io::Read;
+use utils::fs::file_to_string;
+use errors::*;
 
+const ESCAPE_CHAR: char = '\\';
 
-pub fn render_playpen<P: AsRef<Path>>(s: &str, path: P) -> String {
+pub fn replace_all<P: AsRef<Path>>(s: &str, path: P) -> Result<String> {
     // When replacing one thing in a string by something with a different length,
     // the indices after that will not correspond,
     // we therefore have to store the difference to correct this
     let mut previous_end_index = 0;
     let mut replaced = String::new();
 
-    for playpen in find_playpens(s, path) {
+    for playpen in find_links(s) {
-
-        if playpen.escaped {
-            replaced.push_str(&s[previous_end_index..playpen.start_index - 1]);
-            replaced.push_str(&s[playpen.start_index..playpen.end_index]);
-            previous_end_index = playpen.end_index;
-            continue;
-        }
-
-        // Check if the file exists
-        if !playpen.rust_file.exists() || !playpen.rust_file.is_file() {
-            warn!("[-] No file exists for {{{{#playpen }}}}\n    {}", playpen.rust_file.to_str().unwrap());
-            continue;
-        }
-
-        // Open file & read file
-        let mut file = if let Ok(f) = File::open(&playpen.rust_file) {
-            f
-        } else {
-            continue;
-        };
-        let mut file_content = String::new();
-        if file.read_to_string(&mut file_content).is_err() {
-            continue;
-        };
-
-        let editable = if playpen.editable { ",editable" } else { "" };
-        let replacement = String::new() + "``` rust" + editable + "\n" + &file_content + "\n```\n";
-
         replaced.push_str(&s[previous_end_index..playpen.start_index]);
-        replaced.push_str(&replacement);
+        replaced.push_str(&playpen.render_with_path(&path)?);
         previous_end_index = playpen.end_index;
-        // println!("Playpen{{ {}, {}, {:?}, {} }}", playpen.start_index,
-        // playpen.end_index, playpen.rust_file,
-        // playpen.editable);
     }
 
     replaced.push_str(&s[previous_end_index..]);
-
+    Ok(replaced)
-    replaced
 }
 
 #[derive(PartialOrd, PartialEq, Debug)]
-struct Playpen {
+enum LinkType<'a> {
+    Escaped,
+    Include(PathBuf),
+    Playpen(PathBuf, Vec<&'a str>),
+}
+
+#[derive(PartialOrd, PartialEq, Debug)]
+struct Link<'a> {
     start_index: usize,
     end_index: usize,
-    rust_file: PathBuf,
+    link: LinkType<'a>,
-    editable: bool,
+    link_text: &'a str,
-    escaped: bool,
 }
 
-fn find_playpens<P: AsRef<Path>>(s: &str, base_path: P) -> Vec<Playpen> {
+impl<'a> Link<'a> {
-    let base_path = base_path.as_ref();
+    fn from_capture(cap: Captures<'a>) -> Option<Link<'a>> {
-    let mut playpens = vec![];
-    for (i, _) in s.match_indices("{{#playpen") {
-        debug!("[*]: find_playpen");
 
-        let mut escaped = false;
+        let link_type = match (cap.get(0), cap.get(1), cap.get(2)) {
+            (_, Some(typ), Some(rest)) => {
+                let mut path_props = rest.as_str().split_whitespace();
+                let file_path = path_props.next().map(PathBuf::from);
+                let props: Vec<&str> = path_props.collect();
 
-        if i > 0 {
+                match (typ.as_str(), file_path) {
-            if let Some(c) = s[i - 1..].chars().nth(0) {
+                    ("include", Some(pth)) => Some(LinkType::Include(pth)),
-                if c == '\\' {
+                    ("playpen", Some(pth)) => Some(LinkType::Playpen(pth, props)),
-                    escaped = true
+                    _ => None,
                 }
-            }
+            },
-        }
+            (Some(mat), None, None) if mat.as_str().starts_with(ESCAPE_CHAR) => Some(LinkType::Escaped),
-        // DON'T forget the "+ i" else you have an index out of bounds error !!
+            _ => None,
-        let end_i = if let Some(n) = s[i..].find("}}") {
+        };
-            n
-        } else {
-            continue;
-        } + i + 2;
 
-        debug!("s[{}..{}] = {}", i, end_i, s[i..end_i].to_string());
+        link_type.and_then(|lnk| {
-
+            cap.get(0)
-        // If there is nothing between "{{#playpen" and "}}" skip
+                .map(|mat| {
-        if end_i - 2 - (i + 10) < 1 {
+                         Link {
-            continue;
+                             start_index: mat.start(),
-        }
+                             end_index: mat.end(),
-        if s[i + 10..end_i - 2].trim().is_empty() {
+                             link: lnk,
-            continue;
+                             link_text: mat.as_str(),
-        }
+                         }
-
+                     })
-        debug!("{}", s[i + 10..end_i - 2].to_string());
+        })
-
-        // Split on whitespaces
-        let params: Vec<&str> = s[i + 10..end_i - 2].split_whitespace().collect();
-        let editable = params
-            .get(1)
-            .map(|p| p.find("editable").is_some())
-            .unwrap_or(false);
-
-        playpens.push(Playpen {
-                          start_index: i,
-                          end_index: end_i,
-                          rust_file: base_path.join(PathBuf::from(params[0])),
-                          editable: editable,
-                          escaped: escaped,
-                      })
     }
 
-    playpens
+    fn render_with_path<P: AsRef<Path>>(&self, base: P) -> Result<String> {
+        let base = base.as_ref();
+        match self.link {
+            // omit the escape char
+            LinkType::Escaped => Ok((&self.link_text[1..]).to_owned()),
+            LinkType::Include(ref pat) => {
+                file_to_string(base.join(pat)).chain_err(|| format!("Could not read file for link {}", self.link_text))
+            },
+            LinkType::Playpen(ref pat, ref attrs) => {
+                let contents = file_to_string(base.join(pat))
+                    .chain_err(|| format!("Could not read file for link {}", self.link_text))?;
+                let ftype = if !attrs.is_empty() { "rust," } else { "rust" };
+                Ok(format!("```{}{}\n{}\n```\n", ftype, attrs.join(","), contents))
+            },
+        }
+    }
 }
 
+struct LinkIter<'a>(CaptureMatches<'a, 'a>);
 
+impl<'a> Iterator for LinkIter<'a> {
+    type Item = Link<'a>;
+    fn next(&mut self) -> Option<Link<'a>> {
+        for cap in &mut self.0 {
+            if let Some(inc) = Link::from_capture(cap) {
+                return Some(inc);
+            }
+        }
+        None
+    }
+}
 
+fn find_links(contents: &str) -> LinkIter {
+    // lazily compute following regex
+    // r"\\\{\{#.*\}\}|\{\{#([a-zA-Z0-9]+)\s*([a-zA-Z0-9_.\-:/\\\s]+)\}\}")?;
+    lazy_static! {
+        static ref RE: Regex = Regex::new(r"(?x) # insignificant whitespace mode
+                    \\\{\{\#.*\}\}               # match escaped link
+                    |                            # or
+                    \{\{\s*                      # link opening parens and whitespace
+                      \#([a-zA-Z0-9]+)           # link type
+                      \s*                        # separating whitespace
+                      ([a-zA-Z0-9\s_.\-:/\\]+)   # link target path and space separated properties
+                    \s*\}\}                      # whitespace and link closing parens
+                                 ").unwrap();
+    }
+    LinkIter(RE.captures_iter(contents))
+}
 
 // ---------------------------------------------------------------------------------
 //      Tests
 //
 
 #[test]
-fn test_find_playpens_no_playpen() {
+fn test_find_links_no_link() {
-    let s = "Some random text without playpen...";
+    let s = "Some random text without link...";
-    assert!(find_playpens(s, "") == vec![]);
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
 }
 
 #[test]
-fn test_find_playpens_partial_playpen() {
+fn test_find_links_partial_link() {
     let s = "Some random text with {{#playpen...";
-    assert!(find_playpens(s, "") == vec![]);
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
+    let s = "Some random text with {{#include...";
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
+    let s = "Some random text with \\{{#include...";
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
 }
 
 #[test]
-fn test_find_playpens_empty_playpen() {
+fn test_find_links_empty_link() {
-    let s = "Some random text with {{#playpen}} and {{#playpen   }}...";
+    let s = "Some random text with {{#playpen}} and {{#playpen   }} {{}} {{#}}...";
-    assert!(find_playpens(s, "") == vec![]);
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
 }
 
 #[test]
-fn test_find_playpens_simple_playpen() {
+fn test_find_links_unknown_link_type() {
+    let s = "Some random text with {{#playpenz ar.rs}} and {{#incn}} {{baz}} {{#bar}}...";
+    assert!(find_links(s).collect::<Vec<_>>() == vec![]);
+}
+
+#[test]
+fn test_find_links_simple_link() {
     let s = "Some random text with {{#playpen file.rs}} and {{#playpen test.rs }}...";
 
-    println!("\nOUTPUT: {:?}\n", find_playpens(s, ""));
+    let res = find_links(s).collect::<Vec<_>>();
+    println!("\nOUTPUT: {:?}\n", res);
 
-    assert!(find_playpens(s, "") ==
+    assert_eq!(res,
-            vec![Playpen {
+               vec![Link {
-                     start_index: 22,
+                        start_index: 22,
-                     end_index: 42,
+                        end_index: 42,
-                     rust_file: PathBuf::from("file.rs"),
+                        link: LinkType::Playpen(PathBuf::from("file.rs"), vec![]),
-                     editable: false,
+                        link_text: "{{#playpen file.rs}}",
-                     escaped: false,
+                    },
-                 },
+                    Link {
-                 Playpen {
+                        start_index: 47,
-                     start_index: 47,
+                        end_index: 68,
-                     end_index: 68,
+                        link: LinkType::Playpen(PathBuf::from("test.rs"), vec![]),
-                     rust_file: PathBuf::from("test.rs"),
+                        link_text: "{{#playpen test.rs }}",
-                     editable: false,
+                    }]);
-                     escaped: false,
-                 }]);
 }
 
 #[test]
-fn test_find_playpens_complex_playpen() {
+fn test_find_links_escaped_link() {
-    let s = "Some random text with {{#playpen file.rs editable}} and {{#playpen test.rs editable }}...";
-
-    println!("\nOUTPUT: {:?}\n", find_playpens(s, "dir"));
-
-    assert!(find_playpens(s, "dir") ==
-            vec![Playpen {
-                     start_index: 22,
-                     end_index: 51,
-                     rust_file: PathBuf::from("dir/file.rs"),
-                     editable: true,
-                     escaped: false,
-                 },
-                 Playpen {
-                     start_index: 56,
-                     end_index: 86,
-                     rust_file: PathBuf::from("dir/test.rs"),
-                     editable: true,
-                     escaped: false,
-                 }]);
-}
-
-#[test]
-fn test_find_playpens_escaped_playpen() {
     let s = "Some random text with escaped playpen \\{{#playpen file.rs editable}} ...";
 
-    println!("\nOUTPUT: {:?}\n", find_playpens(s, ""));
+    let res = find_links(s).collect::<Vec<_>>();
+    println!("\nOUTPUT: {:?}\n", res);
 
-    assert!(find_playpens(s, "") ==
+    assert_eq!(res,
-            vec![Playpen {
+               vec![Link {
-                     start_index: 39,
+                        start_index: 38,
-                     end_index: 68,
+                        end_index: 68,
-                     rust_file: PathBuf::from("file.rs"),
+                        link: LinkType::Escaped,
-                     editable: true,
+                        link_text: "\\{{#playpen file.rs editable}}",
-                     escaped: true,
+                    }]);
-                 }]);
+}
+
+#[test]
+fn test_find_playpens_with_properties() {
+    let s = "Some random text with escaped playpen {{#playpen file.rs editable }} and some more\n text  {{#playpen my.rs editable no_run should_panic}} ...";
+
+    let res = find_links(s).collect::<Vec<_>>();
+    println!("\nOUTPUT: {:?}\n", res);
+    assert_eq!(res,
+               vec![Link {
+                        start_index: 38,
+                        end_index: 68,
+                        link: LinkType::Playpen(PathBuf::from("file.rs"), vec!["editable"]),
+                        link_text: "{{#playpen file.rs editable }}",
+                    },
+                    Link {
+                        start_index: 90,
+                        end_index: 137,
+                        link: LinkType::Playpen(PathBuf::from("my.rs"), vec!["editable", "no_run", "should_panic"]),
+                        link_text: "{{#playpen my.rs editable no_run should_panic}}",
+                    }]);
+}
+
+#[test]
+fn test_find_all_link_types() {
+    let s = "Some random text with escaped playpen {{#include file.rs}} and \\{{#contents are insignifficant in escaped link}} some more\n text  {{#playpen my.rs editable no_run should_panic}} ...";
+
+    let res = find_links(s).collect::<Vec<_>>();
+    println!("\nOUTPUT: {:?}\n", res);
+    assert_eq!(res.len(), 3);
+    assert_eq!(res[0],
+               Link {
+                   start_index: 38,
+                   end_index: 58,
+                   link: LinkType::Include(PathBuf::from("file.rs")),
+                   link_text: "{{#include file.rs}}",
+               });
+    assert_eq!(res[1],
+               Link {
+                   start_index: 63,
+                   end_index: 112,
+                   link: LinkType::Escaped,
+                   link_text: "\\{{#contents are insignifficant in escaped link}}",
+               });
+    assert_eq!(res[2],
+               Link {
+                   start_index: 130,
+                   end_index: 177,
+                   link: LinkType::Playpen(PathBuf::from("my.rs"), vec!["editable", "no_run", "should_panic"]),
+                   link_text: "{{#playpen my.rs editable no_run should_panic}}",
+               });
 }

src/renderer/html_handlebars/hbs_renderer.rs

@@ -45,9 +45,9 @@ impl HtmlHandlebars {
                     debug!("[*]: Reading file");
                     f.read_to_string(&mut content)?;
 
-                    // Parse for playpen links
+                    // Parse and expand links
                     if let Some(p) = path.parent() {
-                        content = preprocess::links::render_playpen(&content, p);
+                        content = preprocess::links::replace_all(&content, p)?;
                     }
 
                     content = utils::render_markdown(&content, ctx.book.get_curly_quotes());

src/utils/fs.rs

@@ -4,8 +4,8 @@ use std::io::Read;
 use std::fs::{self, File};
 
 /// Takes a path to a file and try to read the file into a String
-
+pub fn file_to_string<P: AsRef<Path>>(path: P) -> Result<String> {
-pub fn file_to_string(path: &Path) -> Result<String> {
+    let path = path.as_ref();
     let mut file = match File::open(path) {
         Ok(f) => f,
         Err(e) => {