chinese search support

Cargo.toml

@@ -44,7 +44,7 @@ tokio = { version = "0.2.18", features = ["macros"], optional = true }
 warp = { version = "0.2.2", default-features = false, features = ["websocket"], optional = true }
 
 # Search feature
-elasticlunr-rs = { version = "2.3", optional = true, default-features = false }
+elasticlunr-rs = { version = "2.3", optional = true }
 ammonia = { version = "3", optional = true }
 
 [dev-dependencies]

src/renderer/html_handlebars/hbs_renderer.rs

@@ -550,7 +550,7 @@ impl Renderer for HtmlHandlebars {
         {
             let search = html_config.search.unwrap_or_default();
             if search.enable {
-                super::search::create_files(&search, &destination, &book)?;
+                super::search::create_files(&search, &ctx.config.book.language, &destination, &book)?;
             }
         }
 

src/renderer/html_handlebars/search.rs

@@ -12,8 +12,23 @@ use crate::theme::searcher;
 use crate::utils;
 
 /// Creates all files required for search.
-pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
+pub fn create_files(
-    let mut index = Index::new(&["title", "body", "breadcrumbs"]);
+    search_config: &Search,
+    lang: &Option<String>,
+    destination: &Path,
+    book: &Book,
+) -> Result<()> {
+    let mut index = match lang {
+        Some(lang_str) => match lang_str.to_lowercase().as_str() {
+            "zh" => Index::with_language(
+                elasticlunr::Language::Chinese,
+                &["title", "body", "breadcrumbs"],
+            ),
+            _ => Index::new(&["title", "body", "breadcrumbs"]),
+        },
+        None => Index::new(&["title", "body", "breadcrumbs"]),
+    };
+
     let mut doc_urls = Vec::with_capacity(book.sections.len());
 
     for item in book.iter() {
@@ -36,6 +51,7 @@ pub fn create_files(search_config: &Search, destination: &Path, book: &Book) ->
         utils::fs::write_file(destination, "searcher.js", searcher::JS)?;
         utils::fs::write_file(destination, "mark.min.js", searcher::MARK_JS)?;
         utils::fs::write_file(destination, "elasticlunr.min.js", searcher::ELASTICLUNR_JS)?;
+        utils::fs::write_file(destination, "lunr.zh.js", searcher::LUNR_ZH_JS)?;
         debug!("Copying search files ✓");
     }
 

src/theme/index.hbs

@@ -271,6 +271,7 @@
 
         {{#if search_js}}
         <script src="{{ path_to_root }}elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}lunr.zh.js" type="text/javascript" charset="utf-8"></script>
         <script src="{{ path_to_root }}mark.min.js" type="text/javascript" charset="utf-8"></script>
         <script src="{{ path_to_root }}searcher.js" type="text/javascript" charset="utf-8"></script>
         {{/if}}

src/theme/searcher/lunr.zh.js

@@ -0,0 +1,89 @@
+(function (root, factory) {
+  if (typeof define === 'function' && define.amd) {
+    // AMD. Register as an anonymous module.
+    define(factory)
+  } else if (typeof exports === 'object') {
+    /**
+     * Node. Does not work with strict CommonJS, but
+     * only CommonJS-like environments that support module.exports,
+     * like Node.
+     */
+    module.exports = factory()
+  } else {
+    // Browser globals (root is window)
+    factory()(root.lunr);
+  }
+}(this, function () {
+  return function (lunr) {
+    if ('undefined' === typeof lunr) {
+      throw new Error('Lunr is not present. Please include / require Lunr before this script.');
+    }
+    
+    /* register specific locale function */
+    lunr.zh = function () {
+      this.pipeline.reset();
+      this.pipeline.add(
+        lunr.zh.trimmer,
+        lunr.zh.stopWordFilter,
+        lunr.zh.stemmer
+      );
+
+      // for lunr version 2
+      // this is necessary so that every searched word is also stemmed before
+      // in lunr <= 1 this is not needed, as it is done using the normal pipeline
+      if (this.searchPipeline) {
+        this.searchPipeline.reset();
+        this.searchPipeline.add(lunr.zh.stemmer)
+      }
+    };
+
+    lunr.zh.tokenizer = function (str) {
+      if (!arguments.length || str === null || str === undefined) return [];
+      if (Array.isArray(str)) {
+        var arr = str.filter(function (token) {
+          if (token === null || token === undefined) {
+            return false;
+          }
+
+          return true;
+        });
+
+        arr = arr.map(function (t) {
+          return lunr.utils.toString(t);
+        });
+
+        var out = [];
+        arr.forEach(function (item) {
+          var tokens = item.split(lunr.tokenizer.seperator);
+          out = out.concat(tokens);
+        }, this);
+
+        return out;
+      }
+
+      return str.toString().trim().split(lunr.tokenizer.seperator);
+    };
+
+
+    /* lunr trimmer function */
+    lunr.zh.trimmer = function (_token) {
+      return _token;
+    }
+
+    lunr.Pipeline.registerFunction(lunr.zh.trimmer, 'trimmer-zh');
+
+    /* lunr stemmer function */
+    lunr.zh.stemmer = (function () {
+      /* and return a function that stems a word for the current locale */
+      return function (token) {
+        return token;
+      }
+    })();
+    lunr.Pipeline.registerFunction(lunr.zh.stemmer, 'stemmer-zh');
+
+    lunr.zh.stopWordFilter = function (token) {
+      return token;
+    };
+    lunr.Pipeline.registerFunction(lunr.zh.stopWordFilter, 'stopWordFilter-zh');
+  };
+}))

src/theme/searcher/mod.rs

@@ -4,3 +4,4 @@
 pub static JS: &[u8] = include_bytes!("searcher.js");
 pub static MARK_JS: &[u8] = include_bytes!("mark.min.js");
 pub static ELASTICLUNR_JS: &[u8] = include_bytes!("elasticlunr.min.js");
+pub static LUNR_ZH_JS: &[u8] = include_bytes!("lunr.zh.js");

src/theme/searcher/searcher.js

@@ -257,6 +257,9 @@ window.search = window.search || {};
         search_options = config.search_options;
         searchbar_outer = config.searchbar_outer;
         doc_urls = config.doc_urls;
+        if (config.index.lang == "Chinese") {
+            elasticlunr.tokenizer = elasticlunr.zh.tokenizer
+        }
         searchindex = elasticlunr.Index.load(config.index);
 
         // Set up events