From 92ec3ddc55fe8137613bb9532cf937a2c47ec0d3 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Wed, 15 Sep 2021 15:25:31 -0700 Subject: [PATCH] [localization] Fixes for latest master --- guide/src/en/SUMMARY.md | 2 +- guide/src/en/format/configuration/README.md | 4 +- .../en/format/configuration/localization.md | 85 ++++++++++++++++ .../en/format/configuration/translations.md | 86 ----------------- guide/src/en/format/localization.md | 1 + src/book/book.rs | 41 ++++---- src/book/mod.rs | 53 +++++++++- src/config.rs | 10 +- src/preprocess/links.rs | 4 +- src/renderer/html_handlebars/hbs_renderer.rs | 96 ++++--------------- src/renderer/mod.rs | 4 - tests/init.rs | 2 +- 12 files changed, 185 insertions(+), 203 deletions(-) delete mode 100644 guide/src/en/format/configuration/translations.md create mode 100644 guide/src/en/format/localization.md diff --git a/guide/src/en/SUMMARY.md b/guide/src/en/SUMMARY.md index c42184cc..db1a14aa 100644 --- a/guide/src/en/SUMMARY.md +++ b/guide/src/en/SUMMARY.md @@ -15,13 +15,13 @@ - [General](format/configuration/general.md) - [Preprocessors](format/configuration/preprocessors.md) - [Renderers](format/configuration/renderers.md) + - [Localization](format/configuration/localization.md) - [Environment Variables](format/configuration/environment-variables.md) - [Theme](format/theme/README.md) - [index.hbs](format/theme/index-hbs.md) - [Syntax highlighting](format/theme/syntax-highlighting.md) - [Editor](format/theme/editor.md) - [MathJax Support](format/mathjax.md) - - [Localization](format/localization.md) - [mdBook-specific features](format/mdbook.md) - [Continuous Integration](continuous-integration.md) - [For Developers](for_developers/README.md) diff --git a/guide/src/en/format/configuration/README.md b/guide/src/en/format/configuration/README.md index 6a23cfa0..c352a758 100644 --- a/guide/src/en/format/configuration/README.md +++ b/guide/src/en/format/configuration/README.md @@ -4,11 +4,11 @@ This section details the configuration options available in the ***book.toml***: - **[General]** configuration including the `book`, `rust`, `build` sections - **[Preprocessor]** configuration for default and custom book preprocessors - **[Renderer]** configuration for the HTML, Markdown and custom renderers -- **[Translations]** configuration for books written in more than one language +- **[Localization]** configuration for books written in more than one language - **[Environment Variable]** configuration for overriding configuration options in your environment [General]: general.md [Preprocessor]: preprocessors.md [Renderer]: renderers.md -[Translations]: translations.md +[Localization]: localization.md [Environment Variable]: environment-variables.md diff --git a/guide/src/en/format/configuration/localization.md b/guide/src/en/format/configuration/localization.md index d0c0355b..62bd730b 100644 --- a/guide/src/en/format/configuration/localization.md +++ b/guide/src/en/format/configuration/localization.md @@ -1 +1,86 @@ # Localization + +It's possible to write your book in more than one language and bundle all of its +translations into a single output folder, with the ability for readers to switch +between each one in the rendered output. The available languages for your book +are defined in the `[language]` table: + +```toml +[language.en] +name = "English" + +[language.ja] +name = "日本語" +title = "本のサンプル" +description = "この本は実例です。" +authors = ["Ruin0x11"] +``` + +Each language must have a human-readable `name` defined. Also, if the +`[language]` table is defined, you must define `book.language` to be a key of +this table, which will indicate the language whose files will be used for +fallbacks if a page is missing in a translation. + +The `title` and `description` fields, if defined, will override the ones set in +the `[book]` section. This way you can translate the book's title and +description. `authors` provides a list of this translation's authors. + +After defining a new language like `[language.ja]`, add a new subdirectory +`src/ja` and create your `SUMMARY.md` and other files there. + +> **Note:** Whether or not the `[language]` table is defined changes the format +> of the `src` directory that mdBook expects to see. If there is no `[language]` +> table, mdBook will treat the `src` directory as a single translation of the +> book, with `SUMMARY.md` at the root: +> +> ``` +> ├── book.toml +> └── src +> ├── chapter +> │ ├── 1.md +> │ ├── 2.md +> │ └── README.md +> ├── README.md +> └── SUMMARY.md +> ``` +> +> If the `[language]` table is defined, mdBook will instead expect to find +> subdirectories under `src` named after the keys in the table: +> +> ``` +> ├── book.toml +> └── src +> ├── en +> │ ├── chapter +> │ │ ├── 1.md +> │ │ ├── 2.md +> │ │ └── README.md +> │ ├── README.md +> │ └── SUMMARY.md +> └── ja +> ├── chapter +> │ ├── 1.md +> │ ├── 2.md +> │ └── README.md +> ├── README.md +> └── SUMMARY.md +> ``` + +If the `[language]` table is used, you can pass the `-l ` argument +to commands like `mdbook build` to build the book for only a single language. In +this example, `` can be `en` or `ja`. + +Some extra notes on translations: + +- In a translation's `SUMMARY.md` or inside Markdown files, you can link to + pages, images or other files that don't exist in the current translation, but + do exist in the default translation. This is so you can have a fallback in + case new pages get added in the default language that haven't been translated + yet. +- Each translation can have its own `SUMMARY.md` with differing content from + other translations. Even if the translation's summary goes out of sync with + the default language, the links will continue to work so long as the pages + exist in either translation. +- Each translation can have its own pages listed in `SUMMARY.md` that don't + exist in the default translation at all, in case extra information specific to + that language is needed. diff --git a/guide/src/en/format/configuration/translations.md b/guide/src/en/format/configuration/translations.md deleted file mode 100644 index 1962552b..00000000 --- a/guide/src/en/format/configuration/translations.md +++ /dev/null @@ -1,86 +0,0 @@ -## Translations - -It's possible to write your book in more than one language and bundle all of its -translations into a single output folder, with the ability for readers to switch -between each one in the rendered output. The available languages for your book -are defined in the `[language]` table: - -```toml -[language.en] -name = "English" - -[language.ja] -name = "日本語" -title = "本のサンプル" -description = "この本は実例です。" -authors = ["Ruin0x11"] -``` - -Each language must have a human-readable `name` defined. Also, if the -`[language]` table is defined, you must define `book.language` to be a key of -this table, which will indicate the language whose files will be used for -fallbacks if a page is missing in a translation. - -The `title` and `description` fields, if defined, will override the ones set in -the `[book]` section. This way you can translate the book's title and -description. `authors` provides a list of this translation's authors. - -After defining a new language like `[language.ja]`, add a new subdirectory -`src/ja` and create your `SUMMARY.md` and other files there. - -> **Note:** Whether or not the `[language]` table is defined changes the format -> of the `src` directory that mdBook expects to see. If there is no `[language]` -> table, mdBook will treat the `src` directory as a single translation of the -> book, with `SUMMARY.md` at the root: -> -> ``` -> ├── book.toml -> └── src -> ├── chapter -> │ ├── 1.md -> │ ├── 2.md -> │ └── README.md -> ├── README.md -> └── SUMMARY.md -> ``` -> -> If the `[language]` table is defined, mdBook will instead expect to find -> subdirectories under `src` named after the keys in the table: -> -> ``` -> ├── book.toml -> └── src -> ├── en -> │ ├── chapter -> │ │ ├── 1.md -> │ │ ├── 2.md -> │ │ └── README.md -> │ ├── README.md -> │ └── SUMMARY.md -> └── ja -> ├── chapter -> │ ├── 1.md -> │ ├── 2.md -> │ └── README.md -> ├── README.md -> └── SUMMARY.md -> ``` - -If the `[language]` table is used, you can pass the `-l ` argument -to commands like `mdbook build` to build the book for only a single language. In -this example, `` can be `en` or `ja`. - -Some extra notes on translations: - -- In a translation's `SUMMARY.md` or inside Markdown files, you can link to - pages, images or other files that don't exist in the current translation, but - do exist in the default translation. This is so you can have a fallback in - case new pages get added in the default language that haven't been translated - yet. -- Each translation can have its own `SUMMARY.md` with differing content from - other translations. Even if the translation's summary goes out of sync with - the default language, the links will continue to work so long as the pages - exist in either translation. -- Each translation can have its own pages listed in `SUMMARY.md` that don't - exist in the default translation at all, in case extra information specific to - that language is needed. diff --git a/guide/src/en/format/localization.md b/guide/src/en/format/localization.md new file mode 100644 index 00000000..d0c0355b --- /dev/null +++ b/guide/src/en/format/localization.md @@ -0,0 +1 @@ +# Localization diff --git a/src/book/book.rs b/src/book/book.rs index fa65fcb5..301a8457 100644 --- a/src/book/book.rs +++ b/src/book/book.rs @@ -61,8 +61,8 @@ fn load_single_book_translation>( let summary = parse_summary(&summary_content) .with_context(|| format!("Summary parsing failed for file={:?}", summary_md))?; - if cfg.create_missing { - create_missing(localized_src_dir, &summary).with_context(|| "Unable to create missing chapters")?; + if cfg.build.create_missing { + create_missing(&localized_src_dir, &summary).with_context(|| "Unable to create missing chapters")?; } load_book_from_disk(&summary, localized_src_dir, fallback_src_dir, cfg) @@ -83,17 +83,7 @@ fn create_missing(src_dir: &Path, summary: &Summary) -> Result<()> { if let Some(ref location) = link.location { let filename = src_dir.join(location); if !filename.exists() { - if let Some(parent) = filename.parent() { - if !parent.exists() { - fs::create_dir_all(parent)?; - } - } - debug!("Creating missing file {}", filename.display()); - - let mut f = File::create(&filename).with_context(|| { - format!("Unable to create missing file: {}", filename.display()) - })?; - writeln!(f, "# {}", link.name)?; + create_missing_link(&filename, link)?; } } @@ -104,6 +94,20 @@ fn create_missing(src_dir: &Path, summary: &Summary) -> Result<()> { Ok(()) } +fn create_missing_link(filename: &Path, link: &Link) -> Result<()> { + if let Some(parent) = filename.parent() { + if !parent.exists() { + fs::create_dir_all(parent)?; + } + } + debug!("Creating missing file {}", filename.display()); + + let mut f = File::create(&filename)?; + writeln!(f, "# {}", link.name)?; + + Ok(()) +} + /// A dumb tree structure representing a book. /// /// For the moment a book is just a collection of [`BookItems`] which are @@ -117,6 +121,8 @@ fn create_missing(src_dir: &Path, summary: &Summary) -> Result<()> { pub struct Book { /// The sections in this book. pub sections: Vec, + /// Chapter title overrides for this book. + pub chapter_titles: HashMap, __non_exhaustive: (), } @@ -360,6 +366,7 @@ pub(crate) fn load_book_from_disk>( Ok(Book { sections: chapters, + chapter_titles: HashMap::new(), __non_exhaustive: (), }) } @@ -410,8 +417,8 @@ fn load_chapter>( ); } if !location.exists() && cfg.build.create_missing { - create_missing(&location, &link) - .with_context(|| "Unable to create missing chapters")?; + create_missing_link(&location, &link) + .with_context(|| "Unable to create missing link reference")?; } let mut f = File::open(&location) @@ -565,6 +572,7 @@ more text. #[test] fn load_a_single_chapter_with_utf8_bom_from_disk() { let temp_dir = TempFileBuilder::new().prefix("book").tempdir().unwrap(); + let cfg = Config::default(); let chapter_path = temp_dir.path().join("chapter_1.md"); File::create(&chapter_path) @@ -581,7 +589,7 @@ more text. Vec::new(), ); - let got = load_chapter(&link, temp_dir.path(), Vec::new()).unwrap(); + let got = load_chapter(&link, temp_dir.path(), temp_dir.path(), Vec::new(), &cfg).unwrap(); assert_eq!(got, should_be); } @@ -832,6 +840,7 @@ more text. name: String::from("Chapter 1"), content: String::from(DUMMY_SRC), path: Some(PathBuf::from("chapter_1.md")), + source_path: Some(PathBuf::from("chapter_1.md")), ..Default::default() })], ..Default::default() diff --git a/src/book/mod.rs b/src/book/mod.rs index 7403ff04..8a5498d0 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -213,6 +213,25 @@ impl MDBook { Ok(()) } + fn preprocess( + &self, + preprocess_ctx: &PreprocessorContext, + renderer: &dyn Renderer, + book: Book, + ) -> Result { + let mut preprocessed_book = book; + for preprocessor in &self.preprocessors { + if preprocessor_should_run(&**preprocessor, renderer, &self.config) { + debug!("Running the {} preprocessor.", preprocessor.name()); + preprocessed_book = preprocessor.run(&preprocess_ctx, preprocessed_book)?; + } + } + preprocessed_book + .chapter_titles + .extend(preprocess_ctx.chapter_titles.borrow_mut().drain()); + Ok(preprocessed_book) + } + /// Run the entire build process for a particular [`Renderer`]. pub fn execute_build_process(&self, renderer: &dyn Renderer) -> Result<()> { let preprocessed_books = match &self.book { @@ -248,19 +267,20 @@ impl MDBook { } }; + self.render(&preprocessed_books, renderer) + } + + fn render(&self, preprocessed_books: &LoadedBook, renderer: &dyn Renderer) -> Result<()> { let name = renderer.name(); let build_dir = self.build_dir_for(name); - let mut render_context = RenderContext::new( + let render_context = RenderContext::new( self.root.clone(), preprocessed_books.clone(), self.build_opts.clone(), self.config.clone(), build_dir, ); - render_context - .chapter_titles - .extend(preprocess_ctx.chapter_titles.borrow_mut().drain()); info!("Running the {} backend", renderer.name()); renderer @@ -297,6 +317,7 @@ impl MDBook { self.config.clone(), "test".to_string(), ); + let book = LinkPreprocessor::new().run(&preprocess_context, book.clone())?; // Index Preprocessor is disabled so that chapter paths continue to point to the // actual markdown files. @@ -354,6 +375,30 @@ impl MDBook { Ok(()) } + /// Run `rustdoc` tests on the book, linking against the provided libraries. + pub fn test(&self, library_paths: Vec<&str>) -> Result<()> { + let library_args: Vec<&str> = (0..library_paths.len()) + .map(|_| "-L") + .zip(library_paths.into_iter()) + .flat_map(|x| vec![x.0, x.1]) + .collect(); + + let temp_dir = TempFileBuilder::new().prefix("mdbook-").tempdir()?; + + match self.book { + LoadedBook::Localized(ref books) => { + for (language_ident, book) in books.0.iter() { + self.test_book(book, &temp_dir, &library_args, Some(language_ident.clone()))?; + } + } + LoadedBook::Single(ref book) => { + self.test_book(&book, &temp_dir, &library_args, None)? + } + } + + Ok(()) + } + /// The logic for determining where a backend should put its build /// artefacts. /// diff --git a/src/config.rs b/src/config.rs index bccc288d..77eb37e1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -479,15 +479,13 @@ impl<'de> Deserialize<'de> for Config { .unwrap_or_default(); if !language.0.is_empty() { - let default_languages = language.0.iter().filter(|(_, lang)| lang.default).count(); - if default_languages != 1 { + if book.language.is_none() { return Err(D::Error::custom( "If the [language] table is specified, then `book.language` must be declared", )); } let language_ident = book.language.clone().unwrap(); if language.0.get(&language_ident).is_none() { - use serde::de::Error; return Err(D::Error::custom(format!( "Expected [language.{}] to be declared in book.toml", language_ident @@ -495,7 +493,6 @@ impl<'de> Deserialize<'de> for Config { } for (ident, language) in language.0.iter() { if language.name.is_empty() { - use serde::de::Error; return Err(D::Error::custom(format!( "`name` property for [language.{}] must be non-empty", ident @@ -910,7 +907,6 @@ mod tests { title = "Some Book" authors = ["Michael-F-Bryan "] description = "A completely useless book" - multilingual = true src = "source" language = "ja" @@ -1369,14 +1365,14 @@ mod tests { } #[test] - #[should_panic(expected = "Invalid configuration file")] fn book_language_without_languages_table() { let src = r#" [book] language = "en" "#; - Config::from_str(src).unwrap(); + let got = Config::from_str(src).unwrap(); + assert_eq!(got.default_language(), None); } #[test] diff --git a/src/preprocess/links.rs b/src/preprocess/links.rs index 94c91528..9282d642 100644 --- a/src/preprocess/links.rs +++ b/src/preprocess/links.rs @@ -327,9 +327,9 @@ impl<'a> Link<'a> { }) } - fn render_with_path>( + fn render_with_path, P2: AsRef>( &self, - base: P, + base: P1, fallback: Option, chapter_title: &mut String, ) -> Result { diff --git a/src/renderer/html_handlebars/hbs_renderer.rs b/src/renderer/html_handlebars/hbs_renderer.rs index bf9bf976..287281c3 100644 --- a/src/renderer/html_handlebars/hbs_renderer.rs +++ b/src/renderer/html_handlebars/hbs_renderer.rs @@ -80,6 +80,7 @@ impl HtmlHandlebars { handlebars: &mut Handlebars<'a>, theme: &Theme, ) -> Result<()> { + let book_config = &ctx.config.book; let build_dir = ctx.root.join(build_dir); let mut data = make_data( &ctx.root, @@ -104,8 +105,10 @@ impl HtmlHandlebars { destination: destination.to_path_buf(), data: data.clone(), is_index, + book_config: book_config.clone(), html_config: html_config.clone(), edition: ctx.config.rust.edition, + chapter_titles: &book.chapter_titles, }; self.render_item( item, @@ -138,19 +141,21 @@ impl HtmlHandlebars { } // Render the handlebars template with the data - debug!("Render template"); - let rendered = handlebars.render("index", &data)?; + if html_config.print.enable { + debug!("Render template"); + let rendered = handlebars.render("index", &data)?; - let rendered = - self.post_process(rendered, &html_config.playground, ctx.config.rust.edition); + let rendered = + self.post_process(rendered, &html_config.playground, ctx.config.rust.edition); - utils::fs::write_file(&destination, "print.html", rendered.as_bytes())?; - debug!("Creating print.html ✓"); + utils::fs::write_file(destination, "print.html", rendered.as_bytes())?; + debug!("Creating print.html ✓"); + } debug!("Copy static files"); - self.copy_static_files(&destination, &theme, &html_config) + self.copy_static_files(destination, &theme, &html_config) .with_context(|| "Unable to copy across static files")?; - self.copy_additional_css_and_js(&html_config, &ctx.root, &destination) + self.copy_additional_css_and_js(&html_config, &ctx.root, destination) .with_context(|| "Unable to copy across additional CSS and JS")?; // Render search index @@ -158,11 +163,11 @@ impl HtmlHandlebars { { let search = html_config.search.clone().unwrap_or_default(); if search.enable { - super::search::create_files(&search, &destination, &book)?; + super::search::create_files(&search, destination, book)?; } } - self.emit_redirects(&ctx.destination, handlebars, &html_config.redirect) + self.emit_redirects(&ctx.destination, &handlebars, &html_config.redirect) .context("Unable to emit redirects")?; // `src_dir` points to the root source directory. If this book @@ -674,7 +679,6 @@ impl Renderer for HtmlHandlebars { } fn render(&self, ctx: &RenderContext) -> Result<()> { - let book_config = &ctx.config.book; let html_config = ctx.config.html_config().unwrap_or_default(); let src_dir = ctx.source_dir(); let destination = &ctx.destination; @@ -720,75 +724,7 @@ impl Renderer for HtmlHandlebars { debug!("Register handlebars helpers"); self.register_hbs_helpers(&mut handlebars, &html_config); - let mut data = make_data(&ctx.root, book, &ctx.config, &html_config, &theme)?; - - // Print version - let mut print_content = String::new(); - - fs::create_dir_all(&destination) - .with_context(|| "Unexpected error when constructing destination path")?; - - let mut is_index = true; - for item in book.iter() { - let ctx = RenderItemContext { - handlebars: &handlebars, - destination: destination.to_path_buf(), - data: data.clone(), - is_index, - book_config: book_config.clone(), - html_config: html_config.clone(), - edition: ctx.config.rust.edition, - chapter_titles: &ctx.chapter_titles, - }; - self.render_item(item, ctx, &mut print_content)?; - is_index = false; - } - - // Render 404 page - if html_config.input_404 != Some("".to_string()) { - self.render_404(ctx, &html_config, &src_dir, &mut handlebars, &mut data)?; - } - - // Print version - self.configure_print_version(&mut data, &print_content); - if let Some(ref title) = ctx.config.book.title { - data.insert("title".to_owned(), json!(title)); - } - - // Render the handlebars template with the data - if html_config.print.enable { - debug!("Render template"); - let rendered = handlebars.render("index", &data)?; - - let rendered = - self.post_process(rendered, &html_config.playground, ctx.config.rust.edition); - - utils::fs::write_file(destination, "print.html", rendered.as_bytes())?; - debug!("Creating print.html ✓"); - } - - debug!("Copy static files"); - self.copy_static_files(destination, &theme, &html_config) - .with_context(|| "Unable to copy across static files")?; - self.copy_additional_css_and_js(&html_config, &ctx.root, destination) - .with_context(|| "Unable to copy across additional CSS and JS")?; - - // Render search index - #[cfg(feature = "search")] - { - let search = html_config.search.unwrap_or_default(); - if search.enable { - super::search::create_files(&search, destination, book)?; - } - } - - self.emit_redirects(&ctx.destination, &handlebars, &html_config.redirect) - .context("Unable to emit redirects")?; - - // Copy all remaining files, avoid a recursive copy from/to the book build dir - utils::fs::copy_files_except_ext(&src_dir, destination, true, Some(&build_dir), &["md"])?; - - Ok(()) + self.render_books(ctx, &src_dir, &html_config, &mut handlebars, &theme) } } diff --git a/src/renderer/mod.rs b/src/renderer/mod.rs index 8922c2f8..0685f338 100644 --- a/src/renderer/mod.rs +++ b/src/renderer/mod.rs @@ -18,7 +18,6 @@ mod html_handlebars; mod markdown_renderer; use shlex::Shlex; -use std::collections::HashMap; use std::fs; use std::io::{self, ErrorKind, Read}; use std::path::{Path, PathBuf}; @@ -70,8 +69,6 @@ pub struct RenderContext { /// guaranteed to be empty or even exist. pub destination: PathBuf, #[serde(skip)] - pub(crate) chapter_titles: HashMap, - #[serde(skip)] __non_exhaustive: (), } @@ -95,7 +92,6 @@ impl RenderContext { version: crate::MDBOOK_VERSION.to_string(), root: root.into(), destination: destination.into(), - chapter_titles: HashMap::new(), __non_exhaustive: (), } } diff --git a/tests/init.rs b/tests/init.rs index c1b3f9ea..362b5e06 100644 --- a/tests/init.rs +++ b/tests/init.rs @@ -107,7 +107,7 @@ fn run_mdbook_init_with_custom_book_and_src_locations() { let contents = fs::read_to_string(temp.path().join("book.toml")).unwrap(); assert_eq!( contents, - "[book]\nauthors = []\nlanguage = \"en\"\nmultilingual = false\nsrc = \"in\"\n\n[build]\nbuild-dir = \"out\"\ncreate-missing = true\nuse-default-preprocessors = true\n" + "[book]\nauthors = []\nlanguage = \"en\"\nsrc = \"in\"\n\n[build]\nbuild-dir = \"out\"\ncreate-missing = true\nuse-default-preprocessors = true\n[language.en]\nname = \"English\"\n" ); }