Support {{#shiftinclude auto}}

As well as allowing explicitly-specified shift amounts, also
support an "auto" option that strips common leftmost whitespace
from an inclusion.

guide/src/format/mdbook.md

@@ -223,6 +223,12 @@ using the following syntax:
 A positive number for the shift will prepend spaces to all lines; a negative number will remove
 the corresponding number of characters from the beginning of each line.
 
+The special `auto` value will remove common initial whitespace from all lines.
+
+```hbs
+\{{#shiftinclude auto:file.rs:indentedanchor}}
+```
+
 ## Including a file but initially hiding all except specified lines
 
 The `rustdoc_include` helper is for including code from external Rust files that contain complete

src/preprocess/links.rs

@@ -266,14 +266,18 @@ fn parse_include_path(path: &str) -> LinkType<'static> {
 fn parse_shift_include_path(params: &str) -> LinkType<'static> {
     let mut params = params.splitn(2, ':');
     let param0 = params.next().unwrap();
+    let shift = if param0 == "auto" {
+        Shift::Auto
+    } else {
         let shift: isize = param0.parse().unwrap_or_else(|e| {
             log::error!("failed to parse shift amount: {e:?}");
             0
         });
-    let shift = match shift.cmp(&0) {
+        match shift.cmp(&0) {
             Ordering::Greater => Shift::Right(shift as usize),
             Ordering::Equal => Shift::None,
             Ordering::Less => Shift::Left(-shift as usize),
+        }
     };
     let mut parts = params.next().unwrap().splitn(2, ':');
 
@@ -1002,6 +1006,19 @@ mod tests {
         );
     }
 
+    #[test]
+    fn parse_with_auto_shifted_anchor() {
+        let link_type = parse_shift_include_path("auto:arbitrary:some-anchor");
+        assert_eq!(
+            link_type,
+            LinkType::Include(
+                PathBuf::from("arbitrary"),
+                RangeOrAnchor::Anchor("some-anchor".to_string()),
+                Shift::Auto
+            )
+        );
+    }
+
     #[test]
     fn parse_with_more_than_three_colons_ignores_everything_after_third_colon() {
         let link_type = parse_include_path("arbitrary:5:10:17:anything:");
@@ -1053,4 +1070,17 @@ mod tests {
             )
         );
     }
+
+    #[test]
+    fn parse_start_and_end_auto_shifted_range() {
+        let link_type = parse_shift_include_path("auto:arbitrary:5:10");
+        assert_eq!(
+            link_type,
+            LinkType::Include(
+                PathBuf::from("arbitrary"),
+                RangeOrAnchor::Range(LineRange::from(4..10)),
+                Shift::Auto
+            )
+        );
+    }
 }

src/utils/string.rs

@@ -10,16 +10,58 @@ pub enum Shift {
     None,
     Left(usize),
     Right(usize),
+    /// Strip leftmost whitespace that is common to all lines.
+    Auto,
 }
 
-fn shift_line(l: &str, shift: Shift) -> Cow<'_, str> {
+#[derive(PartialEq, Eq, Debug, Clone, Copy)]
+enum ExplicitShift {
+    None,
+    Left(usize),
+    Right(usize),
+}
+
+fn common_leading_ws(lines: &[String]) -> String {
+    let mut common_ws: Option<String> = None;
+    for line in lines {
+        if line.is_empty() {
+            // Don't include empty lines in the calculation.
+            continue;
+        }
+        let ws = line.chars().take_while(|c| c.is_whitespace());
+        if let Some(common) = common_ws {
+            common_ws = Some(
+                common
+                    .chars()
+                    .zip(ws)
+                    .take_while(|(a, b)| a == b)
+                    .map(|(a, _b)| a)
+                    .collect(),
+            );
+        } else {
+            common_ws = Some(ws.collect())
+        }
+    }
+    common_ws.unwrap_or_else(String::new)
+}
+
+fn calculate_shift(lines: &[String], shift: Shift) -> ExplicitShift {
     match shift {
-        Shift::None => Cow::Borrowed(l),
+        Shift::None => ExplicitShift::None,
-        Shift::Right(shift) => {
+        Shift::Left(l) => ExplicitShift::Left(l),
+        Shift::Right(r) => ExplicitShift::Right(r),
+        Shift::Auto => ExplicitShift::Left(common_leading_ws(lines).len()),
+    }
+}
+
+fn shift_line(l: &str, shift: ExplicitShift) -> Cow<'_, str> {
+    match shift {
+        ExplicitShift::None => Cow::Borrowed(l),
+        ExplicitShift::Right(shift) => {
             let indent = " ".repeat(shift);
             Cow::Owned(format!("{indent}{l}"))
         }
-        Shift::Left(skip) => {
+        ExplicitShift::Left(skip) => {
             if l.chars().take(skip).any(|c| !c.is_whitespace()) {
                 log::error!("left-shifting away non-whitespace");
             }
@@ -30,6 +72,7 @@ fn shift_line(l: &str, shift: Shift) -> Cow<'_, str> {
 }
 
 fn shift_lines(lines: &[String], shift: Shift) -> Vec<Cow<'_, str>> {
+    let shift = calculate_shift(lines, shift);
     lines.iter().map(|l| shift_line(l, shift)).collect()
 }
 
@@ -160,20 +203,44 @@ pub fn take_rustdoc_include_anchored_lines(s: &str, anchor: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::{
-        shift_line, take_anchored_lines, take_anchored_lines_with_shift, take_lines,
+        common_leading_ws, shift_line, take_anchored_lines, take_anchored_lines_with_shift,
-        take_lines_with_shift, take_rustdoc_include_anchored_lines, take_rustdoc_include_lines,
+        take_lines, take_lines_with_shift, take_rustdoc_include_anchored_lines,
-        Shift,
+        take_rustdoc_include_lines, ExplicitShift, Shift,
     };
 
+    #[test]
+    fn common_leading_ws_test() {
+        let tests = [
+            (["  line1", "    line2", "  line3"], "  "),
+            (["  line1", "    line2", "line3"], ""),
+            (["\t\tline1", "\t\t  line2", "\t\tline3"], "\t\t"),
+            (["\t line1", " \tline2", "  \t\tline3"], ""),
+        ];
+        for (lines, want) in tests {
+            let lines = lines.into_iter().map(|l| l.to_string()).collect::<Vec<_>>();
+            let got = common_leading_ws(&lines);
+            assert_eq!(got, want, "for input {lines:?}");
+        }
+    }
+
     #[test]
     fn shift_line_test() {
         let s = "    Line with 4 space intro";
-        assert_eq!(shift_line(s, Shift::None), s);
+        assert_eq!(shift_line(s, ExplicitShift::None), s);
-        assert_eq!(shift_line(s, Shift::Left(4)), "Line with 4 space intro");
-        assert_eq!(shift_line(s, Shift::Left(2)), "  Line with 4 space intro");
-        assert_eq!(shift_line(s, Shift::Left(6)), "ne with 4 space intro");
         assert_eq!(
-            shift_line(s, Shift::Right(2)),
+            shift_line(s, ExplicitShift::Left(4)),
+            "Line with 4 space intro"
+        );
+        assert_eq!(
+            shift_line(s, ExplicitShift::Left(2)),
+            "  Line with 4 space intro"
+        );
+        assert_eq!(
+            shift_line(s, ExplicitShift::Left(6)),
+            "ne with 4 space intro"
+        );
+        assert_eq!(
+            shift_line(s, ExplicitShift::Right(2)),
             "      Line with 4 space intro"
         );
     }
@@ -207,6 +274,10 @@ mod tests {
             take_lines_with_shift(s, 1..3, Shift::Right(2)),
             "    ipsum\n      dolor"
         );
+        assert_eq!(
+            take_lines_with_shift(s, 1..3, Shift::Auto),
+            "ipsum\n  dolor"
+        );
         assert_eq!(take_lines_with_shift(s, 3.., Shift::None), "  sit\n  amet");
         assert_eq!(
             take_lines_with_shift(s, 3.., Shift::Right(1)),
@@ -217,6 +288,10 @@ mod tests {
             take_lines_with_shift(s, ..3, Shift::None),
             "  Lorem\n  ipsum\n    dolor"
         );
+        assert_eq!(
+            take_lines_with_shift(s, ..3, Shift::Auto),
+            "Lorem\nipsum\n  dolor"
+        );
         assert_eq!(
             take_lines_with_shift(s, ..3, Shift::Right(4)),
             "      Lorem\n      ipsum\n        dolor"
@@ -226,6 +301,10 @@ mod tests {
             "rem\nsum\ndolor"
         );
         assert_eq!(take_lines_with_shift(s, .., Shift::None), s);
+        assert_eq!(
+            take_lines_with_shift(s, .., Shift::Auto),
+            "Lorem\nipsum\n  dolor\nsit\namet"
+        );
         // corner cases
         assert_eq!(take_lines_with_shift(s, 4..3, Shift::None), "");
         assert_eq!(take_lines_with_shift(s, 4..3, Shift::Left(2)), "");
@@ -307,6 +386,10 @@ mod tests {
             take_anchored_lines_with_shift(s, "test", Shift::Left(2)),
             "dolor\nsit\namet"
         );
+        assert_eq!(
+            take_anchored_lines_with_shift(s, "test", Shift::Auto),
+            "dolor\nsit\namet"
+        );
         assert_eq!(
             take_anchored_lines_with_shift(s, "something", Shift::None),
             ""
@@ -333,6 +416,10 @@ mod tests {
             take_anchored_lines_with_shift(s, "test", Shift::Left(2)),
             "dolor\nsit\namet"
         );
+        assert_eq!(
+            take_anchored_lines_with_shift(s, "test", Shift::Auto),
+            "dolor\nsit\namet"
+        );
         assert_eq!(
             take_anchored_lines_with_shift(s, "test", Shift::Left(4)),
             "lor\nt\net"
@@ -346,18 +433,22 @@ mod tests {
             ""
         );
 
-        let s = "  Lorem\n  ANCHOR: test\n  ipsum\n  ANCHOR: test\n  dolor\n  sit\n  amet\n  ANCHOR_END: test\n  lorem\n  ipsum";
+        let s = "  Lorem\n  ANCHOR: test\n  ipsum\n  ANCHOR: test\n  dolor\n\n\n  sit\n  amet\n  ANCHOR_END: test\n  lorem\n  ipsum";
         assert_eq!(
             take_anchored_lines_with_shift(s, "test", Shift::None),
-            "  ipsum\n  dolor\n  sit\n  amet"
+            "  ipsum\n  dolor\n\n\n  sit\n  amet"
         );
         assert_eq!(
             take_anchored_lines_with_shift(s, "test", Shift::Right(2)),
-            "    ipsum\n    dolor\n    sit\n    amet"
+            "    ipsum\n    dolor\n  \n  \n    sit\n    amet"
         );
         assert_eq!(
             take_anchored_lines_with_shift(s, "test", Shift::Left(2)),
-            "ipsum\ndolor\nsit\namet"
+            "ipsum\ndolor\n\n\nsit\namet"
+        );
+        assert_eq!(
+            take_anchored_lines_with_shift(s, "test", Shift::Auto),
+            "ipsum\ndolor\n\n\nsit\namet"
         );
         assert_eq!(
             take_anchored_lines_with_shift(s, "something", Shift::None),
@@ -371,6 +462,10 @@ mod tests {
             take_anchored_lines_with_shift(s, "something", Shift::Left(2)),
             ""
         );
+        assert_eq!(
+            take_anchored_lines_with_shift(s, "something", Shift::Auto),
+            ""
+        );
 
         // Include non-ASCII.
         let s = "  Lorem\n  ANCHOR:    test2\n  ípsum\n  ANCHOR: test\n  dôlor\n  sit\n  amet\n  ANCHOR_END: test\n  lorem\n  ANCHOR_END:test2\n  ipsum";

tests/dummy_book/src/first/nested.md

@@ -24,6 +24,10 @@ assert!($TEST_STATUS);
 {{#shiftinclude +2:nested-test-with-anchors.rs:myanchor}}
 ```
 
+```rust
+{{#shiftinclude auto:nested-test-with-anchors.rs:indentedanchor}}
+```
+
 ## Rustdoc include adds the rest of the file as hidden
 
 ```rust

tests/searchindex_fixture.json

@@ -176,7 +176,7 @@
           "title": 7
         },
         "7": {
-          "body": 21,
+          "body": 27,
           "breadcrumbs": 6,
           "title": 2
         },
@@ -361,7 +361,7 @@
           "title": "Anchors include the part of a file between special comments"
         },
         "7": {
-          "body": "// The next line will cause a `rendered_output` test to fail if the anchor feature is broken in // such a way that the content between anchors isn't included. // unique-string-for-anchor-test assert!(true);",
+          "body": "// The next line will cause a `rendered_output` test to fail if the anchor feature is broken in // such a way that the content between anchors isn't included. // unique-string-for-anchor-test assert!(true); pub fn indented_function() { // This extra indent remains\n}",
           "breadcrumbs": "First Chapter » Nested Chapter » Includes can be shifted",
           "id": "7",
           "title": "Includes can be shifted"
@@ -1347,6 +1347,22 @@
                 },
                 "df": 0,
                 "docs": {}
+              },
+              "t": {
+                "df": 0,
+                "docs": {},
+                "r": {
+                  "a": {
+                    "df": 1,
+                    "docs": {
+                      "7": {
+                        "tf": 1.0
+                      }
+                    }
+                  },
+                  "df": 0,
+                  "docs": {}
+                }
               }
             }
           },
@@ -1456,11 +1472,14 @@
               }
             },
             "n": {
-              "df": 3,
+              "df": 4,
               "docs": {
                 "27": {
                   "tf": 1.0
                 },
+                "7": {
+                  "tf": 1.0
+                },
                 "8": {
                   "tf": 1.4142135623730951
                 },
@@ -1778,6 +1797,54 @@
                 "e": {
                   "df": 0,
                   "docs": {},
+                  "n": {
+                    "df": 0,
+                    "docs": {},
+                    "t": {
+                      "df": 1,
+                      "docs": {
+                        "7": {
+                          "tf": 1.0
+                        }
+                      },
+                      "e": {
+                        "d": {
+                          "_": {
+                            "df": 0,
+                            "docs": {},
+                            "f": {
+                              "df": 0,
+                              "docs": {},
+                              "u": {
+                                "df": 0,
+                                "docs": {},
+                                "n": {
+                                  "c": {
+                                    "df": 0,
+                                    "docs": {},
+                                    "t": {
+                                      "df": 1,
+                                      "docs": {
+                                        "7": {
+                                          "tf": 1.0
+                                        }
+                                      }
+                                    }
+                                  },
+                                  "df": 0,
+                                  "docs": {}
+                                }
+                              }
+                            }
+                          },
+                          "df": 0,
+                          "docs": {}
+                        },
+                        "df": 0,
+                        "docs": {}
+                      }
+                    }
+                  },
                   "x": {
                     "df": 2,
                     "docs": {
@@ -2444,6 +2511,14 @@
               }
             },
             "u": {
+              "b": {
+                "df": 1,
+                "docs": {
+                  "7": {
+                    "tf": 1.0
+                  }
+                }
+              },
               "df": 0,
               "docs": {},
               "t": {
@@ -2514,6 +2589,26 @@
                   }
                 }
               },
+              "m": {
+                "a": {
+                  "df": 0,
+                  "docs": {},
+                  "i": {
+                    "df": 0,
+                    "docs": {},
+                    "n": {
+                      "df": 1,
+                      "docs": {
+                        "7": {
+                          "tf": 1.0
+                        }
+                      }
+                    }
+                  }
+                },
+                "df": 0,
+                "docs": {}
+              },
               "n": {
                 "d": {
                   "df": 0,
@@ -4461,6 +4556,22 @@
                 },
                 "df": 0,
                 "docs": {}
+              },
+              "t": {
+                "df": 0,
+                "docs": {},
+                "r": {
+                  "a": {
+                    "df": 1,
+                    "docs": {
+                      "7": {
+                        "tf": 1.0
+                      }
+                    }
+                  },
+                  "df": 0,
+                  "docs": {}
+                }
               }
             }
           },
@@ -4633,11 +4744,14 @@
               }
             },
             "n": {
-              "df": 3,
+              "df": 4,
               "docs": {
                 "27": {
                   "tf": 1.0
                 },
+                "7": {
+                  "tf": 1.0
+                },
                 "8": {
                   "tf": 1.4142135623730951
                 },
@@ -4958,6 +5072,54 @@
                 "e": {
                   "df": 0,
                   "docs": {},
+                  "n": {
+                    "df": 0,
+                    "docs": {},
+                    "t": {
+                      "df": 1,
+                      "docs": {
+                        "7": {
+                          "tf": 1.0
+                        }
+                      },
+                      "e": {
+                        "d": {
+                          "_": {
+                            "df": 0,
+                            "docs": {},
+                            "f": {
+                              "df": 0,
+                              "docs": {},
+                              "u": {
+                                "df": 0,
+                                "docs": {},
+                                "n": {
+                                  "c": {
+                                    "df": 0,
+                                    "docs": {},
+                                    "t": {
+                                      "df": 1,
+                                      "docs": {
+                                        "7": {
+                                          "tf": 1.0
+                                        }
+                                      }
+                                    }
+                                  },
+                                  "df": 0,
+                                  "docs": {}
+                                }
+                              }
+                            }
+                          },
+                          "df": 0,
+                          "docs": {}
+                        },
+                        "df": 0,
+                        "docs": {}
+                      }
+                    }
+                  },
                   "x": {
                     "df": 2,
                     "docs": {
@@ -5657,6 +5819,14 @@
               }
             },
             "u": {
+              "b": {
+                "df": 1,
+                "docs": {
+                  "7": {
+                    "tf": 1.0
+                  }
+                }
+              },
               "df": 0,
               "docs": {},
               "t": {
@@ -5730,6 +5900,26 @@
                   }
                 }
               },
+              "m": {
+                "a": {
+                  "df": 0,
+                  "docs": {},
+                  "i": {
+                    "df": 0,
+                    "docs": {},
+                    "n": {
+                      "df": 1,
+                      "docs": {
+                        "7": {
+                          "tf": 1.0
+                        }
+                      }
+                    }
+                  }
+                },
+                "df": 0,
+                "docs": {}
+              },
               "n": {
                 "d": {
                   "df": 0,