From 786ed1671f03c64b2f68860ab18e2fb6ca186164 Mon Sep 17 00:00:00 2001 From: Ramon Buckland Date: Tue, 4 Sep 2018 00:30:32 +1000 Subject: [PATCH] test scripts to compare mdbook version output --- tests/mdbook_version_validator/README.md | 155 ++++++++++++++++++ .../build_books_and_compare.sh | 124 ++++++++++++++ .../build_mdbook_branches.sh | 29 ++++ .../test_mdbook_book_builds.sh | 93 +++++++++++ 4 files changed, 401 insertions(+) create mode 100644 tests/mdbook_version_validator/README.md create mode 100755 tests/mdbook_version_validator/build_books_and_compare.sh create mode 100755 tests/mdbook_version_validator/build_mdbook_branches.sh create mode 100644 tests/mdbook_version_validator/test_mdbook_book_builds.sh diff --git a/tests/mdbook_version_validator/README.md b/tests/mdbook_version_validator/README.md new file mode 100644 index 00000000..4b397f50 --- /dev/null +++ b/tests/mdbook_version_validator/README.md @@ -0,0 +1,155 @@ +# Testing Upstream Books for Breakage from mdbook + +- Theory - when mdbook changes, we should be able to coarsely tell if building a book with the new version fails +- In Practice - Build the book twice and compare + + + 1. Build two binaries of mdbook, some "current" (master) and some "new" branch + (Building dependent binaries - mdbook-linkcheck for example from https://github.com/Michael-F-Bryan/mdbook-linkcheck - are not supported "per build" + but rather the same binary is executed for both. Should not be hard to change that) + + This is it's own BASH shell script + + `tests/mdbook_version_validator/build_mdbook_branches.sh ${MDBOOK_BRANCH_1},${MDBOOK_BRANCH_2} ${G_TMPDIR} ${PWD}/target` + + **Now we have two mdbook binaries** + + 2. Use the github API and find all repos under known orgs on github.com (and support extras via the command line) + + 2. Checkout the master branch of each one + 3. Build each book twice + 4. Compare the output directories + + Comparison is difficult, but the following applies + + `OK` - if the two output directories are identical + `WARN` - status==OK, if the two directories are not identical, but the size of the second is +/- 5% of the first. + `FAIL` - if the size of the second is greater than +/- 5% of the first. + +Not bullet proof - but it works. + +## Quick Start Guide + +`sh tests/mdbook_version_validator/test_mdbook_book_builds.sh ` +- This will build both versions of mdbook +- Search Github for `book.toml` files in repos +- Download each book, build them with both binaries, compare and report + +## Output - Failure + +Showing a tolerance failure (within +/- 5%) + +``` +$ sh tests/mdbook_version_validator/test_mdbook_book_builds.sh master smart-preprocessor +:: We have both binaries [/home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master, /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor] +:: Collecting the test book repos + :: https://github.com/rust-lang-nursery/mdBook + :: https://github.com/rust-lang/rust-by-example + :: https://github.com/rust-lang-nursery/api-guidelines + :: https://github.com/rust-lang-nursery/rustc-guide + :: https://github.com/rust-lang-nursery/rust-cookbook +:: Building https://github.com/rust-lang-nursery/mdBook + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.018quaRm/https___github_com_rust_lang_nursery_mdBook'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang-nursery/mdBook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:50:38 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:50:38 [INFO] (mdbook::book): Running the html backend + :: Building rust-lang-nursery/mdBook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:50:40 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:50:40 [INFO] (mdbook::book): Running the html backend + :: /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.018quaRm/https___github_com_rust_lang_nursery_mdBook/book-example/book.toml the directories differ + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.018quaRm/generated_books/https___github_com_rust_lang_nursery_mdBook/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.018quaRm/generated_books/https___github_com_rust_lang_nursery_mdBook/mdbook-smart-preprocessor + :: WARN differed, but inside tolerance +/- 5% +:: Building https://github.com/rust-lang/rust-by-example + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.018quaRm/https___github_com_rust_lang_rust_by_example'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang/rust-by-example with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:50:45 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:50:45 [INFO] (mdbook::book): Running the html backend +... +``` + +## Output - Success + +``` +23:42 $ sh tests/mdbook_version_validator/test_mdbook_book_builds.sh master smart-preprocessor +:: We have both binaries [/home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master, /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor] +:: Collecting the test book repos + :: https://github.com/rust-lang-nursery/mdBook + :: https://github.com/rust-lang/rust-by-example + :: https://github.com/rust-lang-nursery/api-guidelines + :: https://github.com/rust-lang-nursery/rustc-guide + :: https://github.com/rust-lang-nursery/rust-cookbook +:: Building https://github.com/rust-lang-nursery/mdBook + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_mdBook'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang-nursery/mdBook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:42:40 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:42:40 [INFO] (mdbook::book): Running the html backend + :: Building rust-lang-nursery/mdBook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:42:42 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:42:42 [INFO] (mdbook::book): Running the html backend + :: OK /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_mdBook/book-example/book.toml is identical + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_mdBook/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_mdBook/mdbook-smart-preprocessor +:: Building https://github.com/rust-lang/rust-by-example + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_rust_by_example'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang/rust-by-example with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:42:48 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:42:48 [INFO] (mdbook::book): Running the html backend + :: Building rust-lang/rust-by-example with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:43:02 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:02 [INFO] (mdbook::book): Running the html backend + :: OK /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_rust_by_example/book.toml is identical + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_rust_by_example/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_rust_by_example/mdbook-smart-preprocessor +:: Building https://github.com/rust-lang-nursery/api-guidelines + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_api_guidelines'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang-nursery/api-guidelines with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:43:20 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:20 [INFO] (mdbook::book): Running the html backend + :: Building rust-lang-nursery/api-guidelines with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:43:23 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:23 [INFO] (mdbook::book): Running the html backend + :: OK /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_api_guidelines/book.toml is identical + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_api_guidelines/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_api_guidelines/mdbook-smart-preprocessor +:: Building https://github.com/rust-lang-nursery/rustc-guide + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_rustc_guide'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang-nursery/rustc-guide with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:43:29 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:29 [INFO] (mdbook::book): Running the html backend + 2018-09-03 23:43:40 [INFO] (mdbook::book): Running the linkcheck backend + 2018-09-03 23:43:40 [INFO] (mdbook::renderer): Invoking the "linkcheck" renderer + :: Building rust-lang-nursery/rustc-guide with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:43:40 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:40 [INFO] (mdbook::book): Running the html backend + 2018-09-03 23:43:50 [INFO] (mdbook::book): Running the linkcheck backend + 2018-09-03 23:43:50 [INFO] (mdbook::renderer): Invoking the "linkcheck" renderer + :: OK /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_rustc_guide/book.toml is identical + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_rustc_guide/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_rustc_guide/mdbook-smart-preprocessor +:: Building https://github.com/rust-lang-nursery/rust-cookbook + Cloning into '/var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_rust_cookbook'... + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + :: /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + :: Building rust-lang-nursery/rust-cookbook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-master + 2018-09-03 23:43:54 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:43:54 [INFO] (mdbook::book): Running the html backend + :: Building rust-lang-nursery/rust-cookbook with /home/rbuckland/projects/github.com/rust-lang-nursery/mdBook/target/mdbook-smart-preprocessor + 2018-09-03 23:44:01 [INFO] (mdbook::book): Book building has started + 2018-09-03 23:44:01 [INFO] (mdbook::book): Running the html backend + :: OK /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/https___github_com_rust_lang_nursery_rust_cookbook/book.toml is identical + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_rust_cookbook/mdbook-master + - /var/folders/cy/m597b3cj1s71904cxdtxk8dnlthn4x/T/tmp.wtHqLOnn/generated_books/https___github_com_rust_lang_nursery_rust_cookbook/mdbook-smart-preprocessor +``` diff --git a/tests/mdbook_version_validator/build_books_and_compare.sh b/tests/mdbook_version_validator/build_books_and_compare.sh new file mode 100755 index 00000000..5d55fe60 --- /dev/null +++ b/tests/mdbook_version_validator/build_books_and_compare.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +red=$'\e[1;31m' +grn=$'\e[1;32m' +yel=$'\e[0;33m' +blu=$'\e[1;34m' +mag=$'\e[1;35m' +cyn=$'\e[1;36m' +white=$'\e[0m' + + +#-------------------------- +function help() { + +echo "Usage:" +echo "" + + +echo " $0" +awk -F ')' '/\|--.*shift/ && ! /awk/ { print $1} ' $0 + +cat < /dev/null + + echo "$blu:: Building $BOOK_NAME with $binary${white}" + + (cd $dir && $binary build --dest-dir ${output_dir} ) + + retVal=$? + if [ $retVal -ne 0 ]; then + echo "$red:: Failed to build $book_toml with $binary" + fi + return + +} + +#-------------------------- + +books=$(find ${BOOK_PATH} -type f -name book.toml) +basename_1=$(basename $VERSION1) +basename_2=$(basename $VERSION2) + +echo :: $VERSION1 +echo :: $VERSION2 + +for book_toml in $books +do + + build_book_and_check $VERSION1 $book_toml ${WORKING_DIR}/${basename_1} + build_book_and_check $VERSION2 $book_toml ${WORKING_DIR}/${basename_2} + + DIFF=$(diff -rq ${WORKING_DIR}/${basename_1} ${WORKING_DIR}/${basename_2} ) + if [ "$DIFF" != "" ] + then + echo "$red:: $book_toml the directories differ$white" + echo $mag - ${WORKING_DIR}/${basename_1}$white + echo $mag - ${WORKING_DIR}/${basename_2}$white + + dir_1_size=$(du -s ${WORKING_DIR}/${basename_1} | awk '{print $1*512}' ) + dir_2_size=$(du -s ${WORKING_DIR}/${basename_2} | awk '{print $1*512}' ) + + if (( dir_2_size / dir_1_size * 100 > 105 || dir_2_size / dir_1_size * 100 < 95 )); then + echo $red:: FAIL outside Tolerance +/- 5% $white + FAIL=1 + else + echo $yel:: WARN differed, but inside tolerance +/- 5% $white + fi + else + echo $grn:: OK $book_toml is identical$white + echo $cyn - ${WORKING_DIR}/${basename_1}$white + echo $cyn - ${WORKING_DIR}/${basename_2}$white + fi +done + +if [[ $FAIL -eq 1 ]];then + exit 1 +fi diff --git a/tests/mdbook_version_validator/build_mdbook_branches.sh b/tests/mdbook_version_validator/build_mdbook_branches.sh new file mode 100755 index 00000000..110a01c3 --- /dev/null +++ b/tests/mdbook_version_validator/build_mdbook_branches.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# +# Example: +# sh tests/mdbook_version_validator/build_branches.sh master,smart-preprocessor /tmp/fff `pwd`/target +# +# This will generate two binaries, mdbook_master and mdbook_smart-preprocessor +# We can then use both these binaries, to check the "build" of other books to see if they differ. +# + +BRANCHES=$1 +WORKING_DIR=$2 +OUT_PATH=$3 +REPO=${4:-"https://github.com/rust-lang-nursery/mdbook"} + +for branch in $(echo $BRANCHES | sed "s/,/ /g") +do + safe_branch=$(echo $branch | sed "s/[^A-Za-z0-9-]/_/g") + workdir_for_branch=${WORKING_DIR}_${safe_branch} + binary_name=mdbook-${safe_branch} + + git clone --depth=1 --branch=${branch} ${REPO} ${workdir_for_branch} + rm -rf ${workdir_for_branch}/.git + cd ${workdir_for_branch} && \ + cargo build && \ + mv target/debug/mdbook ${OUT_PATH}/${binary_name} && \ + echo "${safe_branch}=${OUT_PATH}/${binary_name}" +done + diff --git a/tests/mdbook_version_validator/test_mdbook_book_builds.sh b/tests/mdbook_version_validator/test_mdbook_book_builds.sh new file mode 100644 index 00000000..1918d22a --- /dev/null +++ b/tests/mdbook_version_validator/test_mdbook_book_builds.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +red=$'\e[1;31m' +grn=$'\e[1;32m' +blu=$'\e[1;34m' +mag=$'\e[1;35m' +cyn=$'\e[1;36m' +white=$'\e[0m' + +indent() { sed 's/^/ /'; } + +MDBOOK_BRANCH_1=$1 +MDBOOK_BRANCH_2=$2 +# comma separated +EXTRA_REPOS=$3 + +# +# Example run +# sh tests/mdbook_version_validator/test_mdbook_book_builds.sh master smart-preprocessor +# +# or, add your own repos +# +# sh tests/mdbook_version_validator/test_mdbook_book_builds.sh master smart-preprocessor http://git/my/repo1,http://git/my/repo2 +# + +# +# Globals +# +G_BOOKS=() +G_TMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') +G_BINARY_BRANCH_NAME_1=$( echo $MDBOOK_BRANCH_1 | sed 's/[^a-zA-Z0-9-]/_/g') +G_BINARY_BRANCH_NAME_2=$( echo $MDBOOK_BRANCH_2 | sed 's/[^a-zA-Z0-9-]/_/g') +G_DIRNAME=$(dirname $(perl -e 'use Cwd "abs_path";print abs_path(shift)' $0)) + +#----------------------------------------- +function collect_book_repos() { + echo $blu:: Collecting the "test" book repos $white + repos=$1 + + for i in $(echo $repos | sed "s/,/ /g") + do + # call your procedure/other scripts here below + G_BOOKS+=( $i ) + done + + # Search for book.toml on github.com in these Org Repos - org:rust-lang-nursery org:rust-lang + for i in $(curl --silent --fail "https://api.github.com/search/code?q=book+description+in:file+filename:book.toml+org:rust-lang-nursery+org:rust-lang" | jq -r '.items[] | .repository.full_name') + do + G_BOOKS+=( "https://github.com/"$i ) + echo ":: https://github.com/$i" | indent + done +} +#----------------------------------------- +function build_books_and_compare() { + for book_repo in "${G_BOOKS[@]}" + do + echo ${blu}:: Building ${book_repo}$white + repo_name_safe=$( echo $book_repo | sed 's/[^a-zA-Z0-9]/_/g') + book_dir=${G_TMPDIR}/${repo_name_safe} + local user_org=$( basename $( dirname $book_repo )) + local repo=$( basename $book_repo ) + git clone --depth=1 --branch=master ${book_repo} ${book_dir} 2>&1 | indent + rm -rf ${book_dir}/.git + ${G_DIRNAME}/build_books_and_compare.sh --version1 ${G_BINARY_PATH_1} --version2 ${G_BINARY_PATH_2} --book-name ${user_org}/${repo} --book-path ${book_dir} --working-dir ${G_TMPDIR}/generated_books/${repo_name_safe} 2>&1 | indent + done +} +#----------------------------------------- + +function compile_mdbook_binaries() { + # + # Shell out to another script to create the mdbook binaries, according to which "two" branches + # We could make this a little bit more loose .. but currently the binaries will be in ${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_1} + # + if [[ ! ( -f ${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_1} && -f ${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_2} ) ]] + then + echo $blu:: Building the mdbook binaries [${MDBOOK_BRANCH_1},${MDBOOK_BRANCH_2}] $white + ${G_DIRNAME}/build_mdbook_branches.sh ${MDBOOK_BRANCH_1},${MDBOOK_BRANCH_2} ${G_TMPDIR} ${PWD}/target 2>&1 | indent + fi + + if [[ ! ( -f ${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_1} && -f ${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_2} ) ]] + then + echo $red:: Failed to build the mdbook binaries$white + exit 1 + else + G_BINARY_PATH_1=${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_1} + G_BINARY_PATH_2=${PWD}/target/mdbook-${G_BINARY_BRANCH_NAME_2} + echo $blu:: We have both binaries [${G_BINARY_PATH_1}, ${G_BINARY_PATH_2}]$white + fi +} + +compile_mdbook_binaries ${MDBOOK_BRANCH_1} ${MDBOOK_BRANCH_2} +collect_book_repos ${EXTRA_REPOS} +build_books_and_compare