From 96e962f7c9e9607505dbf8cf7663ce869c5f1362 Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 11 Aug 2016 15:32:50 -0700 Subject: [PATCH 1/4] Separate out travis tests, one per different component to test. --- .travis.yml | 10 ++++-- test/travis-ruleset-fetch.sh | 64 +++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index c34836a420b0..59a28329f757 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,11 @@ services: before_install: - docker build -t httpse . script: - - docker run --rm -ti -v $(pwd):/opt -e FIREFOX=/$FIREFOX/firefox/firefox --privileged httpse bash -c "service miredo start && test/travis-ruleset-fetch.sh" + - docker run --rm -ti -v $(pwd):/opt -e FIREFOX=/$FIREFOX/firefox/firefox -e TEST=$TEST --privileged httpse bash -c "service miredo start && test/travis-ruleset-fetch.sh" env: - - FIREFOX=firefox-latest - - FIREFOX=firefox-esr-latest + - TEST=firefox FIREFOX=firefox-dev + - TEST=firefox FIREFOX=firefox-latest + - TEST=firefox FIREFOX=firefox-esr-latest + - TEST=chromium + - TEST=rules + - TEST=fetch diff --git a/test/travis-ruleset-fetch.sh b/test/travis-ruleset-fetch.sh index 7e330bf85350..8c48a342a932 100755 --- a/test/travis-ruleset-fetch.sh +++ b/test/travis-ruleset-fetch.sh @@ -29,43 +29,53 @@ git remote remove upstream-for-travis if ! $ONLY_RULESETS_CHANGED; then echo >&2 "Core code changes have been made." - echo >&2 "Running main test suite." - ./test/firefox.sh - ./test/chromium.sh + if [ "$TEST" == "firefox" ]; then + echo >&2 "Running firefox test suite." + ./test/firefox.sh + fi + if [ "$TEST" == "chromium" ]; then + echo >&2 "Running chromium test suite." + ./test/chromium.sh + fi fi # Only run test if something has changed. if [ "$RULESETS_CHANGED" ]; then echo >&2 "Ruleset database has changed." - echo >&2 "Performing comprehensive coverage test." - ./test/rules.sh + if [ "$TEST" == "rules" ]; then + echo >&2 "Performing comprehensive coverage test." + ./test/rules.sh + fi - echo >&2 "Testing test URLs in all changed rulesets." - # Make a list of all changed rulesets, but exclude those - # that do not exist. - for RULESET in $RULESETS_CHANGED; do - # First check if the given ruleset actually exists - if [ ! -f $RULESET ]; then - echo >&2 "Skipped $RULESET; file not found." - continue - fi - TO_BE_TESTED="$TO_BE_TESTED $RULESET" - done + if [ "$TEST" == "fetch" ]; then + echo >&2 "Testing test URLs in all changed rulesets." + + # Make a list of all changed rulesets, but exclude those + # that do not exist. + for RULESET in $RULESETS_CHANGED; do + # First check if the given ruleset actually exists + if [ ! -f $RULESET ]; then + echo >&2 "Skipped $RULESET; file not found." + continue + fi + TO_BE_TESTED="$TO_BE_TESTED $RULESET" + done - if [ "$TO_BE_TESTED" ]; then - # Do the actual test, using https-everywhere-checker. - OUTPUT_FILE=`mktemp` - trap 'rm "$OUTPUT_FILE"' EXIT - python $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE - # Unfortunately, no specific exit codes are available for connection - # failures, so we catch those with grep. - if [[ `cat $OUTPUT_FILE | grep ERROR | wc -l` -ge 1 ]]; then - echo >&2 "Test URL test failed." - exit 1 + if [ "$TO_BE_TESTED" ]; then + # Do the actual test, using https-everywhere-checker. + OUTPUT_FILE=`mktemp` + trap 'rm "$OUTPUT_FILE"' EXIT + python $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE + # Unfortunately, no specific exit codes are available for connection + # failures, so we catch those with grep. + if [[ `cat $OUTPUT_FILE | grep ERROR | wc -l` -ge 1 ]]; then + echo >&2 "Test URL test failed." + exit 1 + fi fi + echo >&2 "Test URL test succeeded." fi - echo >&2 "Test URL test succeeded." fi exit 0 From d4cfeb05eccf223bcde1f30d99b2aa931a8750c4 Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 11 Aug 2016 18:25:22 -0700 Subject: [PATCH 2/4] Make the 'docker build' step conditional based on what test you're running and if it's necessary to test at all, given what files are changed --- .travis.yml | 4 +- test/fetch.sh | 31 +++++++++++++ test/{travis-ruleset-fetch.sh => travis.sh} | 48 ++++++++------------- 3 files changed, 49 insertions(+), 34 deletions(-) create mode 100755 test/fetch.sh rename test/{travis-ruleset-fetch.sh => travis.sh} (56%) diff --git a/.travis.yml b/.travis.yml index 59a28329f757..88c2ac95a54f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,8 @@ sudo: required language: generic services: - docker -before_install: - - docker build -t httpse . script: - - docker run --rm -ti -v $(pwd):/opt -e FIREFOX=/$FIREFOX/firefox/firefox -e TEST=$TEST --privileged httpse bash -c "service miredo start && test/travis-ruleset-fetch.sh" + - test/travis.sh env: - TEST=firefox FIREFOX=firefox-dev - TEST=firefox FIREFOX=firefox-latest diff --git a/test/fetch.sh b/test/fetch.sh new file mode 100755 index 000000000000..c4e0b95e0980 --- /dev/null +++ b/test/fetch.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Run https-everywhere-checker for each changed ruleset +# This is run from test/travis.sh, but should work just as well without travis +# as long as $RULESETS_CHANGED is supplied. + +RULETESTFOLDER="test/rules" + +# Make a list of all changed rulesets, but exclude those +# that do not exist. +for RULESET in $RULESETS_CHANGED; do + # First check if the given ruleset actually exists + if [ ! -f $RULESET ]; then + echo >&2 "Skipped $RULESET; file not found." + continue + fi + TO_BE_TESTED="$TO_BE_TESTED $RULESET" +done + +if [ "$TO_BE_TESTED" ]; then + # Do the actual test, using https-everywhere-checker. + OUTPUT_FILE=`mktemp` + trap 'rm "$OUTPUT_FILE"' EXIT + python $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE + # Unfortunately, no specific exit codes are available for connection + # failures, so we catch those with grep. + if [[ `cat $OUTPUT_FILE | grep ERROR | wc -l` -ge 1 ]]; then + echo >&2 "Test URL test failed." + exit 1 + fi +fi +echo >&2 "Test URL test succeeded." diff --git a/test/travis-ruleset-fetch.sh b/test/travis.sh similarity index 56% rename from test/travis-ruleset-fetch.sh rename to test/travis.sh index 8c48a342a932..054b1ad2595c 100755 --- a/test/travis-ruleset-fetch.sh +++ b/test/travis.sh @@ -1,9 +1,12 @@ #!/bin/bash -# Run https-everywhere-checker for each changed ruleset +# Wrapper for travis tests + +function docker_build { + docker build -t httpse . +} # Folder paths, relative to parent RULESETFOLDER="src/chrome/content/rules" -RULETESTFOLDER="test/rules" # Go to git repo root; taken from ../test.sh. Note that # $GIT_DIR is .git in this case. @@ -19,23 +22,28 @@ fi # Fetch the current GitHub version of HTTPS-E to compare to its master git remote add upstream-for-travis https://github.com/EFForg/https-everywhere.git +trap 'git remote remove upstream-for-travis' EXIT git fetch upstream-for-travis master COMMON_BASE_COMMIT=$(git merge-base upstream-for-travis/master HEAD) RULESETS_CHANGED=$(git diff --name-only $COMMON_BASE_COMMIT | grep $RULESETFOLDER | grep '.xml') if [ "$(git diff --name-only $COMMON_BASE_COMMIT)" != "$RULESETS_CHANGED" ]; then ONLY_RULESETS_CHANGED=false fi -git remote remove upstream-for-travis + +# At this point, if anything fails, the test should fail +set -e if ! $ONLY_RULESETS_CHANGED; then echo >&2 "Core code changes have been made." if [ "$TEST" == "firefox" ]; then echo >&2 "Running firefox test suite." - ./test/firefox.sh + docker_build + docker run --rm -ti -v $(pwd):/opt -e FIREFOX=/$FIREFOX/firefox/firefox httpse bash -c "test/firefox.sh" fi if [ "$TEST" == "chromium" ]; then echo >&2 "Running chromium test suite." - ./test/chromium.sh + docker_build + docker run --rm -ti -v $(pwd):/opt --privileged httpse bash -c "test/chromium.sh" fi fi # Only run test if something has changed. @@ -44,37 +52,15 @@ if [ "$RULESETS_CHANGED" ]; then if [ "$TEST" == "rules" ]; then echo >&2 "Performing comprehensive coverage test." - ./test/rules.sh + docker_build + docker run --rm -ti -v $(pwd):/opt httpse bash -c "test/rules.sh" fi if [ "$TEST" == "fetch" ]; then echo >&2 "Testing test URLs in all changed rulesets." - - # Make a list of all changed rulesets, but exclude those - # that do not exist. - for RULESET in $RULESETS_CHANGED; do - # First check if the given ruleset actually exists - if [ ! -f $RULESET ]; then - echo >&2 "Skipped $RULESET; file not found." - continue - fi - TO_BE_TESTED="$TO_BE_TESTED $RULESET" - done - - if [ "$TO_BE_TESTED" ]; then - # Do the actual test, using https-everywhere-checker. - OUTPUT_FILE=`mktemp` - trap 'rm "$OUTPUT_FILE"' EXIT - python $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE - # Unfortunately, no specific exit codes are available for connection - # failures, so we catch those with grep. - if [[ `cat $OUTPUT_FILE | grep ERROR | wc -l` -ge 1 ]]; then - echo >&2 "Test URL test failed." - exit 1 - fi - fi - echo >&2 "Test URL test succeeded." + docker_build + docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" --privileged httpse bash -c "service miredo start && test/fetch.sh" fi fi From 6d3b022fbff10354c0003f378dba3028bc25766a Mon Sep 17 00:00:00 2001 From: William Budington Date: Mon, 15 Aug 2016 15:25:44 -0700 Subject: [PATCH 3/4] Updating comment --- test/fetch.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/fetch.sh b/test/fetch.sh index c4e0b95e0980..d3d0636afe4a 100755 --- a/test/fetch.sh +++ b/test/fetch.sh @@ -5,8 +5,7 @@ RULETESTFOLDER="test/rules" -# Make a list of all changed rulesets, but exclude those -# that do not exist. +# Exclude those rulesets that do not exist. for RULESET in $RULESETS_CHANGED; do # First check if the given ruleset actually exists if [ ! -f $RULESET ]; then From 95598f5931c96c28c1153da83cbac03941aaecb5 Mon Sep 17 00:00:00 2001 From: William Budington Date: Mon, 15 Aug 2016 15:33:29 -0700 Subject: [PATCH 4/4] Adding comments for why --privileged is needed in test/travis.sh --- test/travis.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/travis.sh b/test/travis.sh index 054b1ad2595c..b80a8890a231 100755 --- a/test/travis.sh +++ b/test/travis.sh @@ -43,6 +43,7 @@ if ! $ONLY_RULESETS_CHANGED; then if [ "$TEST" == "chromium" ]; then echo >&2 "Running chromium test suite." docker_build + # --privileged is required here because chromium requires kernel lxc access docker run --rm -ti -v $(pwd):/opt --privileged httpse bash -c "test/chromium.sh" fi fi @@ -60,6 +61,7 @@ if [ "$RULESETS_CHANGED" ]; then if [ "$TEST" == "fetch" ]; then echo >&2 "Testing test URLs in all changed rulesets." docker_build + # --privileged is required here for miredo to create a network tunnel docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" --privileged httpse bash -c "service miredo start && test/fetch.sh" fi fi