diff --git a/.travis.yml b/.travis.yml index 78f6ae2..fb75878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,21 @@ sudo: required env: matrix: # openssl tests - - COMP=openssl OS_TYPE=centos OS_VERSION=5 + #- COMP=openssl OS_TYPE=centos OS_VERSION=5 - COMP=openssl OS_TYPE=centos OS_VERSION=6 - COMP=openssl OS_TYPE=centos OS_VERSION=7 - - COMP=openssl OS_TYPE=fedora OS_VERSION=24 + - COMP=openssl OS_TYPE=fedora OS_VERSION=latest # nss tests - - COMP=nss OS_TYPE=centos OS_VERSION=5 + #- COMP=nss OS_TYPE=centos OS_VERSION=5 - COMP=nss OS_TYPE=centos OS_VERSION=6 - COMP=nss OS_TYPE=centos OS_VERSION=7 - - COMP=nss OS_TYPE=fedora OS_VERSION=24 + - COMP=nss OS_TYPE=fedora OS_VERSION=latest TEST_GLOB='@(renego*)' + - COMP=nss OS_TYPE=fedora OS_VERSION=latest TEST_GLOB='!(renego*)' # gnutls tests - - COMP=gnutls OS_TYPE=centos OS_VERSION=5 + #- COMP=gnutls OS_TYPE=centos OS_VERSION=5 - COMP=gnutls OS_TYPE=centos OS_VERSION=6 - COMP=gnutls OS_TYPE=centos OS_VERSION=7 - - COMP=gnutls OS_TYPE=fedora OS_VERSION=24 + - COMP=gnutls OS_TYPE=fedora OS_VERSION=latest services: - docker @@ -25,4 +26,4 @@ before_install: - sudo docker pull ${OS_TYPE}:${OS_VERSION} script: - - ./scripts/test-setup.sh ${OS_TYPE} ${OS_VERSION} ${COMP} + - ./scripts/test-setup.sh ${OS_TYPE} ${OS_VERSION} ${COMP} "${TEST_GLOB}" diff --git a/gnutls/Interoperability/TLSv1-2-with-NSS/nss-client.expect b/gnutls/Interoperability/TLSv1-2-with-NSS/nss-client.expect index 12defa5..074cb55 100755 --- a/gnutls/Interoperability/TLSv1-2-with-NSS/nss-client.expect +++ b/gnutls/Interoperability/TLSv1-2-with-NSS/nss-client.expect @@ -4,9 +4,18 @@ spawn /bin/sh -c "$argv" expect { "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue } eof { } - "subject DN" { send "GET / HTTP/1.0\r\r"; - expect -timeout 5 "HTTP/1.0 200 OK" { close; exit 0} - close; exit 1} + "subject DN" { + send "GET / HTTP/1.0\r\r"; + expect -timeout 5 "HTTP/1.0 200 OK" { + expect -re ".+" { + close; + exit 0; + } + } + + close; + exit 1; + } } set info [wait] #puts "Return from wait: $info" diff --git a/gnutls/Interoperability/TLSv1-2-with-NSS/runtest.sh b/gnutls/Interoperability/TLSv1-2-with-NSS/runtest.sh index 9d3edbf..195e30b 100755 --- a/gnutls/Interoperability/TLSv1-2-with-NSS/runtest.sh +++ b/gnutls/Interoperability/TLSv1-2-with-NSS/runtest.sh @@ -38,6 +38,12 @@ CLIENT_UTIL="/usr/lib/nss/unsupported-tools/tstclnt" [ -f /usr/lib64/nss/unsupported-tools/selfserv ] && SERVER_UTIL="/usr/lib64/nss/unsupported-tools/selfserv" [ -f /usr/lib64/nss/unsupported-tools/tstclnt ] && CLIENT_UTIL="/usr/lib64/nss/unsupported-tools/tstclnt" +# Force NSS to ignore crypto policies +# Temporary workaround for BZ#1426267 +if rlIsFedora ">=25"; then + export NSS_IGNORE_SYSTEM_POLICY=1 +fi + rlJournalStart rlPhaseStartSetup rlAssertRpm --all diff --git a/gnutls/Interoperability/TLSv1-2-with-OpenSSL/gnutls-client.expect b/gnutls/Interoperability/TLSv1-2-with-OpenSSL/gnutls-client.expect index 2ff4471..c1b4068 100644 --- a/gnutls/Interoperability/TLSv1-2-with-OpenSSL/gnutls-client.expect +++ b/gnutls/Interoperability/TLSv1-2-with-OpenSSL/gnutls-client.expect @@ -6,6 +6,7 @@ expect { send "client hello\r" expect { "server hello" { + sleep 1 close exit 0 } diff --git a/gnutls/Interoperability/TLSv1-2-with-OpenSSL/openssl-client.expect b/gnutls/Interoperability/TLSv1-2-with-OpenSSL/openssl-client.expect index 798112a..6d5d3c8 100644 --- a/gnutls/Interoperability/TLSv1-2-with-OpenSSL/openssl-client.expect +++ b/gnutls/Interoperability/TLSv1-2-with-OpenSSL/openssl-client.expect @@ -6,8 +6,10 @@ expect { send "client hello\r" expect "client hello" { expect "client hello" { - close - exit 0 + expect -re ".+" { + close + exit 0 + } } } } diff --git a/gnutls/Interoperability/renegotiation-with-NSS/Makefile b/gnutls/Interoperability/renegotiation-with-NSS/Makefile index 6651123..f91467c 100644 --- a/gnutls/Interoperability/renegotiation-with-NSS/Makefile +++ b/gnutls/Interoperability/renegotiation-with-NSS/Makefile @@ -58,5 +58,5 @@ $(METADATA): Makefile @echo "License: GPLv2" >> $(METADATA) @echo "Confidential: no" >> $(METADATA) @echo "Destructive: no" >> $(METADATA) - @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA) + @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora25" >> $(METADATA) diff --git a/gnutls/Interoperability/renegotiation-with-NSS/runtest.sh b/gnutls/Interoperability/renegotiation-with-NSS/runtest.sh index 6a03511..070e670 100755 --- a/gnutls/Interoperability/renegotiation-with-NSS/runtest.sh +++ b/gnutls/Interoperability/renegotiation-with-NSS/runtest.sh @@ -86,3 +86,4 @@ rlJournalStart rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/gnutls/Interoperability/renegotiation-with-OpenSSL/runtest.sh b/gnutls/Interoperability/renegotiation-with-OpenSSL/runtest.sh index 172a594..3aa0b65 100755 --- a/gnutls/Interoperability/renegotiation-with-OpenSSL/runtest.sh +++ b/gnutls/Interoperability/renegotiation-with-OpenSSL/runtest.sh @@ -72,3 +72,4 @@ rlJournalStart rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/gnutls/Interoperability/resumption-with-NSS/Makefile b/gnutls/Interoperability/resumption-with-NSS/Makefile index 50d75bd..afbe3c2 100644 --- a/gnutls/Interoperability/resumption-with-NSS/Makefile +++ b/gnutls/Interoperability/resumption-with-NSS/Makefile @@ -59,5 +59,5 @@ $(METADATA): Makefile @echo "License: GPLv2" >> $(METADATA) @echo "Confidential: no" >> $(METADATA) @echo "Destructive: no" >> $(METADATA) - @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA) + @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5 -Fedora25" >> $(METADATA) diff --git a/gnutls/Interoperability/resumption-with-NSS/runtest.sh b/gnutls/Interoperability/resumption-with-NSS/runtest.sh index 0be9536..f000fec 100755 --- a/gnutls/Interoperability/resumption-with-NSS/runtest.sh +++ b/gnutls/Interoperability/resumption-with-NSS/runtest.sh @@ -91,3 +91,4 @@ rlJournalStart rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/gnutls/Interoperability/resumption-with-OpenSSL/runtest.sh b/gnutls/Interoperability/resumption-with-OpenSSL/runtest.sh index 33e31b0..acdfb72 100755 --- a/gnutls/Interoperability/resumption-with-OpenSSL/runtest.sh +++ b/gnutls/Interoperability/resumption-with-OpenSSL/runtest.sh @@ -77,3 +77,4 @@ rlJournalStart rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/gnutls/Interoperability/softhsm-integration/Makefile b/gnutls/Interoperability/softhsm-integration/Makefile index 0a19e91..c109235 100644 --- a/gnutls/Interoperability/softhsm-integration/Makefile +++ b/gnutls/Interoperability/softhsm-integration/Makefile @@ -57,5 +57,5 @@ $(METADATA): Makefile @echo "License: GPLv2+" >> $(METADATA) @echo "Confidential: yes" >> $(METADATA) @echo "Destructive: no" >> $(METADATA) - @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA) + @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7" >> $(METADATA) diff --git a/gnutls/Interoperability/softhsm-integration/runtest.sh b/gnutls/Interoperability/softhsm-integration/runtest.sh index dc9b647..0db1218 100755 --- a/gnutls/Interoperability/softhsm-integration/runtest.sh +++ b/gnutls/Interoperability/softhsm-integration/runtest.sh @@ -80,3 +80,4 @@ _EOF rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/nss/Interoperability/CC-nss-with-gnutls/gnutls-client.expect b/nss/Interoperability/CC-nss-with-gnutls/gnutls-client.expect index f057852..d3434cb 100755 --- a/nss/Interoperability/CC-nss-with-gnutls/gnutls-client.expect +++ b/nss/Interoperability/CC-nss-with-gnutls/gnutls-client.expect @@ -6,8 +6,9 @@ expect { send "GET / HTTP/1.0\r\r"; expect { "Generic Web Server" { - close - exit 0 + sleep 1 + close + exit 0 } } } diff --git a/nss/Interoperability/CC-nss-with-gnutls/nss-client.expect b/nss/Interoperability/CC-nss-with-gnutls/nss-client.expect index 12defa5..074cb55 100755 --- a/nss/Interoperability/CC-nss-with-gnutls/nss-client.expect +++ b/nss/Interoperability/CC-nss-with-gnutls/nss-client.expect @@ -4,9 +4,18 @@ spawn /bin/sh -c "$argv" expect { "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue } eof { } - "subject DN" { send "GET / HTTP/1.0\r\r"; - expect -timeout 5 "HTTP/1.0 200 OK" { close; exit 0} - close; exit 1} + "subject DN" { + send "GET / HTTP/1.0\r\r"; + expect -timeout 5 "HTTP/1.0 200 OK" { + expect -re ".+" { + close; + exit 0; + } + } + + close; + exit 1; + } } set info [wait] #puts "Return from wait: $info" diff --git a/nss/Interoperability/CC-nss-with-openssl/openssl-client.expect b/nss/Interoperability/CC-nss-with-openssl/openssl-client.expect index fb20cbe..4d078ad 100644 --- a/nss/Interoperability/CC-nss-with-openssl/openssl-client.expect +++ b/nss/Interoperability/CC-nss-with-openssl/openssl-client.expect @@ -5,8 +5,10 @@ expect { "Verify return code: 0 " { send "GET / HTTP/1.0\r\r" expect "Server: Generic Web Server" { - close - exit 0 + expect -re ".+" { + close + exit 0 + } } } } diff --git a/nss/Interoperability/Interoperability-with-OpenSSL/Makefile b/nss/Interoperability/Interoperability-with-OpenSSL/Makefile index 67c4d95..7fa50ed 100644 --- a/nss/Interoperability/Interoperability-with-OpenSSL/Makefile +++ b/nss/Interoperability/Interoperability-with-OpenSSL/Makefile @@ -63,5 +63,5 @@ $(METADATA): Makefile @echo "License: GPLv2" >> $(METADATA) @echo "Confidential: no" >> $(METADATA) @echo "Destructive: no" >> $(METADATA) - @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora24" >> $(METADATA) + @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora25" >> $(METADATA) diff --git a/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/nss-client.expect b/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/nss-client.expect index 6df1d2e..a9bd07a 100755 --- a/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/nss-client.expect +++ b/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/nss-client.expect @@ -4,9 +4,14 @@ spawn /bin/sh -c "$argv" expect { "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue } eof { } - "subject DN" { expect "subject DN" { send "GET / HTTP/1.0\r\r"; - expect -timeout 5 "Verify return code"; - close}} + "subject DN" { + expect "subject DN" { + send "GET / HTTP/1.0\r\r"; + expect -timeout 5 "Verify return code"; + sleep 1; + close; + } + } } set info [wait] #puts "Return from wait: $info" diff --git a/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/runtest.sh b/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/runtest.sh index 7e6185d..53d9cba 100755 --- a/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/runtest.sh +++ b/nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/runtest.sh @@ -599,3 +599,4 @@ rlJournalStart rlPhaseEnd rlJournalPrintText rlJournalEnd +rlGetTestState diff --git a/scripts/test-runner.sh b/scripts/test-runner.sh index 8086ce7..ca35b57 100755 --- a/scripts/test-runner.sh +++ b/scripts/test-runner.sh @@ -21,6 +21,26 @@ function keep_alive() { done } +# Compare test name with glob expression using extended glob patterns +# $1 - test name +# $2 - glob pattern +function test_name_relevancy() { + # See: http://wiki.bash-hackers.org/syntax/pattern#extended_pattern_language + # Save the original state + shopt -q extglob + local STATE=$? + # Enable extglob + shopt -s extglob + [[ $1 == $2 ]] + local RES=$? + # If the extension was originally disabled, disable it + if [[ $STATE -ne 0 ]]; then + shopt -u extglob + fi + + return $RES +} + set +x if [[ $# < 3 ]]; then @@ -31,6 +51,19 @@ fi OS_TYPE="$1" OS_VERSION="$2" COMPONENT="$3" +TEST_GLOB="$4" + +# As the $OS_VERSION is used for relevancy, we need to replace the 'latest' +# version tag with the actual OS version +# So far the 'latest' tag is used only for Fedora +if [[ $OS_VERSION == "latest" ]]; then + OS_VERSION="$(awk '{ print $3; }' < /etc/fedora-release)" + if ! [[ $OS_VERSION =~ ^[0-9]+$ ]]; then + echo >&2 "FATAL: Couldn't determine OS version ($OS_VERSION)" + exit 1 + fi +fi + if [[ $OS_TYPE == "fedora" ]]; then PKG_MAN="dnf" else @@ -51,6 +84,17 @@ fi $PKG_MAN -y install net-tools coreutils gawk expect make beakerlib findutils \ procps-ng +if [[ $OS_TYPE == "fedora" ]]; then + $PKG_MAN --enablerepo updates-testing -y install beakerlib +else + $PKG_MAN --enablerepo epel-testing -y install beakerlib +fi + +# WORKAROUND: Replace all rlIsRHEL calls with rlIsCentos +if [[ $OS_TYPE == "centos" ]]; then + echo 'rlIsRHEL() { rlIsCentOS "$@"; }' >> /usr/share/beakerlib/testing.sh +fi + EC=0 SKIP=0 INDEX=0 @@ -72,12 +116,26 @@ do SKIP=0 echo "Running test: $test" + + # Check if glob pattern is set + if [[ ! -z "$TEST_GLOB" ]]; then + # If so, check if it matches current test name + TEST_NAME="$(basename $(dirname "$test"))" + if ! test_name_relevancy "$TEST_NAME" "$TEST_GLOB"; then + echo "Test '$TEST_NAME' excluded by given glob expression: $TEST_GLOB" + SKIPPED+=("$test") + continue + fi + fi + + # Makefile is necessary for test execution pushd "$(dirname "$test")" if [[ ! -f Makefile ]]; then echo >&2 "Missing Makefile" EC=1 SKIP=1 fi + if [[ $SKIP -eq 0 ]]; then # Check relevancy if relevancy.awk -v os_type=$OS_TYPE -v os_ver=$OS_VERSION Makefile; then diff --git a/scripts/test-setup.sh b/scripts/test-setup.sh index 8d0d4da..0b47897 100755 --- a/scripts/test-setup.sh +++ b/scripts/test-setup.sh @@ -8,10 +8,28 @@ fi OS_TYPE="$1" OS_VERSION="$2" COMPONENT="$3" +TEST_GLOB="$4" CONT_NAME="${OS_TYPE}-${OS_VERSION}-${COMPONENT}" CERTGEN_REPO="https://github.com/redhat-qe-security/certgen" CERTGEN_PATH="openssl/Library/certgen" +# Test sanity check +# Check if all tests have rlGetTestState at their end +FAILED_CHECKS=0 +FAILED_NAMES=() +while read file; do + if ! grep -Pzq "rlGetTestState[[:space:]]*\z" "$file"; then + FAILED_CHECKS=$(($FAILED_CHECKS+1)) + FAILED_NAMES+=("$file") + fi +done <<< "$(find . -type f -name "runtest.sh")" + +if [[ $FAILED_CHECKS -gt 0 ]]; then + echo "Following tests are missing rlGetTestState command:" + printf '%s\n' "${FAILED_NAMES[@]}" + exit 1 +fi + # Prepare necessary libraries # openssl/certgen: TMP_DIR="$(mktemp -d tmp.XXXXX)" @@ -33,4 +51,4 @@ sudo docker run --rm --name "$CONT_NAME" \ -v $PWD:/workspace:rw \ ${OS_TYPE}:${OS_VERSION} \ /bin/bash -c \ - "bash -x $RUNNER $OS_TYPE $OS_VERSION $COMPONENT" + "bash -x $RUNNER $OS_TYPE $OS_VERSION $COMPONENT '$TEST_GLOB'"