Beakerlib fix

.travis.yml

@@ -3,20 +3,21 @@ sudo: required
 env:
     matrix:
         # openssl tests
-        - COMP=openssl OS_TYPE=centos OS_VERSION=5
+        #- COMP=openssl OS_TYPE=centos OS_VERSION=5
         - COMP=openssl OS_TYPE=centos OS_VERSION=6
         - COMP=openssl OS_TYPE=centos OS_VERSION=7
-        - COMP=openssl OS_TYPE=fedora OS_VERSION=24
+        - COMP=openssl OS_TYPE=fedora OS_VERSION=latest
         # nss tests
-        - COMP=nss OS_TYPE=centos OS_VERSION=5
+        #- COMP=nss OS_TYPE=centos OS_VERSION=5
         - COMP=nss OS_TYPE=centos OS_VERSION=6
         - COMP=nss OS_TYPE=centos OS_VERSION=7
-        - COMP=nss OS_TYPE=fedora OS_VERSION=24
+        - COMP=nss OS_TYPE=fedora OS_VERSION=latest TEST_GLOB='@(renego*)'
+        - COMP=nss OS_TYPE=fedora OS_VERSION=latest TEST_GLOB='!(renego*)'
         # gnutls tests
-        - COMP=gnutls OS_TYPE=centos OS_VERSION=5
+        #- COMP=gnutls OS_TYPE=centos OS_VERSION=5
         - COMP=gnutls OS_TYPE=centos OS_VERSION=6
         - COMP=gnutls OS_TYPE=centos OS_VERSION=7
-        - COMP=gnutls OS_TYPE=fedora OS_VERSION=24
+        - COMP=gnutls OS_TYPE=fedora OS_VERSION=latest
 
 services:
     - docker
@@ -25,4 +26,4 @@ before_install:
     - sudo docker pull ${OS_TYPE}:${OS_VERSION}
 
 script:
-    - ./scripts/test-setup.sh ${OS_TYPE} ${OS_VERSION} ${COMP}
+    - ./scripts/test-setup.sh ${OS_TYPE} ${OS_VERSION} ${COMP} "${TEST_GLOB}"

gnutls/Interoperability/TLSv1-2-with-NSS/nss-client.expect

@@ -4,9 +4,18 @@ spawn /bin/sh -c "$argv"
 expect {
     "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue }
     eof { }
-    "subject DN" { send "GET / HTTP/1.0\r\r"; 
-                   expect -timeout 5 "HTTP/1.0 200 OK" { close; exit 0}
-                   close; exit 1}
+    "subject DN" {
+        send "GET / HTTP/1.0\r\r";
+        expect -timeout 5 "HTTP/1.0 200 OK" {
+            expect -re ".+" {
+                close;
+                exit 0;
+            }
+        }
+
+        close;
+        exit 1;
+    }
 }
 set info [wait]
 #puts "Return from wait: $info"

gnutls/Interoperability/TLSv1-2-with-NSS/runtest.sh

@@ -38,6 +38,12 @@ CLIENT_UTIL="/usr/lib/nss/unsupported-tools/tstclnt"
 [ -f /usr/lib64/nss/unsupported-tools/selfserv ] && SERVER_UTIL="/usr/lib64/nss/unsupported-tools/selfserv"
 [ -f /usr/lib64/nss/unsupported-tools/tstclnt ] && CLIENT_UTIL="/usr/lib64/nss/unsupported-tools/tstclnt"
 
+# Force NSS to ignore crypto policies
+# Temporary workaround for BZ#1426267
+if rlIsFedora ">=25"; then
+    export NSS_IGNORE_SYSTEM_POLICY=1
+fi
+
 rlJournalStart
     rlPhaseStartSetup
         rlAssertRpm --all

gnutls/Interoperability/TLSv1-2-with-OpenSSL/gnutls-client.expect

@@ -6,6 +6,7 @@ expect {
         send "client hello\r"
         expect {
             "server hello" {
+                    sleep 1
                     close
                     exit 0
             }

gnutls/Interoperability/TLSv1-2-with-OpenSSL/openssl-client.expect

@@ -6,8 +6,10 @@ expect {
         send "client hello\r"
         expect "client hello" {
             expect "client hello" {
-                close
-                exit 0
+                expect -re ".+" {
+                    close
+                    exit 0
+                }
             }
         }
     }

gnutls/Interoperability/renegotiation-with-NSS/Makefile

@@ -58,5 +58,5 @@ $(METADATA): Makefile
 	@echo "License:         GPLv2" >> $(METADATA)
 	@echo "Confidential:    no" >> $(METADATA)
 	@echo "Destructive:     no" >> $(METADATA)
-	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA)
+	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora25" >> $(METADATA)
 

gnutls/Interoperability/renegotiation-with-NSS/runtest.sh

@@ -86,3 +86,4 @@ rlJournalStart
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

gnutls/Interoperability/renegotiation-with-OpenSSL/runtest.sh

@@ -72,3 +72,4 @@ rlJournalStart
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

gnutls/Interoperability/resumption-with-NSS/Makefile

@@ -59,5 +59,5 @@ $(METADATA): Makefile
 	@echo "License:         GPLv2" >> $(METADATA)
 	@echo "Confidential:    no" >> $(METADATA)
 	@echo "Destructive:     no" >> $(METADATA)
-	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA)
+	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -Fedora25" >> $(METADATA)
 

gnutls/Interoperability/resumption-with-NSS/runtest.sh

@@ -91,3 +91,4 @@ rlJournalStart
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

gnutls/Interoperability/resumption-with-OpenSSL/runtest.sh

@@ -77,3 +77,4 @@ rlJournalStart
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

gnutls/Interoperability/softhsm-integration/Makefile

@@ -57,5 +57,5 @@ $(METADATA): Makefile
 	@echo "License:         GPLv2+" >> $(METADATA)
 	@echo "Confidential:    yes" >> $(METADATA)
 	@echo "Destructive:     no" >> $(METADATA)
-	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA)
+	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7" >> $(METADATA)
 

gnutls/Interoperability/softhsm-integration/runtest.sh

@@ -80,3 +80,4 @@ _EOF
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

nss/Interoperability/CC-nss-with-gnutls/gnutls-client.expect

@@ -6,8 +6,9 @@ expect {
         send "GET / HTTP/1.0\r\r";
         expect {
             "Generic Web Server" {
-                    close
-                    exit 0
+                sleep 1
+                close
+                exit 0
             }
         }
     }

nss/Interoperability/CC-nss-with-gnutls/nss-client.expect

@@ -4,9 +4,18 @@ spawn /bin/sh -c "$argv"
 expect {
     "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue }
     eof { }
-    "subject DN" { send "GET / HTTP/1.0\r\r"; 
-                   expect -timeout 5 "HTTP/1.0 200 OK" { close; exit 0}
-                   close; exit 1}
+    "subject DN" {
+        send "GET / HTTP/1.0\r\r";
+        expect -timeout 5 "HTTP/1.0 200 OK" {
+            expect -re ".+" {
+                close;
+                exit 0;
+            }
+        }
+
+        close;
+        exit 1;
+    }
 }
 set info [wait]
 #puts "Return from wait: $info"

nss/Interoperability/CC-nss-with-openssl/openssl-client.expect

@@ -5,8 +5,10 @@ expect {
     "Verify return code: 0 " {
         send "GET / HTTP/1.0\r\r"
         expect "Server: Generic Web Server" {
-            close
-            exit 0
+            expect -re ".+" {
+                close
+                exit 0
+            }
         }
     }
 }

nss/Interoperability/Interoperability-with-OpenSSL/Makefile

@@ -63,5 +63,5 @@ $(METADATA): Makefile
 	@echo "License:         GPLv2" >> $(METADATA)
 	@echo "Confidential:    no" >> $(METADATA)
 	@echo "Destructive:     no" >> $(METADATA)
-	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora24" >> $(METADATA)
+	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7 -Fedora25" >> $(METADATA)
 

nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/nss-client.expect

@@ -4,9 +4,14 @@ spawn /bin/sh -c "$argv"
 expect {
     "Enter Password" { send "RedHatEnterpriseLinux6.6\r"; exp_continue }
     eof { }
-    "subject DN" { expect "subject DN" { send "GET / HTTP/1.0\r\r";
-                                         expect -timeout 5 "Verify return code";
-                                         close}}
+    "subject DN" {
+        expect "subject DN" {
+            send "GET / HTTP/1.0\r\r";
+            expect -timeout 5 "Verify return code";
+            sleep 1;
+            close;
+        }
+    }
 }
 set info [wait]
 #puts "Return from wait: $info"

nss/Interoperability/renego-and-resumption-NSS-with-OpenSSL/runtest.sh

@@ -599,3 +599,4 @@ rlJournalStart
     rlPhaseEnd
 rlJournalPrintText
 rlJournalEnd
+rlGetTestState

scripts/test-runner.sh

@@ -21,6 +21,26 @@ function keep_alive() {
     done
 }
 
+# Compare test name with glob expression using extended glob patterns
+# $1 - test name
+# $2 - glob pattern
+function test_name_relevancy() {
+    # See: http://wiki.bash-hackers.org/syntax/pattern#extended_pattern_language
+    # Save the original state
+    shopt -q extglob
+    local STATE=$?
+    # Enable extglob
+    shopt -s extglob
+    [[ $1 == $2 ]]
+    local RES=$?
+    # If the extension was originally disabled, disable it
+    if [[ $STATE -ne 0 ]]; then
+        shopt -u extglob
+    fi
+
+    return $RES
+}
+
 set +x
 
 if [[ $# < 3 ]]; then
@@ -31,6 +51,19 @@ fi
 OS_TYPE="$1"
 OS_VERSION="$2"
 COMPONENT="$3"
+TEST_GLOB="$4"
+
+# As the $OS_VERSION is used for relevancy, we need to replace the 'latest'
+# version tag with the actual OS version
+# So far the 'latest' tag is used only for Fedora
+if [[ $OS_VERSION == "latest" ]]; then
+    OS_VERSION="$(awk '{ print $3; }' < /etc/fedora-release)"
+    if ! [[ $OS_VERSION =~ ^[0-9]+$ ]]; then
+        echo >&2 "FATAL: Couldn't determine OS version ($OS_VERSION)"
+        exit 1
+    fi
+fi
+
 if [[ $OS_TYPE == "fedora" ]]; then
     PKG_MAN="dnf"
 else
@@ -51,6 +84,17 @@ fi
 $PKG_MAN -y install net-tools coreutils gawk expect make beakerlib findutils \
                     procps-ng
 
+if [[ $OS_TYPE == "fedora" ]]; then
+    $PKG_MAN --enablerepo updates-testing -y install beakerlib
+else
+    $PKG_MAN --enablerepo epel-testing -y install beakerlib
+fi
+
+# WORKAROUND: Replace all rlIsRHEL calls with rlIsCentos
+if [[ $OS_TYPE == "centos" ]]; then
+    echo 'rlIsRHEL() { rlIsCentOS "$@"; }' >> /usr/share/beakerlib/testing.sh
+fi
+
 EC=0
 SKIP=0
 INDEX=0
@@ -72,12 +116,26 @@ do
     SKIP=0
 
     echo "Running test: $test"
+
+    # Check if glob pattern is set
+    if [[ ! -z "$TEST_GLOB" ]]; then
+        # If so, check if it matches current test name
+        TEST_NAME="$(basename $(dirname "$test"))"
+        if ! test_name_relevancy "$TEST_NAME" "$TEST_GLOB"; then
+            echo "Test '$TEST_NAME' excluded by given glob expression: $TEST_GLOB"
+            SKIPPED+=("$test")
+            continue
+        fi
+    fi
+
+    # Makefile is necessary for test execution
     pushd "$(dirname "$test")"
     if [[ ! -f Makefile ]]; then
         echo >&2 "Missing Makefile"
         EC=1
         SKIP=1
     fi
+
     if [[ $SKIP -eq 0 ]]; then
         # Check relevancy
         if relevancy.awk -v os_type=$OS_TYPE -v os_ver=$OS_VERSION Makefile; then

scripts/test-setup.sh

@@ -8,10 +8,28 @@ fi
 OS_TYPE="$1"
 OS_VERSION="$2"
 COMPONENT="$3"
+TEST_GLOB="$4"
 CONT_NAME="${OS_TYPE}-${OS_VERSION}-${COMPONENT}"
 CERTGEN_REPO="https://github.com/redhat-qe-security/certgen"
 CERTGEN_PATH="openssl/Library/certgen"
 
+# Test sanity check
+# Check if all tests have rlGetTestState at their end
+FAILED_CHECKS=0
+FAILED_NAMES=()
+while read file; do
+    if ! grep -Pzq "rlGetTestState[[:space:]]*\z" "$file"; then
+        FAILED_CHECKS=$(($FAILED_CHECKS+1))
+        FAILED_NAMES+=("$file")
+    fi
+done <<< "$(find . -type f -name "runtest.sh")"
+
+if [[ $FAILED_CHECKS -gt 0 ]]; then
+    echo "Following tests are missing rlGetTestState command:"
+    printf '%s\n' "${FAILED_NAMES[@]}"
+    exit 1
+fi
+
 # Prepare necessary libraries
 # openssl/certgen:
 TMP_DIR="$(mktemp -d tmp.XXXXX)"
@@ -33,4 +51,4 @@ sudo docker run --rm --name "$CONT_NAME" \
                 -v $PWD:/workspace:rw \
                 ${OS_TYPE}:${OS_VERSION} \
                 /bin/bash -c \
-                "bash -x $RUNNER $OS_TYPE $OS_VERSION $COMPONENT"
+                "bash -x $RUNNER $OS_TYPE $OS_VERSION $COMPONENT '$TEST_GLOB'"