SWIFT-17: add evergreen linux support (#241)

- update evergreen config
  - install swift using swiftenv
  - add support for skipping tests by topology
  - add `find-cmake.sh` script
  - use patrick's new numeric helpers to avoid topology issues
  - ensure all tests of a commands response use `doubleValue` for `ok`
  - skip command monitoring tests on sharded topologies
  - reduce to a single mongos if multiple are provided
  - add 4.2 and 5.0 variants
  - disable macos tests until we can figure out how install without
    root permissions

Author: mbroadst

.evergreen/config.yml

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+find_cmake ()
+{
+  if [ ! -z "$CMAKE" ]; then
+    return 0
+  elif [ -f "/Applications/cmake-3.2.2-Darwin-x86_64/CMake.app/Contents/bin/cmake" ]; then
+    CMAKE="/Applications/cmake-3.2.2-Darwin-x86_64/CMake.app/Contents/bin/cmake"
+  elif [ -f "/Applications/Cmake.app/Contents/bin/cmake" ]; then
+    CMAKE="/Applications/Cmake.app/Contents/bin/cmake"
+  elif [ -f "/opt/cmake/bin/cmake" ]; then
+    CMAKE="/opt/cmake/bin/cmake"
+  elif command -v cmake 2>/dev/null; then
+     CMAKE=cmake
+  elif uname -a | grep -iq 'x86_64 GNU/Linux'; then
+     curl --retry 5 https://cmake.org/files/v3.11/cmake-3.11.0-Linux-x86_64.tar.gz -sS --max-time 120 --fail --output cmake.tar.gz
+     mkdir cmake-3.11.0
+     tar xzf cmake.tar.gz -C cmake-3.11.0 --strip-components=1
+     CMAKE=$(pwd)/cmake-3.11.0/bin/cmake
+  else
+     # Some images have no cmake yet, BUILD-4922.
+     echo "-- MAKE CMAKE --"
+     curl --retry 5 https://cmake.org/files/v3.11/cmake-3.11.0.tar.gz -sS --max-time 120 --fail --output cmake.tar.gz
+     tar xzf cmake.tar.gz
+     cd cmake-3.11.0
+     ./bootstrap --prefix="${INSTALL_DIR}"
+     make -j8
+     make install
+     cd ..
+     CMAKE="${INSTALL_DIR}/bin/cmake"
+     echo "-- DONE MAKING CMAKE --"
+  fi
+}
+
+find_cmake

.evergreen/find-cmake.sh

@@ -2,38 +2,33 @@
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 
-DIR=$(dirname $0)
-# Functions to fetch MongoDB binaries
-. $DIR/download-mongodb.sh
-OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-
-get_distro
-
-# See .evergreen/download-mongodb.sh for most possible values
-case "$DISTRO" in
-   cygwin*)
-      echo "Install Windows dependencies"
-      ;;
-
-   darwin*)
-      echo "Install macOS dependencies"
-      ;;
-
-   linux-rhel*)
-      echo "Install RHEL dependencies"
-      ;;
-
-   linux-ubuntu*)
-      echo "Install Ubuntu dependencies"
-      # sudo apt-get install -y awscli || true
-      ;;
-
-   sunos*)
-      echo "Install Solaris dependencies"
-      # sudo /opt/csw/bin/pkgutil -y -i sasl_dev || true
-      ;;
-
-   *)
-      echo "All other platforms..."
-      ;;
-esac
+# variables
+PROJECT_DIRECTORY=${PROJECT_DIRECTORY:-$PWD}
+SWIFT_VERSION=${SWIFT_VERSION:-4.2}
+INSTALL_DIR="${PROJECT_DIRECTORY}/opt"
+BUILD_DIR="${PROJECT_DIRECTORY}/libmongoc-build"
+EVG_DIR=$(dirname $0)
+
+export SWIFTENV_ROOT="${INSTALL_DIR}/swiftenv"
+export PATH=/opt/cmake/bin:${SWIFTENV_ROOT}/bin:$PATH
+
+# should be set by EVG eventuallty
+LIBMONGOC_VERSION="r1.13"
+
+# find cmake and set the path to it in $CMAKE
+. $EVG_DIR/find-cmake.sh
+
+# install libmongoc
+git clone --depth 1 -b "${LIBMONGOC_VERSION}" https://github.com/mongodb/mongo-c-driver "${BUILD_DIR}"
+cd "${BUILD_DIR}"
+$CMAKE -DCMAKE_INSTALL_PREFIX:PATH="${INSTALL_DIR}"
+make -j8 install
+cd "${PROJECT_DIRECTORY}"
+
+# install swiftenv
+git clone --depth 1 https://github.com/kylef/swiftenv.git "${SWIFTENV_ROOT}"
+
+# install swift
+
+eval "$(swiftenv init -)"
+swiftenv install $SWIFT_VERSION

.evergreen/install-dependencies.sh

@@ -2,28 +2,25 @@
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 
-# Supported/used environment variables:
-#       AUTH                    Set to enable authentication. Defaults to "noauth"
-#       SSL                     Set to enable SSL. Defaults to "nossl"
-#       MONGODB_URI             Set the suggested connection MONGODB_URI (including credentials and topology info)
-#       MARCH                   Machine Architecture. Defaults to lowercase uname -m
-
-AUTH=${AUTH:-noauth}
-SSL=${SSL:-nossl}
-MONGODB_URI=${MONGODB_URI:-}
-
+# variables
+PROJECT_DIRECTORY=${PROJECT_DIRECTORY:-$PWD}
+MONGODB_URI=${MONGODB_URI:-"NO_URI_PROVIDED"}
+SWIFT_VERSION=${SWIFT_VERSION:-4.2}
+INSTALL_DIR="${PROJECT_DIRECTORY}/opt"
+TOPOLOGY=${TOPOLOGY:-single}
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-[ -z "$MARCH" ] && MARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
 
-if [ "$AUTH" != "noauth" ]; then
-  export MONGOC_TEST_USER="bob"
-  export MONGOC_TEST_PASSWORD="pwd123"
-fi
+# enable swiftenv
+export SWIFTENV_ROOT="${INSTALL_DIR}/swiftenv"
+export PATH="${SWIFTENV_ROOT}/bin:$PATH"
+eval "$(swiftenv init -)"
+
+# switch swift version, and run tests
+export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig"
 
-if [ "$SSL" != "nossl" ]; then
-   export MONGOC_TEST_SSL_PEM_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
-   export MONGOC_TEST_SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
-fi
+# override where we look for libmongoc
+export LD_LIBRARY_PATH="${INSTALL_DIR}/lib"
+export DYLD_LIBRARY_PATH="${INSTALL_DIR}/lib"
 
-echo "Running $AUTH tests over $SSL, connecting to $MONGODB_URI"
-make test
+swiftenv local $SWIFT_VERSION
+MONGODB_TOPOLOGY=${TOPOLOGY} MONGODB_URI=$MONGODB_URI make test

.evergreen/run-tests.sh

@@ -10,6 +10,11 @@ final class CommandMonitoringTests: MongoSwiftTestCase {
     }
 
     func testCommandMonitoring() throws {
+        guard MongoSwiftTestCase.topologyType != .sharded else {
+            print("Skipping test case because of unsupported topology type \(MongoSwiftTestCase.topologyType)")
+            return
+        }
+
         let decoder = BSONDecoder()
         let client = try MongoClient(options: ClientOptions(eventMonitoring: true))
         client.enableMonitoring(forEvents: .commandMonitoring)

Tests/MongoSwiftTests/CommandMonitoringTests.swift

@@ -312,7 +312,9 @@ final class MongoCollectionTests: MongoSwiftTestCase {
     func testDropIndexByModel() throws {
         let model = IndexModel(keys: ["cat": 1])
         expect(try self.coll.createIndex(model)).to(equal("cat_1"))
-        expect(try self.coll.dropIndex(model)["ok"]).to(bsonEqual(1.0))
+
+        let res = try self.coll.dropIndex(model)
+        expect((res["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // now there should only be _id_ left
         let indexes = try coll.listIndexes()
@@ -324,7 +326,9 @@ final class MongoCollectionTests: MongoSwiftTestCase {
     func testDropIndexByKeys() throws {
         let model = IndexModel(keys: ["cat": 1])
         expect(try self.coll.createIndex(model)).to(equal("cat_1"))
-        expect(try self.coll.dropIndex(["cat": 1])["ok"]).to(bsonEqual(1.0))
+
+        let res = try self.coll.dropIndex(["cat": 1])
+        expect((res["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // now there should only be _id_ left
         let indexes = try coll.listIndexes()
@@ -336,7 +340,9 @@ final class MongoCollectionTests: MongoSwiftTestCase {
     func testDropAllIndexes() throws {
         let model = IndexModel(keys: ["cat": 1])
         expect(try self.coll.createIndex(model)).to(equal("cat_1"))
-        expect(try self.coll.dropIndexes()["ok"]).to(bsonEqual(1.0))
+
+        let res = try self.coll.dropIndexes()
+        expect((res["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // now there should only be _id_ left
         let indexes = try coll.listIndexes()

Tests/MongoSwiftTests/MongoCollectionTests.swift

@@ -12,7 +12,8 @@ final class MongoDatabaseTests: MongoSwiftTestCase {
         let db = client.db(type(of: self).testDatabase)
 
         let command: Document = ["create": self.getCollectionName(suffix: "1")]
-        expect(try db.runCommand(command)).to(equal(["ok": 1.0]))
+        let res = try db.runCommand(command)
+        expect((res["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
         expect(try (Array(db.listCollections()) as [Document]).count).to(equal(1))
 
         // create collection using createCollection

Tests/MongoSwiftTests/MongoDatabaseTests.swift

@@ -253,12 +253,12 @@ final class ReadWriteConcernTests: MongoSwiftTestCase {
         // run command with a valid readConcern
         let options1 = RunCommandOptions(readConcern: ReadConcern(.local))
         let res1 = try db.runCommand(command, options: options1)
-        expect(res1["ok"]).to(bsonEqual(1.0))
+        expect((res1["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // run command with an empty readConcern
         let options2 = RunCommandOptions(readConcern: ReadConcern())
         let res2 = try db.runCommand(command, options: options2)
-        expect(res2["ok"]).to(bsonEqual(1.0))
+        expect((res2["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // running command with an invalid RC level should throw
         let options3 = RunCommandOptions(readConcern: ReadConcern("blah"))
@@ -299,12 +299,12 @@ final class ReadWriteConcernTests: MongoSwiftTestCase {
         // run command with a valid writeConcern
         let options1 = RunCommandOptions(writeConcern: wc1)
         let res1 = try db.runCommand(command, options: options1)
-        expect(res1["ok"]).to(bsonEqual(1.0))
+        expect((res1["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         // run command with an empty writeConcern
         let options2 = RunCommandOptions(writeConcern: wc2)
         let res2 = try db.runCommand(command, options: options2)
-        expect(res2["ok"]).to(bsonEqual(1.0))
+        expect((res2["ok"] as? BSONNumber)?.doubleValue).to(bsonEqual(1.0))
 
         expect(try coll.insertOne(nextDoc(), options: InsertOneOptions(writeConcern: wc1))).toNot(throwError())
         expect(try coll.insertOne(nextDoc(), options: InsertOneOptions(writeConcern: wc3))).toNot(throwError())

Tests/MongoSwiftTests/ReadWriteConcernTests.swift

@@ -28,6 +28,11 @@ final class SDAMTests: MongoSwiftTestCase {
     // https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/tests/monitoring/standalone.json
     // swiftlint:enable line_length
     func testMonitoring() throws {
+        guard MongoSwiftTestCase.topologyType == .single else {
+            print("Skipping test case because of unsupported topology type \(MongoSwiftTestCase.topologyType)")
+            return
+        }
+
         let client = try MongoClient(options: ClientOptions(eventMonitoring: true))
         client.enableMonitoring(forEvents: .serverMonitoring)
 

Tests/MongoSwiftTests/SDAMMonitoringTests.swift

@@ -1,4 +1,5 @@
 import Foundation
+import mongoc
 @testable import MongoSwift
 import Nimble
 import XCTest
@@ -37,10 +38,31 @@ class MongoSwiftTestCase: XCTestCase {
     /// If the environment variable does not exist, this will use a default of "mongodb://127.0.0.1/".
     static var connStr: String {
         if let connStr = ProcessInfo.processInfo.environment["MONGODB_URI"] {
+            if self.topologyType == .sharded {
+                guard let uri = mongoc_uri_new(connStr) else {
+                    return connStr
+                }
+
+                defer {
+                    mongoc_uri_destroy(uri)
+                }
+
+                guard let hosts = mongoc_uri_get_hosts(uri) else {
+                    return connStr
+                }
+
+                let hostAndPort = withUnsafeBytes(of: hosts.pointee.host_and_port) { rawPtr -> String in
+                    let ptr = rawPtr.baseAddress!.assumingMemoryBound(to: CChar.self)
+                    return String(cString: ptr)
+                }
+
+                return "mongodb://\(hostAndPort)/"
+            }
+
             return connStr
-        } else {
-            return "mongodb://127.0.0.1/"
         }
+
+        return "mongodb://127.0.0.1/"
     }
 
     // indicates whether we are running on a 32-bit platform
@@ -55,6 +77,21 @@ class MongoSwiftTestCase: XCTestCase {
         }
         return name.replacingOccurrences(of: "[ \\+\\$]", with: "_", options: [.regularExpression])
     }
+
+    static var topologyType: TopologyDescription.TopologyType {
+        guard let topology = ProcessInfo.processInfo.environment["MONGODB_TOPOLOGY"] else {
+            return .single
+        }
+
+        switch topology {
+        case "sharded_cluster":
+            return .sharded
+        case "replica_set":
+            return .replicaSetWithPrimary
+        default:
+            return .single
+        }
+    }
 }
 
 extension MongoClient {