Skip to content

Commit 4e672bd

Browse files
blink1073blink1073
authored
PYTHON-4540 Convert libmongocrypt download to python (#2148)
1 parent 3e29671 commit 4e672bd

File tree

7 files changed

+85
-67
lines changed

7 files changed

+85
-67
lines changed

‎.evergreen/run-azurekms-fail-test.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,8 @@
22
set -o errexit # Exit the script with error if any of the commands fail
33
HERE=$(dirname ${BASH_SOURCE:-$0})
44
. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
5-
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
65
SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh
7-
PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
8-
KEY_NAME="${AZUREKMS_KEYNAME}" \
6+
KEY_NAME="${AZUREKMS_KEYNAME}" \
97
KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
108
$HERE/just.sh test-eg
119
bash $HERE/scripts/teardown-tests.sh

‎.evergreen/run-azurekms-test.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ echo "Copying files ... begin"
66
export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
77
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
88
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
9-
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
109
# Set up the remote files to test.
1110
git add .
1211
git commit -m "add files" || true
@@ -20,7 +19,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
2019
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
2120
echo "Untarring file ... end"
2221
echo "Running test ... begin"
23-
AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \
22+
AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash .evergreen/just.sh setup-test" \
2423
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
2524
AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
2625
$DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh

‎.evergreen/run-gcpkms-test.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
88
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
99
export GCPKMS_ZONE=${GCPKMS_ZONE}
1010
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
11-
LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
1211
# Set up the remote files to test.
1312
git add .
1413
git commit -m "add files" || true
@@ -19,7 +18,7 @@ echo "Untarring file ... begin"
1918
GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
2019
echo "Untarring file ... end"
2120
echo "Running test ... begin"
22-
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
21+
GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
2322
GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
2423
echo "Running test ... end"
2524
bash $HERE/scripts/teardown-tests.sh

‎.evergreen/scripts/run-gcpkms-fail-test.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,5 @@
22
set -eu
33
HERE=$(dirname ${BASH_SOURCE:-$0})
44
. $HERE/env.sh
5-
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
6-
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
75
SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh
86
bash ./.evergreen/just.sh test-eg

‎.evergreen/scripts/setup-libmongocrypt.sh‎

Lines changed: 0 additions & 52 deletions
This file was deleted.

‎.evergreen/scripts/setup-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,4 @@ fi
5858

5959
. $ROOT_DIR/.evergreen/utils.sh
6060
PYTHON=${PYTHON_BINARY:-$(find_python3)}
61-
$PYTHON $SCRIPT_DIR/setup-tests.py
61+
$PYTHON $SCRIPT_DIR/setup_tests.py

‎.evergreen/scripts/setup-tests.py‎ renamed to ‎.evergreen/scripts/setup_tests.py‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
11
from __future__ import annotations
22

33
import base64
4+
import dataclasses
5+
import io
46
import logging
57
import os
68
import platform
79
import shlex
10+
import shutil
811
import stat
912
import subprocess
1013
import sys
14+
import tarfile
1115
from pathlib import Path
1216
from typing import Any
17+
from urllib import request
1318

1419
HERE = Path(__file__).absolute().parent
1520
ROOT = HERE.parent.parent
1621
ENV_FILE = HERE / "test-env.sh"
1722
DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
23+
PLATFORM = "windows" if os.name == "nt" else sys.platform
1824

1925
logging.basicConfig(level=logging.INFO)
2026
LOGGER = logging.getLogger(__name__)
@@ -74,6 +80,13 @@
7480
)
7581

7682

83+
@dataclasses.dataclass
84+
class Distro:
85+
name: str
86+
version_id: str
87+
arch: str
88+
89+
7790
def write_env(name: str, value: Any) -> None:
7891
with ENV_FILE.open("a", newline="\n") as fid:
7992
# Remove any existing quote chars.
@@ -92,6 +105,69 @@ def run_command(cmd: str) -> None:
92105
LOGGER.info("Running command %s... done.", cmd)
93106

94107

108+
def get_distro() -> Distro:
109+
name = ""
110+
version_id = ""
111+
arch = platform.machine()
112+
with open("/etc/os-release") as fid:
113+
for line in fid.readlines():
114+
line = line.replace('"', "") # noqa: PLW2901
115+
if line.startswith("NAME="):
116+
_, _, name = line.strip().partition("=")
117+
if line.startswith("VERSION_ID="):
118+
_, _, version_id = line.strip().partition("=")
119+
return Distro(name=name, version_id=version_id, arch=arch)
120+
121+
122+
def setup_libmongocrypt():
123+
target = ""
124+
if PLATFORM == "windows":
125+
# PYTHON-2808 Ensure this machine has the CA cert for google KMS.
126+
if is_set("TEST_FLE_GCP_AUTO"):
127+
run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"')
128+
target = "windows-test"
129+
130+
elif PLATFORM == "darwin":
131+
target = "macos"
132+
133+
else:
134+
distro = get_distro()
135+
if distro.name.startswith("Debian"):
136+
target = f"debian{distro.version_id}"
137+
elif distro.name.startswith("Red Hat"):
138+
if distro.version_id.startswith("7"):
139+
target = "rhel-70-64-bit"
140+
elif distro.version_id.startswith("8"):
141+
if distro.arch == "aarch64":
142+
target = "rhel-82-arm64"
143+
else:
144+
target = "rhel-80-64-bit"
145+
146+
if not is_set("LIBMONGOCRYPT_URL"):
147+
if not target:
148+
raise ValueError("Cannot find libmongocrypt target for current platform!")
149+
url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz"
150+
else:
151+
url = os.environ["LIBMONGOCRYPT_URL"]
152+
153+
shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True)
154+
155+
LOGGER.info(f"Fetching {url}...")
156+
with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310
157+
if response.status == 200:
158+
fileobj = io.BytesIO(response.read())
159+
with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid:
160+
fid.extractall(Path.cwd() / "libmongocrypt")
161+
LOGGER.info(f"Fetching {url}... done.")
162+
163+
run_command("ls -la libmongocrypt")
164+
run_command("ls -la libmongocrypt/nocrypto")
165+
166+
if PLATFORM == "windows":
167+
# libmongocrypt's windows dll is not marked executable.
168+
run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
169+
170+
95171
def handle_test_env() -> None:
96172
AUTH = os.environ.get("AUTH", "noauth")
97173
SSL = os.environ.get("SSL", "nossl")
@@ -156,7 +232,7 @@ def handle_test_env() -> None:
156232
write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
157233

158234
if is_set("TEST_ENTERPRISE_AUTH"):
159-
if os.name == "nt":
235+
if PLATFORM == "windows":
160236
LOGGER.info("Setting GSSAPI_PASS")
161237
write_env("GSSAPI_PASS", os.environ["SASL_PASS"])
162238
write_env("GSSAPI_CANONICALIZE", "true")
@@ -214,19 +290,19 @@ def handle_test_env() -> None:
214290
if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
215291
# Check for libmongocrypt download.
216292
if not (ROOT / "libmongocrypt").exists():
217-
run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh")
293+
setup_libmongocrypt()
218294

219295
# TODO: Test with 'pip install pymongocrypt'
220296
UV_ARGS.append("--group pymongocrypt_source")
221297

222298
# Use the nocrypto build to avoid dependency issues with older windows/python versions.
223299
BASE = ROOT / "libmongocrypt/nocrypto"
224-
if sys.platform == "linux":
300+
if PLATFORM == "linux":
225301
if (BASE / "lib/libmongocrypt.so").exists():
226302
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so"
227303
else:
228304
PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so"
229-
elif sys.platform == "darwin":
305+
elif PLATFORM == "darwin":
230306
PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib"
231307
else:
232308
PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll"
@@ -244,7 +320,7 @@ def handle_test_env() -> None:
244320
if is_set("TEST_CRYPT_SHARED"):
245321
CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
246322
LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
247-
if os.name == "nt":
323+
if PLATFORM == "windows":
248324
write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH")
249325
else:
250326
write_env(

0 commit comments

Comments
 (0)