verify pre-existing binaries' SHA512 (#25)

resolves #24

- checks the SHA512 for binaries that are already installed
- checks the SHA512 for binaries that are not already installed (freshly downloaded binaries)
- add unit tests

Author: 2bndy5

.gitignore

@@ -5,6 +5,7 @@ clang_tools/__pycache__
 dist
 clang_tools/llvm-project*
 .coverage
+coverage.xml
 htmlcov/
 .vscode/
 tests/pytestdebug.log

README.rst

@@ -20,6 +20,19 @@ clang-tools Introduction
 
 Install clang-tools binaries (clang-format, clang-tidy) with pip.
 
+Features
+--------
+
+- Binaries are statically linked for improved portability.
+- Binaries are checked with SHA512 checksum. This ensures:
+  
+  1. Downloads are not corrupted.
+  2. Old binary builds can be updated.
+- Installed binaries are symbolically linked for better cross-platform usage.
+  For example (on Windows), the ``clang-tidy-13.exe`` binary executable can
+  also be invoked with the symbolic link titled ``clang-tidy.exe``
+- Customizable install path.
+
 Install
 -------
 

clang_tools/install.py

@@ -7,14 +7,10 @@
 import os
 import shutil
 import sys
-from pathlib import Path
-from pathlib import PurePath
+from pathlib import Path, PurePath
 
-from . import install_os
-from . import RESET_COLOR
-from . import suffix
-from . import YELLOW
-from .util import download_file
+from . import install_os, RESET_COLOR, suffix, YELLOW
+from .util import download_file, verify_sha512, get_sha_checksum
 
 
 def clang_tools_binary_url(
@@ -46,14 +42,21 @@ def install_tool(tool_name: str, version: str, directory: str) -> bool:
     :returns: `True` if the binary had to be downloaded and installed.
         `False` if the binary was not downloaded but is installed in ``directory``.
     """
-    if Path(directory, f"{tool_name}-{version}{suffix}").exists():
-        print(f"{tool_name}-{version}", "already installed")
-        return False
+    destination = Path(directory, f"{tool_name}-{version}{suffix}")
     bin_url = clang_tools_binary_url(tool_name, version)
-    bin_name = str(PurePath(bin_url).stem)
+    if destination.exists():
+        print(f"{tool_name}-{version}", "already installed...")
+        print("   checking SHA512...", end=" ")
+        if verify_sha512(get_sha_checksum(bin_url), destination.read_bytes()):
+            print("valid")
+            return False
+        print("invalid")
     print("downloading", tool_name, f"(version {version})")
+    bin_name = str(PurePath(bin_url).stem)
     download_file(bin_url, bin_name)
     move_and_chmod_bin(bin_name, f"{tool_name}-{version}{suffix}", directory)
+    if not verify_sha512(get_sha_checksum(bin_url), destination.read_bytes()):
+        raise ValueError(f"file was corrupted during download from {bin_url}")
     return True
 
 

clang_tools/util.py

@@ -6,6 +6,7 @@
 """
 import platform
 import math
+import hashlib
 from pathlib import Path
 import urllib.request
 from typing import Optional
@@ -63,3 +64,33 @@ def download_file(url: str, file_name: str) -> Optional[str]:
     file = Path(file_name)
     file.write_bytes(buffer)
     return file.as_posix()
+
+
+def get_sha_checksum(binary_url: str) -> str:
+    """Fetch the SHA512 checksum corresponding to the released binary.
+
+    :param binary_url: The URL used to download the binary.
+
+    :returns: A `str` containing the contents of the SHA512sum file given
+        ``binary_url``.
+    """
+    with urllib.request.urlopen(
+        binary_url.replace(".exe", "") + ".sha512sum"
+    ) as response:
+        response: HTTPResponse
+        return response.read(response.length).decode(encoding="utf-8")
+
+
+def verify_sha512(checksum: str, exe: bytes) -> bool:
+    """Verify the executable binary's SHA512 hash matches the valid checksum.
+
+    :param checksum: The SHA512 checksum.
+    :param exe: The `bytes` content of the binary executable that is to be verified.
+
+    :returns: `True` if the ``exe`` hash matches the ``checksum`` given,
+        otherwise `False`.
+    """
+    if " " in checksum:
+        # released checksum's include the corresponding filename (which we don't need)
+        checksum = checksum.split(" ", 1)[0]
+    return checksum == hashlib.sha512(exe).hexdigest()

docs/_static/extra_css.css

@@ -1,7 +1,3 @@
-.md-typeset table.data:not(.plain) {
-  display:inline-block;
-}
-
 tbody .stub,
 thead {
   background-color: var(--md-accent-bg-color--light);

tests/clang-query-12_linux-amd64.sha512sum

@@ -0,0 +1 @@
+e5ae0dc1df5b259e7ed1bf6887af9d0eeba3a659a1465999cc732e014763e5ce7d38c0bef9f36d9df1a2732ce9fac12419d8a78cc157668ad464094a8f65f066  clang-query-12_linux-amd64

tests/clang-query-12_macosx-amd64.sha512sum

@@ -0,0 +1 @@
+8dca8b2de3637cefd3ec739b1efd4e7430a1c83fccdca90999f1e67c14c8caabaed3df3f5fdc9a3c9ed7ab28952c9c04660c20b1f0c36bfc96d22239cd13a9f9  clang-query-12_macosx-amd64

tests/clang-query-12_windows-amd64.sha512sum

@@ -0,0 +1 @@
+0d7abeea833917074ced1810dbd3bcf94d0b1de15bf83dc051f65163f5e5db1ed88ee56e9000d9a67e17a74886b92a942ab14e6f27c6df04d12fdff64bf1437b *clang-query-12_windows-amd64

tests/test_util.py

@@ -1,15 +1,15 @@
 """Tests related to the utility functions."""
-from pathlib import Path
+from pathlib import Path, PurePath
 import pytest
+from clang_tools import install_os
 from clang_tools.install import clang_tools_binary_url
-from clang_tools.util import check_install_os
-from clang_tools.util import download_file
+from clang_tools.util import check_install_os, download_file, get_sha_checksum
 
 
 def test_check_install_os():
     """Tests the return value of `check_install_os()`."""
-    install_os = check_install_os()
-    assert install_os in ("linux", "windows", "macosx")
+    current_os = check_install_os()
+    assert current_os in ("linux", "windows", "macosx")
 
 
 @pytest.mark.parametrize(
@@ -21,3 +21,14 @@ def test_download_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, tag: str
     url = clang_tools_binary_url("clang-query", "12", release_tag=tag)
     file_name = download_file(url, "file.tar.gz")
     assert file_name is not None
+
+
+def test_get_sha(monkeypatch: pytest.MonkeyPatch):
+    """Test the get_sha() function used to fetch the
+    releases' corresponding SHA512 checksum."""
+    monkeypatch.chdir(PurePath(__file__).parent.as_posix())
+    expected = Path(f"clang-query-12_{install_os}-amd64.sha512sum").read_text(
+        encoding="utf-8"
+    )
+    url = clang_tools_binary_url("clang-query", "12", release_tag="master-208096c1")
+    assert get_sha_checksum(url) == expected