ci: Use geant4 conda in CI on more platforms (#79)

* use geant4 conda in CI on more platforms

* attempt to fix CI

* style: pre-commit fixes

* check datasets

* update datasets to make conda work

* renaming of application data dir

* style: pre-commit fixes

* fix bad getter for geant4 data path

* style: pre-commit fixes

* fix bad getter for geant4 data path

* fix bad getter for geant4 data path

* style: pre-commit fixes

* install datasets

* style: pre-commit fixes

* address pip version warning

* fix bad command

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: lobis

.github/workflows/build-test.yaml

@@ -20,8 +20,13 @@ env:
 
 jobs:
   install-with-geant4-from-conda:
-    name: Install with conda
-    runs-on: ubuntu-latest
+    name: Install dependencies with conda
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest] # Geant4 is not available on conda for windows TODO: update this when it is
     defaults:
       run:
         shell: bash -l {0}
@@ -32,29 +37,50 @@ jobs:
           auto-update-conda: true
           python-version: 3.12
 
-      - name: Install Geant4 using conda
+      - name: Install Geant4 and cmake using conda
+        run: |
+          conda install -c conda-forge cmake ninja geant4=${{ env.GEANT4_VERSION }}
+
+      - name: Install additional dependencies (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
         run: |
-          conda install -c conda-forge geant4=11.2.2 cmake ninja gxx_linux-64
+          conda install -c conda-forge gxx_linux-64
+
+      - name: Install additional dependencies (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          conda install -c conda-forge clang_osx-64
 
       - name: Print versions
         run: |
           conda --version
+          cmake --version
+
+          echo "Geant4: ${{ env.GEANT4_VERSION }}"
+
           geant4-config --version
+          geant4-config --check-datasets
+
+      - name: Install datasets if missing # Shouldn't they be available after the conda install?
+        run: |
+          geant4-config --install-datasets
+          geant4-config --check-datasets
 
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: pip install the package from source
         run: |
-          python -m pip install .
+          python -m pip install .[test]
           python -c "import geant4_python_application; print(geant4_python_application.version)"
 
-      - name: Run example notebooks
+      - name: Check datasets are accessible # Datasets should have been installed by the conda package, and should be accessible
         run: |
-          python -m pip install jupyter nbconvert matplotlib
-          for notebook in $(find examples -name '*.ipynb'); do
-            jupyter nbconvert --to notebook --execute --inplace $notebook
-          done
+          python -c "import geant4_python_application as g4; g4.files.datasets.check_datasets(throw=True)"
+
+      - name: Run tests
+        run: |
+          python -m pytest -vv --reruns 3 --reruns-delay 30 --only-rerun "(?i)http|timeout|connection|socket|resolve"
 
   build-geant4:
     name: Build Geant4 on ${{ matrix.os }}
@@ -200,6 +226,7 @@ jobs:
 
       - name: pip install the package
         run: |
+          python -m pip install --upgrade pip
           python -m pip install .[test]
           git tag
           python -c "import geant4_python_application; print(geant4_python_application.version)"

src/geant4_python_application/__init__.py

@@ -8,7 +8,10 @@
 from geant4_python_application._version import version, version_tuple
 from geant4_python_application.application import Application
 from geant4_python_application.detector import Detector
-from geant4_python_application.files.datasets import data_directory, install_datasets
+from geant4_python_application.files.datasets import (
+    application_data_directory as data_directory,
+)
+from geant4_python_application.files.datasets import install_datasets
 from geant4_python_application.files.directories import application_directory
 from geant4_python_application.gdml import basic_gdml
 

src/geant4_python_application/files/datasets.py

@@ -4,6 +4,7 @@
 import hashlib
 import os
 import shutil
+import subprocess
 import tarfile
 import tempfile
 from collections import namedtuple
@@ -20,7 +21,7 @@
 # It is discouraged to use the python package directory to store data
 # data_dir = os.path.join(os.path.dirname(__file__), "geant4/data")
 # another idea is to use 'platformdirs' to store data in a platform-specific location
-def data_directory() -> str:
+def application_data_directory() -> str:
     return os.path.join(
         geant4_python_application.application_directory(),
         geant4_python_application.__name__,
@@ -31,7 +32,21 @@ def data_directory() -> str:
 
 
 def geant4_data_directory() -> str:
-    return os.environ["GEANT4_DATA_DIR"]
+    try:
+        # capture stdout, save into a str
+        geant4_prefix = subprocess.run(
+            ["geant4-config", "--prefix"], capture_output=True, check=False
+        )
+        geant4_data_path = os.path.join(
+            geant4_prefix.stdout.decode().strip(), "share", "Geant4", "data"
+        )
+        if not os.path.exists(geant4_data_path):
+            return ""
+
+        return geant4_data_path
+
+    except FileNotFoundError:
+        return ""
 
 
 # the datasets versions should be updated with each Geant4 version (remember to update the checksum too!)
@@ -164,14 +179,11 @@ def _download_extract_dataset(dataset: Dataset, pbar: tqdm):
 
         f.seek(0)
         with tarfile.open(fileobj=f, mode="r:gz") as tar:
-            tar.extractall(data_directory())
+            tar.extractall(application_data_directory())
 
 
-def missing_datasets(directory: str | None = None) -> list[Dataset]:
-    if directory is None:
-        directory = data_directory()
+def missing_datasets(directory: str) -> list[Dataset]:
     datasets_to_download = []
-
     for dataset in datasets:
         path = os.path.join(directory, dataset.name + dataset.version)
         if not os.path.exists(path):
@@ -180,7 +192,12 @@ def missing_datasets(directory: str | None = None) -> list[Dataset]:
 
 
 def check_datasets(throw: bool = False) -> bool:
-    datasets_to_download = missing_datasets()
+    datasets_to_download = missing_datasets(directory=application_data_directory())
+    if datasets_to_download:
+        # if not found in the application data directory, try to find them in the Geant4 installation data directory
+        if geant4_data_directory():
+            datasets_to_download = missing_datasets(directory=geant4_data_directory())
+
     if datasets_to_download:
         if throw:
             raise RuntimeError(
@@ -192,27 +209,27 @@ def check_datasets(throw: bool = False) -> bool:
 
 def install_datasets(show_progress: bool = True):
     # first try to see if the datasets are installed in the application directory
-    datasets_to_download = missing_datasets()
+    datasets_to_download = missing_datasets(directory=application_data_directory())
     if not datasets_to_download:
         # datasets are installed in application directory
-        os.environ["GEANT4_DATA_DIR"] = data_directory()
+        os.environ["GEANT4_DATA_DIR"] = application_data_directory()
         return
 
-    # check if the datasets are present in the corresponding Geant4 directory
-    if "GEANT4_DATA_DIR" in os.environ and not bool(
-        missing_datasets(os.environ["GEANT4_DATA_DIR"])
-    ):
-        # return
-        ...
+    if geant4_data_directory():
+        datasets_to_download = missing_datasets(directory=geant4_data_directory())
+        if not datasets_to_download:
+            # datasets are installed in Geant4 data directory, no need to install them in the application data directory
+            os.environ["GEANT4_DATA_DIR"] = geant4_data_directory()
+            return
 
-    # download and extract the datasets
-    os.environ["GEANT4_DATA_DIR"] = data_directory()
+    # download and extract the datasets to the application data directory
+    os.environ["GEANT4_DATA_DIR"] = application_data_directory()
 
-    os.makedirs(data_directory(), exist_ok=True)
+    os.makedirs(application_data_directory(), exist_ok=True)
     if show_progress:
         print(
             f"""
-Geant4 datasets will be installed to "{data_directory()}".
+Geant4 datasets will be installed to "{application_data_directory()}".
 This may take a while but only needs to be done once.
 You can override the default location by calling `application_directory(path)` or `application_directory(temp=True)` to use a temporary directory.
 The following Geant4 datasets will be installed: {", ".join([f"{dataset.name}@v{dataset.version}" for dataset in datasets_to_download])}"""
@@ -236,13 +253,13 @@ def install_datasets(show_progress: bool = True):
 
     if show_progress:
         total_size_gb = sum(
-            fp.stat().st_size for fp in Path(data_directory()).rglob("*")
+            fp.stat().st_size for fp in Path(application_data_directory()).rglob("*")
         ) / (1024**3)
         print(f"Geant4 datasets size on disk after extraction: {total_size_gb:.2f}GB")
 
 
 def uninstall_datasets():
-    dir_to_remove = os.path.dirname(data_directory())
+    dir_to_remove = os.path.dirname(application_data_directory())
     package_dir = os.path.dirname(__file__)
 
     if not os.path.relpath(package_dir, dir_to_remove).startswith(".."):