1110 Migrate Python packages to use pyproject.toml for builds

.github/actions/build-py/action.yml

@@ -19,9 +19,11 @@ runs:
       shell: bash
       run: |
         cd pycode/memilio-${{ inputs.package }}/
-        /opt/python/cp38-cp38/bin/python -m pip install scikit-build
+        /opt/python/cp38-cp38/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp38-cp38/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp38-cp38/bin/python -m build --no-isolation --wheel
-        /opt/python/cp311-cp311/bin/python -m pip install scikit-build
+        /opt/python/cp311-cp311/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp311-cp311/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp311-cp311/bin/python -m build --no-isolation --wheel
         # Exclude memilio-generation, because its a pure python package, cmake is only used in the build process to retrieve data from cpp
         if [[ -f "CMakeLists.txt" ]] && [ "${{ inputs.package }}" != "generation" ]; then 

.github/actions/test-pylint/action.yml

@@ -13,6 +13,10 @@ runs:
         sudo apt-get -qq update
         sudo apt-get -qq -y install python3-pip gnupg
         python -m pip install --upgrade pip
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
     - name: Download Python Wheels
       uses: actions/download-artifact@v4
       with:
@@ -28,7 +32,11 @@ runs:
       run: |
         cd pycode/memilio-${{ inputs.package }}
         mkdir -p build_pylint
-        python setup.py pylint
+        if [ -f tools/run_pylint.py ]; then
+          python tools/run_pylint.py
+        else
+          python -m pylint --rcfile=../pylintrc --load-plugins=pylint_json2html --output-format=jsonextended memilio/ > build_pylint/pylint_extended.json || true
+        fi
         pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
     - name: Upload Pylint Report
       uses: actions/upload-artifact@v4

.readthedocs.yaml

@@ -11,7 +11,7 @@ build:
   jobs:
     install:
       - pip install --exists-action=w --no-cache-dir -r docs/requirements-build.txt
-      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt --no-build-isolation
+      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
 
 sphinx:
   configuration: docs/source/conf.py

cpp/thirdparty/CMakeLists.txt

@@ -37,7 +37,22 @@ if(MEMILIO_USE_BUNDLED_SPDLOG)
 
     if(NOT spdlog_POPULATED)
         FetchContent_Populate(spdlog)
+
+        # Even when the rest builds shared libraries, we sometimes
+        # prefer to keep spdlog static (e.g. for Python wheels where we don't want
+        # an extra shared dependency). Temporarily override BUILD_SHARED_LIBS while
+        # configuring spdlog to force a static build in that case.
+        if(MEMILIO_PREFER_STATIC_SPDLOG)
+            set(_MEMILIO_PREV_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+            set(BUILD_SHARED_LIBS OFF)
+        endif()
+
         add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+        if(MEMILIO_PREFER_STATIC_SPDLOG)
+            set(BUILD_SHARED_LIBS ${_MEMILIO_PREV_BUILD_SHARED_LIBS})
+            unset(_MEMILIO_PREV_BUILD_SHARED_LIBS)
+        endif()
     endif()
 
 else()
@@ -198,6 +213,7 @@ else()
 endif()
 
 find_package(sbml ${MEMILIO_SBML_VERSION} CONFIG)
+
 if(sbml_FOUND)
     set(MEMILIO_HAS_SBML ON)
     message(STATUS "Found libSBML: ${libsbml_INCLUDE_DIRS}")

docs/source/python/m-epidata.rst

@@ -85,7 +85,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- Add an executable to the setup.py in "pycode/memilio-epidata".
+- Add a console script entry to the ``pycode/memilio-epidata/pyproject.toml`` file.
 - Add it to the cli_dict in getDataIntoPandasDataFrame.py.
     - Add a meaningful key for the new script.
     - for the dict value add a list in the form [comment to print when script is started, list of used parser arguments (optional)].

docs/source/python/m-generation.rst

@@ -30,7 +30,7 @@ For a successful build, the development libraries for Python need to be installe
 .. warning::
     Generation currently requires specifically version ``18.1.1`` of `libclang.so`, since the function ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_ generates the abstract syntax tree using `clang-18`. Different versions may lead to unsupported abstractions.
 
-    If you want to try a different version, set your `libclang` version under ``install_requires`` in the `setup.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/setup.py>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
+    If you want to try a different version, set your `libclang` version under ``dependencies`` in the `pyproject.toml <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/pyproject.toml>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
 
 Usage
 -----

docs/source/python/python_packages.rst

@@ -111,9 +111,9 @@ Please see the individual package documentation for more details on the function
 Installation
 ------------
 
-Each package provides a `setup.py` script that installs the package and its dependencies. 
+Each package provides a ``pyproject.toml`` that installs the package and its dependencies with pip.
 The dependencies of the individual packages are denoted in their documentation.
-The installation can be run with the following command (from the directory containing the `setup.py`)
+The installation can be run with the following command (from the directory containing the ``pyproject.toml`` file)
 
 .. code-block:: console 
 
@@ -171,9 +171,11 @@ Run pylint with the commands in the package folder
 
 .. code-block:: console
 
-    python setup.py pylint
+    python tools/run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
+Packages without the helper script can run pylint directly, for example ``python -m pylint memilio``.
+
 Pylint report for actual master:
 
-`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`_
+`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`_

pycode/memilio-epidata/README.rst

@@ -23,9 +23,9 @@ A more detailed description of the sources can be found in the `epidata subfolde
 Installation
 ------------
 
-Use the provided ``setup.py`` script to install the package and its dependencies.
+This project uses ``pyproject.toml`` to install the package and its dependencies.
 
-To install the package, use (from the directory that contains ``setup.py``)
+To install the package, run (from the directory that contains ``pyproject.toml``)
 
 .. code:: sh
 
@@ -116,7 +116,7 @@ Run pylint with the commands
 
 .. code:: sh
 
-    python setup.py pylint
+    python tools/run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
 Pylint report for actual master:
@@ -163,7 +163,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- add a executable to the setup.py in "pycode/memilio-epidata"
+- add a script entry to ``pyproject.toml`` in "pycode/memilio-epidata"
 - add it to the cli_dict in getDataIntoPandasDataFrame.py
     - add a meaningfull key for the new script
     - as the value add a list in the form [comment to print when script is started, list of used parser arguments (optional)]

pycode/memilio-epidata/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "memilio-epidata"
+version = "1.0.0"
+description = "Part of MEmilio project, reads epidemiological data from different official and unofficial sources."
+readme = "README.rst"
+requires-python = ">=3.8"
+license = { text = "Apache-2.0" }
+authors = [{ name = "MEmilio Team" }]
+maintainers = [
+  { email = "martin.kuehn@dlr.de" }
+]
+dependencies = [
+  "pandas>=2.0.0",
+  "pyarrow",
+  "matplotlib",
+  "tables",
+  "numpy>=1.22,<1.25",
+  "openpyxl",
+  "xlrd",
+  "xlsxwriter",
+  "requests",
+  "pyxlsb",
+  "wget",
+  "twill==3.1",
+  "PyQt6-sip<13.9",
+  "PyQt6",
+  "python-calamine",
+  "python-magic-bin; sys_platform == 'win32'",
+  "python-magic; sys_platform != 'win32'"
+]
+
+[project.optional-dependencies]
+dev = [
+  "pyfakefs>=4.6,<5.3.3",
+  "coverage>=7.0.1",
+  "pylint>=2.13.0,<2.16",
+  "pylint_json2html==0.4.0"
+]
+
+[project.scripts]
+getcasedata = "memilio.epidata.getCaseData:main"
+getpopuldata = "memilio.epidata.getPopulationData:main"
+getjhdata = "memilio.epidata.getJHData:main"
+getdividata = "memilio.epidata.getDIVIData:main"
+getsimdata = "memilio.epidata.getSimulationData:main"
+cleandata = "memilio.epidata.cleanData:main"
+getcommutermobility = "memilio.epidata.getCommuterMobility:main"
+getvaccinationdata = "memilio.epidata.getVaccinationData:main"
+gethospitalizationdata = "memilio.epidata.getHospitalizationData:main"
+
+[project.urls]
+Homepage = "https://github.com/SciCompMod/memilio"
+Team = "https://memilio.readthedocs.io/en/latest/team.html"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["memilio*"]

pycode/memilio-epidata/tools/run_pylint.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Generate Pylint reports in jsonextended format for the epidata package."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    project_dir = Path(__file__).resolve().parent.parent
+    repo_root = project_dir.parent
+    build_dir = project_dir / "build_pylint"
+    build_dir.mkdir(exist_ok=True)
+    output_file = build_dir / "pylint_extended.json"
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "pylint",
+        "--rcfile",
+        str(repo_root / "pylintrc"),
+        "--load-plugins",
+        "pylint_json2html",
+        "--output-format=jsonextended",
+        "memilio/",
+    ]
+
+    with output_file.open("w", encoding="utf-8") as stream:
+        result = subprocess.run(
+            cmd,
+            cwd=project_dir,
+            stdout=stream,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+    if result.stderr:
+        sys.stderr.write(result.stderr)
+
+    if result.returncode:
+        sys.stderr.write(
+            "pylint reported issues; see build_pylint/pylint_extended.json for details.\n"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())