Update development tooling to work with semiwrap et al

Author: virtuald

.gitignore

@@ -1,7 +1,8 @@
 
 *.py[ciod]
 /dist
+/dist-other
+/cache
 
 rpy-include
-py.typed
 .venv/

devtools/__main__.py

@@ -1,4 +1,5 @@
 import os
+import pathlib
 import sys
 
 import click
@@ -11,19 +12,24 @@
 # Environment variables for configuring the builds
 #
 
-# Always run robotpy-build in parallel
-os.environ["RPYBUILD_PARALLEL"] = "1"
+# cache downloaded files by default
+if "HATCH_ROBOTPY_CACHE" not in os.environ:
+    os.environ["HATCH_ROBOTPY_CACHE"] = str(
+        (pathlib.Path(__file__).parent.parent / "cache").resolve()
+    )
+
 
 # MACOSX_DEPLOYMENT_TARGET is required for linking to WPILib
 if sys.platform == "darwin":
     os.environ["MACOSX_DEPLOYMENT_TARGET"] = "13.3"
 
 
 @click.group()
+@click.option("-v", "--verbose", default=False, is_flag=True)
 @click.pass_context
-def main(ctx: click.Context):
+def main(ctx: click.Context, verbose: bool):
     """RobotPy development tool"""
-    ctx.obj = Context()
+    ctx.obj = Context(verbose)
 
 
 main.add_command(ci.ci)
@@ -35,7 +41,7 @@ def main(ctx: click.Context):
 def info(ctx: Context):
     """Display information"""
     for project in ctx.subprojects.values():
-        print(project.name, project.requires)
+        print(project.name, project.build_requires)
 
 
 @main.command()
@@ -55,12 +61,31 @@ def develop(ctx: Context, package: str):
             project.develop()
 
 
+@main.command
+@click.argument("package", required=False)
+@click.pass_obj
+def uninstall(ctx: Context, package: str):
+    """Uninstall robotpy packages"""
+    if package:
+        for project in ctx.subprojects.values():
+            if project.name == package:
+                project.uninstall()
+                break
+        else:
+            raise click.BadParameter(f"invalid package {package}")
+    else:
+        for project in ctx.subprojects.values():
+            project.uninstall()
+
+
 @main.command()
 @click.pass_obj
 def update_init(ctx: Context):
     """Update __init__.py in all projects"""
     for project in ctx.subprojects.values():
-        project.update_init()
+        if project.is_semiwrap_project():
+            with ctx.handle_exception(f"update-init {project.name}"):
+                project.update_init()
 
 
 @main.command()

devtools/ci.py

@@ -2,11 +2,12 @@
 # CI commands
 #
 
+import pathlib
 import sys
+import typing as T
 
 import click
 from packaging.requirements import Requirement
-import setuptools_scm
 
 from .ctx import Context
 from .update_pyproject import ProjectUpdater
@@ -39,24 +40,68 @@ def check_pyproject(ctx: Context):
 @ci.command()
 @click.option("--no-test", default=False, is_flag=True)
 @click.pass_obj
-def run(ctx: Context, no_test: bool):
+def build_other_wheels(ctx: Context, no_test: bool):
     """
-    Builds wheels, runs tests
+    Builds wheels that don't use meson, runs tests
     """
 
-    # Get the current build version
-    version = setuptools_scm.get_version()
+    for project in ctx.subprojects.values():
+        if project.is_meson_project():
+            continue
+
+        with ctx.handle_exception(project.name):
+            ctx.install_build_deps(subproject=project)
+
+            project.build_wheel(
+                wheel_path=ctx.wheel_path,
+                other_wheel_path=ctx.other_wheel_path,
+                install=True,
+                config_settings=[],
+            )
+            if not no_test:
+                project.test(install_requirements=True)
+
+
+@ci.command()
+@click.option("--no-test", default=False, is_flag=True)
+@click.option(
+    "--cross",
+    help="meson cross.txt file (installed at ~/.local/share/meson/cross/FILENAME)",
+)
+@click.pass_obj
+def build_meson_wheels(ctx: Context, no_test: bool, cross: T.Optional[str]):
+    """
+    Builds wheels that use meson, runs tests.
+
+    Needs wheels that are in the non-meson builds
+    """
 
     # Fix build dependencies to be == what we are building
     # - install_requires already has this via ==THIS_VERSION in robotpy-build
-    for project in ctx.subprojects.values():
-        for i in range(len(project.requires)):
-            req = project.requires[i]
-            if req.name in ctx.subprojects:
-                project.requires[i] = Requirement(f"{req.name}=={version}")
+    # for project in ctx.subprojects.values():
+    #     for i in range(len(project.build_requires)):
+    #         req = project.build_requires[i]
+    #         if req.name in ctx.subprojects:
+    #             project.build_requires[i] = Requirement(f"{req.name}=={version}")
+
+    # Check that the build dependencies match the versions of the projects
+    # that we're building
+
+    config_settings = []
+    if cross:
+        config_settings.append(f"setup-args=--cross-file={cross}")
 
     for project in ctx.subprojects.values():
-        project.install_build_deps(wheel_path=ctx.wheel_path)
-        project.bdist_wheel(wheel_path=ctx.wheel_path, install=True)
-        if not no_test:
-            project.test(install_requirements=True)
+        if not project.is_meson_project():
+            continue
+
+        with ctx.handle_exception(project.name):
+            ctx.install_build_deps(subproject=project)
+            project.build_wheel(
+                wheel_path=ctx.wheel_path,
+                other_wheel_path=ctx.other_wheel_path,
+                install=True,
+                config_settings=config_settings,
+            )
+            if not no_test:
+                project.test(install_requirements=True)

devtools/config.py

@@ -1,38 +1,38 @@
-import pydantic
+import dataclasses
 import tomlkit
-import typing
+import typing as T
 
+from .util import parse_input
 
-class Model(pydantic.BaseModel):
-    class Config:
-        extra = "forbid"
 
-
-class SubprojectConfig(Model):
-    #: If any managed project has one of these as a dependency, the
-    #: minimum version should be this
-    min_version: str
+@dataclasses.dataclass
+class SubprojectConfig:
+    #: The key in `py_versions` to set the project version from
+    py_version: str
 
     #: Whether this should be built on roborio or not
     roborio: bool
 
 
-class Parameters(Model):
+@dataclasses.dataclass
+class Parameters:
     wpilib_bin_version: str
     wpilib_bin_url: str
 
-    robotpy_build_req: str
+    exclude_artifacts: T.Set[str]
 
-    exclude_artifacts: typing.Set[str]
+    requirements: T.Dict[str, str]
 
 
-class UpdateConfig(Model):
+@dataclasses.dataclass
+class UpdateConfig:
+    py_versions: T.Dict[str, str]
     params: Parameters
-    subprojects: typing.Dict[str, SubprojectConfig]
+    subprojects: T.Dict[str, SubprojectConfig]
 
 
-def load(fname) -> typing.Tuple[UpdateConfig, tomlkit.TOMLDocument]:
+def load(fname) -> T.Tuple[UpdateConfig, tomlkit.TOMLDocument]:
     with open(fname) as fp:
         cfgdata = tomlkit.parse(fp.read())
 
-    return UpdateConfig(**cfgdata), cfgdata
+    return parse_input(cfgdata, UpdateConfig, fname), cfgdata

devtools/ctx.py

@@ -1,18 +1,22 @@
+import contextlib
 import pathlib
 import subprocess
+import sys
 import sysconfig
 import typing
 
 import toposort
 
 from . import config
 from .subproject import Subproject
+from .util import run_pip
 
 
 class Context:
     """Global context used by all rdev commands"""
 
-    def __init__(self) -> None:
+    def __init__(self, verbose: bool) -> None:
+        self.verbose = verbose
         self.root_path = pathlib.Path(__file__).parent.parent
         self.subprojects_path = self.root_path / "subprojects"
         self.cfgpath = self.root_path / "rdev.toml"
@@ -21,6 +25,7 @@ def __init__(self) -> None:
         self.is_roborio = sysconfig.get_platform() == "linux-roborio"
 
         self.wheel_path = self.root_path / "dist"
+        self.other_wheel_path = self.root_path / "dist-other"
 
         subprojects: typing.List[Subproject] = []
         for project, cfg in self.cfg.subprojects.items():
@@ -33,7 +38,7 @@ def __init__(self) -> None:
         # Create a sorted dictionary of subprojects ordered by build order
         si = {p.pyproject_name: i for i, p in enumerate(subprojects)}
         ti = {
-            i: [si[r.name] for r in p.requires if r.name in si]
+            i: [si[r.name] for r in p.build_requires + p.dependencies if r.name in si]
             for i, p in enumerate(subprojects)
         }
 
@@ -61,3 +66,54 @@ def git_is_file_dirty(self, relpath: str) -> bool:
             ["git", "status", "--porcelain", relpath], cwd=self.root_path
         ).decode("utf-8")
         return output != ""
+
+    @contextlib.contextmanager
+    def handle_exception(self, msg: str):
+        try:
+            yield
+        except Exception as e:
+            if self.verbose:
+                raise
+
+            print(f"ERROR: {msg}: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    @property
+    def internal_pyprojects(self):
+        if not hasattr(self, "_internal_pyprojects"):
+            self._internal_pyprojects = [
+                s.pyproject_name for s in self.subprojects.values()
+            ]
+        return self._internal_pyprojects
+
+    def install_build_deps(
+        self,
+        *,
+        subproject: Subproject,
+    ):
+        # separate requirements into internal and external
+        internal = []
+        external = []
+
+        for req in subproject.build_requires:
+            if req.name in self.internal_pyprojects:
+                internal.append(req)
+            else:
+                external.append(req)
+
+        if external:
+            run_pip(
+                "install",
+                *[str(req) for req in external],
+            )
+
+        if internal:
+            run_pip(
+                "install",
+                "--no-index",
+                "--find-links",
+                str(self.wheel_path),
+                "--find-links",
+                str(self.other_wheel_path),
+                *[str(req) for req in internal],
+            )