Merge branch 'master' into goutam/pyarrow_expr

Author: goutamvenkat-anyscale

.buildkite/core.rayci.yml

@@ -185,9 +185,10 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker --
         python/ray/tests/modin/... core
         --install-mask all-ray-libraries
-        --build-name datalbuild
+        --python-version 3.10
+        --build-name datalbuild-py3.10
     depends_on:
-      - datalbuild
+      - datalbuild-multipy
       - forge
 
   - label: ":ray: core: dashboard tests"

.buildkite/data.rayci.yml

@@ -210,7 +210,7 @@ steps:
         --except-tags gpu,post_wheel_build,doctest,dask
         --parallelism-per-worker 2
         --skip-ray-installation
-    depends_on: datalbuild-multipy
+    depends_on: databuild-multipy
 
   - label: ":database: data: dask doc tests"
     tags:
@@ -269,7 +269,7 @@ steps:
     instance_type: small
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... data
-        --build-name databuild-py3.10
+        --build-name datalbuild-py3.10
         --python-version 3.10
         --parallelism-per-worker 3
     depends_on: datalbuild-multipy
@@ -286,7 +286,7 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
         --parallelism-per-worker 3
-        --build-name databuild-py3.10
+        --build-name datalbuild-py3.10
         --python-version 3.10
         --except-tags gpu_only,gpu
     depends_on: datalbuild-multipy

WORKSPACE

@@ -66,6 +66,12 @@ python_register_toolchains(
     register_toolchains = False,
 )
 
+python_register_toolchains(
+    name = "python3_10",
+    python_version = "3.10",
+    register_toolchains = False,
+)
+
 load("@python3_9//:defs.bzl", python39 = "interpreter")
 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 

bazel/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@python3_9//:defs.bzl", python39 = "interpreter")
+load("@python3_10//:defs.bzl", python310 = "interpreter")
 load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_runtime", "py_runtime_pair")
 
@@ -82,3 +83,24 @@ toolchain(
     toolchain = ":py39_runtime_pair",
     toolchain_type = "@bazel_tools//tools/python:toolchain_type",
 )
+
+py_runtime(
+    name = "py310_runtime",
+    interpreter = python310,
+    python_version = "PY3",
+    visibility = ["//visibility:private"],
+)
+
+py_runtime_pair(
+    name = "py310_runtime_pair",
+    py2_runtime = None,
+    py3_runtime = ":py310_runtime",
+    visibility = ["//visibility:private"],
+)
+
+toolchain(
+    name = "py310_toolchain",
+    exec_compatible_with = ["//:hermetic_python"],
+    toolchain = ":py310_runtime_pair",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)

ci/raydepsets/cli.py

@@ -17,7 +17,6 @@
 
 DEFAULT_UV_FLAGS = """
     --generate-hashes
-    --strip-extras
     --unsafe-package setuptools
     --index-url https://pypi.org/simple
     --index-strategy unsafe-best-match
@@ -318,10 +317,10 @@ def compile(
         if append_flags:
             args.extend(_flatten_flags(append_flags))
         if constraints:
-            for constraint in constraints:
+            for constraint in sorted(constraints):
                 args.extend(["-c", constraint])
         if requirements:
-            for requirement in requirements:
+            for requirement in sorted(requirements):
                 args.extend([requirement])
         if packages:
             # need to add a dash to process stdin

ci/raydepsets/tests/test_cli.py

@@ -712,6 +712,66 @@ def test_compile_with_packages_and_requirements(self):
             output_text_valid = output_file_valid.read_text()
             assert output_text == output_text_valid
 
+    @patch("sys.stdout", new_callable=io.StringIO)
+    def test_requirements_ordering(self, mock_stdout):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            copy_data_to_tmpdir(tmpdir)
+            save_packages_to_file(
+                Path(tmpdir) / "requirements_expanded.txt",
+                ["six"],
+            )
+            save_packages_to_file(
+                Path(tmpdir) / "requirements_compiled_test_expand.txt",
+                ["zipp"],
+            )
+            manager = _create_test_manager(tmpdir)
+            manager.compile(
+                constraints=["requirement_constraints_test.txt"],
+                requirements=[
+                    "requirements_test.txt",
+                    "requirements_expanded.txt",
+                    "requirements_compiled_test_expand.txt",
+                ],
+                append_flags=["--no-annotate", "--no-header"],
+                name="requirements_ordering_test_depset",
+                output="requirements_compiled_requirements_ordering.txt",
+            )
+            stdout = mock_stdout.getvalue()
+            assert (
+                "requirements_compiled_test_expand.txt requirements_expanded.txt requirements_test.txt"
+                in stdout
+            )
+
+    @patch("sys.stdout", new_callable=io.StringIO)
+    def test_constraints_ordering(self, mock_stdout):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            copy_data_to_tmpdir(tmpdir)
+            save_packages_to_file(
+                Path(tmpdir) / "requirements_expanded.txt",
+                ["six==1.17.0"],
+            )
+            save_packages_to_file(
+                Path(tmpdir) / "requirements_compiled_test_expand.txt",
+                ["zipp==3.19.2"],
+            )
+            manager = _create_test_manager(tmpdir)
+            manager.compile(
+                requirements=["requirements_test.txt"],
+                constraints=[
+                    "requirement_constraints_test.txt",
+                    "requirements_expanded.txt",
+                    "requirements_compiled_test_expand.txt",
+                ],
+                append_flags=["--no-annotate", "--no-header"],
+                name="constraints_ordering_test_depset",
+                output="requirements_compiled_constraints_ordering.txt",
+            )
+            stdout = mock_stdout.getvalue()
+            assert (
+                "-c requirement_constraints_test.txt -c requirements_compiled_test_expand.txt -c requirements_expanded.txt"
+                in stdout
+            )
+
     @patch("sys.stdout", new_callable=io.StringIO)
     def test_execute_pre_hook(self, mock_stdout):
         with tempfile.TemporaryDirectory() as tmpdir:

doc/source/cluster/kubernetes/examples/mobilenet-rayservice.md

@@ -19,7 +19,7 @@ Note that the YAML file in this example uses `serveConfigV2`. You need KubeRay v
 
 ```sh
 # Create a RayService
-kubectl apply -f https://raw.githubusercontent.com/ray-project/kuberay/v1.4.2/ray-operator/config/samples/ray-service.mobilenet.yaml
+kubectl apply -f https://raw.githubusercontent.com/ray-project/kuberay/v1.5.0/ray-operator/config/samples/ray-service.mobilenet.yaml
 ```
 
 * The [mobilenet.py](https://github.com/ray-project/serve_config_examples/blob/master/mobilenet/mobilenet.py) file needs `tensorflow` as a dependency. Hence, the YAML file uses `rayproject/ray-ml` image instead of `rayproject/ray` image.

doc/source/cluster/kubernetes/examples/rayjob-batch-inference-example.md

@@ -37,12 +37,12 @@ The KubeRay operator Pod must be on the CPU node if you have set up the taint fo
 
 ## Step 2: Submit the RayJob
 
-Create the RayJob custom resource with [ray-job.batch-inference.yaml](https://github.com/ray-project/kuberay/blob/v1.4.2/ray-operator/config/samples/ray-job.batch-inference.yaml).
+Create the RayJob custom resource with [ray-job.batch-inference.yaml](https://github.com/ray-project/kuberay/blob/v1.5.0/ray-operator/config/samples/ray-job.batch-inference.yaml).
 
 Download the file with `curl`:
 
 ```bash
-curl -LO https://raw.githubusercontent.com/ray-project/kuberay/v1.4.2/ray-operator/config/samples/ray-job.batch-inference.yaml
+curl -LO https://raw.githubusercontent.com/ray-project/kuberay/v1.5.0/ray-operator/config/samples/ray-job.batch-inference.yaml
 ```
 
 Note that the `RayJob` spec contains a spec for the `RayCluster`. This tutorial uses a single-node cluster with 4 GPUs. For production use cases, use a multi-node cluster where the head node doesn't have GPUs, so that Ray can automatically schedule GPU workloads on worker nodes which won't interfere with critical Ray processes on the head node.

doc/source/cluster/kubernetes/getting-started/kuberay-operator-installation.md

@@ -17,15 +17,15 @@ kind create cluster --image=kindest/node:v1.26.0
 ```sh
 helm repo add kuberay https://ray-project.github.io/kuberay-helm/
 helm repo update
-# Install both CRDs and KubeRay operator v1.4.2.
-helm install kuberay-operator kuberay/kuberay-operator --version 1.4.2
+# Install both CRDs and KubeRay operator v1.5.0.
+helm install kuberay-operator kuberay/kuberay-operator --version 1.5.0
 ```
 
 ### Method 2: Kustomize
 
 ```sh
 # Install CRD and KubeRay operator.
-kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.4.2"
+kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.5.0"
 ```
 
 ## Step 3: Validate Installation

doc/source/cluster/kubernetes/getting-started/raycluster-quick-start.md

@@ -28,7 +28,7 @@ Once the KubeRay operator is running, you're ready to deploy a RayCluster. Creat
 
 ```sh
 # Deploy a sample RayCluster CR from the KubeRay Helm chart repo:
-helm install raycluster kuberay/ray-cluster --version 1.4.2
+helm install raycluster kuberay/ray-cluster --version 1.5.0
 ```