adding example for a simple remote state

Author: chris-aeviator

configs/pymcs-benchmark/base.yaml

@@ -9,34 +9,48 @@ resources:
     - 6383  # GCS server (Ray head node port)
     - 8263  # Dashboard port (optional, if --include-dashboard is true)
     - 50001  # Ray client server port
+    - 50002  # Ray client server port
 
 num_nodes: 1
 envs:
   LLM_DEVICE: NONE # CPU or CUDA
 
 # this will be synced to the node as `~/sky_workdir`
+file_mounts:
+  # Format: <cluster path>: <local path/cloud object URI>
+  /pymc-server: /tmp/pymc-server
 workdir: ./
 # The setup command.  Will be run under the working directory.
 setup: |
   set -e  # Exit if any command failed.
 
+  # storage layer
+  sudo mkdir /data
+  sudo chown gcpuser -R /data
+  wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20241002175041.0.0_amd64.deb -O minio.deb
+  sudo dpkg -i minio.deb
+
   # install pixi and project dependencies
   curl -fsSL https://pixi.sh/install.sh | bash
   source /home/gcpuser/.bashrc
-  pixi install --manifest-path pyproject.toml -e server
+
+  cp pyproject.toml pyproject.toml.orig
+  cp pyproject.toml.cloud pyproject.toml
+
+  pixi install --manifest-path pyproject.toml -e pymc
 
 
   # FIXME: check why ray client is not installed from pixi, setup is correct according to https://pixi.sh/latest/reference/project_configuration/#version-specification
   pixi run \
-        --environment server \
+        --environment pymc \
         --manifest-path pyproject.toml \
         pip3 install "ray[default,client]==2.37.0"
 
    
   # start separate ray for pymc-server
   # TODO: Launch the head-only command only on the first node in multinode setup
   pixi run \
-        --environment server\
+        --environment pymc\
         --manifest-path pyproject.toml \
         ray start \
             --head \
@@ -45,6 +59,9 @@ setup: |
             --dashboard-host=0.0.0.0 \
             --dashboard-port=8263 \
             --disable-usage-stats
-            
+
+  # this will block
+  nohup minio server /data --address 0.0.0.0:50002 > minio.log 2>&1 &
+  minio server /data 
 
 

examples/traces/produce_trace.py

@@ -105,11 +105,11 @@ def run(self):
         compiled = nutpie.compile_pymc_model(freeze_dims_and_data(model), backend="numba")
 
         # Sample using nutpie
-        tr = nutpie.sample(compiled, store_unconstrained=True, store_gradient=True, seed=123)
+        #tr = nutpie.sample(compiled, store_unconstrained=True, store_gradient=True, seed=123)
         _ = tr.to_zarr("/tmp/output.zarr")
         self.result = tr
-        from ray.util import inspect_serializability
-        self.is_ser = inspect_serializability(tr, name="tr")
+        #  from ray.util import inspect_serializability
+        #  self.is_ser = inspect_serializability(tr, name="tr")
 
         self.done = True
 
@@ -121,6 +121,8 @@ def run(self):
     bar='911'
 )
 
-#print(server.run())
+server.run()
+print(server.result)
+server._kill()
 
-breakpoint()
+#breakpoint()

examples/traces/s3_minio.py

@@ -0,0 +1,22 @@
+import s3fs
+import xarray as xr
+import arviz
+
+# Configure the S3 connection (MinIO)
+s3 = s3fs.S3FileSystem(
+    anon=False,  # Set to True if your bucket allows anonymous access
+    key='minioadmin',   # Replace with your MinIO access key
+    secret='minioadmin', # Replace with your MinIO secret key
+    client_kwargs={'endpoint_url': 'http://34.66.59.65:50002'}  # Replace with your MinIO URL
+)
+
+# Specify the path to the Zarr dataset in the MinIO bucket
+zarr_path = 'upload/out.zarr'
+
+# Open the Zarr dataset with xarray
+store = s3fs.S3Map(root=zarr_path, s3=s3)
+#ds = xr.open_zarr(store, consolidated=True)
+ds = arviz.from_zarr(store)
+print(ds)
+breakpoint()
+