Merge pull request 'loading the yaml files in a more structured way' (#5) from fix/pymcs/module-imports into main

Reviewed-on: https://git.myzel.io/PyMC/PyMC-Server/pulls/5

Author: chris-aeviator

.gitignore

@@ -147,16 +147,16 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
 # pytype static type analyzer
 .pytype/
 
 # Cython debug symbols
 cython_debug/
-
+*pymc.yaml
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+

examples/cluster/dev.yaml

@@ -0,0 +1,15 @@
+num_nodes: 1
+resources:
+  # Optional; if left out, automatically pick the cheapest cloud.
+  cloud: gcp
+  cpus: 1
+  # 1x NVIDIA V100 GPU
+  #accelerators: V200:2
+  ports : 80
+
+workdir: ./src/pymc_server/workDir_test
+pymc_module: pymc-marketing
+run: |
+  echo "Hello, SkyPilot!"
+  uname -a
+  python test.py

examples/cluster/gcp_start_stop.yaml

@@ -0,0 +1,15 @@
+# start and stop GCP instances
+name: gcp-start-stop
+
+resources:
+  cloud: gcp
+  cpus: 1+
+
+num_nodes: 1
+
+# The setup command.  Will be run under the working directory.
+setup: 'echo "hi"'
+workdir: ./src/pymc_server/workDir_test
+
+# The command to run.  Will be run under the working directory.
+run: 'python test.py'

requirements.txt

@@ -5,3 +5,4 @@ numba
 ray[default]
 skypilot
 skypilot[gcp]
+hiyapyco

setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

src/pymc_server/cli.py

@@ -1,49 +1,60 @@
-
 import click
-#import colorama
-#import dotenv
 import sky
-from file_merger import mergeYaml
-from launch_cli import launch as cli_launch
+from typing import Any, Dict, List, Optional, Tuple, Union
+from pymc_server.utils.yaml import merge_yaml
+from pymc_server.launch_cli import launch as cli_launch
+from pymc_server.cli_factory import setup_launch_factory, setup_status_factory
+from sky.usage import usage_lib
+
+
+from sky.cli import (
+    status as sky_status,
+    launch as sky_launch,
+    check as sky_check
+)
+   
+
+# TODO: remove, check pyproject.py for a reference to this function
 @click.group()
 def cli():
     pass
 
-@click.command()
-#@click.option('--count', default=1, help='Number of greetings.')
-#@click.option('--name', prompt='Your name',
-#              help='The person to greet.')
-def status():
-    task = sky.Task(run='echo hello SkyPilot')
-    task.set_resources(sky.Resources(cloud=sky.AWS(), accelerators='V100:4'))
-    sky.launch(task, cluster_name='my-cluster')
-    click.echo(sky.status(cluster_names=None, refresh=False))
-    """Simple program that greets NAME for a total of COUNT times."""
-    #for x in range(count):
-    #    click.echo(f"Hello {name}!")
+@setup_status_factory
+@usage_lib.entrypoint
+def status(*args, **kwargs):
+    """ calls the sky status command by passing the click context"""
+    ctx = click.get_current_context()
+    ctx.invoke(sky_status, *args, **kwargs)
 
-cli.add_command(status)
 
-@click.command()
-#@click.option('--count', default=1, help='Number of greetings.')
-#@click.option('--name', prompt='Your name',
-#              help='The person to greet.')
-def launch():
-    print('Launch Mode ::')
-    '''
-    task = sky.Task(run='echo hello SkyPilot')
-    task.set_resources(sky.Resources(cloud=sky.AWS(), accelerators='V100:4'))
-    sky.launch(task, cluster_name='my-cluster')
-    click.echo(sky.status(cluster_names=None, refresh=False))
-    '''
-    
-   # print(mergeYaml(devPath='dev.yaml',pymcPath='pymc.yaml'))
-    cli_launch()
-
-    """Simple program that greets NAME for a total of COUNT times."""
-    #for x in range(count):
-    #    click.echo(f"Hello {name}!")
+@setup_launch_factory
+@usage_lib.entrypoint
+def launch(*args, **kwargs):
+    """Launch a cluster or task.
+
+    If ENTRYPOINT points to a valid YAML file, it is read in as the task
+    specification. Otherwise, it is interpreted as a bash command.
 
+    In both cases, the commands are run under the task's workdir (if specified)
+    and they undergo job queue scheduling.
+    """
+    #  cli_launch(*args, **kwargs)
+    ctx = click.get_current_context()
+    ctx.invoke(cli_launch, *args, **kwargs)
+
+@setup_status_factory
+@usage_lib.entrypoint
+def check(*args, **kwargs):
+    """ calls the sky status command by passing the click context"""
+    ctx = click.get_current_context()
+    #sky_check(*args, **kwargs)
+    ctx.invoke(sky_check, *args, **kwargs)
+
+
+cli.add_command(status)
 cli.add_command(launch)
+cli.add_command(check)
+
 if __name__ == '__main__':
     cli()
+

src/pymc_server/cli_factory.py

@@ -0,0 +1,172 @@
+import click
+from sky.cli import (
+    _DocumentedCodeCommand,
+    _get_shell_complete_args,
+    _complete_file_name,
+    _complete_cluster_name,
+    _CLUSTER_FLAG_HELP,
+    _TASK_OPTIONS_WITH_NAME,
+    _EXTRA_RESOURCES_OPTIONS,
+    usage_lib,
+    backends,
+    _add_click_options
+   
+)
+
+
+def setup_status_factory(func):
+    options = [
+        click.command(),
+        click.option('--all',
+                      '-a',
+                      default=False,
+                      is_flag=True,
+                      required=False,
+                      help='Show all information in full.'),
+        click.option(
+            '--refresh',
+            '-r',
+            default=False,
+            is_flag=True,
+            required=False,
+            help='Query the latest cluster statuses from the cloud provider(s).'),
+        click.option('--ip',
+                      default=False,
+                      is_flag=True,
+                      required=False,
+                      help=('Get the IP address of the head node of a cluster. This '
+                            'option will override all other options. For Kubernetes '
+                            'clusters, the returned IP address is the internal IP '
+                            'of the head pod, and may not be accessible from outside '
+                            'the cluster.')),
+        click.option('--endpoints',
+                      default=False,
+                      is_flag=True,
+                      required=False,
+                      help=('Get all exposed endpoints and corresponding URLs for a'
+                            'cluster. This option will override all other options.')),
+        click.option('--endpoint',
+                      required=False,
+                      default=None,
+                      type=int,
+                      help=('Get the endpoint URL for the specified port number on the '
+                            'cluster. This option will override all other options.')),
+        click.option('--show-managed-jobs/--no-show-managed-jobs',
+                      default=True,
+                      is_flag=True,
+                      required=False,
+                      help='Also show recent in-progress managed jobs, if any.'),
+        click.option('--show-services/--no-show-services',
+                      default=True,
+                      is_flag=True,
+                      required=False,
+                      help='Also show sky serve services, if any.'),
+        click.argument('clusters',
+                        required=False,
+                        type=str,
+                        nargs=-1,
+                        **_get_shell_complete_args(_complete_cluster_name)),
+    ]
+    for option in reversed(options):
+        func = option(func)
+    return func
+
+def setup_launch_factory(func):
+    options = [
+        click.command(cls=_DocumentedCodeCommand),
+        click.argument('entrypoint',
+                        required=False,
+                        type=str,
+                        nargs=-1,
+                        **_get_shell_complete_args(_complete_file_name)),
+        click.option('--pymc_module',
+                      '-m',
+                      required=False,
+                      type=str,
+                      help=('Define the PyMC module / project you need tu use')),
+        click.option('--cluster',
+                      '-c',
+                      default=None,
+                      type=str,
+                      **_get_shell_complete_args(_complete_cluster_name),
+                      help=_CLUSTER_FLAG_HELP),
+        click.option('--dryrun',
+                      default=False,
+                      is_flag=True,
+                      help='If True, do not actually run the job.'),
+        click.option(
+            '--detach-setup',
+            '-s',
+            default=False,
+            is_flag=True,
+            help=
+            ('If True, run setup in non-interactive mode as part of the job itself. '
+             'You can safely ctrl-c to detach from logging, and it will not interrupt '
+             'the setup process. To see the logs again after detaching, use `sky logs`.'
+             ' To cancel setup, cancel the job via `sky cancel`. Useful for long-'
+             'running setup commands.')),
+        click.option(
+            '--detach-run',
+            '-d',
+            default=False,
+            is_flag=True,
+            help=('If True, as soon as a job is submitted, return from this call '
+                  'and do not stream execution logs.')),
+        click.option('--docker',
+                      'backend_name',
+                      flag_value=backends.LocalDockerBackend.NAME,
+                      default=False,
+                      help='If used, runs locally inside a docker container.'),
+        _add_click_options(_TASK_OPTIONS_WITH_NAME + _EXTRA_RESOURCES_OPTIONS),
+        click.option(
+            '--idle-minutes-to-autostop',
+            '-i',
+            default=None,
+            type=int,
+            required=False,
+            help=('Automatically stop the cluster after this many minutes '
+                  'of idleness, i.e., no running or pending jobs in the cluster\'s job '
+                  'queue. Idleness gets reset whenever setting-up/running/pending jobs '
+                  'are found in the job queue. '
+                  'Setting this flag is equivalent to '
+                  'running ``sky launch -d ...`` and then ``sky autostop -i <minutes>``'
+                  '. If not set, the cluster will not be autostopped.')),
+        click.option(
+            '--retry-until-up',
+            '-r',
+            default=False,
+            is_flag=True,
+            required=False,
+            help=('Whether to retry provisioning infinitely until the cluster is up, '
+                  'if we fail to launch the cluster on any possible region/cloud due '
+                  'to unavailability errors.'),
+        ),
+        click.option(
+            '--yes',
+            '-y',
+            is_flag=True,
+            default=False,
+            required=False,
+            # Disabling quote check here, as there seems to be a bug in pylint,
+            # which incorrectly recognizes the help string as a docstring.
+            # pylint: disable=bad-docstring-quotes
+            help='Skip confirmation prompt.'),
+        click.option('--no-setup',
+                      is_flag=True,
+                      default=False,
+                      required=False,
+                      help='Skip setup phase when (re-)launching cluster.'),
+        click.option(
+            '--clone-disk-from',
+            '--clone',
+            default=None,
+            type=str,
+            **_get_shell_complete_args(_complete_cluster_name),
+            help=('[Experimental] Clone disk from an existing cluster to launch '
+                  'a new one. This is useful when the new cluster needs to have '
+                  'the same data on the boot disk as an existing cluster.'))
+    ]
+
+    for option in reversed(options):
+        func = option(func)
+    return func

src/pymc_server/config/pymc-marketing/base.yaml

@@ -0,0 +1,22 @@
+resources:
+  # Optional; if left out, automatically pick the cheapest cloud.
+  cloud: aws
+  #disk_tier: medium
+  # 1x NVIDIA V100 GPU
+  #accelerators: V100:1
+  ports: 9000
+
+# Working directory (optional) containing the project codebase.
+# Its contents are synced to ~/sky_workdir/ on the cluster.
+workdir: .
+
+# Typical use: pip install -r requirements.txt
+# Invoked under the workdir (i.e., can use its files).
+setup: |
+  echo "Running setup."
+
+# Typical use: make use of resources, such as running training.
+# Invoked under the workdir (i.e., can use its files).
+run: |
+  echo "Hello, SkyPilot!"
+  #conda env list

src/pymc_server/pymc.yaml renamed to src/pymc_server/config/pymc-test/base.yaml

@@ -4,6 +4,7 @@ resources:
   disk_tier: medium
   # 1x NVIDIA V100 GPU
   accelerators: V100:1
+  ports: 9000
 
 # Working directory (optional) containing the project codebase.
 # Its contents are synced to ~/sky_workdir/ on the cluster.
@@ -18,4 +19,4 @@ setup: |
 # Invoked under the workdir (i.e., can use its files).
 run: |
   echo "Hello, SkyPilot!"
-  conda env list
+  conda env list

src/pymc_server/config/pymc-tofu/base.yaml

@@ -0,0 +1,22 @@
+resources:
+  # Optional; if left out, automatically pick the cheapest cloud.
+  cloud: aws
+  disk_tier: medium
+  # 1x NVIDIA V100 GPU
+  accelerators: V100:1
+  ports: 9000
+
+# Working directory (optional) containing the project codebase.
+# Its contents are synced to ~/sky_workdir/ on the cluster.
+workdir: .
+
+# Typical use: pip install -r requirements.txt
+# Invoked under the workdir (i.e., can use its files).
+setup: |
+  echo "Running setup."
+
+# Typical use: make use of resources, such as running training.
+# Invoked under the workdir (i.e., can use its files).
+run: |
+  echo "Hello, SkyPilot!"
+  conda env list