Test out pytask interactively in a Jupyter notebook. This is a demonstration of how to use pytask to manage data processing tasks in a Pythonic way, leveraging the power of decorators and type hints to define tasks and their dependencies

Author: TinasheMTapera

.gitignore

@@ -4,4 +4,5 @@ logs/*
 .DS_Store
 sandbox
 slurm*
-data
+data
+bld

dag.pdf

@@ -435,4 +435,5 @@ dependencies:
       - types-setuptools==76.0.0.20250313
       - uritemplate==4.1.1
       - watchdog==6.0.0
+      - -e .
 prefix: /n/home03/ttapera/.conda/envs/era5_sandbox

environment.yml

@@ -1,3 +1,7 @@
 [build-system]
 requires = ["setuptools>=64.0"]
 build-backend = "setuptools.build_meta"
+
+[tool.pytask.ini_options]
+paths = ["src/era5_sandbox"]
+editor_url_scheme = "vscode"

notes/10_pytask_demo.ipynb

@@ -38,7 +38,7 @@ status = 3
 user = TinasheMTapera
 
 ### Optional ###
-requirements = fastcore pyprojroot ipykernel nbdev cdsapi hydra-core omegaconf ipykernel snakemake geopandas xarray shapely matplotlib cartopy rasterstats rioxarray pydrive2 google-auth numpy tqdm
+requirements = fastcore pyprojroot ipykernel nbdev cdsapi hydra-core omegaconf ipykernel snakemake geopandas xarray shapely matplotlib cartopy rasterstats rioxarray pydrive2 google-auth numpy tqdm pytask pygraphviz
 # dev_requirements = 
 # console_scripts = src
 # conda_user = 

pyproject.toml

@@ -23,6 +23,7 @@
                                                                                             'era5_sandbox/aggregate.py'),
                                         'era5_sandbox.aggregate.resample_netcdf': ( 'aggregate.html#resample_netcdf',
                                                                                     'era5_sandbox/aggregate.py')},
+            'era5_sandbox.config': {},
             'era5_sandbox.core': { 'era5_sandbox.core.ClimateDataFileHandler': ('core.html#climatedatafilehandler', 'era5_sandbox/core.py'),
                                    'era5_sandbox.core.ClimateDataFileHandler.__enter__': ( 'core.html#climatedatafilehandler.__enter__',
                                                                                            'era5_sandbox/core.py'),
@@ -64,4 +65,12 @@
                                        'era5_sandbox.download.main': ('download_raw_data.html#main', 'era5_sandbox/download.py')},
             'era5_sandbox.publish': { 'era5_sandbox.publish.gather_exposure_geodataframes': ( 'publish.html#gather_exposure_geodataframes',
                                                                                               'era5_sandbox/publish.py'),
-                                      'era5_sandbox.publish.main': ('publish.html#main', 'era5_sandbox/publish.py')}}}
+                                      'era5_sandbox.publish.main': ('publish.html#main', 'era5_sandbox/publish.py')},
+            'era5_sandbox.task_data_preparation': { 'era5_sandbox.task_data_preparation.task_add_another_column': ( 'pytask_demo.html#task_add_another_column',
+                                                                                                                    'era5_sandbox/task_data_preparation.py'),
+                                                    'era5_sandbox.task_data_preparation.task_add_one': ( 'pytask_demo.html#task_add_one',
+                                                                                                         'era5_sandbox/task_data_preparation.py'),
+                                                    'era5_sandbox.task_data_preparation.task_create_random_data': ( 'pytask_demo.html#task_create_random_data',
+                                                                                                                    'era5_sandbox/task_data_preparation.py'),
+                                                    'era5_sandbox.task_data_preparation.task_plot_data': ( 'pytask_demo.html#task_plot_data',
+                                                                                                           'era5_sandbox/task_data_preparation.py')}}}