(fix): generate test notebook as part of setup

Author: kameshsampath

.gitignore

@@ -1 +1,2 @@
-work
+work
+notebooks/*.ipynb

notebooks/.gitkeep

@@ -3,6 +3,7 @@
 import os
 import re
 from datetime import datetime
+from pathlib import Path
 
 import docker
 import requests
@@ -358,7 +359,29 @@ def grants_to_catalog_role_on_catalog(
         print(f"An error occurred: {e}")
 
 
-def run_steps():
+def generate_setup_notebook_cells(
+    principal_client_id,
+    principal_client_secret,
+    catalog_name=os.getenv("POLARIS_CATALOG_NAME", "my_catalog"),
+):
+    from jinja2 import Environment, FileSystemLoader
+
+    env = Environment(loader=FileSystemLoader("templates"))
+    template_context = {
+        "polaris_catalog_name": catalog_name,
+        "principal_client_id": principal_client_id,
+        "principal_client_secret": principal_client_secret,
+    }
+    notebook_template = env.get_template("setup_verify_notebook.ipynb.j2")
+    content = notebook_template.render(template_context)
+
+    template_out_dir = Path("notebooks")
+    template_out_dir.mkdir(exist_ok=True)
+    with open(template_out_dir.joinpath("polaris_setup_verify.ipynb"), "w") as f:
+        f.write(content)
+
+
+def run_steps() -> tuple[str, str]:
     # Get the root principal credentials from the Polaris Container logs
     client_id, client_secret = extract_root_principal_credentials()
     LOGGER.info(f"Client ID: {client_id},Client Secret: {client_secret}")
@@ -418,4 +441,9 @@ def run_steps():
 
 if __name__ == "__main__":
     LOGGER.info("Running Polaris Demo Setup")
-    run_steps()
+    principal_client_id, principal_client_secret = run_steps()
+    LOGGER.info("Generating Setup Verification Notebook Cells")
+    generate_setup_notebook_cells(
+        principal_client_id,
+        principal_client_secret,
+    )

setup.py

@@ -11,14 +11,16 @@
     "\n",
     "## DEFINE SENSITIVE VARIABLES\n",
     "POLARIS_URI = 'http://polaris:8181/api/catalog'\n",
-    "POLARIS_CATALOG_NAME = 'polardb'\n",
-    "POLARIS_CREDENTIALS = '6d3fd58497b70df7:c6f1c4c5e914cc66534ebdef2a790e55'\n",
+    "POLARIS_CATALOG_NAME = '{{polaris_catalog_name}}'\n",
+    "POLARIS_CREDENTIALS = '{{principal_client_id}}:{{principal_client_secret}}'\n",
     "POLARIS_SCOPE = 'PRINCIPAL_ROLE:ALL'\n",
     "\n",
     "conf = (\n",
     "    pyspark.SparkConf()\n",
     "        .setAppName('polaris_demo')\n",
+    {% if use_maven_mirror == '3.0' %}
     "        .set('spark.jars.ivySettings','/conf/ivy_settings.xml')\n",
+    {% endif %}
     "  \t\t#packages\n",
     "        .set('spark.jars.packages', 'org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0')\n",
     "  \t\t#SQL Extensions\n",
@@ -29,8 +31,6 @@
     "        .set('spark.sql.catalog.polardb', 'org.apache.iceberg.spark.SparkCatalog')\n",
     "        .set('spark.sql.catalog.polardb.catalog-impl', 'org.apache.iceberg.rest.RESTCatalog')\n",
     "        .set('spark.sql.catalog.polardb.uri', POLARIS_URI)\n",
-    "        # this causes error, TODO need to investigate\n",
-    "        #.set('spark.sql.catalog.my_catalog.oauth2-server-uri', POLARIS_URI)\n",
     "        .set('spark.sql.catalog.polardb.credential', POLARIS_CREDENTIALS)\n",
     "        .set('spark.sql.catalog.polardb.scope', POLARIS_SCOPE)\n",
     "        .set('spark.sql.catalog.polardb.token-refresh-enabled', 'true')\n",