Initial commit

Demo/scripts/my_spark_notebook.ipynb

@@ -0,0 +1,72 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from stackit_spark import get_spark\n",
+    "\n",
+    "from my_tools.catalog_spark import get_nessie_token\n",
+    "\n",
+    "if \"STACKIT__PAPERMILL\" in os.environ:\n",
+    "    # Create Spark Session with Iceberg Rest Credentials for Dremio Enterprise Catalog\n",
+    "    catalog_name_in_spark = \"stackit\"\n",
+    "    spark = get_spark()\n",
+    "    spark.sql(f\"USE {catalog_name_in_spark}\")\n",
+    "\n",
+    "    sdf = spark.sql(f\"SELECT * FROM DEMO.user\")\n",
+    "    sdf.show()\n",
+    "    \n",
+    "else:\n",
+    "    tokenendpoint = os.environ[\"TOKEN_ENDPOINT\"]\n",
+    "    catalogendpoint = os.environ[\"CATALOG_ENDPOINT\"]\n",
+    "    password = os.environ[\"DREMIO_PAT\"]\n",
+    "\n",
+    "\n",
+    "    nessie_token = get_nessie_token(tokenendpoint, password)\n",
+    "\n",
+    "        # Create Spark Session with Iceberg Rest Credentials for Dremio Enterprise Catalog\n",
+    "    catalog_name_in_spark = \"stackit\"\n",
+    "    spark = get_spark(\n",
+    "        additional_config={\n",
+    "            \"spark.jars.packages\": \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.iceberg:iceberg-aws-bundle:1.6.1\",\n",
+    "            f\"spark.sql.catalog.{catalog_name_in_spark}\": \"org.apache.iceberg.spark.SparkCatalog\",\n",
+    "            f\"spark.sql.catalog.{catalog_name_in_spark}.type\": \"rest\",\n",
+    "            f\"spark.sql.catalog.{catalog_name_in_spark}.warehouse\": \"catalog-s3\",\n",
+    "            f\"spark.sql.catalog.{catalog_name_in_spark}.uri\": catalogendpoint,\n",
+    "            f\"spark.sql.catalog.{catalog_name_in_spark}.token\": nessie_token,\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "    spark.sql(f\"USE {catalog_name_in_spark}\")\n",
+    "\n",
+    "    sdf = spark.sql(f\"SELECT * FROM DEMO.user\")\n",
+    "    sdf.show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "stackit-papermill",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}