72 lines
2.4 KiB
Text
72 lines
2.4 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"from stackit_spark import get_spark\n",
|
|
"\n",
|
|
"from my_tools.catalog_spark import get_nessie_token\n",
|
|
"\n",
|
|
"if \"STACKIT__PAPERMILL\" in os.environ:\n",
|
|
" # Create Spark Session with Iceberg Rest Credentials for Dremio Enterprise Catalog\n",
|
|
" catalog_name_in_spark = \"stackit\"\n",
|
|
" spark = get_spark()\n",
|
|
" spark.sql(f\"USE {catalog_name_in_spark}\")\n",
|
|
"\n",
|
|
" sdf = spark.sql(f\"SELECT * FROM DEMO.user\")\n",
|
|
" sdf.show()\n",
|
|
" \n",
|
|
"else:\n",
|
|
" tokenendpoint = os.environ[\"TOKEN_ENDPOINT\"]\n",
|
|
" catalogendpoint = os.environ[\"CATALOG_ENDPOINT\"]\n",
|
|
" password = os.environ[\"DREMIO_PAT\"]\n",
|
|
"\n",
|
|
"\n",
|
|
" nessie_token = get_nessie_token(tokenendpoint, password)\n",
|
|
"\n",
|
|
" # Create Spark Session with Iceberg Rest Credentials for Dremio Enterprise Catalog\n",
|
|
" catalog_name_in_spark = \"stackit\"\n",
|
|
" spark = get_spark(\n",
|
|
" additional_config={\n",
|
|
" \"spark.jars.packages\": \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.iceberg:iceberg-aws-bundle:1.6.1\",\n",
|
|
" f\"spark.sql.catalog.{catalog_name_in_spark}\": \"org.apache.iceberg.spark.SparkCatalog\",\n",
|
|
" f\"spark.sql.catalog.{catalog_name_in_spark}.type\": \"rest\",\n",
|
|
" f\"spark.sql.catalog.{catalog_name_in_spark}.warehouse\": \"catalog-s3\",\n",
|
|
" f\"spark.sql.catalog.{catalog_name_in_spark}.uri\": catalogendpoint,\n",
|
|
" f\"spark.sql.catalog.{catalog_name_in_spark}.token\": nessie_token,\n",
|
|
" }\n",
|
|
" )\n",
|
|
"\n",
|
|
" spark.sql(f\"USE {catalog_name_in_spark}\")\n",
|
|
"\n",
|
|
" sdf = spark.sql(f\"SELECT * FROM DEMO.user\")\n",
|
|
" sdf.show()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "stackit-papermill",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|