Initial commit

Demo/scripts/my_tools/read_write_data.py

@@ -0,0 +1,65 @@
+
+def read_and_write_data():
+    import stackit_spark
+    import pandas as pd
+    import os
+    import boto3
+    import pyarrow.parquet as pq
+    from io import BytesIO
+
+        # Initialize the S3 client
+    s3 = boto3.client(
+        's3',
+        aws_access_key_id='J902D929PQ1BC1HG93ZS',
+        aws_secret_access_key='J7x29/4s6eDQ0T1UywB8xo1byWTetwXrCdvYoudH',
+        endpoint_url='https://object.storage.eu01.onstackit.cloud'
+    )
+    
+    bucket_name = 'data-tpcds'
+    folder_names = [
+        's1000/time_dim', 
+        's1000/item', 
+        's1000/date_dim',
+        's1000/customer_demographics',
+        's1000/web_sales_',
+        's1000/customer1'
+    ]
+    
+    result_dfs = {}
+    spark = stackit_spark.get_spark()
+    spark.sql("USE lakehouse")
+    spark.sql("CREATE NAMESPACE IF NOT EXISTS DEMO_USECASE_AIRFLOW")
+    
+    for folder_name in folder_names:
+        objs = s3.list_objects_v2(Bucket=bucket_name, Prefix=folder_name)['Contents']
+        
+        df_list = []
+        for obj in objs:
+            key = obj['Key']
+            try:
+                # Get object from S3
+                response = s3.get_object(Bucket=bucket_name, Key=key)
+                body = response['Body'].read()  # Read the object's bytes
+                # Read Parquet file using PyArrow
+                table = pq.read_table(BytesIO(body))
+                df_part = table.to_pandas()
+                
+                df_list.append(df_part)
+            except Exception as e:
+                print(f"Error processing file {key} in folder {folder_name}: {e}")
+
+        # Concatenate the DataFrames for the current folder
+        if df_list:
+            df = pd.concat(df_list, ignore_index=True)
+            result_dfs[folder_name] = df
+        else:
+            result_dfs[folder_name] = None
+        
+    for df in result_dfs:
+        table_name=f"DEMO_USECASE_AIRFLOW.{df.replace('/', '_')}"
+        spark_df = spark.createDataFrame(result_dfs[df])
+        # Save Spark DataFrame as an Iceberg table
+        spark_df.write.mode("overwrite").saveAsTable(table_name)
+       
+
+read_and_write_data()