starting a fresh tidy

pipeline/facts.py

@@ -1,19 +1,21 @@
-import polars as pl
-import logging
 import os
 
+import polars as pl
+
+
 class Facts:
-    def __init__(self, config):
+    def __init__(self, config,logger):
         self.config = config
         self.base_file_path = self.config['base_data_path']
+        self.logger = logger
         os.makedirs(self.config['warehouse_data_path'], exist_ok=True)
-        
+
     def get_full_file_path(self, file_name):
         return f"{self.base_file_path}/{file_name}"
-    
+
 class FactTransactions(Facts):
-    def __init__(self, config):
-        super().__init__(config)
+    def __init__(self, config,logger):
+        super().__init__(config,logger)
         self.file_path = self.get_full_file_path('transactions.parquet')
         self.transform()
 
@@ -21,7 +23,7 @@ class FactTransactions(Facts):
         try:
             source_transactions = pl.read_parquet(self.file_path)
         except FileNotFoundError:
-            logging.error("The transactions DataFrame does not exist")
+            self.logger.error("The transactions DataFrame does not exist")
             return
 
         try:
@@ -39,16 +41,16 @@ class FactTransactions(Facts):
                 "transfer_account_id"
             ])
         except Exception as e:
-            logging.error(f"Failed to select columns from the transactions DataFrame: {e}")
+            self.logger.error(f"Failed to select columns from the transactions DataFrame: {e}")
             return
 
-        logging.info("Transforming the transactions DataFrame")
+        self.logger.info("Transforming the transactions DataFrame")
         try:
             resolve_transaction_dates = base_transactions.with_columns([
                 pl.col("date").str.strptime(pl.Date, format="%Y-%m-%d").alias("date")
             ])
         except Exception as e:
-            logging.error(f"Failed to covert the date to date format: {e}")
+            self.logger.error(f"Failed to covert the date to date format: {e}")
             return
 
         try:
@@ -70,22 +72,22 @@ class FactTransactions(Facts):
             drop_transaction_columns = fix_transaction_values.drop([
                 "id", "date", "amount"
             ])
-            
+
         except Exception as e:
-            logging.error(f"Failed to transform the transactions DataFrame: {e}")
+            self.logger.error(f"Failed to transform the transactions DataFrame: {e}")
             return
         # Write the DataFrame to a new parquet file
-        logging.info("Writing the transformed transactions DataFrame to parquet file")
+        self.logger.info("Writing the transformed transactions DataFrame to parquet file")
         try:
             drop_transaction_columns.write_parquet(
                 self.config['warehouse_data_path'] + '/transactions.parquet'
             )
         except Exception as e:
-            logging.error(f"Failed to write the transformed transactions DataFrame: {e}")
+            self.logger.error(f"Failed to write the transformed transactions DataFrame: {e}")
 
 class FactScheduledTransactions(Facts):
-    def __init__(self, config):
-        super().__init__(config)
+    def __init__(self, config,logger):
+        super().__init__(config,logger)
         self.file_path = self.get_full_file_path('scheduled_transactions.parquet')
         self.transform()
 
@@ -93,7 +95,7 @@ class FactScheduledTransactions(Facts):
         try:
             source_scheduled = pl.read_parquet(self.file_path)
         except FileNotFoundError:
-            logging.error("The scheduled transactions DataFrame does not exist")
+            self.logger.error("The scheduled transactions DataFrame does not exist")
             return
 
         try:
@@ -111,7 +113,7 @@ class FactScheduledTransactions(Facts):
                 "transfer_account_id"
             ])
         except Exception as e:
-            logging.error(f"Failed to select columns from the scheduled transactions DataFrame: {e}")
+            self.logger.error(f"Failed to select columns from the scheduled transactions DataFrame: {e}")
             return
 
         try:
@@ -120,10 +122,10 @@ class FactScheduledTransactions(Facts):
                 pl.col("date_next").str.strptime(pl.Date, format="%Y-%m-%d").alias("date_next")
             ])
         except Exception as e:
-            logging.error(f"Failed to covert the date to date format: {e}")
+            self.logger.error(f"Failed to covert the date to date format: {e}")
             return
-        
-        logging.info("Transforming the scheduled transactions DataFrame")
+
+        self.logger.info("Transforming the scheduled transactions DataFrame")
         try:
             add_scheduled_prefix = resolve_scheduled_dates.with_columns([
                 pl.col("id").alias("scheduled_transaction_id")
@@ -141,10 +143,10 @@ class FactScheduledTransactions(Facts):
                 "id", "amount"
             ])
         except Exception as e:
-            logging.error(f"Failed to transform the scheduled transactions DataFrame: {e}")
+            self.logger.error(f"Failed to transform the scheduled transactions DataFrame: {e}")
             return
-        logging.info("Writing the transformed scheduled transactions DataFrame to parquet file")
+        self.logger.info("Writing the transformed scheduled transactions DataFrame to parquet file")
         try:
             drop_scheduled_columns.write_parquet(self.config['warehouse_data_path'] + '/scheduled_transactions.parquet')
         except Exception as e:
-            logging.error(f"Failed to write the transformed scheduled transactions DataFrame: {e}")
+            self.logger.error(f"Failed to write the transformed scheduled transactions DataFrame: {e}")