Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

starting a fresh tidy

Browse Source
This commit is contained in:
Jake
2026-07-26 10:34:18 +01:00
parent 5af82e5753
commit 33cbc5c6ed
14 changed files with 985 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/dimensions.py
+48 -47
View File
@@ -1,21 +1,23 @@
import polars as pl
import logging
import os
from datetime import date
import polars as pl
class Dimensions:
def __init__(self, config):
def __init__(self, config,logger):
self.config = config
self.base_file_path = self.config['base_data_path']
os.makedirs(self.config['warehouse_data_path'], exist_ok=True)
self.logger = logger
def get_full_file_path(self, file_name):
return f"{self.base_file_path}/{file_name}"
class DimAccounts(Dimensions):
def __init__(self, config):
super().__init__(config)
def __init__(self, config,logger):
super().__init__(config,logger)
self.file_path = self.get_full_file_path('accounts.parquet')
self.transform()
@@ -24,10 +26,10 @@ class DimAccounts(Dimensions):
try:
source_accounts = pl.read_parquet(self.file_path)
except Exception as e:
logging.error(f"Failed to read the base accounts parquet file: {e}")
self.logger.error(f"Failed to read the base accounts parquet file: {e}")
return
logging.info("Transforming the accounts DataFrame")
self.logger.info("Transforming the accounts DataFrame")
try:
base_accounts = (
source_accounts.select([
@@ -44,7 +46,7 @@ class DimAccounts(Dimensions):
])
)
except Exception as e:
logging.error(f"Failed to select columns from the categories DataFrame: {e}")
self.logger.error(f"Failed to select columns from the categories DataFrame: {e}")
return
try:
@@ -65,29 +67,29 @@ class DimAccounts(Dimensions):
"id", "name", "type"
])
except Exception as e:
logging.error(f"Failed to transform the accounts DataFrame: {e}")
self.logger.error(f"Failed to transform the accounts DataFrame: {e}")
return
logging.info("Writing the transformed accounts DataFrame to parquet file")
self.logger.info("Writing the transformed accounts DataFrame to parquet file")
try:
drop_accounts_columns.write_parquet(self.config['warehouse_data_path'] + '/accounts.parquet')
except Exception as e:
logging.error(f"Failed to write the transformed accounts DataFrame to parquet file: {e}")
self.logger.error(f"Failed to write the transformed accounts DataFrame to parquet file: {e}")
return
class DimCategories(Dimensions):
def __init__(self, config):
super().__init__(config)
def __init__(self, config,logger):
super().__init__(config,logger)
self.file_path = self.get_full_file_path('categories.parquet')
self.transform()
def transform(self):
try:
source_categories = pl.read_parquet(self.file_path)
except Exception as e:
logging.error(f"Failed to read the base categories parquet file: {e}")
except Exception as e:
self.logger.error(f"Failed to read the base categories parquet file: {e}")
return
logging.info("Transforming the categories DataFrame")
self.logger.info("Transforming the categories DataFrame")
try:
base_categories = source_categories.select([
'id',
@@ -101,9 +103,9 @@ class DimCategories(Dimensions):
'deleted'
])
except Exception as e:
logging.error(f"Failed to select columns from the categories DataFrame: {e}")
self.logger.error(f"Failed to select columns from the categories DataFrame: {e}")
return
try:
add_categories_prefix = base_categories.with_columns([
pl.col('id').alias('category_id'),
@@ -121,29 +123,29 @@ class DimCategories(Dimensions):
'id', 'name'
])
except Exception as e:
logging.error(f"Failed to transform the categories DataFrame: {e}")
self.logger.error(f"Failed to transform the categories DataFrame: {e}")
return
logging.info("Writing the transformed categories DataFrame to parquet file")
self.logger.info("Writing the transformed categories DataFrame to parquet file")
try:
drop_categories_columns.write_parquet(self.config['warehouse_data_path'] + '/categories.parquet')
except Exception as e:
logging.error(f"Failed to write the transformed categories DataFrame to parquet file: {e}")
self.logger.error(f"Failed to write the transformed categories DataFrame to parquet file: {e}")
return
class DimPayees(Dimensions):
def __init__(self, config):
super().__init__(config)
def __init__(self, config,logger):
super().__init__(config,logger)
self.file_path = self.get_full_file_path('payees.parquet')
self.transform()
def transform(self):
try:
source_payees = pl.read_parquet(self.file_path)
except Exception as e:
logging.error(f"Failed to read the base payees parquet file: {e}")
self.logger.error(f"Failed to read the base payees parquet file: {e}")
return
logging.info("Transforming the payees DataFrame")
self.logger.info("Transforming the payees DataFrame")
try:
base_payees = source_payees.select([
'id',
@@ -151,7 +153,7 @@ class DimPayees(Dimensions):
'deleted'
])
except Exception as e:
logging.error(f"Failed to select columns from the payees DataFrame: {e}")
self.logger.error(f"Failed to select columns from the payees DataFrame: {e}")
return
try:
@@ -163,28 +165,28 @@ class DimPayees(Dimensions):
'id', 'name'
])
except Exception as e:
logging.error(f"Failed to rename columns in the payees DataFrame: {e}")
self.logger.error(f"Failed to rename columns in the payees DataFrame: {e}")
return
# Write the DataFrame to a new parquet file
logging.info("Writing the transformed payees DataFrame to parquet file")
self.logger.info("Writing the transformed payees DataFrame to parquet file")
try:
drop_payees_columns.write_parquet(self.config['warehouse_data_path'] + '/payees.parquet')
except Exception as e:
logging.error(f"Failed to write the transformed payees DataFrame to parquet file: {e}")
self.logger.error(f"Failed to write the transformed payees DataFrame to parquet file: {e}")
return
class DimDate(Dimensions):
def __init__(self, config):
super().__init__(config)
def __init__(self, config,logger):
super().__init__(config,logger)
self.transform()
def transform(self):
# Create a DataFrame with dates from 2020-01-01 to 2030-12-31
try:
dates_df = pl.DataFrame({'date':pl.date_range(date(2020, 1, 1), date(2030, 12, 31), "1d", eager=True)})
except Exception as e:
logging.error(f"Failed to create a DataFrame with dates: {e}")
self.logger.error(f"Failed to create a DataFrame with dates: {e}")
return
# Extract year, month, day, and weekday from the date column
try:
@@ -195,33 +197,32 @@ class DimDate(Dimensions):
pl.col('date').dt.weekday().alias('weekday')
])
except Exception as e:
logging.error(f"Failed to extract year, month, day, and weekday from the date column: {e}")
return
self.logger.error(f"Failed to extract year, month, day, and weekday from the date column: {e}")
return
try:
# Create a new column to indicate if the date is a weekday or weekend
dates_df = dates_df.with_columns([
(pl.col('weekday') < 6).alias('is_weekday') # True for weekdays (Monday to Friday), False for weekends (Saturday and Sunday)
])
except Exception as e:
logging.error(f"Failed to create a new column to indicate if the date is a weekday or weekend: {e}")
self.logger.error(f"Failed to create a new column to indicate if the date is a weekday or weekend: {e}")
return
# Create a primary key by concatenating year, month, and day with no separators
try:
dates_df = dates_df.with_columns([
(pl.col('year').cast(pl.Utf8) +
(pl.col('year').cast(pl.Utf8) +
pl.col('month').cast(pl.Utf8).str.zfill(2) +
pl.col('day').cast(pl.Utf8).str.zfill(2)
).alias('date_id')
])
except Exception as e:
logging.error(f"Failed to create the primary key column: {e}")
self.logger.error(f"Failed to create the primary key column: {e}")
return
# Write the DataFrame to a new parquet file
logging.info("Writing the transformed dates DataFrame to parquet file")
self.logger.info("Writing the transformed dates DataFrame to parquet file")
try:
dates_df.write_parquet(self.config['warehouse_data_path'] + '/dates.parquet')
except Exception as e:
logging.error(f"Failed to write the transformed dates DataFrame to parquet file: {e}")
self.logger.error(f"Failed to write the transformed dates DataFrame to parquet file: {e}")
return