starting a fresh tidy
This commit is contained in:
+48
-47
@@ -1,21 +1,23 @@
|
||||
import polars as pl
|
||||
import logging
|
||||
import os
|
||||
from datetime import date
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
||||
class Dimensions:
|
||||
def __init__(self, config):
|
||||
def __init__(self, config,logger):
|
||||
self.config = config
|
||||
self.base_file_path = self.config['base_data_path']
|
||||
os.makedirs(self.config['warehouse_data_path'], exist_ok=True)
|
||||
|
||||
self.logger = logger
|
||||
|
||||
def get_full_file_path(self, file_name):
|
||||
return f"{self.base_file_path}/{file_name}"
|
||||
|
||||
|
||||
|
||||
|
||||
class DimAccounts(Dimensions):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.file_path = self.get_full_file_path('accounts.parquet')
|
||||
self.transform()
|
||||
|
||||
@@ -24,10 +26,10 @@ class DimAccounts(Dimensions):
|
||||
try:
|
||||
source_accounts = pl.read_parquet(self.file_path)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to read the base accounts parquet file: {e}")
|
||||
self.logger.error(f"Failed to read the base accounts parquet file: {e}")
|
||||
return
|
||||
|
||||
logging.info("Transforming the accounts DataFrame")
|
||||
self.logger.info("Transforming the accounts DataFrame")
|
||||
try:
|
||||
base_accounts = (
|
||||
source_accounts.select([
|
||||
@@ -44,7 +46,7 @@ class DimAccounts(Dimensions):
|
||||
])
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to select columns from the categories DataFrame: {e}")
|
||||
self.logger.error(f"Failed to select columns from the categories DataFrame: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -65,29 +67,29 @@ class DimAccounts(Dimensions):
|
||||
"id", "name", "type"
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to transform the accounts DataFrame: {e}")
|
||||
self.logger.error(f"Failed to transform the accounts DataFrame: {e}")
|
||||
return
|
||||
|
||||
logging.info("Writing the transformed accounts DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed accounts DataFrame to parquet file")
|
||||
try:
|
||||
drop_accounts_columns.write_parquet(self.config['warehouse_data_path'] + '/accounts.parquet')
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed accounts DataFrame to parquet file: {e}")
|
||||
self.logger.error(f"Failed to write the transformed accounts DataFrame to parquet file: {e}")
|
||||
return
|
||||
|
||||
class DimCategories(Dimensions):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.file_path = self.get_full_file_path('categories.parquet')
|
||||
self.transform()
|
||||
|
||||
|
||||
def transform(self):
|
||||
try:
|
||||
source_categories = pl.read_parquet(self.file_path)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to read the base categories parquet file: {e}")
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to read the base categories parquet file: {e}")
|
||||
return
|
||||
logging.info("Transforming the categories DataFrame")
|
||||
self.logger.info("Transforming the categories DataFrame")
|
||||
try:
|
||||
base_categories = source_categories.select([
|
||||
'id',
|
||||
@@ -101,9 +103,9 @@ class DimCategories(Dimensions):
|
||||
'deleted'
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to select columns from the categories DataFrame: {e}")
|
||||
self.logger.error(f"Failed to select columns from the categories DataFrame: {e}")
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
add_categories_prefix = base_categories.with_columns([
|
||||
pl.col('id').alias('category_id'),
|
||||
@@ -121,29 +123,29 @@ class DimCategories(Dimensions):
|
||||
'id', 'name'
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to transform the categories DataFrame: {e}")
|
||||
self.logger.error(f"Failed to transform the categories DataFrame: {e}")
|
||||
return
|
||||
|
||||
logging.info("Writing the transformed categories DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed categories DataFrame to parquet file")
|
||||
try:
|
||||
drop_categories_columns.write_parquet(self.config['warehouse_data_path'] + '/categories.parquet')
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed categories DataFrame to parquet file: {e}")
|
||||
self.logger.error(f"Failed to write the transformed categories DataFrame to parquet file: {e}")
|
||||
return
|
||||
|
||||
class DimPayees(Dimensions):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.file_path = self.get_full_file_path('payees.parquet')
|
||||
self.transform()
|
||||
|
||||
|
||||
def transform(self):
|
||||
try:
|
||||
source_payees = pl.read_parquet(self.file_path)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to read the base payees parquet file: {e}")
|
||||
self.logger.error(f"Failed to read the base payees parquet file: {e}")
|
||||
return
|
||||
logging.info("Transforming the payees DataFrame")
|
||||
self.logger.info("Transforming the payees DataFrame")
|
||||
try:
|
||||
base_payees = source_payees.select([
|
||||
'id',
|
||||
@@ -151,7 +153,7 @@ class DimPayees(Dimensions):
|
||||
'deleted'
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to select columns from the payees DataFrame: {e}")
|
||||
self.logger.error(f"Failed to select columns from the payees DataFrame: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -163,28 +165,28 @@ class DimPayees(Dimensions):
|
||||
'id', 'name'
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to rename columns in the payees DataFrame: {e}")
|
||||
self.logger.error(f"Failed to rename columns in the payees DataFrame: {e}")
|
||||
return
|
||||
|
||||
# Write the DataFrame to a new parquet file
|
||||
logging.info("Writing the transformed payees DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed payees DataFrame to parquet file")
|
||||
try:
|
||||
drop_payees_columns.write_parquet(self.config['warehouse_data_path'] + '/payees.parquet')
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed payees DataFrame to parquet file: {e}")
|
||||
self.logger.error(f"Failed to write the transformed payees DataFrame to parquet file: {e}")
|
||||
return
|
||||
|
||||
class DimDate(Dimensions):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.transform()
|
||||
|
||||
|
||||
def transform(self):
|
||||
# Create a DataFrame with dates from 2020-01-01 to 2030-12-31
|
||||
try:
|
||||
dates_df = pl.DataFrame({'date':pl.date_range(date(2020, 1, 1), date(2030, 12, 31), "1d", eager=True)})
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to create a DataFrame with dates: {e}")
|
||||
self.logger.error(f"Failed to create a DataFrame with dates: {e}")
|
||||
return
|
||||
# Extract year, month, day, and weekday from the date column
|
||||
try:
|
||||
@@ -195,33 +197,32 @@ class DimDate(Dimensions):
|
||||
pl.col('date').dt.weekday().alias('weekday')
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to extract year, month, day, and weekday from the date column: {e}")
|
||||
return
|
||||
self.logger.error(f"Failed to extract year, month, day, and weekday from the date column: {e}")
|
||||
return
|
||||
try:
|
||||
# Create a new column to indicate if the date is a weekday or weekend
|
||||
dates_df = dates_df.with_columns([
|
||||
(pl.col('weekday') < 6).alias('is_weekday') # True for weekdays (Monday to Friday), False for weekends (Saturday and Sunday)
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to create a new column to indicate if the date is a weekday or weekend: {e}")
|
||||
self.logger.error(f"Failed to create a new column to indicate if the date is a weekday or weekend: {e}")
|
||||
return
|
||||
|
||||
|
||||
# Create a primary key by concatenating year, month, and day with no separators
|
||||
try:
|
||||
dates_df = dates_df.with_columns([
|
||||
(pl.col('year').cast(pl.Utf8) +
|
||||
(pl.col('year').cast(pl.Utf8) +
|
||||
pl.col('month').cast(pl.Utf8).str.zfill(2) +
|
||||
pl.col('day').cast(pl.Utf8).str.zfill(2)
|
||||
).alias('date_id')
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to create the primary key column: {e}")
|
||||
self.logger.error(f"Failed to create the primary key column: {e}")
|
||||
return
|
||||
# Write the DataFrame to a new parquet file
|
||||
logging.info("Writing the transformed dates DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed dates DataFrame to parquet file")
|
||||
try:
|
||||
dates_df.write_parquet(self.config['warehouse_data_path'] + '/dates.parquet')
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed dates DataFrame to parquet file: {e}")
|
||||
self.logger.error(f"Failed to write the transformed dates DataFrame to parquet file: {e}")
|
||||
return
|
||||
|
||||
|
||||
+27
-25
@@ -1,19 +1,21 @@
|
||||
import polars as pl
|
||||
import logging
|
||||
import os
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
||||
class Facts:
|
||||
def __init__(self, config):
|
||||
def __init__(self, config,logger):
|
||||
self.config = config
|
||||
self.base_file_path = self.config['base_data_path']
|
||||
self.logger = logger
|
||||
os.makedirs(self.config['warehouse_data_path'], exist_ok=True)
|
||||
|
||||
|
||||
def get_full_file_path(self, file_name):
|
||||
return f"{self.base_file_path}/{file_name}"
|
||||
|
||||
|
||||
class FactTransactions(Facts):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.file_path = self.get_full_file_path('transactions.parquet')
|
||||
self.transform()
|
||||
|
||||
@@ -21,7 +23,7 @@ class FactTransactions(Facts):
|
||||
try:
|
||||
source_transactions = pl.read_parquet(self.file_path)
|
||||
except FileNotFoundError:
|
||||
logging.error("The transactions DataFrame does not exist")
|
||||
self.logger.error("The transactions DataFrame does not exist")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -39,16 +41,16 @@ class FactTransactions(Facts):
|
||||
"transfer_account_id"
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to select columns from the transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to select columns from the transactions DataFrame: {e}")
|
||||
return
|
||||
|
||||
logging.info("Transforming the transactions DataFrame")
|
||||
self.logger.info("Transforming the transactions DataFrame")
|
||||
try:
|
||||
resolve_transaction_dates = base_transactions.with_columns([
|
||||
pl.col("date").str.strptime(pl.Date, format="%Y-%m-%d").alias("date")
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to covert the date to date format: {e}")
|
||||
self.logger.error(f"Failed to covert the date to date format: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -70,22 +72,22 @@ class FactTransactions(Facts):
|
||||
drop_transaction_columns = fix_transaction_values.drop([
|
||||
"id", "date", "amount"
|
||||
])
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to transform the transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to transform the transactions DataFrame: {e}")
|
||||
return
|
||||
# Write the DataFrame to a new parquet file
|
||||
logging.info("Writing the transformed transactions DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed transactions DataFrame to parquet file")
|
||||
try:
|
||||
drop_transaction_columns.write_parquet(
|
||||
self.config['warehouse_data_path'] + '/transactions.parquet'
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to write the transformed transactions DataFrame: {e}")
|
||||
|
||||
class FactScheduledTransactions(Facts):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
def __init__(self, config,logger):
|
||||
super().__init__(config,logger)
|
||||
self.file_path = self.get_full_file_path('scheduled_transactions.parquet')
|
||||
self.transform()
|
||||
|
||||
@@ -93,7 +95,7 @@ class FactScheduledTransactions(Facts):
|
||||
try:
|
||||
source_scheduled = pl.read_parquet(self.file_path)
|
||||
except FileNotFoundError:
|
||||
logging.error("The scheduled transactions DataFrame does not exist")
|
||||
self.logger.error("The scheduled transactions DataFrame does not exist")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -111,7 +113,7 @@ class FactScheduledTransactions(Facts):
|
||||
"transfer_account_id"
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to select columns from the scheduled transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to select columns from the scheduled transactions DataFrame: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -120,10 +122,10 @@ class FactScheduledTransactions(Facts):
|
||||
pl.col("date_next").str.strptime(pl.Date, format="%Y-%m-%d").alias("date_next")
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to covert the date to date format: {e}")
|
||||
self.logger.error(f"Failed to covert the date to date format: {e}")
|
||||
return
|
||||
|
||||
logging.info("Transforming the scheduled transactions DataFrame")
|
||||
|
||||
self.logger.info("Transforming the scheduled transactions DataFrame")
|
||||
try:
|
||||
add_scheduled_prefix = resolve_scheduled_dates.with_columns([
|
||||
pl.col("id").alias("scheduled_transaction_id")
|
||||
@@ -141,10 +143,10 @@ class FactScheduledTransactions(Facts):
|
||||
"id", "amount"
|
||||
])
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to transform the scheduled transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to transform the scheduled transactions DataFrame: {e}")
|
||||
return
|
||||
logging.info("Writing the transformed scheduled transactions DataFrame to parquet file")
|
||||
self.logger.info("Writing the transformed scheduled transactions DataFrame to parquet file")
|
||||
try:
|
||||
drop_scheduled_columns.write_parquet(self.config['warehouse_data_path'] + '/scheduled_transactions.parquet')
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write the transformed scheduled transactions DataFrame: {e}")
|
||||
self.logger.error(f"Failed to write the transformed scheduled transactions DataFrame: {e}")
|
||||
|
||||
+30
-27
@@ -1,15 +1,17 @@
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, Any
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
import config.exit_codes as ec
|
||||
|
||||
|
||||
class Ingest:
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
def __init__(self, config: dict[str, Any],logger):
|
||||
"""
|
||||
Initialize the Ingest class with the provided configuration.
|
||||
"""
|
||||
@@ -22,9 +24,10 @@ class Ingest:
|
||||
self.headers = {'Authorization': f'Bearer {self.api_token}'}
|
||||
self.MAX_RETRIES = config['REQUESTS_MAX_RETRIES']
|
||||
self.RETRY_DELAY = config['REQUESTS_RETRY_DELAY']
|
||||
self.logger = logger
|
||||
|
||||
|
||||
def save_entity_data_to_raw(self, entity: str, data: Dict[str, Any]):
|
||||
def save_entity_data_to_raw(self, entity: str, data: dict[str, Any]):
|
||||
"""
|
||||
Save the data for a specific entity to a new cache file.
|
||||
"""
|
||||
@@ -33,15 +36,15 @@ class Ingest:
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
entity_file = f'{directory}/{current_time}.json'
|
||||
logging.info(f"Saving {entity} data to {entity_file}")
|
||||
self.logger.info(f"Saving {entity} data to {entity_file}")
|
||||
try:
|
||||
with open(entity_file, 'w') as f:
|
||||
json.dump(data, f, indent=4)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to save data for {entity} to {entity_file}")
|
||||
self.logger.error(f"Failed to save data for {entity} to {entity_file}")
|
||||
raise e
|
||||
|
||||
def load_knowledge_cache(self) -> Dict[str, Any]:
|
||||
def load_knowledge_cache(self) -> dict[str, Any]:
|
||||
"""
|
||||
Load the knowledge cache from the file if it exists.
|
||||
"""
|
||||
@@ -59,7 +62,7 @@ class Ingest:
|
||||
with open(self.knowledge_file, 'r') as f:
|
||||
knowledge_cache = json.load(f)
|
||||
except FileNotFoundError:
|
||||
logging.info(f"Knowledge file not found. Creating a new one at {self.knowledge_file}. This is normal for the first run.")
|
||||
self.logger.info(f"Knowledge file not found. Creating a new one at {self.knowledge_file}. This is normal for the first run.")
|
||||
os.makedirs(os.path.dirname(self.knowledge_file), exist_ok=True)
|
||||
knowledge_cache = {}
|
||||
|
||||
@@ -74,33 +77,33 @@ class Ingest:
|
||||
with open(self.knowledge_file, 'w') as f:
|
||||
json.dump(knowledge_cache, f, indent=4)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to update knowledge cache for {entity} in {self.knowledge_file}")
|
||||
self.logger.error(f"Failed to update knowledge cache for {entity} in {self.knowledge_file}")
|
||||
raise e
|
||||
|
||||
def handle_response(self, response) -> bool:
|
||||
if response.status_code == 400:
|
||||
logging.error("Bad request. The request could not be understood by the API due to malformed syntax or validation errors.")
|
||||
self.logger.error("Bad request. The request could not be understood by the API due to malformed syntax or validation errors.")
|
||||
sys.exit(ec.BAD_REQUEST)
|
||||
elif response.status_code == 401:
|
||||
logging.error("Unauthorized. Please check your API token.")
|
||||
self.logger.error("Unauthorized. Please check your API token.")
|
||||
sys.exit(ec.UNAUTHORIZED_API_TOKEN)
|
||||
elif response.status_code == 403:
|
||||
logging.error("Forbidden. Access is denied.")
|
||||
self.logger.error("Forbidden. Access is denied.")
|
||||
sys.exit(ec.FORBIDDEN)
|
||||
elif response.status_code == 404:
|
||||
logging.error("Not found. The specified URL does not exist.")
|
||||
self.logger.error("Not found. The specified URL does not exist.")
|
||||
sys.exit(ec.NOT_FOUND)
|
||||
elif response.status_code == 409:
|
||||
logging.error("Conflict. The resource cannot be saved due to a conflict.")
|
||||
self.logger.error("Conflict. The resource cannot be saved due to a conflict.")
|
||||
sys.exit(ec.CONFLICT)
|
||||
elif response.status_code == 429:
|
||||
logging.error("Too many requests. You have made too many requests in a short amount of time.")
|
||||
self.logger.error("Too many requests. You have made too many requests in a short amount of time.")
|
||||
return True
|
||||
elif response.status_code == 500:
|
||||
logging.error("Internal server error. The API experienced an unexpected error.")
|
||||
self.logger.error("Internal server error. The API experienced an unexpected error.")
|
||||
return True
|
||||
elif response.status_code == 503:
|
||||
logging.error("Service unavailable. The API is temporarily disabled or a request timeout occurred.")
|
||||
self.logger.error("Service unavailable. The API is temporarily disabled or a request timeout occurred.")
|
||||
return True
|
||||
else:
|
||||
response.raise_for_status()
|
||||
@@ -113,14 +116,14 @@ class Ingest:
|
||||
for entity in self.entities:
|
||||
file_path = f'data/raw/{entity}'
|
||||
if os.path.exists(file_path) and os.listdir(file_path):
|
||||
logging.warning(f"Raw data exists for {entity} processing any raw data we already have.")
|
||||
self.logger.warning(f"Raw data exists for {entity} processing any raw data we already have.")
|
||||
break # break here instead of continue as we dont want to update our server knowledge cache and potentially miss data.
|
||||
|
||||
knowledge_cache = self.load_knowledge_cache()
|
||||
last_knowledge = knowledge_cache.get(entity, 0)
|
||||
#logging.debug(f'Last Knowledge of {entity}: {last_knowledge}')
|
||||
|
||||
logging.info(f'Fetching {entity} data since last knowledge: {last_knowledge}')
|
||||
self.logger.info(f'Fetching {entity} data since last knowledge: {last_knowledge}')
|
||||
url = f'{self.base_url}/{self.budget_id}/{entity}?last_knowledge_of_server={last_knowledge}'
|
||||
response = None
|
||||
for attempt in range(self.MAX_RETRIES):
|
||||
@@ -130,17 +133,17 @@ class Ingest:
|
||||
if not should_retry:
|
||||
break # Exit the loop if the request is successful
|
||||
except requests.exceptions.RequestException as e:
|
||||
logging.error(f"Error fetching {entity} data (attempt {attempt + 1}/{self.MAX_RETRIES}): {e}")
|
||||
self.logger.error(f"Error fetching {entity} data (attempt {attempt + 1}/{self.MAX_RETRIES}): {e}")
|
||||
if attempt < self.MAX_RETRIES - 1:
|
||||
time.sleep(self.RETRY_DELAY) # Wait before retrying
|
||||
else:
|
||||
logging.error("Max retries reached. Exiting.")
|
||||
self.logger.error("Max retries reached. Exiting.")
|
||||
sys.exit(ec.REQUESTS_ERROR)
|
||||
|
||||
data = response.json()
|
||||
logging.debug(f'response data: {data}')
|
||||
self.logger.debug(f'response data: {data}')
|
||||
server_knowledge = data['data'].get('server_knowledge')
|
||||
logging.debug(f'{entity} new server knowledge: {server_knowledge}')
|
||||
self.logger.debug(f'{entity} new server knowledge: {server_knowledge}')
|
||||
|
||||
if server_knowledge is not None and server_knowledge != last_knowledge:
|
||||
self.update_server_knowledge_cache(entity, server_knowledge)
|
||||
@@ -148,4 +151,4 @@ class Ingest:
|
||||
entity_data.pop('server_knowledge', None)
|
||||
self.save_entity_data_to_raw(entity, entity_data)
|
||||
else:
|
||||
logging.info(f"No new data for {entity}. Skipping cache update.")
|
||||
self.logger.info(f"No new data for {entity}. Skipping cache update.")
|
||||
|
||||
+12
-18
@@ -1,25 +1,19 @@
|
||||
'''Module to run the data pipeline'''
|
||||
|
||||
import logging
|
||||
|
||||
from pipeline.ingest import Ingest
|
||||
from pipeline.raw_to_base import RawToBase
|
||||
from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate
|
||||
from pipeline.facts import FactTransactions, FactScheduledTransactions
|
||||
from pipeline import dimensions, facts, ingest, raw_to_base
|
||||
|
||||
|
||||
def pipeline_main(config):
|
||||
def pipeline_main(config, logger):
|
||||
'''Run the data pipeline'''
|
||||
logging.info('Starting data pipeline')
|
||||
logger.info('Starting data pipeline')
|
||||
|
||||
ingest = Ingest(config)
|
||||
ingest.start_ingestion()
|
||||
RawToBase(config)
|
||||
DimAccounts(config)
|
||||
DimCategories(config)
|
||||
DimPayees(config)
|
||||
DimDate(config)
|
||||
FactTransactions(config)
|
||||
FactScheduledTransactions(config)
|
||||
ingest.Ingest(config,logger).start_ingestion()
|
||||
raw_to_base.RawToBase(config,logger)
|
||||
dimensions.DimAccounts(config,logger)
|
||||
dimensions.DimCategories(config,logger)
|
||||
dimensions.DimPayees(config,logger)
|
||||
dimensions.DimDate(config,logger)
|
||||
facts.FactTransactions(config,logger)
|
||||
facts.FactScheduledTransactions(config,logger)
|
||||
|
||||
logging.info('Data pipeline completed successfully')
|
||||
logger.info('Data pipeline completed successfully')
|
||||
|
||||
+44
-41
@@ -1,14 +1,16 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any
|
||||
import config.exit_codes as ec
|
||||
from typing import Any
|
||||
|
||||
import polars as pl
|
||||
|
||||
import config.exit_codes as ec
|
||||
|
||||
|
||||
class RawToBase:
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
def __init__(self, config: dict[str, Any],logger):
|
||||
self.entities = config['entities']
|
||||
self.primary_keys = config['primary_keys']
|
||||
self.raw_data_path = config['raw_data_path']
|
||||
@@ -16,39 +18,40 @@ class RawToBase:
|
||||
self.base_data_path = config['base_data_path']
|
||||
self.data = {}
|
||||
self.base_data = {}
|
||||
self.logger = logger
|
||||
self.process_entities()
|
||||
|
||||
def process_entities(self):
|
||||
for entity in self.entities:
|
||||
logging.info(f"Processing entity: {entity}")
|
||||
self.logger.info(f"Processing entity: {entity}")
|
||||
# check the file is in the raw data path, if not skip the entity
|
||||
folder_path = os.path.join(self.raw_data_path, entity)
|
||||
folder_contents = os.listdir(folder_path)
|
||||
if not folder_contents:
|
||||
logging.warning(f"The folder {folder_path} is empty skipping {entity}.")
|
||||
self.logger.warning(f"The folder {folder_path} is empty skipping {entity}.")
|
||||
continue
|
||||
if not self._load_raw_data(entity):
|
||||
logging.warning(f"Skipping processing for entity: {entity} due to empty data.")
|
||||
self.logger.warning(f"Skipping processing for entity: {entity} due to empty data.")
|
||||
continue
|
||||
self._load_existing_base_data(entity)
|
||||
self._combine_data(entity)
|
||||
if not self._save_base_data(entity):
|
||||
logging.error(f"Skipping processing for entity: {entity} due to failed saving base data.")
|
||||
self.logger.error(f"Skipping processing for entity: {entity} due to failed saving base data.")
|
||||
continue
|
||||
if not self._move_raw_to_processed(entity):
|
||||
logging.error(f"entity: {entity} has been processed, but we could not move the file out of the raw folder, please clear the raw folder for {entity}.")
|
||||
self.logger.error(f"entity: {entity} has been processed, but we could not move the file out of the raw folder, please clear the raw folder for {entity}.")
|
||||
sys.exit(ec.MOVE_FILE_ERROR)
|
||||
logging.info(f"Successfully processed entity: {entity}")
|
||||
self.logger.info(f"Successfully processed entity: {entity}")
|
||||
|
||||
def _load_raw_data(self, entity):
|
||||
entity_path = os.path.join(self.raw_data_path, entity)
|
||||
self.data[entity] = []
|
||||
logging.debug(f"Loading data for entity: {entity} from path: {entity_path}")
|
||||
self.logger.debug(f"Loading data for entity: {entity} from path: {entity_path}")
|
||||
|
||||
files = [f for f in os.listdir(entity_path) if f.endswith('.json')]
|
||||
|
||||
if len(files) > 1:
|
||||
logging.error(f"""More than one file found in path: {entity_path}. Skipping processing for entity: {entity}.
|
||||
self.logger.error(f"""More than one file found in path: {entity_path}. Skipping processing for entity: {entity}.
|
||||
recommended actions is to move the newest file(s) out, re-run main.py.
|
||||
Then move the files back in one at a time oldest to newest and run again for each file""")
|
||||
return False
|
||||
@@ -56,12 +59,12 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
if len(files) == 1:
|
||||
file_name = files[0]
|
||||
file_path = os.path.join(entity_path, file_name)
|
||||
logging.debug(f"Reading file: {file_path}")
|
||||
self.logger.debug(f"Reading file: {file_path}")
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to load data from file: {file_path}, error: {e}")
|
||||
self.logger.error(f"Failed to load data from file: {file_path}, error: {e}")
|
||||
return False
|
||||
|
||||
if self._is_data_empty(entity, data, file_path):
|
||||
@@ -69,37 +72,37 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
|
||||
modified_data = self._add_ingestion_date(entity, data, file_name)
|
||||
for index, record in enumerate(modified_data):
|
||||
logging.debug(f"processing record: {record}")
|
||||
self.logger.debug(f"processing record: {record}")
|
||||
filtered_record = {k: v for k, v in record.items() if not k.startswith('debt_')}
|
||||
modified_data[index] = filtered_record
|
||||
logging.debug(f"filtered record: {filtered_record}")
|
||||
logging.debug(f"modified data: {modified_data}")
|
||||
self.logger.debug(f"filtered record: {filtered_record}")
|
||||
self.logger.debug(f"modified data: {modified_data}")
|
||||
|
||||
self.data[entity].append(modified_data)
|
||||
logging.debug(f"Successfully loaded data from file: {file_path}")
|
||||
self.logger.debug(f"Successfully loaded data from file: {file_path}")
|
||||
return True
|
||||
|
||||
def _is_data_empty(self, entity, data, file_path):
|
||||
logging.debug(f"Checking if data is empty for entity: {entity}")
|
||||
self.logger.debug(f"Checking if data is empty for entity: {entity}")
|
||||
if entity == "categories":
|
||||
has_categories = any(group.get("categories") for group in data.get("category_groups", []))
|
||||
if not has_categories:
|
||||
logging.warning(f"Received empty data for entity: {entity} in file: {file_path}, deleting file.")
|
||||
self.logger.warning(f"Received empty data for entity: {entity} in file: {file_path}, deleting file.")
|
||||
os.remove(file_path)
|
||||
return True
|
||||
else:
|
||||
if not data.get(entity, []):
|
||||
logging.warning(f"Received empty data for entity: {entity} in file: {file_path}, deleting file.")
|
||||
self.logger.warning(f"Received empty data for entity: {entity} in file: {file_path}, deleting file.")
|
||||
os.remove(file_path)
|
||||
return True
|
||||
logging.debug(f"Data is not empty for entity: {entity}")
|
||||
self.logger.debug(f"Data is not empty for entity: {entity}")
|
||||
return False
|
||||
|
||||
def _add_ingestion_date(self, entity, data, file_name):
|
||||
modified_data = []
|
||||
ingestion_date = datetime.strptime(file_name.split('.')[0], '%Y%m%d%H%M%S').date()
|
||||
|
||||
logging.debug(f"Adding ingestion date to data for entity: {entity}")
|
||||
self.logger.debug(f"Adding ingestion date to data for entity: {entity}")
|
||||
if entity == 'categories':
|
||||
for group in data.get('category_groups', []):
|
||||
for category in group.get('categories', []):
|
||||
@@ -112,22 +115,22 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
modified_data.append(record)
|
||||
else:
|
||||
modified_data.append({'record': record, 'ingestion_date': ingestion_date})
|
||||
logging.debug(f"Successfully added ingestion date to data for entity: {entity}")
|
||||
self.logger.debug(f"Successfully added ingestion date to data for entity: {entity}")
|
||||
return modified_data
|
||||
|
||||
def _load_existing_base_data(self, entity):
|
||||
base_path = os.path.join(self.base_data_path, f'{entity}.parquet')
|
||||
if os.path.exists(base_path):
|
||||
logging.debug(f"Loading existing base data for entity: {entity} from path: {base_path}")
|
||||
self.logger.debug(f"Loading existing base data for entity: {entity} from path: {base_path}")
|
||||
try:
|
||||
self.base_data[entity] = pl.read_parquet(base_path)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to load existing base data for entity: {entity}, error: {e}, Creating an empty DataFrame")
|
||||
self.logger.error(f"Failed to load existing base data for entity: {entity}, error: {e}, Creating an empty DataFrame")
|
||||
self.base_data[entity] = pl.DataFrame()
|
||||
logging.debug(f"Successfully loaded existing base data for entity: {entity}")
|
||||
self.logger.debug(f"Successfully loaded existing base data for entity: {entity}")
|
||||
else:
|
||||
self.base_data[entity] = pl.DataFrame()
|
||||
logging.debug(f"No existing base data found for entity: {entity}, starting with an empty DataFrame")
|
||||
self.logger.debug(f"No existing base data found for entity: {entity}, starting with an empty DataFrame")
|
||||
|
||||
#Function to cast null Struct({'': Null}) columns to String
|
||||
def _cast_struct_to_string(self,df):
|
||||
@@ -142,7 +145,7 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
return df
|
||||
|
||||
def _combine_data(self, entity):
|
||||
logging.debug(f"Combining data for entity: {entity}")
|
||||
self.logger.debug(f"Combining data for entity: {entity}")
|
||||
combined_data = []
|
||||
|
||||
# Combine data from the entity
|
||||
@@ -159,8 +162,8 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
# Ensure the unique id column is preserved
|
||||
unique_id = self.primary_keys[entity]['unique_id']
|
||||
if unique_id not in new_data_df.columns:
|
||||
logging.error(f"Unique ID column '{unique_id}' not found in the combined data for entity: {entity}")
|
||||
exit(ec.UNIQUE_ID_NOT_FOUND)
|
||||
self.logger.error(f"Unique ID column '{unique_id}' not found in the combined data for entity: {entity}")
|
||||
sys.exit(ec.UNIQUE_ID_NOT_FOUND)
|
||||
|
||||
# Cast columns in new_data_df
|
||||
new_data_df = self._cast_struct_to_string(new_data_df)
|
||||
@@ -188,7 +191,7 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
else:
|
||||
self.base_data[entity] = new_data_df
|
||||
|
||||
logging.debug(f"Successfully combined data for entity: {entity}")
|
||||
self.logger.debug(f"Successfully combined data for entity: {entity}")
|
||||
|
||||
def _save_base_data(self, entity):
|
||||
os.makedirs(self.base_data_path, exist_ok=True)
|
||||
@@ -196,9 +199,9 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
try:
|
||||
self.base_data[entity].write_parquet(file_path)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to save base data for entity: {entity}, error: {e}")
|
||||
self.logger.error(f"Failed to save base data for entity: {entity}, error: {e}")
|
||||
return False
|
||||
logging.debug(f"Saved base data for entity: {entity} to path: {file_path}")
|
||||
self.logger.debug(f"Saved base data for entity: {entity} to path: {file_path}")
|
||||
return True
|
||||
|
||||
def _move_raw_to_processed(self, entity):
|
||||
@@ -210,24 +213,24 @@ Then move the files back in one at a time oldest to newest and run again for eac
|
||||
try:
|
||||
files = [f for f in os.listdir(raw_entity_path) if f.endswith('.json')]
|
||||
if len(files) != 1:
|
||||
logging.error(f"Expected exactly one file in path: {raw_entity_path}, but found {len(files)}")
|
||||
self.logger.error(f"Expected exactly one file in path: {raw_entity_path}, but found {len(files)}")
|
||||
return False
|
||||
|
||||
file_name = files[0]
|
||||
raw_file_path = os.path.join(raw_entity_path, file_name)
|
||||
processed_file_path = os.path.join(processed_path, file_name)
|
||||
|
||||
logging.debug(f"Moving file: {raw_file_path} to {processed_file_path}")
|
||||
self.logger.debug(f"Moving file: {raw_file_path} to {processed_file_path}")
|
||||
|
||||
os.rename(raw_file_path, processed_file_path)
|
||||
logging.debug(f"Moved file: {file_name} to processed")
|
||||
self.logger.debug(f"Moved file: {file_name} to processed")
|
||||
|
||||
except FileNotFoundError as e:
|
||||
logging.error(f"File not found: {e}")
|
||||
self.logger.error(f"File not found: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to move file for entity: {entity}, error: {e}")
|
||||
self.logger.error(f"Failed to move file for entity: {entity}, error: {e}")
|
||||
return False
|
||||
|
||||
logging.debug(f"Moved processed file for entity: {entity} to path: {processed_path}")
|
||||
self.logger.debug(f"Moved processed file for entity: {entity} to path: {processed_path}")
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user