Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working callback, known minor visual with total spend

Browse Source
This commit is contained in:
Jake
2025-04-12 16:19:04 +01:00
parent a79c245511
commit 7668f790aa
4 changed files with 120 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_check.py
+14 -5
View File
@@ -53,8 +53,17 @@ spend_per_payee = master_transactions.sql('''
''' '''
) )
# print(spend_per_day) def update_dates(start_date, end_date):
today = date(date.today()) print("start date", start_date)
# Convert the dates to datetime objects that are compatible with Polars print("end date", end_date)
start_date = pl.Date(today) print(master_transactions)
print(start_date) master_data = master_transactions.filter(
pl.col("date").is_between(start_date, end_date)
)
return master_data
today = date.today()
one_year_ago = today - timedelta(days=5)
data = update_dates(start_date=one_year_ago, end_date=today)
print(data)
visuals/components.py
+93 -89
View File
@@ -6,116 +6,120 @@ import sys
import config.exit_codes as ec import config.exit_codes as ec
# import datetime # import datetime
try: class data_components():
accounts = pl.read_parquet('data/warehouse/accounts.parquet') accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet') categories = pl.read_parquet('data/warehouse/categories.parquet')
dates = pl.read_parquet('data/warehouse/dates.parquet') dates = pl.read_parquet('data/warehouse/dates.parquet')
payees = pl.read_parquet('data/warehouse/payees.parquet') payees = pl.read_parquet('data/warehouse/payees.parquet')
scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet') scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
transactions = pl.read_parquet('data/warehouse/transactions.parquet') transactions = pl.read_parquet('data/warehouse/transactions.parquet')
except FileNotFoundError:
logging.error('Data warehouse files not found. Run the data pipeline to create them.')
sys.exit(ec.MISSING_DATA_FILES)
try:
# Join transactions with accounts, categories, and payees to create a master DataFrame
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\ master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\ .join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\ .join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date') .join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
def __init__(self):
logging.info("Initializing data components")
pass
except Exception as e: def update_dates(start_date, end_date):
logging.error(f'Error joining DataFrames: {e}') logging.info("Updating dates")
sys.exit(ec.BAD_JOIN) logging.debug(f"start_date: {start_date}, end_date: {end_date}")
logging.debug(data_components.master_transactions.columns)
try:
master_data = data_components.master_transactions.filter(
pl.col("date").is_between(start_date, end_date)
)
except Exception as e:
logging.error(f"Error updating dates: {e}")
raise e
return master_data
def update_dates(start_date, end_date): def update_data(master_data,callback=0):
master_data = master_transactions.filter( # Create aggregations
pl.col("date").is_between(start_date, end_date) spend_per_day = master_data.sql('''
) SELECT
return master_data date,
year,
month,
day,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day
ORDER BY date DESC
'''
)
def update_data(master_data): spend_per_category = master_data.sql('''
# Create aggregations SELECT
spend_per_day = master_data.sql(''' category_name,
SELECT ABS(SUM(transaction_amount)) as total
date, FROM self
year, WHERE category_name != 'Inflow: Ready to Assign'
month, GROUP BY category_name
day, ORDER BY total DESC
ABS(SUM(transaction_amount)) as total '''
FROM self )
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day
ORDER BY date DESC
'''
)
spend_per_category = master_data.sql(''' spend_per_payee = master_data.sql('''
SELECT SELECT
category_name, payee_name,
ABS(SUM(transaction_amount)) as total ABS(SUM(transaction_amount)) as total
FROM self FROM self
WHERE category_name != 'Inflow: Ready to Assign' WHERE payee_name != 'Starting Balance'
GROUP BY category_name AND transaction_amount < 0
ORDER BY total DESC GROUP BY payee_name
''' ORDER BY total DESC
) '''
)
spend_per_payee = master_data.sql(''' total_spend = master_data.sql('''
SELECT SELECT ABS(SUM(transaction_amount)) AS total
payee_name, FROM self
ABS(SUM(transaction_amount)) as total WHERE payee_name != 'Starting Balance'
FROM self AND transaction_amount < 0
WHERE payee_name != 'Starting Balance' ''').item()
AND transaction_amount < 0
GROUP BY payee_name
ORDER BY total DESC
'''
)
total_spend = master_data.sql('''
SELECT ABS(SUM(transaction_amount)) AS total
FROM self
WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0
''').item()
# Convert DataFrame to list of dictionaries # Convert DataFrame to list of dictionaries
spend_per_day_data = spend_per_day.to_dicts() spend_per_day_data = spend_per_day.to_dicts()
spend_per_category_data = spend_per_category.to_dicts() spend_per_category_data = spend_per_category.to_dicts()
spend_per_payee_data = spend_per_payee.to_dicts() spend_per_payee_data = spend_per_payee.to_dicts()
# Convert list of dictionaries to Pandas DataFrame # Convert list of dictionaries to Pandas DataFrame
spend_per_day_df = pd.DataFrame(spend_per_day_data) spend_per_day_df = pd.DataFrame(spend_per_day_data)
spend_per_category_df = pd.DataFrame(spend_per_category_data) spend_per_category_df = pd.DataFrame(spend_per_category_data)
spend_per_payee_df = pd.DataFrame(spend_per_payee_data) spend_per_payee_df = pd.DataFrame(spend_per_payee_data)
spend_per_day_line = px.line(spend_per_day_df, x="date", y="total") spend_per_day_line = px.line(spend_per_day_df, x="date", y="total")
spend_per_day_line.update_layout( spend_per_day_line.update_layout(
plot_bgcolor='black', plot_bgcolor='black',
paper_bgcolor='black', paper_bgcolor='black',
font_color='white' font_color='white'
) )
spend_per_category_bar = px.bar(spend_per_category_df, x="category_name", y="total") spend_per_category_bar = px.bar(spend_per_category_df, x="category_name", y="total")
spend_per_category_bar.update_layout( spend_per_category_bar.update_layout(
plot_bgcolor='black', plot_bgcolor='black',
paper_bgcolor='black', paper_bgcolor='black',
font_color='white' font_color='white'
) )
spend_per_payee_bar = px.bar(spend_per_payee_df, x="payee_name", y="total") spend_per_payee_bar = px.bar(spend_per_payee_df, x="payee_name", y="total")
spend_per_payee_bar.update_layout( spend_per_payee_bar.update_layout(
plot_bgcolor='black', plot_bgcolor='black',
paper_bgcolor='black', paper_bgcolor='black',
font_color='white' font_color='white'
) )
data = {"spend_per_day_line": spend_per_day_line, data = {"spend_per_day_line": spend_per_day_line,
"spend_per_category_bar": spend_per_category_bar, "spend_per_category_bar": spend_per_category_bar,
"spend_per_payee_bar": spend_per_payee_bar, "spend_per_payee_bar": spend_per_payee_bar,
"total_spend": total_spend} "total_spend": total_spend}
if callback == 0:
return data return data
else:
return spend_per_day_line,spend_per_category_bar,spend_per_payee_bar,total_spend
visuals/dash_app.py
+11 -8
View File
@@ -4,14 +4,15 @@ from dash import dcc, html
from dash.dependencies import Input, Output, State from dash.dependencies import Input, Output, State
import dash_bootstrap_components as dbc import dash_bootstrap_components as dbc
from visuals.layout import create_layout from visuals.layout import create_layout
from visuals.components import update_data, update_dates from visuals.components import data_components
from datetime import date, timedelta from datetime import date, timedelta
import datetime
today = date.today() today = date.today()
one_year_ago = today - timedelta(days=365) one_year_ago = today - timedelta(days=365)
master_data = update_dates(start_date=one_year_ago, end_date=today) master_data = data_components.update_dates(start_date=one_year_ago, end_date=today)
data = update_data(master_data) data = data_components.update_data(master_data)
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY]) app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])
@@ -22,12 +23,14 @@ app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])
Output("spend_per_payee","figure"), Output("spend_per_payee","figure"),
Output("total_spend","children"), Output("total_spend","children"),
[Input('date-picker-range', 'start_date'), Input('date-picker-range', 'start_date'),
Input('date-picker-range', 'end_date')] Input('date-picker-range', 'end_date')
) )
def update_layout(start_date,end_date): def update_layout(start_date,end_date):
master_data = update_dates(start_date, end_date) actual_start_date = datetime.date.fromisoformat(start_date)
data = update_data(master_data) actual_end_date = datetime.date.fromisoformat(end_date)
return create_layout(data) master_data = data_components.update_dates(actual_start_date,actual_end_date)
# spend_per_day_line,spend_per_category_bar,spend_per_payee_bar,total_spend = data_components.update_data(master_data,callback=1)
return data_components.update_data(master_data,callback=1)
app.layout = create_layout(data) app.layout = create_layout(data)
visuals/layout.py
+2
View File
@@ -21,6 +21,8 @@ def create_topbar():
[ [
dbc.Col( dbc.Col(
dcc.DatePickerRange( dcc.DatePickerRange(
first_day_of_week=1,
display_format="YYYY-MM-DD",
id="date-picker-range", id="date-picker-range",
start_date=one_year_ago, start_date=one_year_ago,
end_date=today, end_date=today,