Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

visual for total spend fixed,

Browse Source
This commit is contained in:
Jake
2025-04-13 11:02:57 +01:00
parent 7668f790aa
commit cb5c03ab74
5 changed files with 153 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_check.py
+64 -57
View File
@@ -1,69 +1,76 @@
import polars as pl import polars as pl
from datetime import date, timedelta from datetime import date, timedelta
import yaml
accounts = pl.read_parquet('data/warehouse/accounts.parquet') # accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet') # categories = pl.read_parquet('data/warehouse/categories.parquet')
dates = pl.read_parquet('data/warehouse/dates.parquet') # dates = pl.read_parquet('data/warehouse/dates.parquet')
payees = pl.read_parquet('data/warehouse/payees.parquet') # payees = pl.read_parquet('data/warehouse/payees.parquet')
scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet') # scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
transactions = pl.read_parquet('data/warehouse/transactions.parquet') # transactions = pl.read_parquet('data/warehouse/transactions.parquet')
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\ # master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\ # .join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\ # .join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date') # .join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
# Create aggregations # # Create aggregations
spend_per_day = master_transactions.sql(''' # spend_per_day = master_transactions.sql('''
SELECT # SELECT
date, # date,
year, # year,
month, # month,
day, # day,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE category_name != 'Inflow: Ready to Assign' # WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day # GROUP BY date, year, month, day
ORDER BY date DESC # ORDER BY date DESC
''' # '''
) # )
spend_per_category = master_transactions.sql(''' # spend_per_category = master_transactions.sql('''
SELECT # SELECT
category_name, # category_name,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE category_name != 'Inflow: Ready to Assign' # WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY category_name # GROUP BY category_name
ORDER BY total DESC # ORDER BY total DESC
''' # '''
) # )
spend_per_payee = master_transactions.sql(''' # spend_per_payee = master_transactions.sql('''
SELECT # SELECT
payee_name, # payee_name,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE payee_name != 'Starting Balance' # WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0 # AND transaction_amount < 0
GROUP BY payee_name # GROUP BY payee_name
ORDER BY total DESC # ORDER BY total DESC
''' # '''
) # )
def update_dates(start_date, end_date): # def update_dates(start_date, end_date):
print("start date", start_date) # print("start date", start_date)
print("end date", end_date) # print("end date", end_date)
print(master_transactions) # print(master_transactions)
master_data = master_transactions.filter( # master_data = master_transactions.filter(
pl.col("date").is_between(start_date, end_date) # pl.col("date").is_between(start_date, end_date)
) # )
return master_data # return master_data
today = date.today() # today = date.today()
one_year_ago = today - timedelta(days=5) # one_year_ago = today - timedelta(days=5)
data = update_dates(start_date=one_year_ago, end_date=today) # data = update_dates(start_date=one_year_ago, end_date=today)
print(data) # print(data)
with open('config/config.yaml', 'r') as file:
config = yaml.safe_load(file)
for k,v in config.items():
print(k,v)
main.py
+6
View File
@@ -33,6 +33,12 @@ def load_config():
logging.error('config.yaml file not found') logging.error('config.yaml file not found')
sys.exit(ec.MISSING_CONFIG_FILE) sys.exit(ec.MISSING_CONFIG_FILE)
except yaml.YAMLError as e: except yaml.YAMLError as e:
# Attempt to print a more informative error message
try:
parsed_yaml = yaml.safe_load_all(file)
for key, value in parsed_yaml:
print(f"Error parsing key '{key}': {value}")
except Exception as e:
logging.error(f'Error loading config.yaml: {e}') logging.error(f'Error loading config.yaml: {e}')
sys.exit(ec.CORRUPTED_CONFIG_FILE) sys.exit(ec.CORRUPTED_CONFIG_FILE)
except Exception as e: except Exception as e:
visuals/components.py
+4 -2
View File
@@ -2,7 +2,6 @@ import polars as pl
import plotly.express as px import plotly.express as px
import pandas as pd import pandas as pd
import logging import logging
import sys
import config.exit_codes as ec import config.exit_codes as ec
# import datetime # import datetime
@@ -115,10 +114,13 @@ class data_components():
font_color='white' font_color='white'
) )
total_spend_line = f"### £{total_spend:,.2f}"
data = {"spend_per_day_line": spend_per_day_line, data = {"spend_per_day_line": spend_per_day_line,
"spend_per_category_bar": spend_per_category_bar, "spend_per_category_bar": spend_per_category_bar,
"spend_per_payee_bar": spend_per_payee_bar, "spend_per_payee_bar": spend_per_payee_bar,
"total_spend": total_spend} "total_spend": total_spend_line}
if callback == 0: if callback == 0:
return data return data
else: else:
visuals/dash_app.py
+1 -2
View File
@@ -26,11 +26,10 @@ app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])
Input('date-picker-range', 'start_date'), Input('date-picker-range', 'start_date'),
Input('date-picker-range', 'end_date') Input('date-picker-range', 'end_date')
) )
def update_layout(start_date,end_date): def update_date_range(start_date,end_date):
actual_start_date = datetime.date.fromisoformat(start_date) actual_start_date = datetime.date.fromisoformat(start_date)
actual_end_date = datetime.date.fromisoformat(end_date) actual_end_date = datetime.date.fromisoformat(end_date)
master_data = data_components.update_dates(actual_start_date,actual_end_date) master_data = data_components.update_dates(actual_start_date,actual_end_date)
# spend_per_day_line,spend_per_category_bar,spend_per_payee_bar,total_spend = data_components.update_data(master_data,callback=1)
return data_components.update_data(master_data,callback=1) return data_components.update_data(master_data,callback=1)
app.layout = create_layout(data) app.layout = create_layout(data)
visuals/layout.py
+19 -27
View File
@@ -16,7 +16,15 @@ def create_layout(data):
def create_topbar(): def create_topbar():
return [ return [
dbc.Container( dbc.Container([
dbc.Row(
dbc.Col(
html.Div(html.H1("Data Pipeline For YNAB, Preview Visualisations"),
className="text-center text-light",
),
width=12,
)
),
dbc.Row( dbc.Row(
[ [
dbc.Col( dbc.Col(
@@ -35,6 +43,7 @@ def create_topbar():
), ),
] ]
) )
]
) )
] ]
@@ -43,21 +52,10 @@ def create_main_body(data):
dbc.Container( dbc.Container(
[ [
dbc.Row( dbc.Row(
dbc.Col( [dbc.Col(
html.Div(
"Data Pipeline For YNAB, Preview Visualisations",
className="text-center text-light",
),
width=12,
)
),
dbc.Row(
[
dbc.Col(
dbc.Card( dbc.Card(
dbc.CardBody( dbc.CardBody(
[ [html.H4(
html.H4(
"Spend Per Day", className="card-title" "Spend Per Day", className="card-title"
), ),
dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'), dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'),
@@ -66,16 +64,13 @@ def create_main_body(data):
className="mb-4", className="mb-4",
), ),
width=12, width=12,
) )]
]
), ),
dbc.Row( dbc.Row(
[ [dbc.Col(
dbc.Col(
dbc.Card( dbc.Card(
dbc.CardBody( dbc.CardBody(
[ [html.H4(
html.H4(
"Spend Per Category", className="card-title" "Spend Per Category", className="card-title"
), ),
dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'), dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'),
@@ -89,14 +84,11 @@ def create_main_body(data):
dbc.Card( dbc.Card(
dbc.CardBody( dbc.CardBody(
[ [
dcc.Markdown(f""" dcc.Markdown('## Total Spend:'),
## Total Spend: dcc.Markdown(data['total_spend'],id='total_spend'),
### £{data['total_spend']:,} ],
"""),
]
), ),
id='total_spend', className="text-center text-light",
className="mb-4",
), ),
width=2, width=2,
), ),