Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

visual for total spend fixed,

Browse Source
This commit is contained in:
Jake
2025-04-13 11:02:57 +01:00
parent 7668f790aa
commit cb5c03ab74
5 changed files with 153 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_check.py
+64 -57
View File
@@ -1,69 +1,76 @@
import polars as pl
from datetime import date, timedelta
import yaml
accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet')
dates = pl.read_parquet('data/warehouse/dates.parquet')
payees = pl.read_parquet('data/warehouse/payees.parquet')
scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
transactions = pl.read_parquet('data/warehouse/transactions.parquet')
# accounts = pl.read_parquet('data/warehouse/accounts.parquet')
# categories = pl.read_parquet('data/warehouse/categories.parquet')
# dates = pl.read_parquet('data/warehouse/dates.parquet')
# payees = pl.read_parquet('data/warehouse/payees.parquet')
# scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
# transactions = pl.read_parquet('data/warehouse/transactions.parquet')
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
# master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
# .join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
# .join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
# .join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
# Create aggregations
spend_per_day = master_transactions.sql('''
SELECT
date,
year,
month,
day,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day
ORDER BY date DESC
'''
)
# # Create aggregations
# spend_per_day = master_transactions.sql('''
# SELECT
# date,
# year,
# month,
# day,
# ABS(SUM(transaction_amount)) as total
# FROM self
# WHERE category_name != 'Inflow: Ready to Assign'
# GROUP BY date, year, month, day
# ORDER BY date DESC
# '''
# )
spend_per_category = master_transactions.sql('''
SELECT
category_name,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY category_name
ORDER BY total DESC
'''
)
# spend_per_category = master_transactions.sql('''
# SELECT
# category_name,
# ABS(SUM(transaction_amount)) as total
# FROM self
# WHERE category_name != 'Inflow: Ready to Assign'
# GROUP BY category_name
# ORDER BY total DESC
# '''
# )
spend_per_payee = master_transactions.sql('''
SELECT
payee_name,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0
GROUP BY payee_name
ORDER BY total DESC
'''
)
# spend_per_payee = master_transactions.sql('''
# SELECT
# payee_name,
# ABS(SUM(transaction_amount)) as total
# FROM self
# WHERE payee_name != 'Starting Balance'
# AND transaction_amount < 0
# GROUP BY payee_name
# ORDER BY total DESC
# '''
# )
def update_dates(start_date, end_date):
print("start date", start_date)
print("end date", end_date)
print(master_transactions)
master_data = master_transactions.filter(
pl.col("date").is_between(start_date, end_date)
)
return master_data
# def update_dates(start_date, end_date):
# print("start date", start_date)
# print("end date", end_date)
# print(master_transactions)
# master_data = master_transactions.filter(
# pl.col("date").is_between(start_date, end_date)
# )
# return master_data
today = date.today()
one_year_ago = today - timedelta(days=5)
# today = date.today()
# one_year_ago = today - timedelta(days=5)
data = update_dates(start_date=one_year_ago, end_date=today)
print(data)
# data = update_dates(start_date=one_year_ago, end_date=today)
# print(data)
with open('config/config.yaml', 'r') as file:
config = yaml.safe_load(file)
for k,v in config.items():
print(k,v)
main.py
+6
View File
@@ -33,6 +33,12 @@ def load_config():
logging.error('config.yaml file not found')
sys.exit(ec.MISSING_CONFIG_FILE)
except yaml.YAMLError as e:
# Attempt to print a more informative error message
try:
parsed_yaml = yaml.safe_load_all(file)
for key, value in parsed_yaml:
print(f"Error parsing key '{key}': {value}")
except Exception as e:
logging.error(f'Error loading config.yaml: {e}')
sys.exit(ec.CORRUPTED_CONFIG_FILE)
except Exception as e:
visuals/components.py
+4 -2
View File
@@ -2,7 +2,6 @@ import polars as pl
import plotly.express as px
import pandas as pd
import logging
import sys
import config.exit_codes as ec
# import datetime
@@ -115,10 +114,13 @@ class data_components():
font_color='white'
)
total_spend_line = f"### £{total_spend:,.2f}"
data = {"spend_per_day_line": spend_per_day_line,
"spend_per_category_bar": spend_per_category_bar,
"spend_per_payee_bar": spend_per_payee_bar,
"total_spend": total_spend}
"total_spend": total_spend_line}
if callback == 0:
return data
else:
visuals/dash_app.py
+1 -2
View File
@@ -26,11 +26,10 @@ app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])
Input('date-picker-range', 'start_date'),
Input('date-picker-range', 'end_date')
)
def update_layout(start_date,end_date):
def update_date_range(start_date,end_date):
actual_start_date = datetime.date.fromisoformat(start_date)
actual_end_date = datetime.date.fromisoformat(end_date)
master_data = data_components.update_dates(actual_start_date,actual_end_date)
# spend_per_day_line,spend_per_category_bar,spend_per_payee_bar,total_spend = data_components.update_data(master_data,callback=1)
return data_components.update_data(master_data,callback=1)
app.layout = create_layout(data)
visuals/layout.py
+19 -27
View File
@@ -16,7 +16,15 @@ def create_layout(data):
def create_topbar():
return [
dbc.Container(
dbc.Container([
dbc.Row(
dbc.Col(
html.Div(html.H1("Data Pipeline For YNAB, Preview Visualisations"),
className="text-center text-light",
),
width=12,
)
),
dbc.Row(
[
dbc.Col(
@@ -35,6 +43,7 @@ def create_topbar():
),
]
)
]
)
]
@@ -43,21 +52,10 @@ def create_main_body(data):
dbc.Container(
[
dbc.Row(
dbc.Col(
html.Div(
"Data Pipeline For YNAB, Preview Visualisations",
className="text-center text-light",
),
width=12,
)
),
dbc.Row(
[
dbc.Col(
[dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
[html.H4(
"Spend Per Day", className="card-title"
),
dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'),
@@ -66,16 +64,13 @@ def create_main_body(data):
className="mb-4",
),
width=12,
)
]
)]
),
dbc.Row(
[
dbc.Col(
[dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
[html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'),
@@ -89,14 +84,11 @@ def create_main_body(data):
dbc.Card(
dbc.CardBody(
[
dcc.Markdown(f"""
## Total Spend:
### £{data['total_spend']:,}
"""),
]
dcc.Markdown('## Total Spend:'),
dcc.Markdown(data['total_spend'],id='total_spend'),
],
),
id='total_spend',
className="mb-4",
className="text-center text-light",
),
width=2,
),