Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

visual for total spend fixed,

Browse Source
This commit is contained in:
Jake
2025-04-13 11:02:57 +01:00
parent 7668f790aa
commit cb5c03ab74
5 changed files with 153 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_check.py
+64 -57
View File
@@ -1,69 +1,76 @@
import polars as pl import polars as pl
from datetime import date, timedelta from datetime import date, timedelta
import yaml
accounts = pl.read_parquet('data/warehouse/accounts.parquet') # accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet') # categories = pl.read_parquet('data/warehouse/categories.parquet')
dates = pl.read_parquet('data/warehouse/dates.parquet') # dates = pl.read_parquet('data/warehouse/dates.parquet')
payees = pl.read_parquet('data/warehouse/payees.parquet') # payees = pl.read_parquet('data/warehouse/payees.parquet')
scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet') # scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
transactions = pl.read_parquet('data/warehouse/transactions.parquet') # transactions = pl.read_parquet('data/warehouse/transactions.parquet')
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\ # master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\ # .join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\ # .join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date') # .join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
# Create aggregations # # Create aggregations
spend_per_day = master_transactions.sql(''' # spend_per_day = master_transactions.sql('''
SELECT # SELECT
date, # date,
year, # year,
month, # month,
day, # day,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE category_name != 'Inflow: Ready to Assign' # WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day # GROUP BY date, year, month, day
ORDER BY date DESC # ORDER BY date DESC
''' # '''
) # )
spend_per_category = master_transactions.sql(''' # spend_per_category = master_transactions.sql('''
SELECT # SELECT
category_name, # category_name,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE category_name != 'Inflow: Ready to Assign' # WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY category_name # GROUP BY category_name
ORDER BY total DESC # ORDER BY total DESC
''' # '''
) # )
spend_per_payee = master_transactions.sql(''' # spend_per_payee = master_transactions.sql('''
SELECT # SELECT
payee_name, # payee_name,
ABS(SUM(transaction_amount)) as total # ABS(SUM(transaction_amount)) as total
FROM self # FROM self
WHERE payee_name != 'Starting Balance' # WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0 # AND transaction_amount < 0
GROUP BY payee_name # GROUP BY payee_name
ORDER BY total DESC # ORDER BY total DESC
''' # '''
) # )
def update_dates(start_date, end_date): # def update_dates(start_date, end_date):
print("start date", start_date) # print("start date", start_date)
print("end date", end_date) # print("end date", end_date)
print(master_transactions) # print(master_transactions)
master_data = master_transactions.filter( # master_data = master_transactions.filter(
pl.col("date").is_between(start_date, end_date) # pl.col("date").is_between(start_date, end_date)
) # )
return master_data # return master_data
today = date.today() # today = date.today()
one_year_ago = today - timedelta(days=5) # one_year_ago = today - timedelta(days=5)
data = update_dates(start_date=one_year_ago, end_date=today) # data = update_dates(start_date=one_year_ago, end_date=today)
print(data) # print(data)
with open('config/config.yaml', 'r') as file:
config = yaml.safe_load(file)
for k,v in config.items():
print(k,v)
main.py
+7 -1
View File
@@ -33,7 +33,13 @@ def load_config():
logging.error('config.yaml file not found') logging.error('config.yaml file not found')
sys.exit(ec.MISSING_CONFIG_FILE) sys.exit(ec.MISSING_CONFIG_FILE)
except yaml.YAMLError as e: except yaml.YAMLError as e:
logging.error(f'Error loading config.yaml: {e}') # Attempt to print a more informative error message
try:
parsed_yaml = yaml.safe_load_all(file)
for key, value in parsed_yaml:
print(f"Error parsing key '{key}': {value}")
except Exception as e:
logging.error(f'Error loading config.yaml: {e}')
sys.exit(ec.CORRUPTED_CONFIG_FILE) sys.exit(ec.CORRUPTED_CONFIG_FILE)
except Exception as e: except Exception as e:
logging.error(f'some other problem {e}') logging.error(f'some other problem {e}')
visuals/components.py
+4 -2
View File
@@ -2,7 +2,6 @@ import polars as pl
import plotly.express as px import plotly.express as px
import pandas as pd import pandas as pd
import logging import logging
import sys
import config.exit_codes as ec import config.exit_codes as ec
# import datetime # import datetime
@@ -115,10 +114,13 @@ class data_components():
font_color='white' font_color='white'
) )
total_spend_line = f"### £{total_spend:,.2f}"
data = {"spend_per_day_line": spend_per_day_line, data = {"spend_per_day_line": spend_per_day_line,
"spend_per_category_bar": spend_per_category_bar, "spend_per_category_bar": spend_per_category_bar,
"spend_per_payee_bar": spend_per_payee_bar, "spend_per_payee_bar": spend_per_payee_bar,
"total_spend": total_spend} "total_spend": total_spend_line}
if callback == 0: if callback == 0:
return data return data
else: else:
visuals/dash_app.py
+1 -2
View File
@@ -26,11 +26,10 @@ app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])
Input('date-picker-range', 'start_date'), Input('date-picker-range', 'start_date'),
Input('date-picker-range', 'end_date') Input('date-picker-range', 'end_date')
) )
def update_layout(start_date,end_date): def update_date_range(start_date,end_date):
actual_start_date = datetime.date.fromisoformat(start_date) actual_start_date = datetime.date.fromisoformat(start_date)
actual_end_date = datetime.date.fromisoformat(end_date) actual_end_date = datetime.date.fromisoformat(end_date)
master_data = data_components.update_dates(actual_start_date,actual_end_date) master_data = data_components.update_dates(actual_start_date,actual_end_date)
# spend_per_day_line,spend_per_category_bar,spend_per_payee_bar,total_spend = data_components.update_data(master_data,callback=1)
return data_components.update_data(master_data,callback=1) return data_components.update_data(master_data,callback=1)
app.layout = create_layout(data) app.layout = create_layout(data)
visuals/layout.py
+77 -85
View File
@@ -16,25 +16,34 @@ def create_layout(data):
def create_topbar(): def create_topbar():
return [ return [
dbc.Container( dbc.Container([
dbc.Row( dbc.Row(
[ dbc.Col(
dbc.Col( html.Div(html.H1("Data Pipeline For YNAB, Preview Visualisations"),
dcc.DatePickerRange( className="text-center text-light",
first_day_of_week=1, ),
display_format="YYYY-MM-DD", width=12,
id="date-picker-range", )
start_date=one_year_ago, ),
end_date=today, dbc.Row(
[
dbc.Col(
dcc.DatePickerRange(
first_day_of_week=1,
display_format="YYYY-MM-DD",
id="date-picker-range",
start_date=one_year_ago,
end_date=today,
),
width=4,
), ),
width=4, dbc.Col(
), html.Button("Change Date Range", id="date-range-confirm-button"),
dbc.Col( width=2,
html.Button("Change Date Range", id="date-range-confirm-button"), ),
width=2, ]
),
]
) )
]
) )
] ]
@@ -42,80 +51,63 @@ def create_main_body(data):
return [ return [
dbc.Container( dbc.Container(
[ [
dbc.Row( dbc.Row(
dbc.Col( [dbc.Col(
html.Div( dbc.Card(
"Data Pipeline For YNAB, Preview Visualisations", dbc.CardBody(
className="text-center text-light", [html.H4(
"Spend Per Day", className="card-title"
),
dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'),
]
), ),
width=12, className="mb-4",
) ),
width=12,
)]
),
dbc.Row(
[dbc.Col(
dbc.Card(
dbc.CardBody(
[html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'),
]
),
className="mb-4",
),
width=5,
), ),
dbc.Row( dbc.Col(
[ dbc.Card(
dbc.Col( dbc.CardBody(
dbc.Card( [
dbc.CardBody( dcc.Markdown('## Total Spend:'),
[ dcc.Markdown(data['total_spend'],id='total_spend'),
html.H4( ],
"Spend Per Day", className="card-title" ),
), className="text-center text-light",
dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'), ),
] width=2,
),
className="mb-4",
),
width=12,
)
]
), ),
dbc.Row( dbc.Col(
[ dbc.Card(
dbc.Col( dbc.CardBody(
dbc.Card( [
dbc.CardBody( html.H4(
[ "Spend Per Payee", className="card-title"
html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'),
]
), ),
className="mb-4", dcc.Graph(figure=data['spend_per_payee_bar'],id='spend_per_payee'),
), ]
width=5,
), ),
dbc.Col( className="mb-4",
dbc.Card( ),
dbc.CardBody( width=5,
[
dcc.Markdown(f"""
## Total Spend:
### £{data['total_spend']:,}
"""),
]
),
id='total_spend',
className="mb-4",
),
width=2,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Payee", className="card-title"
),
dcc.Graph(figure=data['spend_per_payee_bar'],id='spend_per_payee'),
]
),
className="mb-4",
),
width=5,
),
]
), ),
]
),
], ],
fluid=True, fluid=True,
), ),