Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

expanding dash

Browse Source
This commit is contained in:
Jake
2025-04-05 11:04:40 +01:00
parent 5af82e5753
commit b573b3b9bc
6 changed files with 206 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_check.py
+46 -8
View File
@@ -1,17 +1,55 @@
import polars as pl
df = pl.read_parquet('data/warehouse/transactions.parquet')
print("Data loaded from Parquet file:")
print(df)
accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet')
dates = pl.read_parquet('data/warehouse/dates.parquet')
payees = pl.read_parquet('data/warehouse/payees.parquet')
scheduled_transactions = pl.read_parquet('data/warehouse/scheduled_transactions.parquet')
transactions = pl.read_parquet('data/warehouse/transactions.parquet')
relevant_data = df.sql('''
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
# Create aggregations
spend_per_day = master_transactions.sql('''
SELECT
date,
sum(transaction_amount) as total
year,
month,
day,
ABS(SUM(transaction_amount)) as total
FROM self
GROUP BY date
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY date, year, month, day
ORDER BY date DESC
'''
)
print("Data after SQL query:")
print(relevant_data)
spend_per_category = master_transactions.sql('''
SELECT
category_name,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE category_name != 'Inflow: Ready to Assign'
GROUP BY category_name
ORDER BY total DESC
'''
)
spend_per_payee = master_transactions.sql('''
SELECT
payee_name,
ABS(SUM(transaction_amount)) as total
FROM self
WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0
GROUP BY payee_name
ORDER BY total DESC
'''
)
print(spend_per_payee)
main.py
+2 -3
View File
@@ -5,10 +5,10 @@ import yaml
import sys
import atexit
import logging.config
import logging.handlers
import config.exit_codes as ec
from pipeline.pipeline_main import pipeline_main
from visuals.dash_app import app
def set_up_logging():
try:
@@ -55,12 +55,11 @@ if __name__ == '__main__':
config['API_TOKEN'] = API_TOKEN
config['BUDGET_ID'] = BUDGET_ID
try:
pipeline_main(config)
# pipeline_main(config)
# Check if the data was successfully created
data_exists = os.path.exists('data/processed') and os.listdir('data/processed')
if data_exists:
from dash_app import app
app.run() # debug=True
else:
logging.error('Data pipeline did not produce any data. Dash app will not run.')
visuals/__init__.py
View File
dash_app.py → visuals/components.py
+15 -66
View File
@@ -1,14 +1,12 @@
'''Module to create a Dash app that displays visualizations of YNAB data.'''
import polars as pl
import plotly.express as px
from dash import Dash, html, dcc
import dash_bootstrap_components as dbc
import pandas as pd
import logging
import sys
import config.exit_codes as ec
try:
accounts = pl.read_parquet('data/warehouse/accounts.parquet')
categories = pl.read_parquet('data/warehouse/categories.parquet')
@@ -25,7 +23,10 @@ try:
master_transactions = transactions.join(categories, left_on='category_id', right_on='category_id', suffix='_category')\
.join(accounts, left_on='account_id', right_on='account_id', suffix='_account')\
.join(payees, left_on='payee_id', right_on='payee_id', suffix='_payee')\
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')
.join(dates, left_on='transaction_date', right_on='date_id', suffix='_date')\
# date filter for callback date range
#.filter(pl.col('transaction_date'))
except Exception as e:
logging.error(f'Error joining DataFrames: {e}')
sys.exit(ec.BAD_JOIN)
@@ -67,17 +68,26 @@ spend_per_payee = master_transactions.sql('''
ORDER BY total DESC
'''
)
total_spend = master_transactions.sql('''
SELECT ABS(SUM(transaction_amount)) AS total
FROM self
WHERE payee_name != 'Starting Balance'
AND transaction_amount < 0
''').item()
# Convert DataFrame to list of dictionaries
spend_per_day_data = spend_per_day.to_dicts()
spend_per_category_data = spend_per_category.to_dicts()
spend_per_payee_data = spend_per_payee.to_dicts()
# Convert list of dictionaries to Pandas DataFrame
spend_per_day_df = pd.DataFrame(spend_per_day_data)
spend_per_category_df = pd.DataFrame(spend_per_category_data)
spend_per_payee_df = pd.DataFrame(spend_per_payee_data)
spend_per_day_line = px.line(spend_per_day_df, x="date", y="total")
spend_per_day_line.update_layout(
plot_bgcolor='black',
@@ -98,64 +108,3 @@ spend_per_payee_bar.update_layout(
paper_bgcolor='black',
font_color='white'
)
# Initialize the app with a dark theme
app = Dash(external_stylesheets=[dbc.themes.DARKLY])
# App layout
app.layout = dbc.Container(
[
dbc.Row(
dbc.Col(
html.Div("Data Pipeline For YNAB, Preview Visualisations",
className="text-center text-light"),
width=12
)
),
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4("Spend Per Day", className="card-title"),
dcc.Graph(figure=spend_per_day_line)
]
),
className="mb-4"
),
width=12
)
]
),
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4("Spend Per Category", className="card-title"),
dcc.Graph(figure=spend_per_category_bar)
]
),
className="mb-4"
),
width=6
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4("Spend Per Payee", className="card-title"),
dcc.Graph(figure=spend_per_payee_bar)
]
),
className="mb-4"
),
width=6
)
]
)
],
fluid=True
)
visuals/dash_app.py
+38
View File
@@ -0,0 +1,38 @@
'''Module to create a Dash app that displays visualizations of YNAB data.'''
import dash
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output
import visuals.layout as layout
# Initialize the app with a dark theme
app = dash.Dash(external_stylesheets=[dbc.themes.DARKLY])
# App layout
app.layout = layout.create_layout()
# Update toggle visibility of off-canvas section
@app.callback(
Output('off-canvas-section', 'style'),
[Input('toggle-button', 'n_clicks')]
)
def update_off_canvas_section(n_clicks):
if n_clicks is not None:
return {'display': 'block'}
else:
return {'display': 'none'}
# Update off-canvas section with date range picker
@app.callback(
Output('off-canvas-section', 'children'),
[Input('toggle-button', 'n_clicks')]
)
def update_off_canvas_content(n_clicks):
if n_clicks is not None:
# Include a date range picker library (e.g., Dash Bootstrap components) for the off-canvas section
return [
dbc.Input(id='date-picker-start', type='date'),
dbc.Input(id='date-picker-end', type='date')
]
visuals/layout.py
+105
View File
@@ -0,0 +1,105 @@
from dash import html, dcc
from datetime import date
import dash_bootstrap_components as dbc
import visuals.components as charts
def create_layout():
main_body = create_main_body()
sidebar = create_sidebar()
return [html.Div(main_body + sidebar)]
def create_sidebar():
return [
html.Div(id='off-canvas-section', style={'display': 'none'}),
dbc.Row(
dbc.Col(
dcc.DatePickerRange(
id='date-picker-range',
start_date=date(2024, 1, 1),
end_date=date(2026, 1, 1)
),
width=4
),
align='center'
)
]
def create_main_body():
return [html.Button('Toggle Off-Canvas', id='toggle-button'),
dbc.Container(
[
dbc.Row(
dbc.Col(
html.Div(
"Data Pipeline For YNAB, Preview Visualisations",
className="text-center text-light",
),
width=12,
)
),
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4("Spend Per Day", className="card-title"),
dcc.Graph(figure=charts.spend_per_day_line),
]
),
className="mb-4",
),
width=12,
)
]
),
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(figure=charts.spend_per_category_bar),
]
),
className="mb-4",
),
width=5,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
dcc.Markdown(f"""
## Total Spend:
### £{charts.total_spend:,}
"""),
]
),
className="mb-4",
),
width=2,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4("Spend Per Payee", className="card-title"),
dcc.Graph(figure=charts.spend_per_payee_bar),
]
),
className="mb-4",
),
width=5,
),
]
),
],
fluid=True,
)
]