Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: 🎨 more code style updates

Browse Source
This commit is contained in:
Jake Pullen
2025-05-30 11:52:44 +01:00
parent 5da9ba4b0b
commit 33199e35cd
3 changed files with 270 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
-11
View File
@@ -1,11 +0,0 @@
python-dotenv
polars
requests
pyyaml
#visualisation requirements below
dash
pandas
pyarrow
dash-bootstrap-components
# testing requirements below
pytest
tests/test_ingest.py
+131 -81
View File
@@ -8,104 +8,119 @@ import config.exit_codes as ec
# Mock configuration for initializing the Ingest class
mock_config = {
'API_TOKEN': 'test_token',
'BUDGET_ID': 'test_budget_id',
'base_url': 'http://test_base_url',
'knowledge_file': 'data/test_knowledge_file.json',
'entities': ['entity1', 'entity2'],
'raw_data_path': 'test_raw_data_path',
'REQUESTS_MAX_RETRIES': 3,
'REQUESTS_RETRY_DELAY': 1
"API_TOKEN": "test_token",
"BUDGET_ID": "test_budget_id",
"base_url": "http://test_base_url",
"knowledge_file": "data/test_knowledge_file.json",
"entities": ["entity1", "entity2"],
"raw_data_path": "test_raw_data_path",
"REQUESTS_MAX_RETRIES": 3,
"REQUESTS_RETRY_DELAY": 1,
}
# Test for load_knowledge_cache method
def test_load_knowledge_cache_file_exists():
mock_data = {"key": "value"}
with patch('os.path.exists', return_value=True), \
patch('builtins.open', mock_open(read_data=json.dumps(mock_data))) as mock_file:
with (
patch("os.path.exists", return_value=True),
patch("builtins.open", mock_open(read_data=json.dumps(mock_data))) as mock_file,
):
ingest_instance = Ingest(mock_config)
result = ingest_instance.load_knowledge_cache()
mock_file.assert_called_once_with(mock_config['knowledge_file'], 'r')
mock_file.assert_called_once_with(mock_config["knowledge_file"], "r")
assert result == mock_data
def test_load_knowledge_cache_file_not_exists():
with patch('os.path.exists', return_value=False):
def test_load_knowledge_cache_file_not_exists():
with patch("os.path.exists", return_value=False):
ingest_instance = Ingest(mock_config)
result = ingest_instance.load_knowledge_cache()
assert result == {}
# Test for save_entity_data_to_raw method
def test_save_entity_data_to_raw_success():
entity = 'entity1'
entity = "entity1"
data = {"key": "value"}
current_time = '20230101123000'
directory = os.path.join(mock_config['raw_data_path'], entity)
entity_file = f'{directory}/{current_time}.json'
with patch('os.path.exists', return_value=False), \
patch('os.makedirs') as mock_makedirs, \
patch('builtins.open', mock_open()) as mock_file, \
patch('time.strftime', return_value=current_time), \
patch('logging.info') as mock_logging_info:
current_time = "20230101123000"
directory = os.path.join(mock_config["raw_data_path"], entity)
entity_file = f"{directory}/{current_time}.json"
with (
patch("os.path.exists", return_value=False),
patch("os.makedirs") as mock_makedirs,
patch("builtins.open", mock_open()) as mock_file,
patch("time.strftime", return_value=current_time),
patch("logging.info") as mock_logging_info,
):
ingest_instance = Ingest(mock_config)
ingest_instance.save_entity_data_to_raw(entity, data)
mock_makedirs.assert_called_once_with(directory)
mock_file.assert_called_once_with(entity_file, 'w')
mock_file.assert_called_once_with(entity_file, "w")
# Get the file handle and check the written content
handle = mock_file()
handle.write.assert_called()
written_content = ''.join(call.args[0] for call in handle.write.call_args_list)
written_content = "".join(call.args[0] for call in handle.write.call_args_list)
assert written_content == json.dumps(data, indent=4)
mock_logging_info.assert_called_once_with(f"Saving {entity} data to {entity_file}")
mock_logging_info.assert_called_once_with(
f"Saving {entity} data to {entity_file}"
)
def test_save_entity_data_to_raw_existing_directory():
entity = 'entity1'
entity = "entity1"
data = {"key": "value"}
current_time = '20230101123000'
directory = os.path.join(mock_config['raw_data_path'], entity)
entity_file = f'{directory}/{current_time}.json'
with patch('os.path.exists', return_value=True), \
patch('os.makedirs') as mock_makedirs, \
patch('builtins.open', mock_open()) as mock_file, \
patch('time.strftime', return_value=current_time), \
patch('logging.info') as mock_logging_info:
current_time = "20230101123000"
directory = os.path.join(mock_config["raw_data_path"], entity)
entity_file = f"{directory}/{current_time}.json"
with (
patch("os.path.exists", return_value=True),
patch("os.makedirs") as mock_makedirs,
patch("builtins.open", mock_open()) as mock_file,
patch("time.strftime", return_value=current_time),
patch("logging.info") as mock_logging_info,
):
ingest_instance = Ingest(mock_config)
ingest_instance.save_entity_data_to_raw(entity, data)
mock_makedirs.assert_not_called()
mock_file.assert_called_once_with(entity_file, 'w')
mock_file.assert_called_once_with(entity_file, "w")
# Get the file handle and check the written content
handle = mock_file()
handle.write.assert_called()
written_content = ''.join(call.args[0] for call in handle.write.call_args_list)
written_content = "".join(call.args[0] for call in handle.write.call_args_list)
assert written_content == json.dumps(data, indent=4)
mock_logging_info.assert_called_once_with(f"Saving {entity} data to {entity_file}")
mock_logging_info.assert_called_once_with(
f"Saving {entity} data to {entity_file}"
)
def test_save_entity_data_to_raw_error():
entity = 'entity1'
entity = "entity1"
data = {"key": "value"}
current_time = '20230101123000'
directory = os.path.join(mock_config['raw_data_path'], entity)
entity_file = f'{directory}/{current_time}.json'
with patch('os.path.exists', return_value=True), \
patch('builtins.open', mock_open()) as mock_file, \
patch('time.strftime', return_value=current_time), \
patch('logging.info') as mock_logging_info, \
patch('logging.error') as mock_logging_error:
current_time = "20230101123000"
directory = os.path.join(mock_config["raw_data_path"], entity)
entity_file = f"{directory}/{current_time}.json"
with (
patch("os.path.exists", return_value=True),
patch("builtins.open", mock_open()) as mock_file,
patch("time.strftime", return_value=current_time),
patch("logging.info") as mock_logging_info,
patch("logging.error") as mock_logging_error,
):
mock_file.side_effect = Exception("Test error")
ingest_instance = Ingest(mock_config)
@@ -113,39 +128,47 @@ def test_save_entity_data_to_raw_error():
with pytest.raises(Exception, match="Test error"):
ingest_instance.save_entity_data_to_raw(entity, data)
mock_logging_error.assert_called_once_with(f"Failed to save data for {entity} to {entity_file}")
mock_logging_error.assert_called_once_with(
f"Failed to save data for {entity} to {entity_file}"
)
def test_update_server_knowledge_cache_file_exists():
entity = 'entity1'
entity = "entity1"
server_knowledge = {"key": "value"}
existing_cache = {"entity2": {"key": "old_value"}}
updated_cache = {"entity2": {"key": "old_value"}, "entity1": {"key": "value"}}
with patch('builtins.open', mock_open(read_data=json.dumps(existing_cache))) as mock_file, \
patch('os.path.exists', return_value=True), \
patch('logging.error') as mock_logging_error:
with (
patch(
"builtins.open", mock_open(read_data=json.dumps(existing_cache))
) as mock_file,
patch("os.path.exists", return_value=True),
patch("logging.error") as mock_logging_error,
):
ingest_instance = Ingest(mock_config)
ingest_instance.update_server_knowledge_cache(entity, server_knowledge)
mock_file.assert_called_with(mock_config['knowledge_file'], 'w')
mock_file.assert_called_with(mock_config["knowledge_file"], "w")
handle = mock_file()
handle.write.assert_called()
written_content = ''.join(call.args[0] for call in handle.write.call_args_list)
written_content = "".join(call.args[0] for call in handle.write.call_args_list)
assert json.loads(written_content) == updated_cache
mock_logging_error.assert_not_called()
def test_update_server_knowledge_cache_file_not_exists():
entity = 'entity1'
entity = "entity1"
server_knowledge = {"key": "value"}
updated_cache = {"entity1": {"key": "value"}}
with patch('builtins.open', mock_open()) as mock_file, \
patch('os.path.exists', return_value=False), \
patch('os.makedirs') as mock_makedirs, \
patch('logging.info') as mock_logging_info, \
patch('logging.error') as mock_logging_error:
with (
patch("builtins.open", mock_open()) as mock_file,
patch("os.path.exists", return_value=False),
patch("os.makedirs") as mock_makedirs,
patch("logging.info") as mock_logging_info,
patch("logging.error") as mock_logging_error,
):
# Ensure the side_effect list has enough elements to cover all calls to open
mock_file.side_effect = [FileNotFoundError(), mock_open().return_value]
@@ -154,17 +177,25 @@ def test_update_server_knowledge_cache_file_not_exists():
with pytest.raises(FileNotFoundError):
ingest_instance.update_server_knowledge_cache(entity, server_knowledge)
mock_makedirs.assert_called_once_with(os.path.dirname(mock_config['knowledge_file']), exist_ok=True)
mock_file.assert_called_with(mock_config['knowledge_file'], 'w')
mock_logging_error.assert_called_once_with(f"Failed to update knowledge cache for {entity} in {mock_config['knowledge_file']}")
mock_makedirs.assert_called_once_with(
os.path.dirname(mock_config["knowledge_file"]), exist_ok=True
)
mock_file.assert_called_with(mock_config["knowledge_file"], "w")
mock_logging_error.assert_called_once_with(
f"Failed to update knowledge cache for {entity} in {
mock_config['knowledge_file']
}"
)
def test_update_server_knowledge_cache_write_error():
entity = 'entity1'
entity = "entity1"
server_knowledge = {"key": "value"}
with patch('builtins.open', mock_open()) as mock_file, \
patch('logging.error') as mock_logging_error:
with (
patch("builtins.open", mock_open()) as mock_file,
patch("logging.error") as mock_logging_error,
):
mock_file.side_effect = Exception("Test error")
ingest_instance = Ingest(mock_config)
@@ -172,35 +203,43 @@ def test_update_server_knowledge_cache_write_error():
with pytest.raises(Exception, match="Test error"):
ingest_instance.update_server_knowledge_cache(entity, server_knowledge)
mock_logging_error.assert_called_once_with(f"Failed to update knowledge cache for {entity} in {mock_config['knowledge_file']}")
mock_logging_error.assert_called_once_with(
f"Failed to update knowledge cache for {entity} in {
mock_config['knowledge_file']
}"
)
def test_check_rate_limit_above_threshold():
response = MagicMock()
response.headers = {'X-Rate-Limit': '10/100'}
response.headers = {"X-Rate-Limit": "10/100"}
ingest_instance = Ingest(mock_config)
result = ingest_instance.check_rate_limit(response)
assert result is None
def test_check_rate_limit_below_threshold():
response = MagicMock()
response.headers = {'X-Rate-Limit': '90/100'}
response.headers = {"X-Rate-Limit": "90/100"}
ingest_instance = Ingest(mock_config)
result = ingest_instance.check_rate_limit(response)
assert result is None
def test_check_rate_limit_exceeded():
response = MagicMock()
response.headers = {'X-Rate-Limit': '100/100'}
response.headers = {"X-Rate-Limit": "100/100"}
ingest_instance = Ingest(mock_config)
result = ingest_instance.check_rate_limit(response)
assert result is True
def test_check_rate_limit_header_missing():
response = MagicMock()
response.headers = {}
@@ -210,6 +249,7 @@ def test_check_rate_limit_header_missing():
assert result is None
def test_handle_response_bad_request():
response = MagicMock()
response.status_code = 400
@@ -218,9 +258,10 @@ def test_handle_response_bad_request():
with pytest.raises(SystemExit) as e:
ingest_instance.handle_response(response)
assert e.type == SystemExit
assert e.type is SystemExit
assert e.value.code == ec.BAD_REQUEST
def test_handle_response_unauthorized():
response = MagicMock()
response.status_code = 401
@@ -229,9 +270,10 @@ def test_handle_response_unauthorized():
with pytest.raises(SystemExit) as e:
ingest_instance.handle_response(response)
assert e.type == SystemExit
assert e.type is SystemExit
assert e.value.code == ec.UNAUTHORIZED_API_TOKEN
def test_handle_response_forbidden():
response = MagicMock()
response.status_code = 403
@@ -240,9 +282,10 @@ def test_handle_response_forbidden():
with pytest.raises(SystemExit) as e:
ingest_instance.handle_response(response)
assert e.type == SystemExit
assert e.type is SystemExit
assert e.value.code == ec.FORBIDDEN
def test_handle_response_not_found():
response = MagicMock()
response.status_code = 404
@@ -251,9 +294,10 @@ def test_handle_response_not_found():
with pytest.raises(SystemExit) as e:
ingest_instance.handle_response(response)
assert e.type == SystemExit
assert e.type is SystemExit
assert e.value.code == ec.NOT_FOUND
def test_handle_response_conflict():
response = MagicMock()
response.status_code = 409
@@ -262,9 +306,10 @@ def test_handle_response_conflict():
with pytest.raises(SystemExit) as e:
ingest_instance.handle_response(response)
assert e.type == SystemExit
assert e.type is SystemExit
assert e.value.code == ec.CONFLICT
def test_handle_response_too_many_requests():
response = MagicMock()
response.status_code = 429
@@ -274,6 +319,7 @@ def test_handle_response_too_many_requests():
result = ingest_instance.handle_response(response)
assert result is True
def test_handle_response_internal_server_error():
response = MagicMock()
response.status_code = 500
@@ -283,6 +329,7 @@ def test_handle_response_internal_server_error():
result = ingest_instance.handle_response(response)
assert result is True
def test_handle_response_service_unavailable():
response = MagicMock()
response.status_code = 503
@@ -292,6 +339,7 @@ def test_handle_response_service_unavailable():
result = ingest_instance.handle_response(response)
assert result is True
def test_handle_response_ok():
response = MagicMock()
response.status_code = 200
@@ -301,5 +349,7 @@ def test_handle_response_ok():
result = ingest_instance.handle_response(response)
assert result is False
if __name__ == "__main__":
pytest.main()
visuals/layout.py
+100 -79
View File
@@ -16,98 +16,119 @@ def create_layout(data):
def create_topbar():
return [
dbc.Container([
dbc.Row(
dbc.Col(
html.Div(html.H1("Data Pipeline For YNAB, Preview Visualisations"),
className="text-center text-light",
),
width=12,
)
),
dbc.Row(
[
dbc.Col(
dcc.DatePickerRange(
first_day_of_week=1,
display_format="YYYY-MM-DD",
id="date-picker-range",
start_date=one_year_ago,
end_date=today,
dbc.Container(
[
dbc.Row(
dbc.Col(
html.Div(
html.H1("Data Pipeline For YNAB, Preview Visualisations"),
className="text-center text-light",
),
width=4,
width=12,
)
),
dbc.Col(
html.Button("Change Date Range", id="date-range-confirm-button"),
width=2,
dbc.Row(
[
dbc.Col(
dcc.DatePickerRange(
first_day_of_week=1,
display_format="YYYY-MM-DD",
id="date-picker-range",
start_date=one_year_ago,
end_date=today,
),
width=4,
),
dbc.Col(
html.Button(
"Change Date Range", id="date-range-confirm-button"
),
width=2,
),
]
),
]
)
]
]
)
]
def create_main_body(data):
return [
dbc.Container(
[
dbc.Row(
[dbc.Col(
dbc.Card(
dbc.CardBody(
[html.H4(
"Spend Per Day", className="card-title"
),
dcc.Graph(figure=data['spend_per_day_line'],id='spend_per_day'),
]
),
className="mb-4",
),
width=12,
)]
),
dbc.Row(
[dbc.Col(
dbc.Card(
dbc.CardBody(
[html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(figure=data['spend_per_category_bar'],id='spend_per_category'),
]
),
className="mb-4",
),
width=5,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
dcc.Markdown('## Total Spend:'),
dcc.Markdown(data['total_spend'],id='total_spend'),
],
),
className="text-center text-light",
),
width=2,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Payee", className="card-title"
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Day", className="card-title"
),
dcc.Graph(
figure=data["spend_per_day_line"],
id="spend_per_day",
),
]
),
dcc.Graph(figure=data['spend_per_payee_bar'],id='spend_per_payee'),
]
className="mb-4",
),
width=12,
)
]
),
dbc.Row(
[
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Category", className="card-title"
),
dcc.Graph(
figure=data["spend_per_category_bar"],
id="spend_per_category",
),
]
),
className="mb-4",
),
width=5,
),
className="mb-4",
),
width=5,
dbc.Col(
dbc.Card(
dbc.CardBody(
[
dcc.Markdown("## Total Spend:"),
dcc.Markdown(
data["total_spend"], id="total_spend"
),
],
),
className="text-center text-light",
),
width=2,
),
dbc.Col(
dbc.Card(
dbc.CardBody(
[
html.H4(
"Spend Per Payee", className="card-title"
),
dcc.Graph(
figure=data["spend_per_payee_bar"],
id="spend_per_payee",
),
]
),
className="mb-4",
),
width=5,
),
]
),
]
),
],
fluid=True,
),