Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

starting a fresh tidy

Browse Source
This commit is contained in:
Jake
2026-07-26 10:34:18 +01:00
parent 5af82e5753
commit 33cbc5c6ed
14 changed files with 985 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/ingest.py
+30 -27
View File
@@ -1,15 +1,17 @@
import os
import time
import json
import logging
import requests
import os
import sys
from typing import Dict, Any
import time
from typing import Any
import requests
import config.exit_codes as ec
class Ingest:
def __init__(self, config: Dict[str, Any]):
def __init__(self, config: dict[str, Any],logger):
"""
Initialize the Ingest class with the provided configuration.
"""
@@ -22,9 +24,10 @@ class Ingest:
self.headers = {'Authorization': f'Bearer {self.api_token}'}
self.MAX_RETRIES = config['REQUESTS_MAX_RETRIES']
self.RETRY_DELAY = config['REQUESTS_RETRY_DELAY']
self.logger = logger
def save_entity_data_to_raw(self, entity: str, data: Dict[str, Any]):
def save_entity_data_to_raw(self, entity: str, data: dict[str, Any]):
"""
Save the data for a specific entity to a new cache file.
"""
@@ -33,15 +36,15 @@ class Ingest:
if not os.path.exists(directory):
os.makedirs(directory)
entity_file = f'{directory}/{current_time}.json'
logging.info(f"Saving {entity} data to {entity_file}")
self.logger.info(f"Saving {entity} data to {entity_file}")
try:
with open(entity_file, 'w') as f:
json.dump(data, f, indent=4)
except Exception as e:
logging.error(f"Failed to save data for {entity} to {entity_file}")
self.logger.error(f"Failed to save data for {entity} to {entity_file}")
raise e
def load_knowledge_cache(self) -> Dict[str, Any]:
def load_knowledge_cache(self) -> dict[str, Any]:
"""
Load the knowledge cache from the file if it exists.
"""
@@ -59,7 +62,7 @@ class Ingest:
with open(self.knowledge_file, 'r') as f:
knowledge_cache = json.load(f)
except FileNotFoundError:
logging.info(f"Knowledge file not found. Creating a new one at {self.knowledge_file}. This is normal for the first run.")
self.logger.info(f"Knowledge file not found. Creating a new one at {self.knowledge_file}. This is normal for the first run.")
os.makedirs(os.path.dirname(self.knowledge_file), exist_ok=True)
knowledge_cache = {}
@@ -74,33 +77,33 @@ class Ingest:
with open(self.knowledge_file, 'w') as f:
json.dump(knowledge_cache, f, indent=4)
except Exception as e:
logging.error(f"Failed to update knowledge cache for {entity} in {self.knowledge_file}")
self.logger.error(f"Failed to update knowledge cache for {entity} in {self.knowledge_file}")
raise e
def handle_response(self, response) -> bool:
if response.status_code == 400:
logging.error("Bad request. The request could not be understood by the API due to malformed syntax or validation errors.")
self.logger.error("Bad request. The request could not be understood by the API due to malformed syntax or validation errors.")
sys.exit(ec.BAD_REQUEST)
elif response.status_code == 401:
logging.error("Unauthorized. Please check your API token.")
self.logger.error("Unauthorized. Please check your API token.")
sys.exit(ec.UNAUTHORIZED_API_TOKEN)
elif response.status_code == 403:
logging.error("Forbidden. Access is denied.")
self.logger.error("Forbidden. Access is denied.")
sys.exit(ec.FORBIDDEN)
elif response.status_code == 404:
logging.error("Not found. The specified URL does not exist.")
self.logger.error("Not found. The specified URL does not exist.")
sys.exit(ec.NOT_FOUND)
elif response.status_code == 409:
logging.error("Conflict. The resource cannot be saved due to a conflict.")
self.logger.error("Conflict. The resource cannot be saved due to a conflict.")
sys.exit(ec.CONFLICT)
elif response.status_code == 429:
logging.error("Too many requests. You have made too many requests in a short amount of time.")
self.logger.error("Too many requests. You have made too many requests in a short amount of time.")
return True
elif response.status_code == 500:
logging.error("Internal server error. The API experienced an unexpected error.")
self.logger.error("Internal server error. The API experienced an unexpected error.")
return True
elif response.status_code == 503:
logging.error("Service unavailable. The API is temporarily disabled or a request timeout occurred.")
self.logger.error("Service unavailable. The API is temporarily disabled or a request timeout occurred.")
return True
else:
response.raise_for_status()
@@ -113,14 +116,14 @@ class Ingest:
for entity in self.entities:
file_path = f'data/raw/{entity}'
if os.path.exists(file_path) and os.listdir(file_path):
logging.warning(f"Raw data exists for {entity} processing any raw data we already have.")
self.logger.warning(f"Raw data exists for {entity} processing any raw data we already have.")
break # break here instead of continue as we dont want to update our server knowledge cache and potentially miss data.
knowledge_cache = self.load_knowledge_cache()
last_knowledge = knowledge_cache.get(entity, 0)
#logging.debug(f'Last Knowledge of {entity}: {last_knowledge}')
logging.info(f'Fetching {entity} data since last knowledge: {last_knowledge}')
self.logger.info(f'Fetching {entity} data since last knowledge: {last_knowledge}')
url = f'{self.base_url}/{self.budget_id}/{entity}?last_knowledge_of_server={last_knowledge}'
response = None
for attempt in range(self.MAX_RETRIES):
@@ -130,17 +133,17 @@ class Ingest:
if not should_retry:
break # Exit the loop if the request is successful
except requests.exceptions.RequestException as e:
logging.error(f"Error fetching {entity} data (attempt {attempt + 1}/{self.MAX_RETRIES}): {e}")
self.logger.error(f"Error fetching {entity} data (attempt {attempt + 1}/{self.MAX_RETRIES}): {e}")
if attempt < self.MAX_RETRIES - 1:
time.sleep(self.RETRY_DELAY) # Wait before retrying
else:
logging.error("Max retries reached. Exiting.")
self.logger.error("Max retries reached. Exiting.")
sys.exit(ec.REQUESTS_ERROR)
data = response.json()
logging.debug(f'response data: {data}')
self.logger.debug(f'response data: {data}')
server_knowledge = data['data'].get('server_knowledge')
logging.debug(f'{entity} new server knowledge: {server_knowledge}')
self.logger.debug(f'{entity} new server knowledge: {server_knowledge}')
if server_knowledge is not None and server_knowledge != last_knowledge:
self.update_server_knowledge_cache(entity, server_knowledge)
@@ -148,4 +151,4 @@ class Ingest:
entity_data.pop('server_knowledge', None)
self.save_entity_data_to_raw(entity, entity_data)
else:
logging.info(f"No new data for {entity}. Skipping cache update.")
self.logger.info(f"No new data for {entity}. Skipping cache update.")