diff --git a/ingest.py b/ingest.py index ea796a4..6756c8b 100644 --- a/ingest.py +++ b/ingest.py @@ -6,15 +6,15 @@ import requests from typing import Dict, Any class Ingest: - def __init__(self, ingest_info: Dict[str, Any]): + def __init__(self, config: Dict[str, Any]): """ Initialize the Ingest class with the provided configuration. """ - self.api_token = ingest_info['API_TOKEN'] - self.budget_id = ingest_info['BUDGET_ID'] - self.base_url = ingest_info['base_url'] - self.knowledge_file = ingest_info['knowledge_file'] - self.entities = ingest_info['entities'] + self.api_token = config['API_TOKEN'] + self.budget_id = config['BUDGET_ID'] + self.base_url = config['base_url'] + self.knowledge_file = config['knowledge_file'] + self.entities = config['entities'] self.headers = {'Authorization': f'Bearer {self.api_token}'} self.knowledge_cache = self.load_knowledge_cache() self.fetch_and_cache_entity_data() @@ -88,6 +88,10 @@ class Ingest: url = url + f'?last_knowledge_of_server={last_knowledge}' response = requests.get(url, headers=self.headers) + if response.status_code == 401: + logging.error("Unauthorized. Please check your API token.") + break + self.check_rate_limit(response) if response.status_code == 429: diff --git a/test.py b/test.py index b2d514c..f02d1b4 100644 --- a/test.py +++ b/test.py @@ -1,14 +1,15 @@ import polars as pl + entities = ['accounts', 'categories', 'months', 'payees', 'transactions', 'scheduled_transactions'] -# Define the path to the transactions parquet file - -#file_path = 'data/base/categories.parquet' -#file_path = 'data/base/accounts.parquet' -file_path = 'data/base/transactions.parquet' - -# Read the parquet file into a polars DataFrame -transactions_df = pl.read_parquet(file_path) - -# Display the DataFrame -print(transactions_df) \ No newline at end of file +for entity in entities: + print(f"Processing entity: {entity}") + file_path = f'data/base/{entity}.parquet' + # Read the parquet file into a polars DataFrame + entity_df = pl.read_parquet(file_path) + # Print the schema of the DataFrame + print(f"Schema of {entity} DataFrame:") + print(entity_df.schema) + # Display the first few rows of the DataFrame + print(f"First few rows of {entity} DataFrame:") + print(entity_df.head())