Added more error handling and logic to ingest

2024-08-10 17:11:57 +01:00
parent d25136fbb0
commit 24cc006ef5
3 changed files with 83 additions and 23 deletions
@@ -23,4 +23,6 @@ primary_keys:
 raw_data_path: data/raw
 processed_data_path: data/processed
 base_data_path: data/base
-warehouse_data_path: data/warehouse
+warehouse_data_path: data/warehouse
+REQUESTS_MAX_RETRIES: 3
+REQUESTS_RETRY_DELAY: 5
@@ -3,3 +3,8 @@ MISSING_ENV_VARS = 1
 MISSING_CONFIG_FILE = 2
 CORRUPTED_CONFIG_FILE = 3
 UNAUTHORIZED_API_TOKEN = 4
+REQUESTS_ERROR = 5
+BAD_REQUEST = 6
+FORBIDDEN = 7
+NOT_FOUND = 8
+CONFLICT = 9