added better logging and moved config to its own folder

main.py

@@ -2,28 +2,49 @@ import os
 import dotenv
 import logging
 import yaml
+import sys
+import atexit
+import logging.config
+import logging.handlers
 
+import config.exit_codes as ec
 from pipeline.ingest import Ingest
 from pipeline.raw_to_base import RawToBase
 from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate
 from pipeline.facts import FactTransactions, FactScheduledTransactions
 
+def set_up_logging():
+    with open('config/logging_config.yaml', 'r') as f:
+        try:
+            log_config = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            print(e)
+    logging.config.dictConfig(log_config)
+    queue_handler = logging.getHandlerByName('queue_handler')
+    if queue_handler is not None:
+        queue_handler.listener.start()
+        atexit.register(queue_handler.listener.stop)
+
+logger = logging.getLogger("data_pipeline_for_ynab")
+os.makedirs('logs', exist_ok=True)
+set_up_logging()
+# Load environment variables
 dotenv.load_dotenv()
 
 API_TOKEN = os.getenv('API_TOKEN')
 BUDGET_ID = os.getenv('BUDGET_ID')
-logging.basicConfig(level=logging.DEBUG)
-if not API_TOKEN or not BUDGET_ID:
-    logging.error('API_TOKEN or BUDGET_ID is not set in .env file')
-    exit(1)
 
-with open('config.yaml', 'r') as file:
-    config = yaml.safe_load(file)
+def main():
+    if not API_TOKEN or not BUDGET_ID:
+        logging.error('API_TOKEN or BUDGET_ID is not set in .env file')
+        sys.exit(ec.MISSING_ENV_VARS)
 
-config['API_TOKEN'] = API_TOKEN
-config['BUDGET_ID'] = BUDGET_ID
+    with open('config/config.yaml', 'r') as file:
+        config = yaml.safe_load(file)
+
+    config['API_TOKEN'] = API_TOKEN
+    config['BUDGET_ID'] = BUDGET_ID
 
-if __name__ == '__main__':
     Ingest(config)
     RawToBase(config)
     DimAccounts(config)
@@ -32,3 +53,11 @@ if __name__ == '__main__':
     DimDate(config)
     FactTransactions(config)
     FactScheduledTransactions(config)
+
+if __name__ == '__main__':
+    try:
+        main()
+    except SystemExit as e:
+        exit_code = e.code
+        logging.error(f'Program exited with code {exit_code}')
+        raise