separation of areas

dash_app.py

@@ -6,7 +6,6 @@ from dash import Dash, html, dcc
 import dash_bootstrap_components as dbc
 import pandas as pd
 
-# Load data
 accounts = pl.read_parquet('data/warehouse/accounts.parquet')
 categories = pl.read_parquet('data/warehouse/categories.parquet')
 dates = pl.read_parquet('data/warehouse/dates.parquet')
@@ -149,7 +148,3 @@ app.layout = dbc.Container(
     ],
     fluid=True
 )
-
-# Run the app
-if __name__ == '__main__':
-    app.run(debug=True)

main.py

@@ -8,10 +8,8 @@ import logging.config
 import logging.handlers
 
 import config.exit_codes as ec
-from pipeline.ingest import Ingest
+from dash_app import app
-from pipeline.raw_to_base import RawToBase
+from pipeline.pipeline_main import pipeline_main
-from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate
-from pipeline.facts import FactTransactions, FactScheduledTransactions
 
 def set_up_logging():
     try:
@@ -37,41 +35,30 @@ dotenv.load_dotenv()
 API_TOKEN = os.getenv('API_TOKEN')
 BUDGET_ID = os.getenv('BUDGET_ID')
 
-def main():
+
-    if not API_TOKEN or not BUDGET_ID:
+if not API_TOKEN or not BUDGET_ID:
     logging.error('API_TOKEN or BUDGET_ID is not set in .env file')
     sys.exit(ec.MISSING_ENV_VARS)
 
-    try:
+try:
     with open('config/config.yaml', 'r') as file:
         config = yaml.safe_load(file)
-    except FileNotFoundError:
+except FileNotFoundError:
     logging.error('config.yaml file not found')
     sys.exit(ec.MISSING_CONFIG_FILE)
-    except yaml.YAMLError as e:
+except yaml.YAMLError as e:
     logging.error(f'Error loading config.yaml: {e}')
     sys.exit(ec.CORRUPTED_CONFIG_FILE)
 
-    config['API_TOKEN'] = API_TOKEN
+config['API_TOKEN'] = API_TOKEN
-    config['BUDGET_ID'] = BUDGET_ID
+config['BUDGET_ID'] = BUDGET_ID
 
-    logging.info('Starting data pipeline')
+    #sys.exit(ec.SUCCESS)
-
-    Ingest(config)
-    RawToBase(config)
-    DimAccounts(config)
-    DimCategories(config)
-    DimPayees(config)
-    DimDate(config)
-    FactTransactions(config)
-    FactScheduledTransactions(config)
-
-    logging.info('Data pipeline completed successfully')
-    sys.exit(ec.SUCCESS)
 
 if __name__ == '__main__':
     try:
-        main()
+        pipeline_main(config)
+        app.run() #debug=True)
     except SystemExit as e:
         exit_code = e.code
         if exit_code == ec.SUCCESS:

pipeline/pipeline_main.py

@@ -0,0 +1,24 @@
+'''Module to run the data pipeline'''
+
+import logging
+
+from pipeline.ingest import Ingest
+from pipeline.raw_to_base import RawToBase
+from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate
+from pipeline.facts import FactTransactions, FactScheduledTransactions
+
+
+def pipeline_main(config):
+    '''Run the data pipeline'''
+    logging.info('Starting data pipeline')
+
+    Ingest(config)
+    RawToBase(config)
+    DimAccounts(config)
+    DimCategories(config)
+    DimPayees(config)
+    DimDate(config)
+    FactTransactions(config)
+    FactScheduledTransactions(config)
+
+    logging.info('Data pipeline completed successfully')