diff --git a/dash_app.py b/dash_app.py index 36df2b4..9533285 100644 --- a/dash_app.py +++ b/dash_app.py @@ -6,7 +6,6 @@ from dash import Dash, html, dcc import dash_bootstrap_components as dbc import pandas as pd -# Load data accounts = pl.read_parquet('data/warehouse/accounts.parquet') categories = pl.read_parquet('data/warehouse/categories.parquet') dates = pl.read_parquet('data/warehouse/dates.parquet') @@ -149,7 +148,3 @@ app.layout = dbc.Container( ], fluid=True ) - -# Run the app -if __name__ == '__main__': - app.run(debug=True) diff --git a/main.py b/main.py index 33e686b..834983c 100644 --- a/main.py +++ b/main.py @@ -8,10 +8,8 @@ import logging.config import logging.handlers import config.exit_codes as ec -from pipeline.ingest import Ingest -from pipeline.raw_to_base import RawToBase -from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate -from pipeline.facts import FactTransactions, FactScheduledTransactions +from dash_app import app +from pipeline.pipeline_main import pipeline_main def set_up_logging(): try: @@ -37,41 +35,30 @@ dotenv.load_dotenv() API_TOKEN = os.getenv('API_TOKEN') BUDGET_ID = os.getenv('BUDGET_ID') -def main(): - if not API_TOKEN or not BUDGET_ID: - logging.error('API_TOKEN or BUDGET_ID is not set in .env file') - sys.exit(ec.MISSING_ENV_VARS) - try: - with open('config/config.yaml', 'r') as file: - config = yaml.safe_load(file) - except FileNotFoundError: - logging.error('config.yaml file not found') - sys.exit(ec.MISSING_CONFIG_FILE) - except yaml.YAMLError as e: - logging.error(f'Error loading config.yaml: {e}') - sys.exit(ec.CORRUPTED_CONFIG_FILE) +if not API_TOKEN or not BUDGET_ID: + logging.error('API_TOKEN or BUDGET_ID is not set in .env file') + sys.exit(ec.MISSING_ENV_VARS) - config['API_TOKEN'] = API_TOKEN - config['BUDGET_ID'] = BUDGET_ID +try: + with open('config/config.yaml', 'r') as file: + config = yaml.safe_load(file) +except FileNotFoundError: + logging.error('config.yaml file not found') + sys.exit(ec.MISSING_CONFIG_FILE) +except yaml.YAMLError as e: + logging.error(f'Error loading config.yaml: {e}') + sys.exit(ec.CORRUPTED_CONFIG_FILE) - logging.info('Starting data pipeline') +config['API_TOKEN'] = API_TOKEN +config['BUDGET_ID'] = BUDGET_ID - Ingest(config) - RawToBase(config) - DimAccounts(config) - DimCategories(config) - DimPayees(config) - DimDate(config) - FactTransactions(config) - FactScheduledTransactions(config) - - logging.info('Data pipeline completed successfully') - sys.exit(ec.SUCCESS) + #sys.exit(ec.SUCCESS) if __name__ == '__main__': try: - main() + pipeline_main(config) + app.run() #debug=True) except SystemExit as e: exit_code = e.code if exit_code == ec.SUCCESS: diff --git a/pipeline/pipeline_main.py b/pipeline/pipeline_main.py new file mode 100644 index 0000000..05d3a9b --- /dev/null +++ b/pipeline/pipeline_main.py @@ -0,0 +1,24 @@ +'''Module to run the data pipeline''' + +import logging + +from pipeline.ingest import Ingest +from pipeline.raw_to_base import RawToBase +from pipeline.dimensions import DimAccounts, DimCategories, DimPayees, DimDate +from pipeline.facts import FactTransactions, FactScheduledTransactions + + +def pipeline_main(config): + '''Run the data pipeline''' + logging.info('Starting data pipeline') + + Ingest(config) + RawToBase(config) + DimAccounts(config) + DimCategories(config) + DimPayees(config) + DimDate(config) + FactTransactions(config) + FactScheduledTransactions(config) + + logging.info('Data pipeline completed successfully')