from dagster import job, op,get_dagster_logger import os import sys import yaml from data_pipeline_for_YNAB.config import exit_codes as ec from data_pipeline_for_YNAB.pipeline.pipeline_main import pipeline_main logger = get_dagster_logger() def _load_config(logger): try: with open('jobs/data_pipeline_for_YNAB/config/config.yaml', 'r') as file: config = yaml.safe_load(file) return config except FileNotFoundError: logger.error('config.yaml file not found') sys.exit(ec.MISSING_CONFIG_FILE) except yaml.YAMLError as e: logger.error(f'Error loading config.yaml: {e}') sys.exit(ec.CORRUPTED_CONFIG_FILE) @op def refresh_ynab(): API_TOKEN = os.getenv('API_TOKEN') BUDGET_ID = os.getenv('BUDGET_ID') config = _load_config(logger) config['API_TOKEN'] = API_TOKEN config['BUDGET_ID'] = BUDGET_ID try: pipeline_main(config, logger) data_exists = os.path.exists('jobs/data_pipeline_for_YNAB/data/processed') and os.listdir('jobs/data_pipeline_for_YNAB/data/processed') if data_exists: logger.info('Processing Successful') # sys.exit(ec.SUCCESS) else: logger.error('Data pipeline did not produce any data. Dash app will not run.') sys.exit(ec.NO_DATA_PRODUCED) except SystemExit as e: exit_code = e.code if exit_code == ec.SUCCESS: logger.info('Program exited successfully') else: logger.error(f'Program exited with code {exit_code}') raise @job def dagster_ynab_runner(): refresh_ynab() if __name__ == '__main__': refresh_ynab() # test comment