57 lines
1.7 KiB
Python
57 lines
1.7 KiB
Python
from dagster import job, op,get_dagster_logger
|
|
import os
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
from data_pipeline_for_YNAB.config import exit_codes as ec
|
|
from data_pipeline_for_YNAB.pipeline.pipeline_main import pipeline_main
|
|
|
|
logger = get_dagster_logger()
|
|
|
|
|
|
def _load_config(logger):
|
|
try:
|
|
with open('jobs/data_pipeline_for_YNAB/config/config.yaml', 'r') as file:
|
|
config = yaml.safe_load(file)
|
|
return config
|
|
except FileNotFoundError:
|
|
logger.error('config.yaml file not found')
|
|
sys.exit(ec.MISSING_CONFIG_FILE)
|
|
except yaml.YAMLError as e:
|
|
logger.error(f'Error loading config.yaml: {e}')
|
|
sys.exit(ec.CORRUPTED_CONFIG_FILE)
|
|
|
|
@op
|
|
def refresh_ynab():
|
|
API_TOKEN = os.getenv('API_TOKEN')
|
|
BUDGET_ID = os.getenv('BUDGET_ID')
|
|
config = _load_config(logger)
|
|
config['API_TOKEN'] = API_TOKEN
|
|
config['BUDGET_ID'] = BUDGET_ID
|
|
try:
|
|
pipeline_main(config, logger)
|
|
|
|
data_exists = os.path.exists('jobs/data_pipeline_for_YNAB/data/processed') and os.listdir('jobs/data_pipeline_for_YNAB/data/processed')
|
|
if data_exists:
|
|
logger.info('Processing Successful')
|
|
# sys.exit(ec.SUCCESS)
|
|
else:
|
|
logger.error('Data pipeline did not produce any data. Dash app will not run.')
|
|
sys.exit(ec.NO_DATA_PRODUCED)
|
|
except SystemExit as e:
|
|
exit_code = e.code
|
|
if exit_code == ec.SUCCESS:
|
|
logger.info('Program exited successfully')
|
|
else:
|
|
logger.error(f'Program exited with code {exit_code}')
|
|
raise
|
|
|
|
@job
|
|
def dagster_ynab_runner():
|
|
refresh_ynab()
|
|
|
|
if __name__ == '__main__':
|
|
refresh_ynab()
|
|
# test comment
|