migration commit
This commit is contained in:
@@ -10,3 +10,4 @@ __pycache__/*
|
|||||||
/logs/*
|
/logs/*
|
||||||
.vscode/*
|
.vscode/*
|
||||||
*.coverage
|
*.coverage
|
||||||
|
.ruff_cache/*
|
||||||
|
|||||||
@@ -9,27 +9,30 @@ import logging.config
|
|||||||
import config.exit_codes as ec
|
import config.exit_codes as ec
|
||||||
from visuals.dash_app import app
|
from visuals.dash_app import app
|
||||||
|
|
||||||
|
|
||||||
def set_up_logging():
|
def set_up_logging():
|
||||||
try:
|
try:
|
||||||
with open('config/logging_config.yaml', 'r') as f:
|
with open("config/logging_config.yaml", "r") as f:
|
||||||
log_config = yaml.safe_load(f)
|
log_config = yaml.safe_load(f)
|
||||||
logging.config.dictConfig(log_config)
|
logging.config.dictConfig(log_config)
|
||||||
except yaml.YAMLError as e:
|
except yaml.YAMLError as e:
|
||||||
print(f"Error parsing logging configuration file: {e}")
|
print(f"Error parsing logging configuration file: {e}")
|
||||||
log_config = {} # Initialize log_config to an empty dictionary
|
log_config = {} # Initialize log_config to an empty dictionary
|
||||||
logging.basicConfig(level=logging.INFO) # Fallback to a basic configuration
|
# Fallback to a basic configuration
|
||||||
queue_handler = logging.getHandlerByName('queue_handler')
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
queue_handler = logging.getHandlerByName("queue_handler")
|
||||||
if queue_handler is not None:
|
if queue_handler is not None:
|
||||||
queue_handler.listener.start()
|
queue_handler.listener.start()
|
||||||
atexit.register(queue_handler.listener.stop)
|
atexit.register(queue_handler.listener.stop)
|
||||||
|
|
||||||
|
|
||||||
def load_config():
|
def load_config():
|
||||||
try:
|
try:
|
||||||
with open('config/config.yaml', 'r') as file:
|
with open("config/config.yaml", "r") as file:
|
||||||
config = yaml.safe_load(file)
|
config = yaml.safe_load(file)
|
||||||
return config
|
return config
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
logging.error('config.yaml file not found')
|
logging.error("config.yaml file not found")
|
||||||
sys.exit(ec.MISSING_CONFIG_FILE)
|
sys.exit(ec.MISSING_CONFIG_FILE)
|
||||||
except yaml.YAMLError:
|
except yaml.YAMLError:
|
||||||
# Attempt to print a more informative error message
|
# Attempt to print a more informative error message
|
||||||
@@ -38,43 +41,46 @@ def load_config():
|
|||||||
for key, value in parsed_yaml:
|
for key, value in parsed_yaml:
|
||||||
print(f"Error parsing key '{key}': {value}")
|
print(f"Error parsing key '{key}': {value}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f'Error loading config.yaml: {e}')
|
logging.error(f"Error loading config.yaml: {e}")
|
||||||
sys.exit(ec.CORRUPTED_CONFIG_FILE)
|
sys.exit(ec.CORRUPTED_CONFIG_FILE)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f'some other problem {e}')
|
logging.error(f"some other problem {e}")
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger("data_pipeline_for_ynab")
|
logger = logging.getLogger("data_pipeline_for_ynab")
|
||||||
os.makedirs('logs', exist_ok=True)
|
os.makedirs("logs", exist_ok=True)
|
||||||
set_up_logging()
|
set_up_logging()
|
||||||
|
|
||||||
# Load environment variables
|
# Load environment variables
|
||||||
dotenv.load_dotenv()
|
dotenv.load_dotenv()
|
||||||
|
|
||||||
API_TOKEN = os.getenv('API_TOKEN')
|
API_TOKEN = os.getenv("API_TOKEN")
|
||||||
BUDGET_ID = os.getenv('BUDGET_ID')
|
BUDGET_ID = os.getenv("BUDGET_ID")
|
||||||
|
|
||||||
if not API_TOKEN or not BUDGET_ID:
|
if not API_TOKEN or not BUDGET_ID:
|
||||||
logging.error('API_TOKEN or BUDGET_ID is not set in .env file')
|
logging.error("API_TOKEN or BUDGET_ID is not set in .env file")
|
||||||
sys.exit(ec.MISSING_ENV_VARS)
|
sys.exit(ec.MISSING_ENV_VARS)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
config = load_config()
|
config = load_config()
|
||||||
config['API_TOKEN'] = API_TOKEN
|
config["API_TOKEN"] = API_TOKEN
|
||||||
config['BUDGET_ID'] = BUDGET_ID
|
config["BUDGET_ID"] = BUDGET_ID
|
||||||
try:
|
try:
|
||||||
# pipeline_main(config)
|
# pipeline_main(config)
|
||||||
|
|
||||||
# Check if the data was successfully created
|
# Check if the data was successfully created
|
||||||
data_exists = os.path.exists('data/processed') and os.listdir('data/processed')
|
data_exists = os.path.exists("data/processed") and os.listdir("data/processed")
|
||||||
if data_exists:
|
if data_exists:
|
||||||
app.run(debug=True)
|
app.run(debug=True)
|
||||||
else:
|
else:
|
||||||
logging.error('Data pipeline did not produce any data. Dash app will not run.')
|
logging.error(
|
||||||
|
"Data pipeline did not produce any data. Dash app will not run."
|
||||||
|
)
|
||||||
sys.exit(ec.NO_DATA_PRODUCED)
|
sys.exit(ec.NO_DATA_PRODUCED)
|
||||||
except SystemExit as e:
|
except SystemExit as e:
|
||||||
exit_code = e.code
|
exit_code = e.code
|
||||||
if exit_code == ec.SUCCESS:
|
if exit_code == ec.SUCCESS:
|
||||||
logging.info('Program exited successfully')
|
logging.info("Program exited successfully")
|
||||||
else:
|
else:
|
||||||
logging.error(f'Program exited with code {exit_code}')
|
logging.error(f"Program exited with code {exit_code}")
|
||||||
raise
|
raise
|
||||||
|
|||||||
Reference in New Issue
Block a user