20 lines
487 B
Python
20 lines
487 B
Python
'''Module to run the data pipeline'''
|
|
|
|
from pipeline import duckdb_layer, ingest, raw_to_base
|
|
|
|
|
|
def pipeline_main(config, logger):
|
|
'''Run the data pipeline'''
|
|
logger.info('Starting data pipeline')
|
|
|
|
ingest.Ingest(config, logger).start_ingestion()
|
|
raw_to_base.RawToBase(config, logger)
|
|
|
|
duckdb_layer.get_duckdb_layer(
|
|
config['base_data_path'],
|
|
config['warehouse_data_path'],
|
|
logger
|
|
)
|
|
|
|
logger.info('Data pipeline completed successfully')
|