Migrating to duckdb

pipeline/pipeline_main.py

@@ -1,19 +1,19 @@
 '''Module to run the data pipeline'''
 
-from pipeline import dimensions, facts, ingest, raw_to_base
+from pipeline import duckdb_layer, ingest, raw_to_base
 
 
 def pipeline_main(config, logger):
     '''Run the data pipeline'''
     logger.info('Starting data pipeline')
 
-    ingest.Ingest(config,logger).start_ingestion()
-    raw_to_base.RawToBase(config,logger)
-    dimensions.DimAccounts(config,logger)
-    dimensions.DimCategories(config,logger)
-    dimensions.DimPayees(config,logger)
-    dimensions.DimDate(config,logger)
-    facts.FactTransactions(config,logger)
-    facts.FactScheduledTransactions(config,logger)
+    ingest.Ingest(config, logger).start_ingestion()
+    raw_to_base.RawToBase(config, logger)
+
+    duckdb_layer.get_duckdb_layer(
+        config['base_data_path'],
+        config['warehouse_data_path'],
+        logger
+    )
 
     logger.info('Data pipeline completed successfully')