From 861ccd0dc46c49ceb71bdfab2d744c2a35704e13 Mon Sep 17 00:00:00 2001 From: Jake Pullen Date: Sat, 10 Aug 2024 18:47:06 +0100 Subject: [PATCH] bug fix, a sucessful pipeline completion resulted in error message... bug fix, a successful dimension save resulted in failed save and skip the rest of processing --- main.py | 5 ++++- pipeline/raw_to_base.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index b897ce8..33e686b 100644 --- a/main.py +++ b/main.py @@ -74,5 +74,8 @@ if __name__ == '__main__': main() except SystemExit as e: exit_code = e.code - logging.error(f'Program exited with code {exit_code}') + if exit_code == ec.SUCCESS: + logging.info('Program exited successfully') + else: + logging.error(f'Program exited with code {exit_code}') raise diff --git a/pipeline/raw_to_base.py b/pipeline/raw_to_base.py index 1652b38..1e2072a 100644 --- a/pipeline/raw_to_base.py +++ b/pipeline/raw_to_base.py @@ -35,7 +35,7 @@ class RawToBase: logging.error(f"entity: {entity} failed duplicate resolution.") sys.exit(ec.DUPLICATE_RESOLUTION_ERROR) if not self._save_base_data(entity): - logging.warning(f"Skipping processing for entity: {entity} due to failed saving base data.") + logging.error(f"Skipping processing for entity: {entity} due to failed saving base data.") continue if not self._move_raw_to_processed(entity): logging.error(f"entity: {entity} has been processed, but we could not move the file out of the raw folder, please clear the raw folder for {entity}.") @@ -165,9 +165,9 @@ Then move the files back in one at a time oldest to newest and run again for eac except Exception as e: logging.error(f"Failed to save base data for entity: {entity}, error: {e}") return False - logging.debug(f"Saved base data for entity: {entity} to path: {file_path}") - + return True + def _move_raw_to_processed(self, entity): raw_entity_path = os.path.join(self.raw_data_path, entity) processed_path = os.path.join(self.processed_data_path, entity)