Jake/data_pipeline_for_YNAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bug fix, a sucessful pipeline completion resulted in error message...

Browse Source 
bug fix, a successful dimension save resulted in failed save and skip the rest of processing
This commit is contained in:
Jake Pullen
2024-08-10 18:47:06 +01:00
parent 812e107013
commit 861ccd0dc4
2 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+4 -1
View File
@@ -74,5 +74,8 @@ if __name__ == '__main__':
main() main()
except SystemExit as e: except SystemExit as e:
exit_code = e.code exit_code = e.code
logging.error(f'Program exited with code {exit_code}') if exit_code == ec.SUCCESS:
logging.info('Program exited successfully')
else:
logging.error(f'Program exited with code {exit_code}')
raise raise
pipeline/raw_to_base.py
+2 -2
View File
@@ -35,7 +35,7 @@ class RawToBase:
logging.error(f"entity: {entity} failed duplicate resolution.") logging.error(f"entity: {entity} failed duplicate resolution.")
sys.exit(ec.DUPLICATE_RESOLUTION_ERROR) sys.exit(ec.DUPLICATE_RESOLUTION_ERROR)
if not self._save_base_data(entity): if not self._save_base_data(entity):
logging.warning(f"Skipping processing for entity: {entity} due to failed saving base data.") logging.error(f"Skipping processing for entity: {entity} due to failed saving base data.")
continue continue
if not self._move_raw_to_processed(entity): if not self._move_raw_to_processed(entity):
logging.error(f"entity: {entity} has been processed, but we could not move the file out of the raw folder, please clear the raw folder for {entity}.") logging.error(f"entity: {entity} has been processed, but we could not move the file out of the raw folder, please clear the raw folder for {entity}.")
@@ -165,8 +165,8 @@ Then move the files back in one at a time oldest to newest and run again for eac
except Exception as e: except Exception as e:
logging.error(f"Failed to save base data for entity: {entity}, error: {e}") logging.error(f"Failed to save base data for entity: {entity}, error: {e}")
return False return False
logging.debug(f"Saved base data for entity: {entity} to path: {file_path}") logging.debug(f"Saved base data for entity: {entity} to path: {file_path}")
return True
def _move_raw_to_processed(self, entity): def _move_raw_to_processed(self, entity):
raw_entity_path = os.path.join(self.raw_data_path, entity) raw_entity_path = os.path.join(self.raw_data_path, entity)