Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
85deee7843
|
@@ -9,7 +9,6 @@ The project consists of a main pipeline workflow that processes multiple modules
|
|||||||
- `main.py`: Main pipeline orchestrator that calls on the modules as needed
|
- `main.py`: Main pipeline orchestrator that calls on the modules as needed
|
||||||
- `batch_nimrod.py`: Module for batch processing multiple NIMROD files with configurable bounding boxes
|
- `batch_nimrod.py`: Module for batch processing multiple NIMROD files with configurable bounding boxes
|
||||||
- `generate_timeseries.py`: Module for extracting cropped rain data and creating rainfall timeseries
|
- `generate_timeseries.py`: Module for extracting cropped rain data and creating rainfall timeseries
|
||||||
- `combine_timeseries.py`: Module for combining grouped timeseries CSVs into consolidated datasets
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@@ -31,10 +30,6 @@ The project consists of a main pipeline workflow that processes multiple modules
|
|||||||
- Extract cropped rain data based on specified locations
|
- Extract cropped rain data based on specified locations
|
||||||
- Create rainfall timeseries CSVs for each location
|
- Create rainfall timeseries CSVs for each location
|
||||||
- Parse datetime from filename and create proper datetime index
|
- Parse datetime from filename and create proper datetime index
|
||||||
|
|
||||||
### combine_timeseries.py
|
|
||||||
|
|
||||||
- Combine multiple timeseries CSV files into grouped datasets
|
|
||||||
- Group locations by specified output groups
|
- Group locations by specified output groups
|
||||||
- Create consolidated CSV files for each group
|
- Create consolidated CSV files for each group
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,16 @@ logging.basicConfig(
|
|||||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def process_pipeline(dat_file):
|
||||||
|
# 1. Process DAT to ASC
|
||||||
|
asc_file = batch._process_single_file(dat_file)
|
||||||
|
if not asc_file:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 2. Extract data from ASC
|
||||||
|
file_results = timeseries.process_asc_file(asc_file, locations)
|
||||||
|
return file_results
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
os.makedirs(Path(Config.ASC_TOP_FOLDER), exist_ok=True)
|
os.makedirs(Path(Config.ASC_TOP_FOLDER), exist_ok=True)
|
||||||
os.makedirs(Path(Config.COMBINED_FOLDER), exist_ok=True)
|
os.makedirs(Path(Config.COMBINED_FOLDER), exist_ok=True)
|
||||||
@@ -45,16 +55,6 @@ if __name__ == "__main__":
|
|||||||
# Initialize results structure
|
# Initialize results structure
|
||||||
results = {loc[0]: {"dates": [], "values": []} for loc in locations}
|
results = {loc[0]: {"dates": [], "values": []} for loc in locations}
|
||||||
|
|
||||||
def process_pipeline(dat_file):
|
|
||||||
# 1. Process DAT to ASC
|
|
||||||
asc_file = batch._process_single_file(dat_file)
|
|
||||||
if not asc_file:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# 2. Extract data from ASC
|
|
||||||
file_results = timeseries.process_asc_file(asc_file, locations)
|
|
||||||
return file_results
|
|
||||||
|
|
||||||
# Get list of DAT files
|
# Get list of DAT files
|
||||||
dat_files = [
|
dat_files = [
|
||||||
f for f in os.listdir(Path(Config.DAT_TOP_FOLDER)) if not f.startswith(".")
|
f for f in os.listdir(Path(Config.DAT_TOP_FOLDER)) if not f.startswith(".")
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "met-office"
|
name = "met-office"
|
||||||
version = "1.1.0"
|
version = "1.1.1"
|
||||||
description = "Convert .dat nimrod files to .asc files"
|
description = "Convert .dat nimrod files to .asc files"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.14"
|
requires-python = ">=3.14"
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ requires-python = ">=3.14"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "met-office"
|
name = "met-office"
|
||||||
version = "1.1.0"
|
version = "1.1.1"
|
||||||
source = { virtual = "." }
|
source = { virtual = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "numpy" },
|
{ name = "numpy" },
|
||||||
|
|||||||
Reference in New Issue
Block a user