diff --git a/.gitignore b/.gitignore index 6fdf2830..2f2ab84c 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,4 @@ tutorials/pnw_data # Jupyterbook _build/ +log.txt diff --git a/src/noisepy/seis/datatypes.py b/src/noisepy/seis/datatypes.py index 71434a5c..05274a62 100644 --- a/src/noisepy/seis/datatypes.py +++ b/src/noisepy/seis/datatypes.py @@ -230,6 +230,8 @@ class ConfigParameters(BaseModel): description="Storage options to pass to fsspec, keyed by protocol (local files are ''))", ) + stations_file: Optional[str] = Field(default=None) + def get_storage_options(self, path: str) -> Dict[str, Any]: """The storage options for the given path""" url = urlparse(path) @@ -239,6 +241,29 @@ def get_storage_options(self, path: str) -> Dict[str, Any]: def dt(self) -> float: return 1.0 / self.samp_freq + def load_stations(self, stations_list=None) -> Optional[List[str]]: + if stations_list is None and self.stations_file: + # Use get_filesystem to get the filesystem associated with the filename + fs = get_filesystem(self.stations_file, storage_options=self.storage_options) + + # Load the list from the file + with fs.open(self.stations_file, "r") as file: + self.stations = file.read().splitlines() + else: + self.stations = stations_list + + return self.stations if self.stations else None + + def save_stations(self, value: List[str]): + if self.stations_file: + fs = get_filesystem(self.stations_file, storage_options=self.storage_options) + # Save the list to the file + with fs.open(self.stations_file, "w") as file: + file.write("\n".join(value) + "\n") + # Set stations field to Empty List + # self.stations = [] + return None + @model_validator(mode="after") def validate(cls, m: ConfigParameters) -> ConfigParameters: def validate_date(d: datetime, name: str): @@ -251,6 +276,14 @@ def validate_date(d: datetime, name: str): validate_date(m.end_date, "end_date") if m.substack_len % m.cc_len != 0: raise ValueError(f"substack_len ({m.substack_len}) must be a multiple of cc_len ({m.cc_len})") + + if m.stations_file: + fs = get_filesystem(m.stations_file, storage_options=m.storage_options) + + # Check if the file exists + if not fs.exists(m.stations_file): + raise ValueError(f"{m.stations_file} is not a valid file path in stations_file.") + return m # TODO: Remove once all uses of ConfigParameters have been converted to use strongly typed access diff --git a/tests/data/stations.txt b/tests/data/stations.txt new file mode 100644 index 00000000..c3040643 --- /dev/null +++ b/tests/data/stations.txt @@ -0,0 +1,3 @@ +RPV +SVD +BBR diff --git a/tests/data/stations1.txt b/tests/data/stations1.txt new file mode 100644 index 00000000..5e0325ca --- /dev/null +++ b/tests/data/stations1.txt @@ -0,0 +1,2 @@ +new_station1 +new_station2 diff --git a/tests/test_stations_file.py b/tests/test_stations_file.py new file mode 100644 index 00000000..27d2047a --- /dev/null +++ b/tests/test_stations_file.py @@ -0,0 +1,17 @@ +import os + +from noisepy.seis.datatypes import ConfigParameters + + +def test_stations_file_behavior(): + # Test loading from stations_file + config = ConfigParameters(stations_file=os.path.join(os.path.dirname(__file__), "./data/stations.txt")) + assert config.stations == ["*"] + + config.load_stations() + + assert config.stations == ["RPV", "SVD", "BBR"] + + new_stations = ["new_station1", "new_station2"] + config.stations_file = os.path.join(os.path.dirname(__file__), "./data/stations1.txt") + ConfigParameters.save_stations(config, new_stations) diff --git a/tutorials/noisepy_scedc_tutorial.ipynb b/tutorials/noisepy_scedc_tutorial.ipynb index 8fc56af4..d03e4e5c 100644 --- a/tutorials/noisepy_scedc_tutorial.ipynb +++ b/tutorials/noisepy_scedc_tutorial.ipynb @@ -72,6 +72,7 @@ "from noisepy.seis.datatypes import CCMethod, ConfigParameters, FreqNorm, RmResp, StackMethod, TimeNorm # Main configuration object\n", "from noisepy.seis.channelcatalog import XMLStationChannelCatalog # Required stationXML handling object\n", "import os\n", + "import shutil\n", "from datetime import datetime, timezone\n", "from datetimerange import DateTimeRange\n", "\n", @@ -242,7 +243,9 @@ "outputs": [], "source": [ "# For this tutorial make sure the previous run is empty\n", - "os.system(f\"rm -rf {cc_data_path}\")\n" + "#os.system(f\"rm -rf {cc_data_path}\")\n", + "if os.path.exists(cc_data_path):\n", + " shutil.rmtree(cc_data_path)" ] }, { @@ -274,8 +277,19 @@ "#stations = \"RPV,STS,LTP,LGB,WLT,CPP,PDU,CLT,SVD,BBR\".split(\",\") # filter to these stations\n", "stations = \"RPV,SVD,BBR\".split(\",\") # filter to these stations\n", "# stations = \"DGR,DEV,DLA,DNR,FMP,HLL,LGU,LLS,MLS,PDU,PDR,RIN,RIO,RVR,SMS,BBR,CHN,MWC,RIO,BBS,RPV,ADO,DEV\".split(\",\") # filter to these stations\n", + "\n", + "# There are 2 ways to load stations: You can either pass a list of stations or load the stations from a text file.\n", + "# TODO : will be removed with issue #270\n", + "config.load_stations(stations)\n", + "\n", + "# For loading it from a text file, write the path of the file in stations_file field of config instance as below\n", + "# config.stations_file = os.path.join(os.path.dirname(__file__), \"path/my_stations.txt\")\n", + "\n", + "# TODO : will be removed with issue #270\n", + "# config.load_stations()\n", + "\n", "catalog = XMLStationChannelCatalog(S3_STATION_XML, storage_options=S3_STORAGE_OPTIONS) # Station catalog\n", - "raw_store = SCEDCS3DataStore(S3_DATA, catalog, channel_filter(stations, \"BH\"), timerange, storage_options=S3_STORAGE_OPTIONS) # Store for reading raw data from S3 bucket\n", + "raw_store = SCEDCS3DataStore(S3_DATA, catalog, channel_filter(config.stations, \"BH\"), timerange, storage_options=S3_STORAGE_OPTIONS) # Store for reading raw data from S3 bucket\n", "cc_store = ASDFCCStore(cc_data_path) # Store for writing CC data" ] },