feat: Read station list from a file (#269)

* Fixed Issue #174

* Incorporating the helper functions

* Adding TODO

* Adding TODO

.gitignore

@@ -176,3 +176,4 @@ tutorials/pnw_data
 
 # Jupyterbook
 _build/
+log.txt

src/noisepy/seis/datatypes.py

@@ -230,6 +230,8 @@ class ConfigParameters(BaseModel):
         description="Storage options to pass to fsspec, keyed by protocol (local files are ''))",
     )
 
+    stations_file: Optional[str] = Field(default=None)
+
     def get_storage_options(self, path: str) -> Dict[str, Any]:
         """The storage options for the given path"""
         url = urlparse(path)
@@ -239,6 +241,29 @@ def get_storage_options(self, path: str) -> Dict[str, Any]:
     def dt(self) -> float:
         return 1.0 / self.samp_freq
 
+    def load_stations(self, stations_list=None) -> Optional[List[str]]:
+        if stations_list is None and self.stations_file:
+            # Use get_filesystem to get the filesystem associated with the filename
+            fs = get_filesystem(self.stations_file, storage_options=self.storage_options)
+
+            # Load the list from the file
+            with fs.open(self.stations_file, "r") as file:
+                self.stations = file.read().splitlines()
+        else:
+            self.stations = stations_list
+
+        return self.stations if self.stations else None
+
+    def save_stations(self, value: List[str]):
+        if self.stations_file:
+            fs = get_filesystem(self.stations_file, storage_options=self.storage_options)
+            # Save the list to the file
+            with fs.open(self.stations_file, "w") as file:
+                file.write("\n".join(value) + "\n")
+            # Set stations field to Empty List
+            # self.stations = []
+        return None
+
     @model_validator(mode="after")
     def validate(cls, m: ConfigParameters) -> ConfigParameters:
         def validate_date(d: datetime, name: str):
@@ -251,6 +276,14 @@ def validate_date(d: datetime, name: str):
         validate_date(m.end_date, "end_date")
         if m.substack_len % m.cc_len != 0:
             raise ValueError(f"substack_len ({m.substack_len}) must be a multiple of cc_len ({m.cc_len})")
+
+        if m.stations_file:
+            fs = get_filesystem(m.stations_file, storage_options=m.storage_options)
+
+            # Check if the file exists
+            if not fs.exists(m.stations_file):
+                raise ValueError(f"{m.stations_file} is not a valid file path in stations_file.")
+
         return m
 
     # TODO: Remove once all uses of ConfigParameters have been converted to use strongly typed access

tests/data/stations.txt

@@ -0,0 +1,3 @@
+RPV
+SVD
+BBR

tests/data/stations1.txt

@@ -0,0 +1,2 @@
+new_station1
+new_station2

tests/test_stations_file.py

@@ -0,0 +1,17 @@
+import os
+
+from noisepy.seis.datatypes import ConfigParameters
+
+
+def test_stations_file_behavior():
+    # Test loading from stations_file
+    config = ConfigParameters(stations_file=os.path.join(os.path.dirname(__file__), "./data/stations.txt"))
+    assert config.stations == ["*"]
+
+    config.load_stations()
+
+    assert config.stations == ["RPV", "SVD", "BBR"]
+
+    new_stations = ["new_station1", "new_station2"]
+    config.stations_file = os.path.join(os.path.dirname(__file__), "./data/stations1.txt")
+    ConfigParameters.save_stations(config, new_stations)

tutorials/noisepy_scedc_tutorial.ipynb

@@ -72,6 +72,7 @@
         "from noisepy.seis.datatypes import CCMethod, ConfigParameters, FreqNorm, RmResp, StackMethod, TimeNorm        # Main configuration object\n",
         "from noisepy.seis.channelcatalog import XMLStationChannelCatalog        # Required stationXML handling object\n",
         "import os\n",
+        "import shutil\n",
         "from datetime import datetime, timezone\n",
         "from datetimerange import DateTimeRange\n",
         "\n",
@@ -242,7 +243,9 @@
       "outputs": [],
       "source": [
         "# For this tutorial make sure the previous run is empty\n",
-        "os.system(f\"rm -rf {cc_data_path}\")\n"
+        "#os.system(f\"rm -rf {cc_data_path}\")\n",
+        "if os.path.exists(cc_data_path):\n",
+        "    shutil.rmtree(cc_data_path)"
       ]
     },
     {
@@ -274,8 +277,19 @@
         "#stations = \"RPV,STS,LTP,LGB,WLT,CPP,PDU,CLT,SVD,BBR\".split(\",\") # filter to these stations\n",
         "stations = \"RPV,SVD,BBR\".split(\",\") # filter to these stations\n",
         "# stations = \"DGR,DEV,DLA,DNR,FMP,HLL,LGU,LLS,MLS,PDU,PDR,RIN,RIO,RVR,SMS,BBR,CHN,MWC,RIO,BBS,RPV,ADO,DEV\".split(\",\") # filter to these stations\n",
+        "\n",
+        "# There are 2 ways to load stations: You can either pass a list of stations or load the stations from a text file.\n",
+        "# TODO : will be removed with issue #270\n",
+        "config.load_stations(stations)\n",
+        "\n",
+        "# For loading it from a text file, write the path of the file in stations_file field of config instance as below\n",
+        "# config.stations_file = os.path.join(os.path.dirname(__file__), \"path/my_stations.txt\")\n",
+        "\n",
+        "# TODO : will be removed with issue #270\n",
+        "# config.load_stations()\n",
+        "\n",
         "catalog = XMLStationChannelCatalog(S3_STATION_XML, storage_options=S3_STORAGE_OPTIONS) # Station catalog\n",
-        "raw_store = SCEDCS3DataStore(S3_DATA, catalog, channel_filter(stations, \"BH\"), timerange, storage_options=S3_STORAGE_OPTIONS) # Store for reading raw data from S3 bucket\n",
+        "raw_store = SCEDCS3DataStore(S3_DATA, catalog, channel_filter(config.stations, \"BH\"), timerange, storage_options=S3_STORAGE_OPTIONS) # Store for reading raw data from S3 bucket\n",
         "cc_store = ASDFCCStore(cc_data_path) # Store for writing CC data"
       ]
     },