Convert marine LETKF task to use JCB #3381
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,26 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import copy | ||
| import f90nml | ||
| from jcb import render | ||
| from logging import getLogger | ||
| import os | ||
| from pygfs.jedi import Jedi | ||
| from pygfs.task.analysis import Analysis | ||
| import pygfs.utils.marine_da_utils as mdau | ||
| from typing import Dict | ||
| from wxflow import (add_to_datetime, | ||
| AttrDict, | ||
| Executable, | ||
| FileHandler, | ||
| logit, | ||
| parse_j2yaml, | ||
| save_as_yaml, | ||
| Template, | ||
| TemplateConstants, | ||
| to_timedelta, | ||
| to_YMDH, | ||
| YAMLFile) | ||
|
|
||
| logger = getLogger(__name__.split('.')[-1]) | ||
|
|
||
|
|
@@ -37,23 +45,42 @@ def __init__(self, config: Dict) -> None: | |
| logger.info("init") | ||
| super().__init__(config) | ||
|
|
||
| _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) | ||
| _half_assim_freq = to_timedelta(f"{self.task_config.assim_freq}H") / 2 | ||
| _letkf_yaml_file = 'letkf.yaml' | ||
| _letkf_exec_args = [self.task_config.MARINE_LETKF_EXEC, | ||
| 'soca', | ||
| 'localensembleda', | ||
| _letkf_yaml_file] | ||
| # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert | ||
| _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) | ||
| _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) | ||
|
|
||
| self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle | ||
| self.task_config.MARINE_WINDOW_MIDDLE = self.task_config.current_cycle | ||
| self.task_config.WINDOW_BEGIN = _window_begin | ||
| self.task_config.WINDOW_END = _window_end | ||
| self.task_config.MARINE_WINDOW_BEGIN = _window_begin | ||
| self.task_config.MARINE_WINDOW_END = _window_end | ||
| self.task_config.letkf_exec_args = _letkf_exec_args | ||
| self.task_config.letkf_yaml_file = _letkf_yaml_file | ||
| self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') | ||
| self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') | ||
| self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') | ||
| self.task_config.ENSPERT_RELPATH = _enspert_relpath | ||
| self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') | ||
|
|
||
| # Create dictionary of JEDI objects | ||
| expected_keys = ['marineanlletkf'] | ||
| self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) | ||
| self.jedi_dict.PARMgfs = self.task_config.PARMgfs | ||
| #self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin.strftime('%Y-%m-%dT%H:%M:%SZ') | ||
| self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin | ||
| self.jedi_dict.MARINE_WINDOW_END = _window_end | ||
| #self.jedi_dict.MARINE_WINDOW_LENGTH = _window_end - _window_begin | ||
| self.jedi_dict.MARINE_WINDOW_LENGTH = f"PT{self.task_config['assim_freq']}H" | ||
| print("jedi_dict: ",self.jedi_dict) | ||
|
|
||
|
|
||
| @logit(logger) | ||
| def initialize(self): | ||
|
|
@@ -66,43 +93,17 @@ def initialize(self): | |
| None | ||
| """ | ||
|
|
||
| # NOTE: This task requires the ensemble background to be staged, which is done in the | ||
| # BMAT task (which is a dependency of this task) | ||
|
|
||
| logger.info("initialize") | ||
|
|
||
| # make directories and stage ensemble background files | ||
| soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) | ||
| FileHandler(soca_fix_stage_list).sync() | ||
| # stage letkf-specific files and directories | ||
| letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) | ||
| FileHandler(letkf_stage_list).sync() | ||
| obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) | ||
|
|
||
| # get the list of observations | ||
|
|
@@ -127,13 +128,54 @@ def initialize(self): | |
| else: | ||
| logger.warning(f"{obs_file} is not available in {self.task_config.COMIN_OBS}") | ||
|
|
||
| # set aside for LETKF configuration | ||
| observers = {'observers': obs_to_use} | ||
|
|
||
| # stage the desired obs files | ||
| FileHandler({'copy': obs_files_to_copy}).sync() | ||
|
|
||
| #################################################################################################### | ||
|
|
||
|
|
||
| # initialize JEDI LETKF observer application | ||
| #self.task_config['jcb_algo'] = 'marine_letkf' | ||
| logger.info(f"Initializing JEDI LETKF observer application") | ||
| #self.jedi_dict['marineanlletkf'].initialize(self.task_config) | ||
| self.jedi_dict['marineanlletkf'].initialize(self.jedi_dict) | ||
|
|
||
| envconfig_jcb = copy.deepcopy(self.task_config) | ||
| envconfig_jcb['cyc'] = int(os.getenv('cyc')) | ||
|
Contributor
There was a problem hiding this comment.
|
||
| envconfig_jcb['PDY'] = self.task_config.current_cycle.strftime('%Y%m%d') | ||
| envconfig_jcb['window_length'] = f"PT{self.task_config['assim_freq']}H" | ||
|
Contributor
There was a problem hiding this comment. In general, if you use |
||
|
|
||
| # Render the JCB configuration files | ||
| jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') | ||
| jcb_algo_yaml = os.path.join(self.task_config.PARMsoca, 'letkf/marine-jcb-lektf.yaml.j2') | ||
|
|
||
| jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=envconfig_jcb) | ||
| jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) | ||
| jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) | ||
| jcb_algo_config = parse_j2yaml(path=jcb_algo_yaml, data=envconfig_jcb) | ||
| jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) | ||
|
Contributor
There was a problem hiding this comment. If I understand this correctly, the need for the |
||
| jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) | ||
|
|
||
| # Override base with the application specific config | ||
| jcb_config = {**jcb_base_config, **jcb_algo_config} | ||
|
|
||
| # convert datetime to string | ||
| jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') | ||
|
Contributor
There was a problem hiding this comment. Any JCB variables like this should be set in the JCB base yaml. In fact, |
||
| jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') | ||
| jcb_config['window_length'] = f"PT{self.task_config['assim_freq']}H" | ||
|
|
||
| # Render the full JEDI configuration file using JCB | ||
| jedi_config = render(jcb_config) | ||
| jedi_config['observations'] = observers | ||
|
Contributor
There was a problem hiding this comment. I think by the time the JCB render method is called, this |
||
|
|
||
| # Save the JEDI configuration file | ||
| letkf_yaml_jcb = 'letkf.yaml' | ||
| save_as_yaml(jedi_config, letkf_yaml_jcb) | ||
|
|
||
| ###################################### | ||
|
|
||
| # swap date and stack size in mom_input.nml | ||
| domain_stack_size = self.task_config.DOMAIN_STACK_SIZE | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't understand the reason to make a copy of
task_config. I know this is whatmarine_analysis.pydoes too, but with the atmosphere, aerosols, and snow, the JCB base and algo yamls are all parsed usingtask_config. I think at least for consistency we should do it this way for the marine jobs too.