From 31c34397f28b96e5862bba57d33316dd12d60d3e Mon Sep 17 00:00:00 2001
From: lkugler <lukas.kugler@gmail.com>
Date: Tue, 21 Feb 2023 18:58:21 +0100
Subject: [PATCH] docs
---
analysis_only.py | 7 +--
docs/source/index.rst | 4 +-
docs/source/tutorial1.ipynb | 122 +++++++++++++++++++++++-------------
docs/source/tutorial2.ipynb | 85 ++++++++++++-------------
docs/source/tutorial3.ipynb | 33 +++++-----
5 files changed, 139 insertions(+), 112 deletions(-)
diff --git a/analysis_only.py b/analysis_only.py
index c1ae8b9..27f8cf6 100755
--- a/analysis_only.py
+++ b/analysis_only.py
@@ -5,7 +5,6 @@ running the forecast model without assimilation
import os, sys, shutil
import datetime as dt
-from dartwrf import utils
from dartwrf.workflows import WorkFlows
@@ -16,8 +15,6 @@ assim_time = prior_valid_time
w = WorkFlows(exp_config='cfg.py', server_config='srvx1.py')
-w.assimilate(assim_time, prior_init_time, prior_valid_time, prior_path_exp)
+id = w.assimilate(assim_time, prior_init_time, prior_valid_time, prior_path_exp)
-
-
-# id_sat = create_satimages(time, depends_on=id)
+# w.create_satimages(time, depends_on=id)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index ef7c001..703f525 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -1,7 +1,9 @@
Welcome to the DART-WRF documentation!
======================================
-DART-WRF is a python package to run an Ensemble Data Assimilation system using the data assimilation suite `DART <https://docs.dart.ucar.edu/en/latest/README.html>`_ and the weather research and forecast model `WRF <https://www2.mmm.ucar.edu/wrf/users/docs/docs_and_pubs.html>`_.
+**DART-WRF** is a python package to run an Ensemble Data Assimilation system using the data assimilation suite `DART <https://docs.dart.ucar.edu/en/latest/README.html>`_ and the weather research and forecast model `WRF <https://www2.mmm.ucar.edu/wrf/users/docs/docs_and_pubs.html>`_.
+
+DART-WRF is available at `github.com/lkugler/DART-WRF <https://github.com/lkugler/DART-WRF>`.
Installation
------------
diff --git a/docs/source/tutorial1.ipynb b/docs/source/tutorial1.ipynb
index 256149a..018a319 100644
--- a/docs/source/tutorial1.ipynb
+++ b/docs/source/tutorial1.ipynb
@@ -7,36 +7,97 @@
"source": [
"# Tutorial 1: The assimilation step\n",
"DART-WRF is a python package which automates many things like configuration, saving configuration and output, handling computing resources, etc.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "93d59d4d-c514-414e-81fa-4ff390290811",
+ "metadata": {},
+ "source": [
+ "### Configuring the experiment\n",
+ "Firstly, you need to configure the experiment in `config/cfg.py`.\n",
"\n",
- "**Goal**: Using a predefined configuration file, run an example of Data Assimilation.\n",
+ "Let's go through the most important settings:\n",
"\n",
- "**What you need to know**:\n",
+ "```python\n",
+ "exp = utils.Experiment()\n",
+ "exp.expname = \"test_newcode\"\n",
+ "exp.model_dx = 2000\n",
+ "exp.n_ens = 10\n",
+ "exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC']\n",
+ "exp.filter_kind = 1\n",
+ "exp.prior_inflation = 0\n",
+ "exp.post_inflation = 4\n",
+ "exp.sec = True\n",
+ "exp.cov_loc_vert_km_horiz_km = (3, 20)\n",
+ "```\n",
+ "In case you want to generate new observations (observing system simulations experiment, OSSE), set `sxp.use_existing_obsseq = False`.\n",
"\n",
- "- There is a config/ folder with experimental configuration in config/cfg.py.\n",
- "- There is an example script analysis_only.py which handles the DA programs.\n",
+ "`exp.nature` defines where observations will be drawn from, e.g.:\n",
+ "```python\n",
+ "exp.nature = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1'\n",
+ "```\n",
+ "\n",
+ "`exp.input_profile` is used, if you create initial conditions from a so called wrf_profile (see WRF guide).\n",
+ "```python\n",
+ "exp.input_profile = '/mnt/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'\n",
+ "```\n",
+ "\n",
+ "Vertical localization is tricky to set.\n",
+ "For horizontal localization half-width of 20 km and 3 km vertically, set\n",
+ "`exp.cov_loc_vert_km_horiz_km = (3, 20)`\n",
+ "You can also set it to zero for no vertical localization.\n",
+ "\n",
+ "\n",
+ "Set you desired observations like this. \n",
+ "```python\n",
+ "t = dict(plotname='Temperature', plotunits='[K]',\n",
+ " kind='RADIOSONDE_TEMPERATURE', \n",
+ " n_obs=1, obs_locations=[(45., 0.)],\n",
+ " error_generate=0.2, error_assimilate=0.2,\n",
+ " heights=[1000,], # range(1000, 17001, 2000),\n",
+ " cov_loc_radius_km=50)\n",
"\n",
- "**To-Do**:\n",
+ "exp.observations = [t,]\n",
+ "```\n",
"\n",
- "- Go into the DART-WRF folder that we created and installed last time.\n",
- "- Copy the configuration file from `` to your config/ folder.\n"
+ "To generate a grid of observations, use\n",
+ "```python\n",
+ "vis = dict(plotname='VIS 0.6µm', plotunits='[1]',\n",
+ " kind='MSG_4_SEVIRI_BDRF', sat_channel=1, \n",
+ " n_obs=961, obs_locations='square_array_evenly_on_grid',\n",
+ " error_generate=0.03, error_assimilate=0.03,\n",
+ " cov_loc_radius_km=20)\n",
+ "```\n",
+ "But caution, n_obs should only be one of the following:\n",
+ "- 22500 for 2km observation density/resolution \n",
+ "- 5776 for 4km; \n",
+ "- 961 for 10km; \n",
+ "- 256 for 20km; \n",
+ "- 121 for 30km\n",
+ "For vertically resolved data, like radar, n_obs is the number of observations at each observation height level."
]
},
{
"cell_type": "markdown",
- "id": "24b23d8c-29e6-484c-ad1f-f2bc07e66c66",
+ "id": "16bd3521-f98f-4c4f-8019-31029fd678ae",
"metadata": {},
"source": [
+ "### Configuring the hardware\n",
+ "In case you use a cluster which is not supported, configure paths inside `config/clusters.py`.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Assimilate observations with a prior given by previous forecasts\n",
"We start by importing some modules:\n",
"```python\n",
- "import os, sys, shutil\n",
"import datetime as dt\n",
- "\n",
- "from dartwrf import utils\n",
- "from config.cfg import exp\n",
- "from config.clusters import cluster\n",
+ "from dartwrf.workflows import WorkFlows\n",
"```\n",
"\n",
- "Then, we set the directory paths and times of the prior ensemble forecasts:\n",
+ "To assimilate observations at dt.datetime `time` we set the directory paths and times of the prior ensemble forecasts:\n",
"\n",
"```python\n",
"prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'\n",
@@ -47,43 +108,14 @@
"\n",
"Finally, we run the data assimilation by calling\n",
"```python\n",
- "cluster.setup()\n",
- "\n",
- "os.system(\n",
- " cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '\n",
- " +assim_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_init_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_valid_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_path_exp\n",
- " )\n",
+ "w = WorkFlows(exp_config='cfg.py', server_config='srvx1.py')\n",
"\n",
- "create_satimages(time)\n",
+ "w.assimilate(assim_time, prior_init_time, prior_valid_time, prior_path_exp)\n",
"```\n",
"\n",
"Congratulations! You're done!"
]
},
- {
- "cell_type": "markdown",
- "id": "31b23faf-0986-407f-b07f-d635a71ec2c6",
- "metadata": {},
- "source": [
- "---\n",
- "#### Queueing systems\n",
- "Note: In case you have to use a queueing system, use the builtin job scheduler, e.g.:\n",
- "```python\n",
- "id = cluster.create_job(\"Assim\", \n",
- " cfg_update={\"ntasks\": \"12\", \"time\": \"60\", \"mem\": \"200G\", \n",
- " \"ntasks-per-node\": \"12\", \"ntasks-per-core\": \"2\"}\n",
- " ).run(cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '\n",
- " +assim_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_init_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_valid_time.strftime('%Y-%m-%d_%H:%M ')\n",
- " +prior_path_exp, depends_on=[depends_on])\n",
- "```\n",
- "where `depends_on` is either `None` or `int` (a previous job's SLURM id)."
- ]
- },
{
"cell_type": "code",
"execution_count": null,
diff --git a/docs/source/tutorial2.ipynb b/docs/source/tutorial2.ipynb
index 79f0df0..df58b78 100644
--- a/docs/source/tutorial2.ipynb
+++ b/docs/source/tutorial2.ipynb
@@ -7,36 +7,24 @@
"tags": []
},
"source": [
- "# Tutorial 2: Cycled experiment\n",
+ "# Tutorial 2: Forecast after DA\n",
"\n",
"\n",
"**Goal**: To run a cycled data assimilation experiment.\n",
"[`cycled_exp.py`](https://github.com/lkugler/DART-WRF/blob/master/generate_free.py) contains an example which will be explained here:\n",
"\n",
- "#### Configure your experiment\n",
- "We start again by configuring `config/cfg.py`.\n",
+ "Now there are two options:\n",
+ "1) To start a forecast from an existing forecast, i.e. from WRF restart files\n",
+ "2) To start a forecast from defined thermodynamic profiles, i.e. from a `wrf_profile`\n",
"\n",
- "Then we write a script (or edit an existing one) in the main directory `DART-WRF/`.\n",
- "`nano new_experiment.py`\n",
"\n",
- "---\n",
- "Any script needs to import some modules:\n",
+ "### Restart a forecast\n",
+ "To run a forecast from initial conditions of a previous forecasts, we import these modules\n",
"```python\n",
- "import os, sys, shutil\n",
"import datetime as dt\n",
- "\n",
- "from dartwrf import utils\n",
- "from config.cfg import exp\n",
- "from config.clusters import cluster\n",
+ "from dartwrf.workflows import WorkFlows\n",
"```\n",
"\n",
- "---\n",
- "Now there are two options:\n",
- "- To start a forecast from an existing forecast, i.e. from WRF restart files\n",
- "- To start a forecast from defined thermodynamic profiles, i.e. from a `wrf_profile`\n",
- "\n",
- "\n",
- "#### Run a forecast from initial conditions of a previous forecasts\n",
"Let's say you want to run a forecast starting at 9 UTC until 12 UTC.\n",
"Initial conditions shall be taken from a previous experiment in `/user/test/data/sim_archive/exp_abc` which was initialized at 6 UTC and there are WRF restart files for 9 UTC.\n",
"Then the code would be\n",
@@ -46,50 +34,59 @@
"prior_init_time = dt.datetime(2008,7,30,6)\n",
"prior_valid_time = dt.datetime(2008,7,30,9)\n",
"\n",
- "cluster.setup()\n",
+ "w = WorkFlows(exp_config='cfg.py', server_config='srvx1.py')\n",
"\n",
"begin = dt.datetime(2008, 7, 30, 9)\n",
"end = dt.datetime(2008, 7, 30, 12)\n",
"\n",
- "prepare_WRFrundir(begin)\n",
+ "w.prepare_WRFrundir(begin)\n",
"\n",
- "prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time)\n",
+ "w.prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time)\n",
"\n",
- "run_ENS(begin=begin, # start integration from here\n",
- " end=end, # integrate until here\n",
- " output_restart_interval=9999, # do not write WRF restart files\n",
- " )\n",
+ "w.run_ENS(begin=begin, # start integration from here\n",
+ " end=end, # integrate until here\n",
+ " output_restart_interval=9999, # do not write WRF restart files\n",
+ " )\n",
"```\n",
"Note that we use predefined workflow functions like `run_ENS`.\n",
"\n",
"\n",
- "#### Assimilate observations with a prior given by previous forecasts\n",
- "To assimilate observations at dt.datetime `time` use this command:\n",
+ "### Forecast run after Data Assimilation\n",
+ "In order to continue after assimilation you need the posterior = prior (1) + increments (2)\n",
"\n",
+ "1. Set posterior = prior\n",
"```python\n",
- "prior_path_exp = '/user/test/data/sim_archive/exp_abc'\n",
- "prior_init_time = dt.datetime(2008,7,30,6)\n",
- "prior_valid_time = dt.datetime(2008,7,30,9)\n",
- "time = dt.datetime(2008,7,30,9) # time of assimilation\n",
- "\n",
- "assimilate(time, prior_init_time, prior_valid_time, prior_path_exp)\n",
+ "id = w.prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, depends_on=id)\n",
"```\n",
"\n",
+ "2) Update posterior with increments from assimilation\n",
+ "After this, the wrfrst files are updated with assimilation increments from DART output and copied to the WRF's run directories so you can continue to run the forecast ensemble.\n",
+ "```python\n",
+ "id = w.update_IC_from_DA(time, depends_on=id)\n",
+ "```\n",
"\n",
- "#### Update initial conditions from Data Assimilation\n",
- "In order to continue after assimilation you need the posterior = prior (1) + increments (2)\n",
- "\n",
- "1. Set prior with this function:\n",
- "\n",
- "`id = prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, depends_on=id)`\n",
+ "3) Define how long you want to run the forecast and when you want WRF restart files. Since they take a lot of space, we want as few as possible.\n",
"\n",
- "where path is `str`, times are `dt.datetime`.\n",
+ "```python\n",
+ "timedelta_integrate = dt.timedelta(hours=5)\n",
+ "output_restart_interval = 9999 # any value larger than the forecast duration\n",
+ "```\n",
"\n",
- "2. To update the model state with assimilation increments, you need to update the WRF restart files by running\n",
+ "If you run DA in cycles of 15 minutes, it will be\n",
+ "```python\n",
+ "timedelta_integrate = dt.timedelta(hours=5)\n",
+ "timedelta_btw_assim = dt.timedelta(minutes=15)\n",
+ "output_restart_interval = timedelta_btw_assim.total_seconds()/60\n",
+ "```\n",
"\n",
- "`id = update_IC_from_DA(time, depends_on=id)`\n",
"\n",
- "After this, the wrfrst files are updated with assimilation increments (filter_restart) and copied to the WRF's run directories so you can continue to run the ENS after assimilation using function `run_ENS()`."
+ "3) Run WRF ensemble\n",
+ "```python\n",
+ "id = w.run_ENS(begin=time, # start integration from here\n",
+ " end=time + timedelta_integrate, # integrate until here\n",
+ " output_restart_interval=output_restart_interval,\n",
+ " depends_on=id)\n",
+ "```\n"
]
},
{
diff --git a/docs/source/tutorial3.ipynb b/docs/source/tutorial3.ipynb
index 9769038..d3018ab 100644
--- a/docs/source/tutorial3.ipynb
+++ b/docs/source/tutorial3.ipynb
@@ -4,17 +4,16 @@
"cell_type": "markdown",
"id": "fd5c3005-f237-4495-9185-2d4d474cafd5",
"metadata": {
- "jp-MarkdownHeadingCollapsed": true,
"tags": []
},
"source": [
- "# Tutorial 3: Cycled experiment\n",
+ "# Tutorial 3: Cycle forecast and assimilation\n",
"\n",
"\n",
"**Goal**: To run a cycled data assimilation experiment.\n",
"[`cycled_exp.py`](https://github.com/lkugler/DART-WRF/blob/master/generate_free.py) contains an example which will be explained here:\n",
"\n",
- "After configuring your experiment the loop looks like\n",
+ "For example, your experiment can look like this\n",
"\n",
"```python\n",
"prior_path_exp = '/jetfs/home/lkugler/data/sim_archive/exp_v1.19_P2_noDA'\n",
@@ -24,8 +23,8 @@
"last_assim_time = dt.datetime(2008, 7, 30, 14)\n",
"forecast_until = dt.datetime(2008, 7, 30, 14, 15)\n",
"\n",
- "prepare_WRFrundir(init_time)\n",
- "id = run_ideal(depends_on=id)\n",
+ "w.prepare_WRFrundir(init_time)\n",
+ "id = w.run_ideal(depends_on=id)\n",
"\n",
"prior_init_time = init_time\n",
"prior_valid_time = time\n",
@@ -37,13 +36,13 @@
" # i.e. 13z as a prior to assimilate 12z observations\n",
" prior_valid_time = time\n",
"\n",
- " id = assimilate(time, prior_init_time, prior_valid_time, prior_path_exp, depends_on=id)\n",
+ " id = w.assimilate(time, prior_init_time, prior_valid_time, prior_path_exp, depends_on=id)\n",
"\n",
" # 1) Set posterior = prior\n",
- " id = prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, depends_on=id)\n",
+ " id = w.prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, depends_on=id)\n",
"\n",
" # 2) Update posterior += updates from assimilation\n",
- " id = update_IC_from_DA(time, depends_on=id)\n",
+ " id = w.update_IC_from_DA(time, depends_on=id)\n",
"\n",
" # How long shall we integrate?\n",
" timedelta_integrate = timedelta_btw_assim\n",
@@ -53,15 +52,15 @@
" output_restart_interval = 9999 # no restart file after last assim\n",
"\n",
" # 3) Run WRF ensemble\n",
- " id = run_ENS(begin=time, # start integration from here\n",
- " end=time + timedelta_integrate, # integrate until here\n",
- " output_restart_interval=output_restart_interval,\n",
- " depends_on=id)\n",
+ " id = w.run_ENS(begin=time, # start integration from here\n",
+ " end=time + timedelta_integrate, # integrate until here\n",
+ " output_restart_interval=output_restart_interval,\n",
+ " depends_on=id)\n",
"\n",
" # as we have WRF output, we can use own exp path as prior\n",
" prior_path_exp = cluster.archivedir \n",
"\n",
- " id_sat = create_satimages(time, depends_on=id)\n",
+ " id_sat = w.create_satimages(time, depends_on=id)\n",
"\n",
" # increment time\n",
" time += timedelta_btw_assim\n",
@@ -69,13 +68,13 @@
" # update time variables\n",
" prior_init_time = time - timedelta_btw_assim\n",
" \n",
- "verify_sat(id_sat)\n",
- "verify_wrf(id)\n",
- "verify_fast(id)\n",
+ "w.verify_sat(id_sat)\n",
+ "w.verify_wrf(id)\n",
+ "w.verify_fast(id)\n",
"```\n",
"\n",
"#### Job scheduling status\n",
- "The script submits jobs into the SLURM queue with dependencies so that SLURM starts the jobs itself as soon as resources are available. Most jobs need only a few cores, but model integration is done across many nodes:\n",
+ "If you work on a server with a queueing system, the script submits jobs into the SLURM queue with dependencies so that SLURM starts the jobs itself as soon as resources are available. Most jobs need only a few cores, but model integration is done across many nodes. You can look at the status with\n",
"```bash\n",
"$ squeue -u `whoami` --sort=i\n",
" JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)\n",
--
GitLab