Skip to content
Snippets Groups Projects
Commit e18f4b5f authored by skomo's avatar skomo
Browse files

initial git repo of version 7.0.3

parents
No related branches found
No related tags found
No related merge requests found
Showing
with 1882 additions and 0 deletions
DAY1
DAY2
DTIME 1
M_TYPE AN FC FC FC FC FC FC FC FC FC FC FC AN FC FC FC FC FC FC FC FC FC FC FC
M_TIME 00 00 00 00 00 00 00 00 00 00 00 00 12 12 12 12 12 12 12 12 12 12 12 12
M_STEP 00 01 02 03 04 05 06 07 08 09 10 11 00 01 02 03 04 05 06 07 08 09 10 11
M_CLASS OD
M_STREAM OPER
M_NUMBER OFF
M_EXPVER 1
M_GRID 200
M_LEFT -10000
M_LOWER 30000
M_UPPER 60000
M_RIGHT 30000
M_LEVEL 137
M_RESOL 799
M_GAUSS 0
M_ACCURACY 24
M_OMEGA 0
M_OMEGADIFF 0
M_ETA 1
M_ETADIFF 0
M_DPDETA 1
M_SMOOTH 0
M_FORMAT GRIB2
M_ADDPAR /186/187/188/235/139/39
PREFIX ENH
ECSTORAGE 0
ECTRANS 1
ECFSDIR ectmp:/${USER}/econdemand/
MAILOPS ${USER}
MAILFAIL ${USER}
EOF
DAY1
DAY2
DTIME 3
M_TYPE AN FC FC FC FC FC FC FC FC FC FC FC AN FC FC FC FC FC FC FC FC FC FC FC
M_TIME 00 00 00 00 00 00 00 00 00 00 00 00 12 12 12 12 12 12 12 12 12 12 12 12
M_STEP 00 01 02 03 04 05 06 07 08 09 10 11 00 01 02 03 04 05 06 07 08 09 10 11
ACCTIME 00/12
ACCTYPE FC
ACCMAXSTEP 12
M_CLASS OD
M_STREAM OPER
M_NUMBER OFF
M_EXPVER 1
M_GRID 200
M_LEFT -10000
M_LOWER 30000
M_UPPER 60000
M_RIGHT 30000
M_LEVEL 137
M_RESOL 1279
M_GAUSS 1
M_ACCURACY 24
M_OMEGA 0
M_OMEGADIFF 0
M_ETA 0
M_ETADIFF 0
M_DPDETA 1
M_SMOOTH 0
M_FORMAT GRIB2
M_ADDPAR 186/187/188/235/139/39
PREFIX ENHG
ECSTORAGE 0
ECTRANS 1
ECFSDIR ectmp:/${USER}/econdemand/
MAILOPS ${USER}
MAILFAIL ${USER}
EOF
DAY1
DAY2
DTIME 1
M_TYPE AN FC FC FC FC FC AN FC FC 4V FC FC 4V FC FC FC FC FC AN FC FC 4V FC FC 4V
M_TIME 00 00 00 00 00 00 06 00 00 09 00 00 09 12 12 12 12 12 18 12 12 21 12 12 21
M_STEP 00 01 02 03 04 05 00 07 08 00 10 11 03 01 02 03 04 05 00 07 08 00 10 11 03
PREFIX OP
M_UPPER 60000
M_LOWER 10000
M_LEFT -25000
M_RIGHT 60000
M_CLASS OD
M_STREAM OPER
M_NUMBER OFF
M_GRID 1000
M_RESOL 255
M_SMOOTH 0
M_GAUSS 1
M_ETA 0
M_ETAPAR 77
M_DPDETA 1
M_LEVEL 137
M_LEVELIST 136/to/137
M_ADDPAR 186/187/188/235/139/39
M_FORMAT GRIB1
ECSTORAGE 0
ECTRANS 1
ECFSDIR ectmp:/${USER}/ecops
MAILOPS ${USER}
MAILFAIL ${USER}
This diff is collapsed.
#
# (C) Copyright 2014 UIO.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
#
# Creation: July 2014 - Anne Fouilloux - University of Oslo
#
#
from gribapi import *
import traceback
import sys,os
##############################################################
# GRIB utilities
##############################################################
class GribTools:
'class for GRIB API with new methods'
def __init__(self,filename):
self.filename=filename
# get keyvalues for a given list of keynames
# a where statment can be given (list of key and list of values)
def getkeys(self,keynames,wherekeynames=[],wherekeyvalues=[]):
fileid=open(self.filename,'r')
return_list=[]
while 1:
gid_in = grib_new_from_file(fileid)
if gid_in is None: break
select=True
i=0
if len(wherekeynames) != len(wherekeyvalues): raise Exception("Give a value for each keyname!")
for wherekey in wherekeynames:
if not grib_is_defined(gid_in, wherekey): raise Exception("where Key was not defined")
select=select and (str(wherekeyvalues[i])==str(grib_get(gid_in, wherekey)))
i=i+1
if select:
llist = []
for key in keynames:
llist.extend([str(grib_get(gid_in, key))])
return_list.append(llist)
grib_release(gid_in)
fileid.close()
return return_list
# set keyvalues for a given list of keynames
# a where statment can be given (list of key and list of values)
# an input file must be given as an input for reading grib messages
# note that by default all messages are written out
# if you want to get only those meeting the where statement, use
# strict=true
def setkeys(self,fromfile,keynames,keyvalues, wherekeynames=[],wherekeyvalues=[], strict=False, filemode='w'):
fout=open(self.filename,filemode)
fin=open(fromfile)
while 1:
gid_in = grib_new_from_file(fin)
if gid_in is None: break
select=True
i=0
if len(wherekeynames) != len(wherekeyvalues): raise Exception("Give a value for each keyname!")
for wherekey in wherekeynames:
if not grib_is_defined(gid_in, wherekey): raise Exception("where Key was not defined")
select=select and (str(wherekeyvalues[i])==str(grib_get(gid_in, wherekey)))
i=i+1
if select:
i=0
for key in keynames:
grib_set(gid_in, key, keyvalues[i])
i=i+1
if strict:
if select:
grib_write(gid_in,fout)
else:
grib_write(gid_in,fout)
grib_release(gid_in)
fin.close()
fout.close()
# Add the content of a grib file but only messages
# corresponding to keys/values
# if selectWhere is False select fields that are different from keynames/keyvalues
def copy(self,filename_in, selectWhere=True, keynames=[], keyvalues=[],filemode='w'):
fin=open(filename_in)
fout=open(self.filename,filemode)
while 1:
gid_in = grib_new_from_file(fin)
if gid_in is None: break
select=True
i=0
if len(keynames) != len(keyvalues): raise Exception("Give a value for each keyname!")
for key in keynames:
if not grib_is_defined(gid_in, key): raise Exception("Key was not defined")
if selectWhere:
select=select and (str(keyvalues[i])==str(grib_get(gid_in, key)))
else:
select=select and (str(keyvalues[i])!=str(grib_get(gid_in, key)))
i=i+1
if select:
grib_write(gid_in,fout)
grib_release(gid_in)
fin.close()
fout.close()
# Create index from a list of files if it does not exist or read it
def index(self,index_keys=["mars"], index_file = "my.idx"):
print "index to be done"
self.iid = None
if (os.path.exists(index_file)):
self.iid = grib_index_read(index_file)
print "Use existing index file: %s "%(index_file)
else:
for file in self.filename:
print "Inputfile: %s "%(file)
if self.iid == None:
self.iid = grib_index_new_from_file(file,index_keys)
else:
grib_index_add_file(self.iid,file)
if self.iid != None:
grib_index_write(self.iid,index_file)
return self.iid
# README #
This documentation shows how to use these python scripts to extract ECMWF data and generate the ***FLEXPART*** specific input files.
### Introduction ###
To run ***FLEXPART*** with ECMWF data, you first need to retrieve ECMWF GRIB fields and generate ***FLEXPART*** specific input files afterwards. ***FLEXPART*** needs the GRIB data seperated by time so that flex_extract will combine all data in output files seperated for each time step. The format of these output filenames looks like: `prefixYYMMDDHH`, where prefix can be defined in ***flex_extract*** (usually it should be limited to "2" characters because of further processing reasons).
For setting up a complete ***FLEXPART*** environment it is recommended to build a directory structure such as:
![FLEXPART directory structure](../docs/Fp_extract_strukt-pngnew.png)
For installation instructions and usage of ***FLEXPART*** please see ***FLEXPART***-Documentation!
### Overview ###
%-- list all files in python dir and shortly explain structure
##### CONTROL file #####
For specifying which ECMWF dataset you would like to retrieve and which time and spatial resolution you would like to retrieve you have to prepare a `CONTROL` file. In the directory you can find a couple of example `CONTROL` files:
```
CONTROL_CERA
CONTROL_CV
CONTROL_EA5.public
CONTROL_FC.pure
CONTROL_OD.highres.gauss
CONTROL_CERA.public
CONTROL_EA5
CONTROL_EI.global
CONTROL_FC.twiceaday
CONTROL_OPS.4V
CONTROL_CF
CONTROL_EA5.highres
CONTROL_EI.public
CONTROL_OD.highres.eta
CONTROL.temp
```
#### Python scripts ####
The main program for doing both, retrieving ECMWF data and generate the FLEXPART input files, is `submit.py`. It combines the two sub-programs `getMARSdata.py` and `prepareFLEXPART.py` which can also be run by themselves for example for debugging purposes.
To get the usage of the program, use the `-h` option:
```
usage: submit.py [-h] [--start_date START_DATE] [--end_date END_DATE]
[--date_chunk DATE_CHUNK] [--basetime BASETIME] [--step STEP]
[--levelist LEVELIST] [--area AREA] [--inputdir INPUTDIR]
[--outputdir OUTPUTDIR]
[--flexpart_root_scripts FLEXPART_ROOT_SCRIPTS] [--ppid PPID]
[--job_template JOB_TEMPLATE] [--queue QUEUE]
[--controlfile CONTROLFILE] [--debug DEBUG] [--public PUBLIC]
Retrieve FLEXPART input from ECMWF MARS archive
optional arguments:
-h, --help show this help message and exit
--start_date START_DATE
start date YYYYMMDD (default: None)
--end_date END_DATE end_date YYYYMMDD (default: None)
--date_chunk DATE_CHUNK
# of days to be retrieved at once (default: None)
--basetime BASETIME base such as 00/12 (for half day retrievals) (default:
None)
--step STEP steps such as 00/to/48 (default: None)
--levelist LEVELIST Vertical levels to be retrieved, e.g. 30/to/60
(default: None)
--area AREA area defined as north/west/south/east (default: None)
--inputdir INPUTDIR root directory for storing intermediate files
(default: None)
--outputdir OUTPUTDIR
root directory for storing output files (default:
None)
--flexpart_root_scripts FLEXPART_ROOT_SCRIPTS
FLEXPART root directory (to find grib2flexpart and
COMMAND file) Normally flex_extract resides in the
scripts directory of the FLEXPART distribution
(default: None)
--ppid PPID Specify parent process id for rerun of prepareFLEXPART
(default: None)
--job_template JOB_TEMPLATE
job template file for submission to ECMWF (default:
job.temp)
--queue QUEUE queue for submission to ECMWF (e.g. ecgate or cca )
(default: None)
--controlfile CONTROLFILE
file with control parameters (default: CONTROL.temp)
--debug DEBUG Debug mode - leave temporary files intact (default: 0)
--public PUBLIC Public mode - retrieves the public datasets (default:
False)
```
Optional arguments are listed in squared brackets.
The sub-programs `getMARSdata.py` and `prepareFLEXPART.py` are the two main parts of this software.
* `getMARSdata.py`
This program retrieves the ECMWF data from ECMWF servers using [ECMWF WebApi](https://software.ecmwf.int/wiki/display/WEBAPI/ECMWF+Web+API+Home) or [Mars](https://software.ecmwf.int/wiki/display/UDOC/MARS+user+documentation), depending on your user status and your selection of the running mode. It requires the `ECMWF WebApi` python library (see Requirements below). Check with your local IT group as it may be already available.
* `prepareFLEXPART.py`
This program generates FLEXPART input files. It requires python interface to grib_api and the Fortran program `CONVERT2` (located in `src` directory with instruction on how to compile it). `CONVERT2` needs a namelist which is generated from the python program.
### Requirements ###
| Python Support | Link | Notes |
| --------------------- |:----------------------:|:-------------------------|
| python | [http://www.python.org](http://www.python.org) | We have used [Anaconda python](https://store.continuum.io/cshop/anaconda/) for our testing |
| python-numpy | [http://www.numpy.org/](http://www.numpy.org/) | Not necessary if you have installed python Anaconda
| ecmwfapi | [https://software.ecmwf.int/wiki/display/WEBAPI/ECMWF+Web+API+Home](https://software.ecmwf.int/wiki/display/WEBAPI/ECMWF+Web+API+Home) | You also need to install your API key (as explained in the documentation)
| Utilities | Link | Notes |
| --------------------- |:----------------------:|:-------------------------|
|gfortran|[https://gcc.gnu.org/wiki/GFortran](https://gcc.gnu.org/wiki/GFortran)| Make sure that you are using the same compiler version for all libraries in connection with flex_extract, GRIB_API and FLEXPART, otherwise you will have problems.|
| grib-api | [https://software.ecmwf.int/wiki/display/GRIB/Home](https://software.ecmwf.int/wiki/display/GRIB/Home) | Make sure you install GRIB-API with JPEG support and python GRIB-API.|
|FFTW|[http://www.fftw.org](http://www.fftw.org)|This is needed by the Emos library, as is mentioned in the installation instructions for Emos|
| Emos | [https://software.ecmwf.int/wiki/display/EMOS/Emoslib](https://software.ecmwf.int/wiki/display/EMOS/Emoslib) | Make sure you carefully read the instructions and have installed the FFTW package before installing Emos.|
| CONVERT2 | |to run prepareFLEXPART.py, you need to compile the Fortragn program CONVERT2 (located in the src directory). See separate README file in this directory to get instructions on how to compile this code.|
### Installation ###
* Environment
At UIO, Red Hat 6 Linux systems (64 bits) were used for testing. We use the [Module package](http://modules.sourceforge.net/) to set-up user environment.
* Getting the source code
In the directory of your choice:
`
git clone git@bitbucket.org:flexpart/flexpart.git
`
This command will create a subdirectory called flexpart: it contains the latest FLEXPART version.
Then set the environment variable `FLEXPART_ROOT`:
** Korn-shell or Bash users:
`
cd flexpart
export FLEXPART_ROOT=$PWD
`
** C-shell users:
`
cd flexpart
setenv FLEXPART_ROOT=$PWD
`
* Installation
Make sure you first generate the `CONVERT2` program (see separate instructions in `preproc/src`).
Users need to be able to execute prepareFLEXPART.py and getMARSdata.py so make sure they have the correct unix permissions:
`
cd preproc/python
chmod uog+rx getMARSdata.py prepareFLEXPART.py
`
These two programs must be in the user PATH. At UIO this is done automatically when loading flexpart. If not, you would need to do the following:
** Korn-shell or Bash users:
`
export PATH=$FLEXPART_HOME/preproc/python:$PATH
`
** C-shell users:
`
setenv PATH $FLEXPART_HOME/preproc/python:$PATH
`
Where `$FLEXPART_HOME` is the directory where FLEXPART
* Testing your installation
First check that grib-api python interface is correctly installed on your platform:
`
python
>>> from gribapi import *
>>>
`
Use `CTRL-D` to quit python.
Then check that `ecmwfapi` is properly installed:
`
python
>>> from ecmwfapi import *
>>>
`
If the two previous tests were successful, you can run `tests/preproc` (See separate instructions in `tests/preproc`).
If any of these two tests fail, this probably means that either `ecmwfapi` or `grib-api` have not been installed properly.
Please report any problems.
### Installation FAQ ###
#!/usr/bin/env python
#
# (C) Copyright 2014 UIO.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Creation: October 2014 - Anne Fouilloux - University of Oslo
#
import os
import glob
class UIOFiles:
'class to manipulate files'
def __init__(self,suffix):
# type of files to manipulate such as ['.grib', 'grb', 'grib1', 'grib2', 'grb1','grb2']
self.suffix=suffix
def listFiles(self,pathname,pattern):
''' list files (suffix previously given) within this directory. '''
# Get the absolute path of the pathname parameter
pathname = os.path.abspath(pathname)
# Get a list of files in pathname
filesInCurDir0 = glob.glob(pathname+'/'+pattern)
filesInCurDir=[]
for f in filesInCurDir0:
filesInCurDir.append(f.split('/')[-1])
self.counter = 0
self.files = []
# Traverse through all files
for file in filesInCurDir:
curFile = os.path.join(pathname, file)
# Check if it's a normal file or directory
if os.path.isfile(curFile):
# Get the file extension
fileNoExt,curFileExtension = os.path.splitext(curFile)
# Check if the file has an extension of typical video files
if curFileExtension in self.suffix:
# We have got a file file! Increment the counter
self.counter += 1
# add this filename in the list
self.files.append(curFile)
else:
# We got a directory, enter into it for further processing
self.listFiles(curFile)
return
#!/usr/bin/env python
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
#
# Creation: October 2014 - Anne Fouilloux - University of Oslo
# Extension November 2015 - Leopold Haimberger - University of Vienna for:
# - using the WebAPI also for general MARS retrievals
# - job submission on ecgate and cca
# - job templates suitable for twice daily operational dissemination
# - dividing retrievals of longer periods into digestable chunks
# - retrieve also longer term forecasts, not only analyses and short term forecast data
# - conversion into GRIB2
# - conversion into .fp format for faster execution of FLEXPART
#
#
# Further documentation may be obtained from www.flexpart.eu
#
# Requirements:
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
#
# Get MARS GRIB fields from ECMWF for FLEXPART
#
#import socket
#hostname=socket.gethostname()
#ecapi= 'ecmwf' not in hostname
try:
ecapi=True
import ecmwfapi
except ImportError:
ecapi=False
import calendar
import shutil
import datetime
import time
import os,glob,sys,inspect
#from string import strip
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
if localpythonpath not in sys.path:
sys.path.append(localpythonpath)
from FlexpartTools import MARSretrieval, EIFlexpart, silentremove, \
Control,myerror,normalexit, interpret_args_and_control
def getMARSdata(args,c):
if not os.path.exists(c.inputdir):
os.makedirs(c.inputdir)
print "start date %s "%(c.start_date)
print "end date %s "%(c.end_date)
if ecapi:
if int(c.public):
server = ecmwfapi.ECMWFDataServer()
else:
server = ecmwfapi.ECMWFService("mars")
else:
server = False
c.ecapi=ecapi
print 'ecapi:',c.ecapi
# Retrieve ERA interim data for running flexpart
syear=int(c.start_date[:4])
smonth=int(c.start_date[4:6])
sday=int(c.start_date[6:])
start = datetime.date( year = syear, month = smonth, day = sday )
startm1=start- datetime.timedelta(days=1)
if c.basetime=='00':
start=startm1
eyear=int(c.end_date[:4])
emonth=int(c.end_date[4:6])
eday=int(c.end_date[6:])
end = datetime.date( year = eyear, month = emonth, day = eday )
if c.basetime=='00' or c.basetime=='12':
endp1=end+ datetime.timedelta(days=1)
else:
endp1=end+ datetime.timedelta(days=2)
datechunk=datetime.timedelta(days=int(c.date_chunk))
print 'removing content of '+c.inputdir
tobecleaned=glob.glob(c.inputdir+'/*_acc_*.'+str(os.getppid())+'.*.grb')
for f in tobecleaned:
os.remove(f)
times=None
if c.maxstep<=24:
day=startm1
while day<endp1:
# we need to retrieve MARS data for this period (maximum one month)
flexpart = EIFlexpart(c,fluxes=True)
if day+datechunk-datetime.timedelta(days=1)<endp1:
dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
else:
dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
print "retrieve " + dates + " in dir " + c.inputdir
try:
flexpart.retrieve(server, c.public, dates, times, c.inputdir)
except IOError:
myerror(c,'MARS request failed')
day+=datechunk
else:
day=start
while day<=end:
# we need to retrieve MARS data for this period (maximum one month)
flexpart = EIFlexpart(c,fluxes=True)
if day+datechunk-datetime.timedelta(days=1)<end:
dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
else:
dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
print "retrieve " + dates + " in dir " + c.inputdir
flexpart.retrieve(server, c.public, dates, times, c.inputdir)
day+=datechunk
tobecleaned=glob.glob(c.inputdir+'/*__*.'+str(os.getppid())+'.*.grb')
for f in tobecleaned:
os.remove(f)
day=start
times=None
while day<=end:
# we need to retrieve MARS data for this period (maximum one month)
flexpart = EIFlexpart(c)
if day+datechunk-datetime.timedelta(days=1)<end:
dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
else:
dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
print "retrieve " + dates + " in dir " + c.inputdir
flexpart.retrieve(server, c.public, dates, times, c.inputdir)
day+=datechunk
if __name__ == "__main__":
args,c=interpret_args_and_control()
getMARSdata(args,c)
normalexit(c)
#!/usr/bin/env python
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
#
# Creation: October 2014 - Anne Fouilloux - University of Oslo
# Extension November 2015 - Leopold Haimberger - University of Vienna for:
# - using the WebAPI also for general MARS retrievals
# - job submission on ecgate and cca
# - job templates suitable for twice daily operational dissemination
# - dividing retrievals of longer periods into digestable chunks
# - retrieve also longer term forecasts, not only analyses and short term forecast data
# - conversion into GRIB2
# - conversion into .fp format for faster execution of FLEXPART
#
#
# Further documentation may be obtained from www.flexpart.eu
#
# Requirements:
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
#
#
import calendar
import shutil
import datetime
import time
import os,sys,glob
import subprocess
import inspect
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.append(localpythonpath)
from UIOTools import UIOFiles
from string import strip
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
from GribTools import GribTools
from FlexpartTools import EIFlexpart, Control, install_args_and_control
from getMARSdata import getMARSdata
from prepareFLEXPART import prepareFLEXPART
def main():
calledfromdir=os.getcwd()
os.chdir(localpythonpath)
args,c=install_args_and_control()
# if c.outputdir[0]!='/':
# c.outputdir=os.path.join(calledfromdir,c.outputdir)
# c.inputdir=c.outputdir
if args.install_target!=None:
install_via_gateway(c,args.install_target)
else:
print 'Please specify installation target (local|ecgate|cca)'
print 'use -h or --help for help'
sys.exit()
def install_via_gateway(c,target):
ecd=c.flexextractdir
template=ecd+'python/compilejob.temp'
job=ecd+'python/compilejob.ksh'
fo=open(job,'w')
with open(template) as f:
fdata = f.read().split('\n')
for data in fdata:
if 'MAKEFILE=' in data:
if c.makefile is not None:
data='export MAKEFILE='+c.makefile
if 'FLEXPART_ROOT_SCRIPTS=' in data:
if c.flexpart_root_scripts!='../':
data='export FLEXPART_ROOT_SCRIPTS='+c.flexpart_root_scripts
else:
data='export FLEXPART_ROOT_SCRIPTS=$HOME'
if target.lower()!='local':
if '--workdir' in data:
data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid
if '##PBS -o' in data:
data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out'
if 'FLEXPART_ROOT_SCRIPTS=' in data:
if c.ec_flexpart_root_scripts!='../':
data='export FLEXPART_ROOT_SCRIPTS='+c.ec_flexpart_root_scripts
else:
data='export FLEXPART_ROOT_SCRIPTS=$HOME'
fo.write(data+'\n')
f.close()
fo.close()
if target.lower()!='local':
template=ecd+'python/job.temp.o'
with open(template) as f:
fdata = f.read().split('\n')
f.close()
fo=open(template[:-2],'w')
for data in fdata:
if '--workdir' in data:
data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid
if '##PBS -o' in data:
data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out'
if 'export PATH=${PATH}:' in data:
data+=c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3/python'
if 'cat>>' in data or 'cat >>' in data:
i=data.index('>')
fo.write(data[:i]+data[i+1:]+'\n')
fo.write('GATEWAY '+c.gateway+'\n')
fo.write('DESTINATION '+c.destination+'\n')
fo.write('EOF\n')
fo.write(data+'\n')
fo.close()
job=ecd+'python/ECMWF_ENV'
fo=open(job,'w')
fo.write('ECUID '+c.ecuid+'\n')
fo.write('ECGID '+c.ecgid+'\n')
fo.write('GATEWAY '+c.gateway+'\n')
fo.write('DESTINATION '+c.destination+'\n')
fo.close()
if target.lower()=='local':
# compile CONVERT2
if c.flexpart_root_scripts==None or c.flexpart_root_scripts=='../':
print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified'
print 'Only CONVERT2 will be compiled in '+ecd+'/../src'
else:
c.flexpart_root_scripts=os.path.expandvars(os.path.expanduser(c.flexpart_root_scripts))
if os.path.abspath(ecd)!=os.path.abspath(c.flexpart_root_scripts):
os.chdir('/')
p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
try:
os.makedirs(c.flexpart_root_scripts+'/flex_extract_v7.0.3')
except:
pass
os.chdir(c.flexpart_root_scripts+'/flex_extract_v7.0.3')
p=subprocess.check_call(['tar','-xvf',ecd+'../flex_extract_v7.0.3.tar'])
os.chdir(c.flexpart_root_scripts+'/flex_extract_v7.0.3/src')
os.chdir('../src')
print 'install flex_extract_v7.0.3 software on '+target+' in directory '+os.getcwd()
if c.makefile==None:
makefile='Makefile.local.ifort'
else:
makefile=c.makefile
flist=glob.glob('*.mod')+glob.glob('*.o')
if flist:
p=subprocess.check_call(['rm']+flist)
try:
print 'Using makefile: '+makefile
p=subprocess.check_call(['make','-f',makefile])
p=subprocess.check_call(['ls','-l','CONVERT2'])
except:
print 'compile failed - please edit '+makefile+' or try another Makefile in the src directory.'
print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB and EMOSLIB must be adapted.'
print 'Available Makefiles:'
print glob.glob('Makefile*')
elif target.lower()=='ecgate':
os.chdir('/')
p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
try:
p=subprocess.check_call(['ecaccess-file-put',ecd+'../flex_extract_v7.0.3.tar','ecgate:/home/ms/'+c.ecgid+'/'+c.ecuid+'/flex_extract_v7.0.3.tar'])
except:
print 'ecaccess-file-put failed! Probably the eccert key has expired.'
exit(1)
p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh'])
print 'compilejob.ksh has been submitted to ecgate for installation in '+c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3'
print 'You should get an email with subject flexcompile within the next few minutes'
elif target.lower()=='cca':
os.chdir('/')
p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
try:
p=subprocess.check_call(['ecaccess-file-put',ecd+'../flex_extract_v7.0.3.tar','cca:/home/ms/'+c.ecgid+'/'+c.ecuid+'/flex_extract_v7.0.3.tar'])
except:
print 'ecaccess-file-put failed! Probably the eccert key has expired.'
exit(1)
p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh'])
print 'compilejob.ksh has been submitted to cca for installation in '+c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3'
print 'You should get an email with subject flexcompile within the next few minutes'
else:
print 'ERROR: unknown installation target ',target
print 'Valid targets: ecgate, cca, local'
return
if __name__ == "__main__":
main()
#!/usr/bin/env python
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Functionality provided: Simple tool for creating maps and time series of retrieved fields.
#
# Requirements:
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
import datetime
import time
import os,inspect,sys,glob
import socket
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
if localpythonpath not in sys.path:
sys.path.append(localpythonpath)
from matplotlib.pylab import *
import matplotlib.patches as mpatches
from mpl_toolkits.basemap import Basemap,addcyclic
import matplotlib.colors as mcolors
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Polygon
import matplotlib.cm as cmx
import matplotlib.colors as colors
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
from FlexpartTools import interpret_args_and_control,silentremove,product,Control
from GribTools import GribTools
from gribapi import *
from rasotools.utils import stats
def plot_retrieved(args,c):
start = datetime.datetime.strptime(c.start_date,'%Y%m%d%H')
end = datetime.datetime.strptime(c.end_date,'%Y%m%d%H')
c.paramIds=asarray(c.paramIds,dtype='int')
c.levels=asarray(c.levels,dtype='int')
c.area=asarray(c.area)
index_keys=["date","time","step"]
indexfile=c.inputdir+"/date_time_stepRange.idx"
silentremove(indexfile)
files=glob.glob(c.inputdir+'/'+c.prefix+'*')
grib=GribTools(files)
iid=grib.index(index_keys=index_keys, index_file = indexfile)
gdict=dict(Ni = 360, Nj = 181, iScansNegatively = 0, jScansPositively = 0,
jPointsAreConsecutive = 0, alternativeRowScanning = 0,
latitudeOfFirstGridPointInDegrees = 90,
longitudeOfFirstGridPointInDegrees = 181,
latitudeOfLastGridPointInDegrees = -90,
longitudeOfLastGridPointInDegrees = 180,
iDirectionIncrementInDegrees = 1,
jDirectionIncrementInDegrees = 1
)
index_vals = []
for key in index_keys:
key_vals = grib_index_get(iid,key)
print key_vals
index_vals.append(key_vals)
fdict=dict()
fmeta=dict()
fstamp=dict()
for p in c.paramIds:
for l in c.levels:
key='{:0>3}_{:0>3}'.format(p,l)
fdict[key]=[]
fmeta[key]=[]
fstamp[key]=[]
for prod in product(*index_vals):
for i in range(len(index_keys)):
grib_index_select(iid,index_keys[i],prod[i])
gid = grib_new_from_index(iid)
# if gid is not None:
while(gid is not None):
date = grib_get(gid, 'date')
fdate= datetime.datetime(date/10000,mod(date,10000)/100,mod(date,100))
gtime = grib_get(gid, 'time')
step = grib_get(gid, 'step')
fdatetime=fdate+datetime.timedelta(hours=gtime/100)
# if fdatetime<start or fdatetime>end:
# break
gtype = grib_get(gid, 'type')
paramId = grib_get(gid, 'paramId')
parameterName = grib_get(gid, 'parameterName')
level=grib_get(gid,'level')
if step>=c.start_step and step <=c.end_step and fdatetime>=start and fdatetime<=end and paramId in c.paramIds and level in c.levels:
key='{:0>3}_{:0>3}'.format(paramId,level)
print key
fdatetimestep=fdatetime+datetime.timedelta(hours=step)
if len(fstamp)==0:
fstamp[key].append(fdatetimestamp)
fmeta[key].append((paramId,parameterName,gtype,fdatetime,gtime,step,level))
fdict[key].append(flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']])))
else:
i=0
inserted=False
for i in range(len(fstamp[key])):
if fdatetimestep<fstamp[key][i]:
fstamp[key][i:i]=[fdatetimestep]
fmeta[key][i:i]=[(paramId,parameterName,gtype,fdatetime,gtime,step,level)]
fdict[key][i:i]=[flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']]))]
inserted=True
break
if not inserted:
fstamp[key].append(fdatetimestep)
fmeta[key].append((paramId,parameterName,gtype,fdatetime,gtime,step,level))
fdict[key].append(flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']])))
grib_release(gid)
gid = grib_new_from_index(iid)
for k in fdict.keys():
fml=fmeta[k]
fdl=fdict[k]
for fd,fm in zip(fdl,fml):
ftitle=fm[1]+' {} '.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+' '+stats(fd)
pname='_'.join(fm[1].split())+'_{}_'.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+'.{:0>3}'.format(fm[5])
plotmap(fd, fm,gdict,ftitle,pname+'.eps')
for k in fdict.keys():
fml=fmeta[k]
fdl=fdict[k]
fsl=fstamp[k]
if fdl:
fm=fml[0]
fd=fdl[0]
ftitle=fm[1]+' {} '.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+' '+stats(fd)
pname='_'.join(fm[1].split())+'_{}_'.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+'.{:0>3}'.format(fm[5])
lat=-20
lon=20
plottimeseries(fdl,fml,fsl,lat,lon, gdict, ftitle, pname+'.eps')
def plottimeseries(flist,fmetalist,ftimestamps,lat,lon,gdict,ftitle,filename):
t1=time.time()
latindex=(lat+90)*180/(gdict['Nj']-1)
lonindex=(lon+179)*360/gdict['Ni']
farr=asarray(flist)
ts=farr[:,latindex,lonindex]
f=plt.figure(figsize=(12,6.7))
plt.plot(ftimestamps,ts)
plt.title(ftitle)
savefig(c.outputdir+'/'+filename)
print 'created ',c.outputdir+'/'+filename
plt.close(f)
print time.time()-t1,'s'
def plotmap(flist,fmetalist,gdict,ftitle,filename):
t1=time.time()
f=plt.figure(figsize=(12,6.7))
mbaxes = f.add_axes([0.05, 0.15, 0.8, 0.7])
m =Basemap(llcrnrlon=-180.,llcrnrlat=-90.,urcrnrlon=180,urcrnrlat=90.)
#if bw==0 :
#fill_color=rgb(0.6,0.8,1)
#else:
#fill_color=rgb(0.85,0.85,0.85)
lw=0.3
m.drawmapboundary()
parallels = arange(-90.,91,90.)
# labels = [left,right,top,bottom]
m.drawparallels(parallels,labels=[True,True,True,True],linewidth=lw)
meridians = arange(-180.,181.,60.)
m.drawmeridians(meridians,labels=[True,True,True,True],linewidth=lw)
m.drawcoastlines(linewidth=lw)
xleft=gdict['longitudeOfFirstGridPointInDegrees']
if xleft>180.0:
xleft-=360.
x=linspace(xleft,gdict['longitudeOfLastGridPointInDegrees'],gdict['Ni'])
y=linspace(gdict['latitudeOfLastGridPointInDegrees'],gdict['latitudeOfFirstGridPointInDegrees'],gdict['Nj'])
xx, yy = m(*meshgrid(x,y))
s=m.contourf(xx,yy, flist)
title(ftitle,y=1.1)
cbaxes = f.add_axes([0.9, 0.2, 0.04, 0.6])
cb=colorbar(cax=cbaxes)
savefig(c.outputdir+'/'+filename)
print 'created ',c.outputdir+'/'+filename
plt.close(f)
print time.time()-t1,'s'
def interpret_plotargs(*args,**kwargs):
parser = ArgumentParser(description='Retrieve FLEXPART input from ECMWF MARS archive',
formatter_class=ArgumentDefaultsHelpFormatter)
# the most important arguments
parser.add_argument("--start_date", dest="start_date",
help="start date YYYYMMDD")
parser.add_argument( "--end_date", dest="end_date",
help="end_date YYYYMMDD")
parser.add_argument("--start_step", dest="start_step",
help="start step in hours")
parser.add_argument( "--end_step", dest="end_step",
help="end_step in hours")
# some arguments that override the default in the control file
parser.add_argument("--levelist", dest="levelist",help="Vertical levels to be retrieved, e.g. 30/to/60")
parser.add_argument("--area", dest="area", help="area defined as north/west/south/east")
parser.add_argument("--paramIds", dest="paramIds", help="parameter IDs")
parser.add_argument("--prefix", dest="prefix",default='EN', help="output file name prefix")
# set the working directories
parser.add_argument("--inputdir", dest="inputdir",default=None,
help="root directory for storing intermediate files")
parser.add_argument("--outputdir", dest="outputdir",default=None,
help="root directory for storing output files")
parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
help="FLEXPART root directory (to find grib2flexpart and COMMAND file)\n\
Normally flex_extract resides in the scripts directory of the FLEXPART distribution")
parser.add_argument("--controlfile", dest="controlfile",default='CONTROL.temp',
help="file with control parameters")
args = parser.parse_args()
try:
c=Control(args.controlfile)
except IOError:
try:
c=Control(localpythonpath+args.controlfile)
except:
print 'Could not read control file "'+args.controlfile+'"'
print 'Either it does not exist or its syntax is wrong.'
print 'Try "'+sys.argv[0].split('/')[-1]+' -h" to print usage information'
exit(1)
if args.levelist:
c.levels=args.levelist.split('/')
else:
c.levels=[0]
if args.area:
c.area=args.area.split('/')
else:
c.area='[0,0]'
c.paramIds=args.paramIds.split('/')
if args.start_step:
c.start_step=int(args.start_step)
else:
c.start_step=0
if args.end_step:
c.end_step=int(args.end_step)
else:
c.end_step=0
c.start_date=args.start_date
c.end_date=args.end_date
c.prefix=args.prefix
c.inputdir=args.inputdir
if args.outputdir:
c.outputdir=args.outputdir
else:
c.outputdir=c.inputdir
return args,c
if __name__ == "__main__":
args,c=interpret_plotargs()
plot_retrieved(args,c)
#!/usr/bin/env python
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
#
# Creation: October 2014 - Anne Fouilloux - University of Oslo
# Extension November 2015 - Leopold Haimberger - University of Vienna for:
# - using the WebAPI also for general MARS retrievals
# - job submission on ecgate and cca
# - job templates suitable for twice daily operational dissemination
# - dividing retrievals of longer periods into digestable chunks
# - retrieve also longer term forecasts, not only analyses and short term forecast data
# - conversion into GRIB2
# - conversion into .fp format for faster execution of FLEXPART
#
# Requirements:
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
#
import calendar
import shutil
import datetime
import time
import os,inspect,sys
import socket
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
if localpythonpath not in sys.path:
sys.path.append(localpythonpath)
from UIOTools import UIOFiles
#from string import strip
from GribTools import GribTools
from FlexpartTools import EIFlexpart, Control,interpret_args_and_control, cleanup
hostname=socket.gethostname()
ecapi= 'ecmwf' not in hostname
try:
if ecapi:
import ecmwfapi
except ImportError:
ecapi=False
def prepareFLEXPART(args,c):
namelist='fort.4'
if not args.ppid:
c.ppid=str(os.getppid())
else:
c.ppid=args.ppid
c.ecapi=ecapi
syear=int(c.start_date[:4])
smonth=int(c.start_date[4:6])
sday=int(c.start_date[6:])
start = datetime.date( year = syear, month = smonth, day = sday )
eyear=int(c.end_date[:4])
emonth=int(c.end_date[4:6])
eday=int(c.end_date[6:])
end = datetime.date( year = eyear, month = emonth, day = eday )
inputfiles=UIOFiles(['.grib', '.grb', '.grib1', '.grib2', '.grb1','.grb2'])
startm1=start- datetime.timedelta(days=1)
endp1=end + datetime.timedelta(days=1)
inputfiles.listFiles(c.inputdir, '*OG_acc_SL*.'+c.ppid+'.*')
if not os.path.exists(c.outputdir):
os.makedirs(c.outputdir)
flexpart = EIFlexpart(c,fluxes=True)
flexpart.create_namelist(c,'fort.4')
flexpart.deacc_fluxes(inputfiles, c)
inputfiles=UIOFiles(['.grib', '.grb', '.grib1', '.grib2', '.grb1','.grb2'])
print 'Prepare '+start.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
# we will make the list of files from the root inputdir
inputfiles.listFiles(c.inputdir, '????__??.*'+c.ppid+'.*')
if c.basetime=='00':
start=startm1
flexpart = EIFlexpart(c)
flexpart.create(inputfiles, c) # produce FLEXPART-ready GRIB files
flexpart.process_output(c) # process GRIB files - copy/transfer/interpolate them or make them GRIB2
if int(c.debug)!=0:
print('Temporary files left intact')
else:
cleanup(c)
if __name__ == "__main__":
args,c=interpret_args_and_control()
prepareFLEXPART(args,c)
#!/usr/bin/env python
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
#
# Creation: October 2014 - Anne Fouilloux - University of Oslo
# Extension November 2015 - Leopold Haimberger - University of Vienna for:
# - using the WebAPI also for general MARS retrievals
# - job submission on ecgate and cca
# - job templates suitable for twice daily operational dissemination
# - dividing retrievals of longer periods into digestable chunks
# - retrieve also longer term forecasts, not only analyses and short term forecast data
# - conversion into GRIB2
# - conversion into .fp format for faster execution of FLEXPART
#
# Further documentation may be obtained from www.flexpart.eu
#
# Requirements:
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
#
#
import calendar
import shutil
import datetime
import time
import os,sys,glob
import subprocess
import inspect
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.append(localpythonpath)
from UIOTools import UIOFiles
from string import strip
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
from GribTools import GribTools
from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, normalexit,myerror
from getMARSdata import getMARSdata
from prepareFLEXPART import prepareFLEXPART
def main():
calledfromdir=os.getcwd()
# os.chdir(localpythonpath)
args,c=interpret_args_and_control()
if args.queue==None:
if c.inputdir[0]!='/':
c.inputdir=os.path.join(calledfromdir,c.inputdir)
if c.outputdir[0]!='/':
c.outputdir=os.path.join(calledfromdir,c.outputdir)
getMARSdata(args,c)
prepareFLEXPART(args,c)
normalexit(c)
else:
submit(args.job_template,c,args.queue)
def submit(jtemplate,c,queue):
f=open(jtemplate)
lftext=f.read().split('\n')
insert_point=lftext.index('EOF')
f.close()
clist=c.tolist()
colist=[]
mt=0
for elem in clist:
if 'maxstep' in elem:
mt=int(elem.split(' ')[1])
for elem in clist:
if 'start_date' in elem:
elem='start_date '+'${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
if 'end_date' in elem:
elem='start_date '+'${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
if 'base_time' in elem:
elem='base_time '+'${MSJ_BASETIME}'
if 'time' in elem and mt>24:
elem='time '+'${MSJ_BASETIME} {MSJ_BASETIME}'
colist.append(elem)
lftextondemand=lftext[:insert_point]+clist+lftext[insert_point+2:]
lftextoper=lftext[:insert_point]+colist+lftext[insert_point+2:]
h=open('job.ksh','w')
h.write('\n'.join(lftextondemand))
h.close()
h=open('joboper.ksh','w')
h.write('\n'.join(lftextoper))
h.close()
try:
p=subprocess.check_call(['ecaccess-job-submit','-queueName',queue,'job.ksh'])
except:
print 'ecaccess-job-submit failed, probably eccert has expired'
exit(1)
#pout=p.communicate(input=s)[0]
print 'You should get an email with subject flex.hostname.pid'
if __name__ == "__main__":
main()
{
"install": {
"script": "install.py",
"control": null,
"inputdir": null,
"ecuid":"lh0",
"ecgid":"spatlh00",
"gateway":"srvx7.img.univie.ac.at",
"destination":"leo@genericSftp",
"local": {"target":"local"},
"local_flexpart": {"target":"local","flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
"ecgate": {"target":"ecgate"},
"cca": {"target":"cca"}
},
"default": {
"script": "submit.py",
"control": null,
"inputdir": null,
"start_date": "20131107",
"local": {},
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
"ecgate": {"queue":"ecgate"},
"cca": {"queue":"cca"}
},
"work": {
"script": "submit.py",
"control": null,
"start_date": "20131107",
"inputdir": "$SCRATCH/work",
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
},
"fc": {
"script": "submit.py",
"control": "CONTROL_FC",
"inputdir": "$SCRATCH/workfc",
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
},
"cv": {
"script": "submit.py",
"control": "CONTROL_CV",
"inputdir": "$SCRATCH/workcv",
"start_date": "20131107",
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
},
"hires": {
"script": "submit.py",
"control": "CONTROL_HIRES",
"inputdir": "$SCRATCH/workhires",
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
"ecgate": {"queue":"ecgate"}
},
"hiresgauss": {
"script": "submit.py",
"control": "CONTROL_HIRESGAUSS",
"inputdir": "$SCRATCH/workhiresgauss",
"basetime": "00",
"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
"ecgate": {"queue":"ecgate"},
"cca": {"queue":"cca"}
},
"ops": {
"script": "submit.py",
"start_date": "20131108",
"control": "CONTROL_OPS_V6.0_4V.temp",
"inputdir": "$SCRATCH/workops2",
"local_flexpart": {"basetime":"00"},
"local_flexpart12": {"basetime":"12"},
"ecgate": {"queue":"ecgate","basetime":"00"}
},
"opsfc": {
"script": "submit.py",
"start_date": "20131108",
"control": "CONTROL_OPS_V6.0",
"inputdir": "$SCRATCH/workopsfc",
"local_flexpart": {"basetime":"00"},
"local_flexpart12": {"basetime":"12"},
"ecgate": {"queue":"ecgate","basetime":"00"}
}
}
\ No newline at end of file
#!/usr/bin/env python
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# Leopold Haimberger, Dec 2015
#
# Functionality provided: This script triggers the flex_extract test suite. Call with
# testsuite.py [test group]
#
#
# Further documentation may be obtained from www.flexpart.eu
#
# Test groups are specified in testsuite.json
# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
# dateutils
# matplotlib (optional, for debugging)
import os,sys
import json
import subprocess
try:
taskfile=open('testsuite.json')
except:
print 'could not open suite definition file testsuite.json'
exit()
if not os.path.isfile('../src/CONVERT2'):
print '../src/CONVERT2 could not be found'
print 'please run "install.py --target=local" first'
exit()
fprs=os.getenv('FLEXPART_ROOT_SCRIPTS')
if fprs is None:
print 'FLEXPART_ROOT_SCRIPTS not set .. some test jobs may fail'
tasks=json.load(taskfile,encoding='latin-1')
taskfile.close()
if not os.path.exists('../test'):
os.makedirs('../test')
if len(sys.argv)>1:
groups=sys.argv[1:]
else:
groups=['xinstall','default','ops','work','cv','fc']#,'hires']
jobcounter=0
jobfailed=0
for g in groups:
try:
tk,tv=g,tasks[g]
except:
continue
garglist=[]
for ttk,ttv in tv.iteritems():
if isinstance(ttv,basestring):
if ttk!='script':
garglist.append('--'+ttk)
if '$'==ttv[0]:
garglist.append(os.path.expandvars(ttv))
else:
garglist.append(ttv)
for ttk,ttv in tv.iteritems():
if isinstance(ttv,dict):
arglist=[]
for tttk,tttv in ttv.iteritems():
if isinstance(tttv,basestring):
arglist.append('--'+tttk)
if '$' in tttv[0]:
arglist.append(os.path.expandvars(tttv))
else:
arglist.append(tttv)
print 'Command: ',' '.join([tv['script']]+garglist+arglist)
o='../test/'+tk+'_'+ttk+'_'+'_'.join(ttv.keys())
print 'Output will be sent to ',o
f=open(o,'w')
try:
p=subprocess.check_call([tv['script']]+garglist+arglist,stdout=f,stderr=f)
except:
f.write('\nFAILED\n')
print 'FAILED'
jobfailed+=1
jobcounter+=1
f.close()
print 'Test suite tasks completed'
print str(jobcounter-jobfailed)+' successful, '+str(jobfailed)+' failed'
print 'If tasks have been submitted via ECACCESS please check emails'
#print tasks
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
#GRIB_API_INCLUDE_DIR=/usr/local/gcc-4.9.3/grib1.12.3//include
#GRIB_API_LIB=-openmp -L/usr/local/gcc-4.9.3/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
#EMOSLIB=-lemosR64
OPT =
DEBUG = -g
LIB = $(GRIB_API_LIBS) $(EMOSLIB)
FC=ftn $(F90FLAGS)
F90C=ftn $(F90FLAGS)
FFLAGS = $(OPT) -I. -r8 -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -r8 -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
#GRIB_API_INCLUDE_DIR=/usr/local/gcc-4.9.3/grib1.12.3//include
#GRIB_API_LIB=-openmp -L/usr/local/gcc-4.9.3/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
#EMOSLIB=-lemosR64
OPT = -g
DEBUG = -g
LIB = $(GRIB_API_LIB) $(EMOSLIB)
FC=gfortran -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore -ffixed-line-length-132 -fopenmp -fconvert=big-endian
F90C=gfortran -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore -ffixed-line-length-132 -fopenmp -fconvert=big-endian
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 -fopenmp phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 -fopenmp grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 -fopenmp ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
#GRIB_API_INCLUDE_DIR=/usr/local/ifort/grib1.12.3//include
#GRIB_API_LIB=-openmp -L/usr/local/ifort/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
OPT = -g
DEBUG = -g
LIB = $(GRIB_API_LIBS) -lemosR64 -lgfortran
FC=ifort -132 -traceback -r8
F90C=ifort -132 -traceback -r8
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 -fopenmp phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 -fopenmp grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 -fopenmp ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
GRIB_API_INCLUDE_DIR=/usr/local/gcc-4.9.3/grib1.12.3//include
GRIB_API_LIB= -L/usr/local/gcc-4.9.3/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
EMOSLIB=-lemosR64
OPT = -g -O3 -fopenmp
DEBUG = -g
LIB = $(GRIB_API_LIB) $(EMOSLIB)
FC=gfortran -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore -ffixed-line-length-132 -fopenmp -fconvert=big-endian
F90C=gfortran -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore -ffixed-line-length-132 -fopenmp -fconvert=big-endian
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 -fopenmp phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 -fopenmp grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 -fopenmp ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
GRIB_API_INCLUDE_DIR=/usr/local/ifort/grib_api-1.14.3//include
GRIB_API_LIB=-openmp -L/usr/local/ifort/grib_api-1.14.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper #-lopenjpeg
OPT = -g
DEBUG = -g
LIB = $(GRIB_API_LIB) -lemosR64 -lgfortran
FC=ifort -132 -traceback -r8
F90C=ifort -132 -traceback -r8
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 -fopenmp phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 -fopenmp grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 -fopenmp ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
###############################################################################
#
# Top level Makefile for ECMWFDATA7.0 software
#
# Last modified: December 1, 2015
#
###############################################################################
.SUFFIXES: .o .c .c~ .f .f~ .F90 .f90 .f90~ .f95 .f95~ .F .F~ .y .y~ .l .l~ \
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
GRIB_API_INCLUDE_DIR=/usr/local/ifort/grib1.12.3//include
GRIB_API_LIBS=-openmp -L/usr/local/ifort/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
OPT = -g
DEBUG = -g
LIB = $(GRIB_API_LIBS) -lemosR64
FC=ifort -132 -traceback -r8
F90C=ifort -132 -traceback -r8
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
LDFLAGS = $(OPT)
BINDIR = .
EXE = CONVERT2
.f.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f
.f90.o:
$(F90C) -c $(F90FLAGS) -132 $(DEBUG) $*.f90
all: ${EXE}
clean:
rm *.o
phgrreal.o: phgrreal.f
$(F90C) -c -g -O3 -fopenmp phgrreal.f
grphreal.o: grphreal.f
$(F90C) -c -g -O3 -fopenmp grphreal.f
ftrafo.o: ftrafo.f
$(F90C) -c -g -O3 -fopenmp ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
###############################################################################
#
# End of the Makefile
#
###############################################################################
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment