Index: examples/wflow_rhine_sbm/wflow_sbm_NC.ini =================================================================== diff -u -rced522d9cb1398cc92a7bbbcc69d23b88708d088 -rf48b5321292915b892132463b750dbda2998e3cb --- examples/wflow_rhine_sbm/wflow_sbm_NC.ini (.../wflow_sbm_NC.ini) (revision ced522d9cb1398cc92a7bbbcc69d23b88708d088) +++ examples/wflow_rhine_sbm/wflow_sbm_NC.ini (.../wflow_sbm_NC.ini) (revision f48b5321292915b892132463b750dbda2998e3cb) @@ -75,6 +75,7 @@ netcdfstatesoutput = states.nc netcdfinput= inmaps.nc netcdfstatesinput = instates.nc +netcdfstaticinput = staticmaps.nc netcdfwritebuffer=100 netcdf_least_significant_digit=2 Index: wflow-py/Scripts/pcr2netcdf.py =================================================================== diff -u -rced522d9cb1398cc92a7bbbcc69d23b88708d088 -rf48b5321292915b892132463b750dbda2998e3cb --- wflow-py/Scripts/pcr2netcdf.py (.../pcr2netcdf.py) (revision ced522d9cb1398cc92a7bbbcc69d23b88708d088) +++ wflow-py/Scripts/pcr2netcdf.py (.../pcr2netcdf.py) (revision f48b5321292915b892132463b750dbda2998e3cb) @@ -20,9 +20,16 @@ """ syntax: + For mapstacks: + pcr2netcdf -S date -E date -N mapstackname -I mapstack_folder -O netcdf_name [-b buffersize] [-c inifile][-s start][-d digit][-Y][-P EPSG] + For single maps + pcr2netcdf -M -S date -N mapname -I mapstack_folder + -O netcdf_name [-b buffersize] [-c inifile][-s start][-d digit][-Y][-P EPSG] + + -M single map mode -S startdate in "%d-%m-%Y %H:%M:%S" e.g. 31-12-1990 00:00:00 -E endDate in "%d-%m-%Y %H:%M:%S" -s startstep (in the mapstack, default = 1) @@ -31,6 +38,7 @@ -N Mapstack-name (prefix) You can sepcify multiple input mapstack to merge them into one netcdf e.g. -N P -N TEMP -N PET + -I input mapstack folder -O output netcdf file -b maxbuf - maximum number of timesteps to buffer before writing (default = 600) @@ -67,6 +75,8 @@ import wflow.pcrut as _pcrut import osgeo.gdal as gdal import wflow.wf_netcdfio as ncdf +import glob + def usage(*args): sys.stdout = sys.stderr for msg in args: print msg @@ -379,6 +389,7 @@ clonemap=None Format="NETCDF4" EPSG="EPSG:4326" + Singlemap = False zlib=True least_significant_digit=None startstep = 1 @@ -392,7 +403,7 @@ ## Main model starts here ######################################################################## try: - opts, args = getopt.getopt(argv, 'c:S:E:N:I:O:b:t:F:zs:d:YP:') + opts, args = getopt.getopt(argv, 'c:S:E:N:I:O:b:t:F:zs:d:YP:M') except getopt.error, msg: usage(msg) @@ -408,14 +419,21 @@ if o == '-Y': perYear = True if o == '-z': zlib=True if o == '-P': EPSG = a + if o == '-M': Singlemap = True if o == '-F': Format=a if o == '-d': least_significant_digit = int(a) if o == '-t': timestepsecs = int(a) - if o == '-N': - mapstackname.append(a) - var.append(a) - varname.append(a) + if o == '-N': + flst = glob.glob(a) + if len(flst) ==1: + mapstackname.append(flst) + var.append(flst) + varname.append(flst) + else: + mapstackname = flst + var =flst + varname =flst # Use first timestep as clone-map @@ -426,14 +444,23 @@ above_thousand = count / 1000 clonemapname = str(mapstackname[0] + '%0' + str(8-len(mapstackname[0])) + '.f.%03.f') % (above_thousand, below_thousand) clonemap = os.path.join(mapstackfolder, clonemapname) + + if Singlemap: + clonemap = mapstackname[0] + _pcrut.setclone(clonemap) x = _pcrut.pcr2numpy(_pcrut.xcoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[0,:] y = _pcrut.pcr2numpy(_pcrut.ycoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[:,0] start=dt.datetime.strptime(startstr,"%d-%m-%Y %H:%M:%S") - end=dt.datetime.strptime(endstr,"%d-%m-%Y %H:%M:%S") + + if Singlemap: + end = start + else: + end=dt.datetime.strptime(endstr,"%d-%m-%Y %H:%M:%S") + if timestepsecs == 86400: if perYear: timeList = date_range_peryear(start, end, tdelta="days") @@ -452,40 +479,55 @@ # break up into separate years - varmeta = {} - startmapstack = startstep + if not Singlemap: + varmeta = {} - if perYear: - for yr_timelist in timeList: - ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1] - ncdf.prepare_nc(ncoutfile_yr, yr_timelist, x, y, metadata, logger,Format=Format,zlib=zlib,EPSG=EPSG) + startmapstack = startstep - idx = 0 - for mname in mapstackname: + if perYear: + for yr_timelist in timeList: + ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1] + ncdf.prepare_nc(ncoutfile_yr, yr_timelist, x, y, metadata, logger,Format=Format,zlib=zlib,EPSG=EPSG) + + idx = 0 + for mname in mapstackname: + logger.info("Converting mapstack: " + mname + " to " + ncoutfile) + # get variable attributes from ini file here + if os.path.exists(inifile): + varmeta = getvarmetadatafromini(inifile,var[idx]) + + write_netcdf_timeseries(mapstackfolder, mname, ncoutfile_yr, var[idx], unit, varname[idx], yr_timelist, varmeta, logger,maxbuf=mbuf,Format=Format,zlib=zlib,least_significant_digit=least_significant_digit,startidx=startmapstack,EPSG=EPSG) + idx = idx + 1 + + startmapstack = startmapstack + len(yr_timelist) + else: + #ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1] + ncdf.prepare_nc(ncoutfile, timeList, x, y, metadata, logger,Format=Format,zlib=zlib,EPSG=EPSG) + idx = 0 + for mname in mapstackname: logger.info("Converting mapstack: " + mname + " to " + ncoutfile) # get variable attributes from ini file here if os.path.exists(inifile): varmeta = getvarmetadatafromini(inifile,var[idx]) - write_netcdf_timeseries(mapstackfolder, mname, ncoutfile_yr, var[idx], unit, varname[idx], yr_timelist, varmeta, logger,maxbuf=mbuf,Format=Format,zlib=zlib,least_significant_digit=least_significant_digit,startidx=startmapstack,EPSG=EPSG) + write_netcdf_timeseries(mapstackfolder, mname, ncoutfile, var[idx], unit, varname[idx], timeList, varmeta, logger,maxbuf=mbuf,Format=Format,zlib=zlib,least_significant_digit=least_significant_digit,startidx=startmapstack,EPSG=EPSG) idx = idx + 1 - - startmapstack = startmapstack + len(yr_timelist) else: - #ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1] - ncdf.prepare_nc(ncoutfile, timeList, x, y, metadata, logger,Format=Format,zlib=zlib,EPSG=EPSG) - idx = 0 - for mname in mapstackname: - logger.info("Converting mapstack: " + mname + " to " + ncoutfile) - # get variable attributes from ini file here - if os.path.exists(inifile): - varmeta = getvarmetadatafromini(inifile,var[idx]) + NcOutput = ncdf.netcdfoutputstatic(ncoutfile, logger, timeList[0],1,timestepsecs=timestepsecs, + maxbuf=1, metadata=metadata, EPSG=EPSG,Format=Format, + zlib=zlib) - write_netcdf_timeseries(mapstackfolder, mname, ncoutfile, var[idx], unit, varname[idx], timeList, varmeta, logger,maxbuf=mbuf,Format=Format,zlib=zlib,least_significant_digit=least_significant_digit,startidx=startmapstack,EPSG=EPSG) - idx = idx + 1 + for file in mapstackname: + pcrdata = _pcrut.readmap(file) + thevar = os.path.basename(file) + NcOutput.savetimestep(1, pcrdata, unit="mm", var=thevar, name=file) + + + + if __name__ == "__main__": main() Index: wflow-py/wflow/wf_DynamicFramework.py =================================================================== diff -u -rced522d9cb1398cc92a7bbbcc69d23b88708d088 -rf48b5321292915b892132463b750dbda2998e3cb --- wflow-py/wflow/wf_DynamicFramework.py (.../wf_DynamicFramework.py) (revision ced522d9cb1398cc92a7bbbcc69d23b88708d088) +++ wflow-py/wflow/wf_DynamicFramework.py (.../wf_DynamicFramework.py) (revision f48b5321292915b892132463b750dbda2998e3cb) @@ -691,13 +691,17 @@ self.logger.debug("Found following input variables to get from netcdf file: " + str(varlst)) self.NcInput = netcdfinput(os.path.join(caseName, self.ncfile), self.logger, varlst) - # Setup all the netCDF files that may be used for input/output + if self.ncfilestates != "None": smaps = self._userModel().stateVariables() maps = [s + ".map" for s in smaps] self.logger.debug("Found following input states to get from netcdf file: " + str(maps)) self.NcInputStates = netcdfinputstates(os.path.join(caseName, self.ncfilestates), self.logger, maps) + + if self.ncfilestatic != "None": + self.NcInputStatic = netcdfinputstatic(os.path.join(caseName, self.ncfilestatic), self.logger) + if self.ncoutfile != 'None': # Ncoutput buffer = int(configget(self._userModel().config, 'framework', 'netcdfwritebuffer', "50")) meta = {} @@ -735,14 +739,8 @@ zlib=self.ncfilecompression,least_significant_digit=self.ncfiledigits) - if self.ncfilestatic != 'None': # Ncoutput - meta = {} - meta['caseName'] = caseName - meta['runId'] = runId - meta['wflow_version'] =__version__ - meta['wflow_release'] =__release__ - self.NcInputStatic = netcdfinput(os.path.join(caseName, self.ncfilestatic), self.logger, varlst) + # Fill the summary (stat) list from the ini file self.statslst = [] _type = wf_sumavg(None) @@ -2008,13 +2006,18 @@ self.logger.error("Required map: " + os.path.abspath(path) + " not found, exiting..") sys.exit(1) return scalar(default) + if self._userModel()._inInitial(): - if ncfilesource == self.ncfilestatic: + if self.ncfilestatic is not 'None': retval, succ = self.NcInputStatic.gettimestep(1, self.logger, var=varname) if succ: return retval else: - return scalar(default) + if fail: + self.logger.error("Required map: " + os.path.abspath(path) + " not found in " + self.ncfilestatic + " exiting..") + sys.exit(1) + else: + return scalar(default) if os.path.isfile(path): mapje = readmap(path) Index: wflow-py/wflow/wf_netcdfio.py =================================================================== diff -u -rced522d9cb1398cc92a7bbbcc69d23b88708d088 -rf48b5321292915b892132463b750dbda2998e3cb --- wflow-py/wflow/wf_netcdfio.py (.../wf_netcdfio.py) (revision ced522d9cb1398cc92a7bbbcc69d23b88708d088) +++ wflow-py/wflow/wf_netcdfio.py (.../wf_netcdfio.py) (revision f48b5321292915b892132463b750dbda2998e3cb) @@ -267,7 +267,7 @@ self.nc_trg.sync() miss = float(nc_var._FillValue) - data = pcr2numpy(pcrdata, miss) + data = pcr2numpy(scalar(pcrdata), miss) if self.bufflst.has_key(var): self.bufflst[var][bufpos, :, :] = data @@ -382,7 +382,7 @@ self.nc_trg.sync() miss = float(nc_var._FillValue) - data = pcr2numpy(pcrdata, miss) + data = pcr2numpy(scalar(pcrdata), miss) if self.bufflst.has_key(var): self.bufflst[var][bufpos, :, :] = data @@ -496,7 +496,6 @@ self.fstep = 0 self.lstep = self.fstep + self.maxsteps - for var in vars: try: self.alldat[var] = self.dataset.variables[var][self.fstep:self.maxsteps] @@ -526,3 +525,42 @@ else: logging.debug("Var (" + var + ") not found returning 0") return cover(scalar(0.0)), False + + +class netcdfinputstatic(): + def __init__(self, netcdffile, logging): + """ + First try to setup a class read netcdf files + (converted with pcr2netcdf.py) + + netcdffile: file to read the forcing data from + logging: python logging object + vars: list of variables to get from file + """ + + if os.path.exists(netcdffile): + self.dataset = netCDF4.Dataset(netcdffile, mode='r') + else: + logging.error(os.path.abspath(netcdffile) + " not found!") + exit(ValueError) + + logging.info("Reading static input from netCDF file: " + netcdffile + ": " + str(self.dataset).replace('\n', ' ')) + + + + def gettimestep(self, timestep, logging, var='P'): + """ + Gets a map for a single timestep. reads data in blocks assuming sequential access + + timestep: framework timestep (1-based) + logging: python logging object + var: variable to get from the file + """ + + if self.dataset.variables.has_key(var): + np_step = self.dataset.variables[var][timestep-1,:,:] + miss = float(self.dataset.variables[var]._FillValue) + return numpy2pcr(Scalar, np_step, miss), True + else: + logging.debug("Var (" + var + ") not found returning 0") + return cover(scalar(0.0)), False