Index: wflow-py/wflow/wf_DynamicFramework.py =================================================================== diff -u -rfaafd5e1b8eee13781bd50c2d05d4a264b9047c3 -rbcfa3db98112744a480253f1f934f647f5f7cd17 --- wflow-py/wflow/wf_DynamicFramework.py (.../wf_DynamicFramework.py) (revision faafd5e1b8eee13781bd50c2d05d4a264b9047c3) +++ wflow-py/wflow/wf_DynamicFramework.py (.../wf_DynamicFramework.py) (revision bcfa3db98112744a480253f1f934f647f5f7cd17) @@ -1168,7 +1168,7 @@ return 0 - def _reportNew(self, variable, name, style=1): + def _reportNew(self, variable, name, style=1,gzipit=False): """ outputformat: (set in the [framework] section of the init file). 1: pcraster @@ -1220,7 +1220,9 @@ import PCRaster as PCRaster else: import PCRaster - PCRaster.report(variable, path) + PCRaster.report(variable, path) + if gzipit: + Gzip(path,storePath=True) elif self.outputFormat == 2: numpy.savez(path,pcr2numpy(variable,-999)) elif self.outputFormat == 3: Index: wflow-py/wflow/wflow_adapt.py =================================================================== diff -u -rf6aa63f193021cb049e58f87512f74a028ac7b0a -rbcfa3db98112744a480253f1f934f647f5f7cd17 --- wflow-py/wflow/wflow_adapt.py (.../wflow_adapt.py) (revision f6aa63f193021cb049e58f87512f74a028ac7b0a) +++ wflow-py/wflow/wflow_adapt.py (.../wflow_adapt.py) (revision bcfa3db98112744a480253f1f934f647f5f7cd17) @@ -103,22 +103,19 @@ fewsNamespace="http://www.wldelft.nl/fews/PI" -def setlogger(logfilename): +def setlogger(logfilename,loggername,thelevel=logging.INFO): """ Set-up the logging system and return a logger object. Exit if this fails - - Input: - - filename - - Output: - - Logger object """ + try: #create logger - logger = logging.getLogger("wflow_adapt") - logger.setLevel(logging.DEBUG) - ch = logging.handlers.RotatingFileHandler(logfile,maxBytes=200000, backupCount=5) - #ch = logging.handlers.FileHandler(logfiles,mode='w') + logger = logging.getLogger(loggername) + if not isinstance(thelevel, int): + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(thelevel) + ch = logging.FileHandler(logfilename,mode='w') console = logging.StreamHandler() console.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) @@ -202,6 +199,7 @@ - Others may follow """ + if os.path.exists(nname): file = open(nname, "r") tree = parse(file) @@ -287,6 +285,7 @@ i = i+1 nrevents = i + for par in uniqueParList: f = open(outputdir + '/' + par + '.tss','w') # write the header @@ -558,7 +557,7 @@ timestepsecs = int(wflow_lib.configget(config,"model","timestepsecs",str(timestepsecs))) - logger = setlogger(logfile) + logger = setlogger(logfile,"wflow_adapt") if mode =="Pre": logger.info("Starting preadapter") @@ -581,7 +580,7 @@ for a in mstacks: var = config.get("outputmaps",a) logger.debug("Creating mapstack xml: " + workdir + "/" + case +"/" +runId + "/" + var + ".xml" ) - mapstackxml(workdir + "/" + case +"/" + runId + "/outmaps/" + var +".xml",var + "?????.???",var,var,getStartTimefromRuninfo(runinfofile),getEndTimefromRuninfo(runinfofile),timestepsecs) + mapstackxml(workdir + "/" + case +"/" + runId + "/outmaps/" + var +".xml",var + "?????.???",var,var,getStartTimefromRuninfo(runinfofile,logger),getEndTimefromRuninfo(runinfofile),timestepsecs) # Back hack to work around the 0 based FEWS problem and create a double timestep zo that we have connection between subsequent runs in FEWS Index: wflow-py/wflow/wflow_lib.py =================================================================== diff -u -r5d7f50da692bc50d992d911c4275fd17dc7ec02b -rbcfa3db98112744a480253f1f934f647f5f7cd17 --- wflow-py/wflow/wflow_lib.py (.../wflow_lib.py) (revision 5d7f50da692bc50d992d911c4275fd17dc7ec02b) +++ wflow-py/wflow/wflow_lib.py (.../wflow_lib.py) (revision bcfa3db98112744a480253f1f934f647f5f7cd17) @@ -57,6 +57,9 @@ import scipy import netCDF4 as nc4 +import gzip, zipfile +import osgeo.gdal as gdal +from osgeo.gdalconst import * @@ -697,3 +700,144 @@ return slope +def Gzip(fileName, storePath=False, chunkSize=1024*1024): + """ + Usage: Gzip(fileName, storePath=False, chunksize=1024*1024) + Gzip the given file to the given storePath and then remove the file. + A chunk size may be selected. Default is 1 megabyte + Input: + fileName: file to be GZipped + storePath: destination folder. Default is False, meaning the file will be zipped to its own folder + chunkSize: size of chunks to write. If set too large, GZip will fail with memory problems + """ + import gzip + if not storePath: + pathName = os.path.split(fileName)[0] + fileName = os.path.split(fileName)[1] + curdir = os.path.curdir + os.chdir(pathName) + # open files for reading / writing + r_file = open(fileName, 'rb') + w_file = gzip.GzipFile(fileName + '.gz', 'wb', 9) + dataChunk = r_file.read(chunkSize) + while dataChunk: + w_file.write(dataChunk) + dataChunk = r_file.read(chunkSize) + w_file.flush() + w_file.close() + r_file.close() + os.unlink(fileName) #We don't need the file now + if not storePath: + os.chdir(curdir) + + + +# These come from GLOFRIS_Utils + +def Gzip(fileName, storePath=False, chunkSize=1024*1024): + """ + Usage: Gzip(fileName, storePath=False, chunksize=1024*1024) + Gzip the given file to the given storePath and then remove the file. + A chunk size may be selected. Default is 1 megabyte + Input: + fileName: file to be GZipped + storePath: destination folder. Default is False, meaning the file will be zipped to its own folder + chunkSize: size of chunks to write. If set too large, GZip will fail with memory problems + """ + if not storePath: + pathName = os.path.split(fileName)[0] + fileName = os.path.split(fileName)[1] + curdir = os.path.curdir + os.chdir(pathName) + # open files for reading / writing + r_file = open(fileName, 'rb') + w_file = gzip.GzipFile(fileName + '.gz', 'wb', 9) + dataChunk = r_file.read(chunkSize) + while dataChunk: + w_file.write(dataChunk) + dataChunk = r_file.read(chunkSize) + w_file.flush() + w_file.close() + r_file.close() + os.unlink(fileName) #We don't need the file now + if not storePath: + os.chdir(curdir) + +def zipFiles(fileList, fileTarget): + """ + Usage: zipFiles(fileList, fileTarget) + zip the given list of files to the given target file + Input: + fileList: list of files to be zipped + fileTarget: target zip-file + """ + zout = zipfile.ZipFile(fileTarget, "w", compression=zipfile.ZIP_DEFLATED) + for fname in fileList: + zout.write(fname, arcname=os.path.split(fname)[1]) + zout.close() + + + +def readMap(fileName, fileFormat): + """ Read geographical file into memory + """ + # Open file for binary-reading + mapFormat = gdal.GetDriverByName(fileFormat) + mapFormat.Register() + ds = gdal.Open(fileName) + if ds is None: + print 'Could not open ' + fileName + '. Something went wrong!! Shutting down' + sys.exit(1) + # Retrieve geoTransform info + geotrans = ds.GetGeoTransform() + originX = geotrans[0] + originY = geotrans[3] + resX = geotrans[1] + resY = geotrans[5] + cols = ds.RasterXSize + rows = ds.RasterYSize + x = linspace(originX+resX/2,originX+resX/2+resX*(cols-1),cols) + y = linspace(originY+resY/2,originY+resY/2+resY*(rows-1),rows) + # Retrieve raster + RasterBand = ds.GetRasterBand(1) # there's only 1 band, starting from 1 + data = RasterBand.ReadAsArray(0,0,cols,rows) + FillVal = RasterBand.GetNoDataValue() + RasterBand = None + ds = None + return x, y, data, FillVal + +def writeMap(fileName, fileFormat, x, y, data, FillVal): + """ Write geographical data into file""" + + verbose = False + gdal.AllRegister() + driver1 = gdal.GetDriverByName('GTiff') + driver2 = gdal.GetDriverByName(fileFormat) + + # Processing + if verbose: + print 'Writing to temporary file ' + fileName + '.tif' + # Create Output filename from (FEWS) product name and data and open for writing + TempDataset = driver1.Create(fileName + '.tif',data.shape[1],data.shape[0],1,gdal.GDT_Float32) + # Give georeferences + xul = x[0]-(x[1]-x[0])/2 + yul = y[0]+(y[0]-y[1])/2 + TempDataset.SetGeoTransform( [ xul, x[1]-x[0], 0, yul, 0, y[1]-y[0] ] ) + # get rasterband entry + TempBand = TempDataset.GetRasterBand(1) + # fill rasterband with array + TempBand.WriteArray(data,0,0) + TempBand.FlushCache() + TempBand.SetNoDataValue(FillVal) + # Create data to write to correct format (supported by 'CreateCopy') + if verbose: + print 'Writing to ' + fileName + '.map' + outDataset = driver2.CreateCopy(fileName, TempDataset, 0) + TempDataset = None + outDataset = None + if verbose: + print 'Removing temporary file ' + fileName + '.tif' + os.remove(fileName + '.tif'); + + if verbose: + print 'Writing to ' + fileName + ' is done!' Index: wflow-py/wflow/wflow_wave.py =================================================================== diff -u -r6d0167e9bc24d8ed8afe4d6fdfb09e539314cb6d -rbcfa3db98112744a480253f1f934f647f5f7cd17 --- wflow-py/wflow/wflow_wave.py (.../wflow_wave.py) (revision 6d0167e9bc24d8ed8afe4d6fdfb09e539314cb6d) +++ wflow-py/wflow/wflow_wave.py (.../wflow_wave.py) (revision bcfa3db98112744a480253f1f934f647f5f7cd17) @@ -318,7 +318,8 @@ self.dynHBoundary = pcrut.readmapSave(os.path.join(self.Dir, wflow_dynhboun),0.0) self.lowerflowbound=configget(self.config,"dynamicwave","lowerflowbound","0") self.fixed_h_tss=configget(self.config,"dynamicwave","levelTss","intss/Hboun.tss") - self.logger.debug("Dynamic wave timestep is: " + str(self.timestepsecs/self.dynsubsteps/self.TsliceDyn) + " seconds") + self.logger.info("Dynamic wave timestep is: " + str(self.timestepsecs/self.dynsubsteps/self.TsliceDyn) + " seconds") + self.logger.info("Lower boundary file: " + self.fixed_h_tss) self.AdaptiveTimeStepping = int(configget(self.config,"dynamicwave","AdaptiveTimeStepping","0"))