Index: trunk/SDToolBox/data_processing.py =================================================================== diff -u -r89 -r112 --- trunk/SDToolBox/data_processing.py (.../data_processing.py) (revision 89) +++ trunk/SDToolBox/data_processing.py (.../data_processing.py) (revision 112) @@ -4,18 +4,9 @@ """ # region // imports -import sys -import os -from typing import List, Set, Dict, Tuple, Optional, Any - -from datetime import datetime, timedelta - from SDToolBox import output_messages as om from SDToolBox.output_data import OutputData -from sklearn.decomposition import PCA as skPCA -from sklearn.preprocessing import StandardScaler -import math import numpy as np import xarray as xr @@ -80,7 +71,7 @@ def _time_resampling( dataset: xr.Dataset, scale: int, - frequency_string: str) -> xr.core.resample: + frequency_string: str) -> xr.DataArray: """Resamples a given xarray dataset. Arguments: @@ -92,7 +83,7 @@ Exception: When no arguments are given. Returns: - xarray.core.resample -- Resample object. + xarray.DataArray -- Resample object. """ if dataset is None or \ scale is None or \ Index: trunk/tests/test_extract_data.py =================================================================== diff -u -r109 -r112 --- trunk/tests/test_extract_data.py (.../test_extract_data.py) (revision 109) +++ trunk/tests/test_extract_data.py (.../test_extract_data.py) (revision 112) @@ -95,14 +95,14 @@ self, input_variables: List[str], input_scenarios: List[str]): # 1. Given # When using local data you can just replace the comment in these lines - dir_test_data = TestUtils.get_local_test_data_dir('earth_test_data') - # dir_test_data = 'P:\\metocean-data\\open\\ERA5\\data\\Global' + # dir_test_data = TestUtils.get_local_test_data_dir('earth_test_data') + dir_test_data = 'P:\\metocean-data\\open\\ERA5\\data\\Global' input_data = InputData() input_data.input_variables = input_variables input_data.input_coordinates = { - input_data.longitude_key: 4.2, - input_data.latitude_key: 2.4} + input_data.longitude_key: [4.2], + input_data.latitude_key: [2.4]} input_data.input_years = [1981, 1982] input_data.input_scenarios = input_scenarios Index: trunk/tests/test_acceptance.py =================================================================== diff -u --- trunk/tests/test_acceptance.py (revision 0) +++ trunk/tests/test_acceptance.py (revision 112) @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Feb 3 14:12:50 2020 + +@author: alvareza +""" +import pytest +import os +import numpy as np +import pandas as pd +import xarray as xr +from netCDF4 import Dataset +from matplotlib import pyplot as plt + +# import sys +# sys.path.insert(0,r'C:\Users\alvareza\checkouts\SD_Toolbox\trunk') +from SDToolBox import extract_data as sd_eda +from SDToolBox.output_data import OutputData +from tests.TestUtils import TestUtils + +# %% Paths + + +class Paths: + def __init__(self, pth): + self.base = pth + self.data = os.path.join(self.base, 'data') + self.out = os.path.join(self.base, 'out') + self.plots = os.path.join(self.out, 'plots') + + def _makedirs(self): + for path in vars(self).values(): + if not os.path.exists(path): + os.mkdir(path) + + +@pytest.mark.acceptance +class Tests_Acceptance: + + def test_antolinezcase_nongridded_data(self): + pth = Paths( + TestUtils.get_local_test_data_dir(dir_name='acceptance_tests')) + pth.era5 = 'P:\\metocean-data\\open\\ERA5\\data\\Global' + + # %% Get the coordinates + with Dataset(os.path.join(pth.data, 'NS_WAM_ERA5.nc'), 'r') as ds: + lat = ds['lat'][:] + lon = ds['lon'][:] + + # %% Extract data + # 1st initialize the class + inda = sd_eda.InputData( + input_coordinates={'LON': lon, 'LAT': lat}, + input_variables=['swh'], + input_scenarios=['era5'], + input_years=[1986]) + + try: + ds = sd_eda.ExtractData.get_era_5( + directory_path=os.path.join(pth.era5), + input_data=inda) + dsxr = ds.generate_netcdf( + dir_path=pth.out, + base_name='test', + dataset_code='era5') + except Exception as e_info: + pytest.fail('Test failed {}'.format(str(e_info))) + + assert ds is not None + assert ds.data_dict is not None + assert ds.data_dict[OutputData.var_lat_key] is not None + assert ds.data_dict[OutputData.var_lon_key] is not None + assert ds.data_dict[OutputData.var_station_idx_key] is not None + assert dsxr is not None + assert os.path.exists(dsxr) + + def test_antolinezcase_gridded_data(self): + pth = Paths( + TestUtils.get_local_test_data_dir(dir_name='acceptance_tests')) + pth.era5 = 'P:\\metocean-data\\open\\ERA5\\data\\Global' + + # %% Get the coordinates + with Dataset(os.path.join(pth.data, 'NS_WAM_ERA5.nc'), 'r') as ds: + lat = ds['lat'][:] + lon = ds['lon'][:] + + # %% Extract data + lonmin = -5 + lonmax = 11 + dlon = 0.5 + latmin = 50 + latmax = 62 + dlat = 0.5 + lon = np.arange(latmin, latmax+dlat, dlat) + lat = np.arange(lonmin, lonmax+dlon, dlon) + + inda = sd_eda.InputData( + input_coordinates={'LON': lon, 'LAT': lat}, + input_variables=['msl'], + input_scenarios=['era5'], + input_years=[1986], + is_gridded=True) + + try: + ds = sd_eda.ExtractData.get_era_5( + directory_path=os.path.join(pth.era5), + input_data=inda) + dsxr = ds.generate_netcdf( + dir_path=pth.out, + base_name='test', + dataset_code='era5') + except Exception as e_info: + pytest.fail('Test failed {}'.format(str(e_info))) + + assert ds is not None + assert ds.data_dict is not None + assert ds.data_dict[OutputData.var_lat_key] is not None + assert ds.data_dict[OutputData.var_lon_key] is not None + assert ds.data_dict[OutputData.var_station_idx_key] is not None + assert dsxr is not None + assert os.path.exists(dsxr) Index: trunk/SDToolBox/extract_data.py =================================================================== diff -u -r110 -r112 --- trunk/SDToolBox/extract_data.py (.../extract_data.py) (revision 110) +++ trunk/SDToolBox/extract_data.py (.../extract_data.py) (revision 112) @@ -4,10 +4,9 @@ """ # region // imports -import sys import os -from typing import List, Set, Dict, Tuple, Optional, Any +from typing import List, Dict, Tuple, Any from abc import ABC, abstractmethod import itertools @@ -108,7 +107,7 @@ netcdf_format = 'netCDF4' stations_idx_key = 'stations_idx' - __file_iterator = [] + __file_iterator: list = [] # Region Abstract methods / properties. @property @@ -166,17 +165,14 @@ def set_file_iterator( self, input_data: InputData, - dir_path: str) -> List[Dict[str, str]]: + dir_path: str): """Sets the file iterator based on the possible file combinations. Arguments: file_combinations {List[List[Any]]} -- File combinations for the dataset type. dir_path {str} -- Path to the parent directory. - - Returns: - List[Dict[str, str]] -- [description] """ # Get combinations file_combinations = self.get_file_combinations(input_data) @@ -504,7 +500,7 @@ index_abs = possea[index].squeeze() if(index_abs.size <= 0): - return None + return None, None indexes = np.unravel_index(index_abs, LON.shape) unique_values, unique_idx = \ np.unique(vals, axis=0, return_index=True) @@ -586,15 +582,15 @@ def get_filepath( self, dir_path: str, - file_entry: Dict[str, str]) -> str: - """Gets the earth filepath. + file_entry: List[List[str]]) -> str: + """Gets the Era5 filepath. Arguments: dir_path {str} -- Parent directory. - file_entry {Dict[str]} -- Dict of file attributes. + file_entry {List[List[str]]} -- List of lists with file attributes. Returns: - str -- File path location based on EARTH format. + str -- File path location based on Era5 format. """ # Find the matching file base_file_name = '' + \ @@ -703,12 +699,12 @@ def get_filepath( self, dir_path: str, - file_entry: Dict[str, str]) -> str: + file_entry: List[List[str]]) -> str: """Gets the Earth filepath. Arguments: dir_path {str} -- Parent directory. - file_entry {Dict[str]} -- Dict of file attributes. + file_entry {List[List[str]]} -- List of list containing file attributes. Returns: str -- File path location based on ERA5 format.