Index: trunk/SDToolBox/extract_data.py =================================================================== diff -u -r91 -r94 --- trunk/SDToolBox/extract_data.py (.../extract_data.py) (revision 91) +++ trunk/SDToolBox/extract_data.py (.../extract_data.py) (revision 94) @@ -162,15 +162,6 @@ def get_new_file_iter(self, file_entry, file_path) -> Dict[str, str]: raise Exception(om.error_needs_to_be_in_subclass) - @abstractmethod - def get_masked_nn_lat_lon( - self, - input_data: InputData, - ref_dataset: Dataset, - cases_dict: dict, - mask_dir: str) -> Tuple[List[int], List[int]]: - raise Exception(om.error_needs_to_be_in_subclass) - def set_file_iterator( self, input_data: InputData, @@ -350,7 +341,8 @@ input_data=input_data, ref_dataset=ref_dataset, cases_dict=cases_dict, - mask_dir=os.path.dirname(os.path.dirname(ref_file_path))) + mask_dir=os.path.dirname( + os.path.dirname(ref_file_path))) def _get_filtered_dict( self, @@ -454,10 +446,53 @@ """ kd = BallTree(data_array, leaf_size=10) # k=2 nearest neighbors where k1 = identity - _, index_found = kd.query(values) + _, index_found = kd.query(values, k=2) value_found = data_array[index_found.squeeze(), :] return index_found, value_found + def get_masked_nn_lat_lon( + self, + input_data: InputData, + ref_dataset: Dataset, + cases_dict: dict, + mask_dir: str) -> Tuple[List[int], List[int]]: + mask_filepath = os.path.join(mask_dir, 'ERA5_landsea_mask.nc') + with Dataset(mask_filepath, 'r', self.netcdf_format) \ + as dsmask: + mask = dsmask['lsm'][0, 0::2, 0::2] + lon = dsmask['longitude'][0::2] + lon[lon > 180] += -360 + lat = dsmask['latitude'][0::2] + loni = np.where(lon == 180)[0][0] + LON, LAT = np.meshgrid(lon, lat) + LON = np.concatenate( + (LON[:, loni+1:], LON[:, 0:loni+1]), + axis=1) + mask = np.concatenate( + (mask[:, loni+1:], mask[:, 0:loni+1]), + axis=1) + # positions where we are in the sea + possea = np.where(mask.ravel() == 0.)[0] + index, vals = self.get_nearest_neighbor_extended( + np.vstack( + ( + input_data._input_lon, + input_data._input_lat + )).T, + np.vstack( + ( + LON.ravel()[possea], + LAT.ravel()[possea])).T) + index_abs = possea[index].squeeze() + + if(index_abs.size <= 0): + return None + indexes = np.array( + [ + np.unravel_index(ii, LON.shape) + for ii in index_abs]).T.tolist() + return indexes[1], indexes[0] + @staticmethod def __get_case_subset( dataset: Dataset, @@ -505,7 +540,7 @@ } __era5_wind_mslp_dict = { - 'msl': 'msl', + 'msl_p': 'msl', 'u10': 'wind_u', 'v10': 'wind_v' } @@ -596,46 +631,6 @@ self.file_fpath_key: file_path } - def get_masked_nn_lat_lon( - self, - input_data: InputData, - ref_dataset: Dataset, - cases_dict: dict, - mask_dir: str) -> Tuple[List[int], List[int]]: - mask_filepath = os.path.join(mask_dir, 'ERA5_landsea_mask.nc') - with Dataset(mask_filepath, 'r', self.netcdf_format) \ - as dsmask: - mask = dsmask['lsm'][0, 0::2, 0::2] - lon = dsmask['longitude'][0::2] - lon[lon > 180] += -360 - lat = dsmask['latitude'][0::2] - loni = np.where(lon == 180)[0][0] - LON, LAT = np.meshgrid(lon, lat) - LON = np.concatenate( - (LON[:, loni+1:], LON[:, 0:loni+1]), - axis=1) - mask = np.concatenate( - (mask[:, loni+1:], mask[:, 0:loni+1]), - axis=1) - # positions where we are in the sea - possea = np.where(mask.ravel() == 0.)[0] - index, vals = self.get_nearest_neighbor_extended( - np.vstack( - ( - input_data._input_lon, - input_data._input_lat - )).T, - np.vstack( - ( - LON.ravel()[possea], - LAT.ravel()[possea])).T) - index_abs = possea[index].squeeze() - indexes = np.array( - [ - np.unravel_index(ii, LON.shape) - for ii in index_abs]) - return indexes - class __EarthExtractor(BaseExtractor): __earth_lon_key = 'lon' __earth_lat_key = 'lat' @@ -741,11 +736,3 @@ self.file_scenario_key: file_entry[3], self.file_fpath_key: file_path } - - def get_masked_nn_lat_lon( - self, - input_data: InputData, - ref_dataset: Dataset, - cases_dict: dict, - mask_dir: str) -> Tuple[List[int], List[int]]: - raise Exception(om.error_function_not_implemented)