Index: trunk/tests/testing_Etienne/.ipynb_checkpoints/SD_ToolBox_Testing-checkpoint.ipynb =================================================================== diff -u -r115 -r117 --- trunk/tests/testing_Etienne/.ipynb_checkpoints/SD_ToolBox_Testing-checkpoint.ipynb (.../SD_ToolBox_Testing-checkpoint.ipynb) (revision 115) +++ trunk/tests/testing_Etienne/.ipynb_checkpoints/SD_ToolBox_Testing-checkpoint.ipynb (.../SD_ToolBox_Testing-checkpoint.ipynb) (revision 117) @@ -140,6 +140,26 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [], + "source": [ + "# get the coordinates of the WAM locations\n", + "lonWAM = DS.variables['lon'][:] # list with longitude positions\n", + "latWAM = DS.variables['lat'][:] # list with latitude positions \n", + "\n", + "coordsWAM = {'LON': [], 'LAT': []}\n", + "for i, j in zip(lonWAM, latWAM):\n", + " coordsWAM['LON'].append(i)\n", + " coordsWAM['LAT'].append(j)\n", + "\n", + "# set other data variables \n", + "varWAM = [v.lower() for v in variables[5:]] # list with the five wave parameters -> it automatically only uses swh, pp1d, mwd\n", + "timeWAMy = [1986] # 1987 to 2018" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -149,26 +169,13 @@ "Extracting variable: Tp, year 1986.\n", "Extracting variable: MWD, year 1986.\n", "Writing C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets\\WAM_ERA5_WAVE.nc\n", - "writing variable pp1d\n", + "writing variable mwd\n", "writing variable swh\n", - "writing variable mwd\n" + "writing variable pp1d\n" ] } ], "source": [ - "# get the coordinates of the WAM locations\n", - "lonWAM = DS.variables['lon'][:] # list with longitude positions\n", - "latWAM = DS.variables['lat'][:] # list with latitude positions \n", - "\n", - "coordsWAM = {'LON': [], 'LAT': []}\n", - "for i, j in zip(lonWAM, latWAM):\n", - " coordsWAM['LON'].append(i)\n", - " coordsWAM['LAT'].append(j)\n", - "\n", - "# set other data variables \n", - "varWAM = [v.lower() for v in variables[5:]] # list with the five wave parameters -> ind automatically only uses swh, pp1d, mwd\n", - "timeWAMy = [1986] # 1987 to 2018\n", - "\n", "# use the SDToolBox function to create input data\n", "Input_DataWAM = ind.InputData(\n", " input_coordinates=coordsWAM,\n", @@ -189,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -244,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -266,7 +273,7 @@ "folder2 = r'C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets'\n", "\n", "# open the data in this script to get the info\n", - "DS_WAM_ERA5 = Dataset(os.path.join(folder2,'WAM_ERA5.nc'))\n", + "DS_WAM_ERA5 = Dataset(os.path.join(folder2,'WAM_ERA5_WAVE.nc'))\n", "\n", "# get the relevant data from the netcdf files\n", "time_WAM_ERA5 = list(DS_WAM_ERA5.variables['time'])\n", @@ -326,49 +333,56 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['msl']\n", - "Extracting variable: Hs, year 1986.\n" + "['swh', 'pp1d', 'mwd']\n", + "Extracting variable: Hs, year 1986.\n", + "Extracting variable: Tp, year 1986.\n", + "Extracting variable: MWD, year 1986.\n", + "Writing C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets\\BOX_ERA5_WAVE.nc\n", + "writing variable swh\n" ] }, { - "ename": "KeyError", - "evalue": "'swh'", + "ename": "ValueError", + "evalue": "shape mismatch: objects cannot be broadcast to a single shape", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;31m# use the SDToolBox function to extract data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mInput_DataBOX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_variables\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mExtract_Data_ERA5BOX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExtractData\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_era_5\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdir_ERA5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mInput_DataBOX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 25\u001b[0m \u001b[1;31m# generate output data for waves\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\extract_data.py\u001b[0m in \u001b[0;36mget_era_5\u001b[1;34m(directory_path, input_data)\u001b[0m\n\u001b[0;32m 55\u001b[0m return data_extractor.extract_subset(\n\u001b[0;32m 56\u001b[0m \u001b[0mdirectory_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdirectory_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m \u001b[0minput_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 58\u001b[0m )\n\u001b[0;32m 59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\extract_data.py\u001b[0m in \u001b[0;36mextract_subset\u001b[1;34m(self, directory_path, input_data)\u001b[0m\n\u001b[0;32m 245\u001b[0m \u001b[0mvar_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvariable_key\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 246\u001b[0m value=self.__get_variable_subset(\n\u001b[1;32m--> 247\u001b[1;33m \u001b[0moutput_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_from_var_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvariable_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 248\u001b[0m \u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 249\u001b[0m \u001b[0minput_dataset\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36mget_from_var_dict\u001b[1;34m(self, var_name)\u001b[0m\n\u001b[0;32m 187\u001b[0m \u001b[0mAny\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;33m-\u001b[0m \u001b[0mValue\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mrequested\u001b[0m \u001b[0mvariable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 188\u001b[0m \"\"\"\n\u001b[1;32m--> 189\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__data_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_val_key\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 190\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 191\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mset_in_var_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvar_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mKeyError\u001b[0m: 'swh'" + "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 29\u001b[0m \u001b[0mdir_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34mr'C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 30\u001b[0m \u001b[0mbase_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'BOX'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m dataset_code='ERA5_WAVE')\n\u001b[0m", + "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36mgenerate_netcdf\u001b[1;34m(self, dir_path, base_name, dataset_code)\u001b[0m\n\u001b[0;32m 170\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_geo_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 171\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_dataset_variables\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_variables_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 173\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_global_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 174\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36m__set_variables_data\u001b[1;34m(self, netcdf, dataset_code)\u001b[0m\n\u001b[0;32m 597\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_gridded\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 598\u001b[0m \u001b[0mnetcdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 599\u001b[1;33m \u001b[0mvariables\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 600\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 601\u001b[0m \u001b[1;31m# When non-gridded we need to transpose the dimensions.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mnetCDF4\\_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable.__setitem__\u001b[1;34m()\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\SDToolBox_env\\lib\\site-packages\\netCDF4\\utils.py\u001b[0m in \u001b[0;36m_StartCountStride\u001b[1;34m(elem, shape, dimensions, grp, datashape, put, use_get_vars)\u001b[0m\n\u001b[0;32m 363\u001b[0m \u001b[0mfullslice\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 364\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mfullslice\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdatashape\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mput\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mhasunlim\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m \u001b[0mdatashape\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbroadcasted_shape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdatashape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 367\u001b[0m \u001b[1;31m# pad datashape with zeros for dimensions not being sliced (issue #906)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\SDToolBox_env\\lib\\site-packages\\netCDF4\\utils.py\u001b[0m in \u001b[0;36mbroadcasted_shape\u001b[1;34m(shp1, shp2)\u001b[0m\n\u001b[0;32m 971\u001b[0m \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mas_strided\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshp1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshp1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 972\u001b[0m \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mas_strided\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshp2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshp2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 973\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbroadcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;31mValueError\u001b[0m: shape mismatch: objects cannot be broadcast to a single shape" ] } ], "source": [ "# select a spatial range (box) for which data is to be extracted\n", - "steplon = 0.5 # degrees\n", - "lonl = -5 # left lon value\n", - "lonr = 11 # right lon value\n", - "steplat = 0.5\n", - "latl = 50 # lower lat value\n", - "latu = 62 # upper lat value\n", - "xrang = np.arange(lonl, lonr+steplon, steplon).tolist() \n", - "yrang = np.arange(latl, latu+steplat, steplat).tolist() \n", + "steplon = 1 # degrees\n", + "lonl = 3 # left lon value\n", + "lonr = 4 # right lon value\n", + "steplat = 1 # degrees\n", + "latl = 53 # lower lat value\n", + "latu = 54 # upper lat value\n", + "xrange = np.arange(lonl, lonr+steplon, steplon).tolist() \n", + "yrange = np.arange(latl, latu+steplat, steplat).tolist() \n", + "varBOX = ['swh', 'pp1d', 'mwd']\n", "\n", - "coordsBOX = {'LON': xrang, 'LAT': yrang}\n", + "coordsBOX = {'LON': xrange, 'LAT': yrange}\n", " \n", "# use the SDToolBox function to create input data\n", - "Input_DataBOX = ind.InputData(\n", + "Input_DataBOX= ind.InputData(\n", " input_coordinates=coordsBOX, \n", - " input_variables=['msl'], \n", + " input_variables=varBOX, \n", " input_scenarios=['era5'],\n", " input_years=timeWAMy, \n", " is_gridded = True)\n", @@ -406,7 +420,16 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['msl']\n", + "Extracting variable: msl_p, year 1986.\n" + ] + } + ], "source": [ "# select a spatial range (box) for which data is to be extracted\n", "steplon = 0.5 # degrees\n", @@ -421,10 +444,9 @@ "coordsBOX = {'LON': xrang, 'LAT': yrang}\n", " \n", "# use the SDToolBox function to create input data\n", - "del(Input_DataBOX) # to clear up the previous one that was defined\n", "Input_DataBOX = ind.InputData(\n", " input_coordinates=coordsBOX, \n", - " input_variables=['msl_p'], \n", + " input_variables=['msl'], \n", " input_scenarios=['era5'],\n", " input_years=timeWAMy, \n", " is_gridded = True)\n", Index: trunk/tests/testing_Etienne/SD_ToolBox_Testing.ipynb =================================================================== diff -u -r115 -r117 --- trunk/tests/testing_Etienne/SD_ToolBox_Testing.ipynb (.../SD_ToolBox_Testing.ipynb) (revision 115) +++ trunk/tests/testing_Etienne/SD_ToolBox_Testing.ipynb (.../SD_ToolBox_Testing.ipynb) (revision 117) @@ -140,6 +140,26 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [], + "source": [ + "# get the coordinates of the WAM locations\n", + "lonWAM = DS.variables['lon'][:] # list with longitude positions\n", + "latWAM = DS.variables['lat'][:] # list with latitude positions \n", + "\n", + "coordsWAM = {'LON': [], 'LAT': []}\n", + "for i, j in zip(lonWAM, latWAM):\n", + " coordsWAM['LON'].append(i)\n", + " coordsWAM['LAT'].append(j)\n", + "\n", + "# set other data variables \n", + "varWAM = [v.lower() for v in variables[5:]] # list with the five wave parameters -> it automatically only uses swh, pp1d, mwd\n", + "timeWAMy = [1986] # 1987 to 2018" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -149,26 +169,13 @@ "Extracting variable: Tp, year 1986.\n", "Extracting variable: MWD, year 1986.\n", "Writing C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets\\WAM_ERA5_WAVE.nc\n", - "writing variable pp1d\n", + "writing variable mwd\n", "writing variable swh\n", - "writing variable mwd\n" + "writing variable pp1d\n" ] } ], "source": [ - "# get the coordinates of the WAM locations\n", - "lonWAM = DS.variables['lon'][:] # list with longitude positions\n", - "latWAM = DS.variables['lat'][:] # list with latitude positions \n", - "\n", - "coordsWAM = {'LON': [], 'LAT': []}\n", - "for i, j in zip(lonWAM, latWAM):\n", - " coordsWAM['LON'].append(i)\n", - " coordsWAM['LAT'].append(j)\n", - "\n", - "# set other data variables \n", - "varWAM = [v.lower() for v in variables[5:]] # list with the five wave parameters -> ind automatically only uses swh, pp1d, mwd\n", - "timeWAMy = [1986] # 1987 to 2018\n", - "\n", "# use the SDToolBox function to create input data\n", "Input_DataWAM = ind.InputData(\n", " input_coordinates=coordsWAM,\n", @@ -189,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -244,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -266,7 +273,7 @@ "folder2 = r'C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets'\n", "\n", "# open the data in this script to get the info\n", - "DS_WAM_ERA5 = Dataset(os.path.join(folder2,'WAM_ERA5.nc'))\n", + "DS_WAM_ERA5 = Dataset(os.path.join(folder2,'WAM_ERA5_WAVE.nc'))\n", "\n", "# get the relevant data from the netcdf files\n", "time_WAM_ERA5 = list(DS_WAM_ERA5.variables['time'])\n", @@ -326,49 +333,56 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['msl']\n", - "Extracting variable: Hs, year 1986.\n" + "['swh', 'pp1d', 'mwd']\n", + "Extracting variable: Hs, year 1986.\n", + "Extracting variable: Tp, year 1986.\n", + "Extracting variable: MWD, year 1986.\n", + "Writing C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets\\BOX_ERA5_WAVE.nc\n", + "writing variable swh\n" ] }, { - "ename": "KeyError", - "evalue": "'swh'", + "ename": "ValueError", + "evalue": "shape mismatch: objects cannot be broadcast to a single shape", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;31m# use the SDToolBox function to extract data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mInput_DataBOX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_variables\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mExtract_Data_ERA5BOX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExtractData\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_era_5\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdir_ERA5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mInput_DataBOX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 25\u001b[0m \u001b[1;31m# generate output data for waves\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\extract_data.py\u001b[0m in \u001b[0;36mget_era_5\u001b[1;34m(directory_path, input_data)\u001b[0m\n\u001b[0;32m 55\u001b[0m return data_extractor.extract_subset(\n\u001b[0;32m 56\u001b[0m \u001b[0mdirectory_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdirectory_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m \u001b[0minput_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 58\u001b[0m )\n\u001b[0;32m 59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\extract_data.py\u001b[0m in \u001b[0;36mextract_subset\u001b[1;34m(self, directory_path, input_data)\u001b[0m\n\u001b[0;32m 245\u001b[0m \u001b[0mvar_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvariable_key\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 246\u001b[0m value=self.__get_variable_subset(\n\u001b[1;32m--> 247\u001b[1;33m \u001b[0moutput_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_from_var_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvariable_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 248\u001b[0m \u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 249\u001b[0m \u001b[0minput_dataset\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36mget_from_var_dict\u001b[1;34m(self, var_name)\u001b[0m\n\u001b[0;32m 187\u001b[0m \u001b[0mAny\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;33m-\u001b[0m \u001b[0mValue\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mrequested\u001b[0m \u001b[0mvariable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 188\u001b[0m \"\"\"\n\u001b[1;32m--> 189\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__data_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_val_key\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 190\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 191\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mset_in_var_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvar_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mKeyError\u001b[0m: 'swh'" + "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 29\u001b[0m \u001b[0mdir_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34mr'C:\\checkouts\\trunk\\tests\\testing_Etienne\\datasets'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 30\u001b[0m \u001b[0mbase_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'BOX'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m dataset_code='ERA5_WAVE')\n\u001b[0m", + "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36mgenerate_netcdf\u001b[1;34m(self, dir_path, base_name, dataset_code)\u001b[0m\n\u001b[0;32m 170\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_geo_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 171\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_dataset_variables\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_variables_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 173\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__set_global_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 174\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mC:\\checkouts\\trunk\\SDToolBox\\output_data.py\u001b[0m in \u001b[0;36m__set_variables_data\u001b[1;34m(self, netcdf, dataset_code)\u001b[0m\n\u001b[0;32m 597\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_gridded\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 598\u001b[0m \u001b[0mnetcdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 599\u001b[1;33m \u001b[0mvariables\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 600\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 601\u001b[0m \u001b[1;31m# When non-gridded we need to transpose the dimensions.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mnetCDF4\\_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable.__setitem__\u001b[1;34m()\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\SDToolBox_env\\lib\\site-packages\\netCDF4\\utils.py\u001b[0m in \u001b[0;36m_StartCountStride\u001b[1;34m(elem, shape, dimensions, grp, datashape, put, use_get_vars)\u001b[0m\n\u001b[0;32m 363\u001b[0m \u001b[0mfullslice\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 364\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mfullslice\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdatashape\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mput\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mhasunlim\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m \u001b[0mdatashape\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbroadcasted_shape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdatashape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 367\u001b[0m \u001b[1;31m# pad datashape with zeros for dimensions not being sliced (issue #906)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\envs\\SDToolBox_env\\lib\\site-packages\\netCDF4\\utils.py\u001b[0m in \u001b[0;36mbroadcasted_shape\u001b[1;34m(shp1, shp2)\u001b[0m\n\u001b[0;32m 971\u001b[0m \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mas_strided\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshp1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshp1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 972\u001b[0m \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mas_strided\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshp2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshp2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 973\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbroadcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;31mValueError\u001b[0m: shape mismatch: objects cannot be broadcast to a single shape" ] } ], "source": [ "# select a spatial range (box) for which data is to be extracted\n", - "steplon = 0.5 # degrees\n", - "lonl = -5 # left lon value\n", - "lonr = 11 # right lon value\n", - "steplat = 0.5\n", - "latl = 50 # lower lat value\n", - "latu = 62 # upper lat value\n", - "xrang = np.arange(lonl, lonr+steplon, steplon).tolist() \n", - "yrang = np.arange(latl, latu+steplat, steplat).tolist() \n", + "steplon = 1 # degrees\n", + "lonl = 3 # left lon value\n", + "lonr = 4 # right lon value\n", + "steplat = 1 # degrees\n", + "latl = 53 # lower lat value\n", + "latu = 54 # upper lat value\n", + "xrange = np.arange(lonl, lonr+steplon, steplon).tolist() \n", + "yrange = np.arange(latl, latu+steplat, steplat).tolist() \n", + "varBOX = ['swh', 'pp1d', 'mwd']\n", "\n", - "coordsBOX = {'LON': xrang, 'LAT': yrang}\n", + "coordsBOX = {'LON': xrange, 'LAT': yrange}\n", " \n", "# use the SDToolBox function to create input data\n", - "Input_DataBOX = ind.InputData(\n", + "Input_DataBOX= ind.InputData(\n", " input_coordinates=coordsBOX, \n", - " input_variables=['msl'], \n", + " input_variables=varBOX, \n", " input_scenarios=['era5'],\n", " input_years=timeWAMy, \n", " is_gridded = True)\n", @@ -406,7 +420,16 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['msl']\n", + "Extracting variable: msl_p, year 1986.\n" + ] + } + ], "source": [ "# select a spatial range (box) for which data is to be extracted\n", "steplon = 0.5 # degrees\n", @@ -421,10 +444,9 @@ "coordsBOX = {'LON': xrang, 'LAT': yrang}\n", " \n", "# use the SDToolBox function to create input data\n", - "del(Input_DataBOX) # to clear up the previous one that was defined\n", "Input_DataBOX = ind.InputData(\n", " input_coordinates=coordsBOX, \n", - " input_variables=['msl_p'], \n", + " input_variables=['msl'], \n", " input_scenarios=['era5'],\n", " input_years=timeWAMy, \n", " is_gridded = True)\n",