Skip to content

Commit cfc43e8

Browse files
kafitzgeraldkafitzgerald
authored
Merge pull request #3 from aashishpanta0/test-encoding
Add notebooks from NCAR Global Hackathon
2 parents fa6dc43 + 15c2e6c commit cfc43e8

File tree

6 files changed

+4720
-6
lines changed

6 files changed

+4720
-6
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ share/python-wheels/
3030
.installed.cfg
3131
*.egg
3232
MANIFEST
33-
33+
**.DS_Store
3434
# PyInstaller
3535
# Usually these files are written by a python script from a template
3636
# before PyInstaller builds the exe, so as to inject date/other infos into it.

‎CITATION.cff‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ cff-version: 1.2.0
22
message: "If you use this cookbook, please cite it as below."
33
authors:
44
# add additional entries for each author -- see https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md
5-
- family-names: Last
6-
given-names: First
7-
orcid: https://orcid.org/xxxx-xxxx-xxxx-xxxx # optional
5+
- family-names: Panta
6+
given-names: Aashish
7+
orcid: https://orcid.org/0009-0001-5843-2098
88
website: URL # optional
9-
affiliation: Affiliation # optional
9+
affiliation: University of Utah
1010
- family-names: Last 2
1111
given-names: First 2
1212
orcid: https://orcid.org/yyyy-yyyy-yyyy-yyyy
@@ -15,4 +15,4 @@ authors:
1515
- name: "Cloud Microphysics Cookbook contributors" # use the 'name' field to acknowledge organizations
1616
website: "https://github.com/ProjectPythia/land-atmosphere-interactions-and-hydrology-cookbook/graphs/contributors"
1717
title: "Land-Atmosphere and Hydrology Applications Cookbook"
18-
abstract: "The cookbook description."
18+
abstract: "The cookbook description."

‎notebooks/healpix_wrf_conus_wind.ipynb‎

Lines changed: 3107 additions & 0 deletions
Large diffs are not rendered by default.

‎notebooks/read_visualize_wind_idx.ipynb‎

Lines changed: 1396 additions & 0 deletions
Large diffs are not rendered by default.

‎notebooks/reencode_wrf_conus.ipynb‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "653324a3-e972-46d1-a80e-57bcf64c8a7a",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import uxarray as ux\n",
11+
"import matplotlib.pyplot as plt\n",
12+
"import cartopy.crs as ccrs\n",
13+
"import cartopy.feature as cfeature\n",
14+
"import os\n",
15+
"import OpenVisus as ov\n",
16+
"import xarray as xr\n",
17+
"from tqdm import tqdm\n",
18+
"import time\n",
19+
"from sys import getsizeof"
20+
]
21+
},
22+
{
23+
"cell_type": "code",
24+
"execution_count": null,
25+
"id": "1c1b063e-b8fb-49ba-b1bb-7b0883307ed6",
26+
"metadata": {},
27+
"outputs": [],
28+
"source": [
29+
"# !pip install OpenVisus openvisuspy boto3 xmltodict panel bokeh\n",
30+
"# !pip install tqdm"
31+
]
32+
},
33+
{
34+
"cell_type": "code",
35+
"execution_count": null,
36+
"id": "41d3f40d-6771-457b-b67a-6ffae1627420",
37+
"metadata": {},
38+
"outputs": [],
39+
"source": [
40+
"main_dir='/glade/derecho/scratch/digital-earths-hackathon/conus404/native_grid'\n",
41+
"home_dir='/glade/u/home/dpanta'\n",
42+
"idx_dir='/glade/u/home/dpanta/idx'"
43+
]
44+
},
45+
{
46+
"cell_type": "code",
47+
"execution_count": null,
48+
"id": "b3b2a300-7496-4945-8932-06f8058a0b40",
49+
"metadata": {},
50+
"outputs": [],
51+
"source": [
52+
"st_file='/glade/derecho/scratch/digital-earths-hackathon/conus404/native_grid/2020-10_eastward_wind.nc'\n",
53+
"\n",
54+
"\n",
55+
"files=[\n",
56+
"'2020-12_eastward_wind.nc',\n",
57+
"'2020-12_northward_wind.nc',\n",
58+
"'2020-12_precipitation_flux.nc',\n",
59+
"'2020-12_specific_humidity.nc',\n",
60+
"'2020-12_surface_air_pressure.nc',\n",
61+
"'2020-12_surface_temperature.nc',\n",
62+
"'2020-12_toa_outgoing_longwave_flux.nc',\n",
63+
"'2020-12_toa_outgoing_shortwave_flux.nc',\n",
64+
"]"
65+
]
66+
},
67+
{
68+
"cell_type": "code",
69+
"execution_count": null,
70+
"id": "f5a5464a-4ab9-4844-a5f7-9449437e1d70",
71+
"metadata": {
72+
"scrolled": true
73+
},
74+
"outputs": [],
75+
"source": [
76+
"ds=xr.open_dataset(st_file)\n",
77+
"ds"
78+
]
79+
},
80+
{
81+
"cell_type": "code",
82+
"execution_count": null,
83+
"id": "fd8061d0-ae23-416b-b20a-e947b61e8673",
84+
"metadata": {},
85+
"outputs": [],
86+
"source": [
87+
"import datetime\n",
88+
"\n",
89+
"idx_filename=f\"/glade/derecho/scratch/dpanta/idx/conus404_all.idx\"\n",
90+
"fields=['eastward_wind','northward_wind','precipitation_flux','specific_humidity','surface_air_pressure','surface_temperature','toa_outgoing_longwave_flux', 'toa_outgoing_shortwave_flux']\n",
91+
"arco=\"4mb\"\n",
92+
"fld_idx=[]\n",
93+
"for fld in fields:\n",
94+
" print(f'Setting Field::: {fld}')\n",
95+
" st_file=f'/glade/derecho/scratch/digital-earths-hackathon/conus404/native_grid/2020-10_{fld}.nc'\n",
96+
"\n",
97+
" ds = xr.open_dataset(st_file, group=\"/\", mask_and_scale=False) \n",
98+
" data=ds[f\"{fld}\"][0,:,:].values\n",
99+
" if int(len(data.shape))>1:\n",
100+
" \n",
101+
" print(f\"{fld}::::{data.shape}\")\n",
102+
" try:\n",
103+
" vmin,vmax=np.min(data),np.max(data)\n",
104+
" except:\n",
105+
" vmin,vmax=0,0\n",
106+
" fld=ov.Field.fromString(f\"\"\"{fld} {str(data.dtype)} format(row_major) min({vmin}) max({vmax})\"\"\")\n",
107+
" fld_idx.append(fld)"
108+
]
109+
},
110+
{
111+
"cell_type": "code",
112+
"execution_count": null,
113+
"id": "e86db120-d1bb-4ad6-b987-b7db41d8bb45",
114+
"metadata": {},
115+
"outputs": [],
116+
"source": [
117+
"T=744\n",
118+
"W,H=data.shape\n"
119+
]
120+
},
121+
{
122+
"cell_type": "code",
123+
"execution_count": null,
124+
"id": "f9f092b3-d793-40c4-872b-dbef1daea331",
125+
"metadata": {},
126+
"outputs": [],
127+
"source": [
128+
"print('Creating IDX Metadata file now...')\n",
129+
"db=ov.CreateIdx(\n",
130+
" url=idx_filename, \n",
131+
"\tdims=[H,W], \n",
132+
"\tfields=fld_idx, \n",
133+
"\tcompression=\"raw\", \n",
134+
"\tarco=arco,\n",
135+
"\ttime=[0,T,\"time_%d/\"])\n",
136+
"print('Created...')\n",
137+
"\n",
138+
"# db=ov.LoadDataset(idx_filename)"
139+
]
140+
},
141+
{
142+
"cell_type": "code",
143+
"execution_count": null,
144+
"id": "89c82d00-1c6a-4271-9c67-1c4da16cc900",
145+
"metadata": {},
146+
"outputs": [],
147+
"source": [
148+
"print('Writing to idx...')\n",
149+
"for i in range(len(fields)):\n",
150+
" print(f'Writing field {fields[i]}...')\n",
151+
" st_file=f'/glade/derecho/scratch/digital-earths-hackathon/conus404/native_grid/2020-10_{fields[i]}.nc'\n",
152+
" ds = xr.open_dataset(st_file, group=\"/\", mask_and_scale=False) \n",
153+
" data=ds[f\"{fields[i]}\"][...].values\n",
154+
" for t in range(data.shape[0]):\n",
155+
" data_to_write=data[t,:,:]\n",
156+
" try:\n",
157+
" db.write(data_to_write,time=t,field=fields[i])\n",
158+
" except:\n",
159+
" pass\n",
160+
"print('Writing data completed...')\n",
161+
"print('Compressing data now...')\n",
162+
"db.compressDataset(['zip'])\n",
163+
"print('Data compressed!!!')"
164+
]
165+
},
166+
{
167+
"cell_type": "code",
168+
"execution_count": null,
169+
"id": "640d1cf3-6f2f-46a3-8630-474f7225dc3b",
170+
"metadata": {},
171+
"outputs": [],
172+
"source": []
173+
}
174+
],
175+
"metadata": {
176+
"kernelspec": {
177+
"display_name": "Python 3 (ipykernel)",
178+
"language": "python",
179+
"name": "python3"
180+
},
181+
"language_info": {
182+
"codemirror_mode": {
183+
"name": "ipython",
184+
"version": 3
185+
},
186+
"file_extension": ".py",
187+
"mimetype": "text/x-python",
188+
"name": "python",
189+
"nbconvert_exporter": "python",
190+
"pygments_lexer": "ipython3",
191+
"version": "3.10.13"
192+
}
193+
},
194+
"nbformat": 4,
195+
"nbformat_minor": 5
196+
}

‎notebooks/setup_dashboard.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
PORT=8989
2+
git clone --single-branch -b test_region_extraction https://github.com/sci-visus/openvisuspy.git
3+
cd openvisuspy
4+
git checkout b2c63396ea3c7f2ab32efa3bbe464dd2918543d6
5+
python -m venv .venv
6+
source .venv/bin/activate
7+
python -m pip install --verbose --no-cache --no-warn-script-location boto3 colorcet fsspec numpy imageio pympler==1.0.1 urllib3 pillow xarray xmltodict plotly requests scikit-image scipy seaborn tifffile pandas tqdm matplotlib zarr altair cartopy dash fastparquet folium geodatasets geopandas geoviews lxml numexpr scikit-learn sqlalchemy statsmodels vega_datasets xlrd yfinance pyarrow pydeck h5py hdf5plugin netcdf4 nexpy nexusformat nbgitpuller intake ipysheet ipywidgets bokeh ipywidgets-bokeh panel holoviews hvplot datashader vtk pyvista trame trame-vtk trame-vuetify notebook "jupyterlab==3.6.6" jupyter_bokeh jupyter-server-proxy jupyterlab-system-monitor "pyviz_comms>=2.0.0,<3.0.0" "jupyterlab-pygments>=0.2.0,<0.3.0"
8+
python -m pip install OpenVisus
9+
python -m pip install -U --force-reinstall scipy
10+
export VISUS_CACHE="./"
11+
export BOKEH_ALLOW_WS_ORIGIN="*"
12+
export BOKEH_RESOURCES="cdn"
13+
export PYTHONPATH=$PWD/src
14+
15+
panel serve ./src/openvisuspy/dashboards --allow-websocket-origin='*' --address=0.0.0.0 --port $PORT --args '/glade/derecho/scratch/dpanta/idx/conus404_all.idx'

0 commit comments

Comments
 (0)