Skip to content

Commit

Permalink
deploy: 2e318bf
Browse files Browse the repository at this point in the history
  • Loading branch information
rcjackson committed May 17, 2024
1 parent c261de6 commit 0306cbc
Show file tree
Hide file tree
Showing 299 changed files with 1,719,304 additions and 0 deletions.
4 changes: 4 additions & 0 deletions _preview/41/.buildinfo
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 13793b38e4c882e659fc13f8e405345c
tags: 645f666f9bcd5a90fca523b33c5a78b7
734 changes: 734 additions & 0 deletions _preview/41/README.html

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
"""
This code stages LASSO-ShCu data files on Cumulus's wolf2 that are needed for the LASSO tutorial portion of the 2024 ARM Summer School.
It is assumed the LASSO-ShCu tar files are pre-copied into a single directory to draw from to build out an organized tree structure.
NOTE: Make sure to set the correct folder locations at the top of main().
Author: William.Gustafson@pnnl.gov
Date: 6-May-2024
"""

from glob import glob
import os
import tarfile


#-----------------------------------------------------------------------
def stage_cogsdiagobsmod(tarfilename, path_stage):
"""
Stage tar file of type sgplassocogsdiagobsmod*tar.
This tar contain info for all sim IDs within the given case date and
already has the case date in the tar file. So, it's untar path starts
at the very top of the tree structure.
Example: ./sgplassocogsdiagobsmodC1.m1.20190404.120000.tar
0123456789012345678901234567890
987654321098765432109876543210987654321
"""

# Untar the source tar into the destination folder...
fh = tarfile.open(tarfilename, mode='r')
fh.extractall(path=path_stage)
fh.close()

# end stage_cogsdiagobsmod()


#-----------------------------------------------------------------------
def stage_confobsmod(tarfilename, path_stage):
"""
Stage tar file of type sgplassoconfobsmod*tar.
This tar does not contain the parent directory information, so we
have to build it.
Example: ./sgplassodiagconfobsmod4C1.m1.20190404.000000.tar
0123456789012345678901234567890
987654321098765432109876543210987654321
"""

# Determine path for untarring and create it if it does not exist...
aa = os.path.basename(tarfilename) # strip off path
datestr = aa[-19:-11] # date comes from the end of the filename
sim_id = int( os.path.basename(aa)[22:(aa.index("C1.m1"))] ) # sim_id is the character(s) prior to C1.m1
path_untar = f"{path_stage}/{datestr}/sim{sim_id:04d}"
os.makedirs(path_untar, exist_ok=True)

# Untar the source tar into the destination folder...
fh = tarfile.open(tarfilename, mode='r')
fh.extractall(path=path_untar)
fh.close()
# end stage_confobsmod()


#-----------------------------------------------------------------------
def stage_diagraw(tarfilename, path_stage):
"""
Stage tar file of type sgplassodiagraw*.tar.
This tar does not contain the parent directory information, so we
have to build it.
Example: ./sgplassodiagraw1C1.m1.20190506.000000.tar
0123456789012345678901234567890
987654321098765432109876543210987654321
"""

# Determine path for untarring and create it if it does not exist...
aa = os.path.basename(tarfilename) # strip off path
datestr = aa[-19:-11] # date comes from the end of the filename
sim_id = int( os.path.basename(aa)[15:(aa.index("C1.m1"))] ) # sim_id is the character(s) prior to C1.m1
path_untar = f"{path_stage}/{datestr}/sim{sim_id:04d}"
os.makedirs(path_untar, exist_ok=True)

# Untar the source tar into the destination folder...
fh = tarfile.open(tarfilename, mode='r')
fh.extractall(path=path_untar)
fh.close()
# end stage_diagraw()


#-----------------------------------------------------------------------
def stage_highfreqobs(tarfilename, path_stage):
"""
Stage tar file of type sgphighfreqobs*tar.
This tar contain observation data organized by folder with each tar
containing the obs for a single date. These get untarred into a
high_res_obs folder inside the case date.
Example: ./sgplassohighfreqobsC1.m1.20190404.120000.tar
0123456789012345678901234567890
987654321098765432109876543210987654321
"""
# Determine path for untarring and create it if it does not exist...
aa = os.path.basename(tarfilename) # strip off path
datestr = aa[-19:-11] # date comes from the end of the filename
path_untar = f"{path_stage}/{datestr}/high_res_obs"
os.makedirs(path_untar, exist_ok=True)

# Untar the source tar into the destination folder...
fh = tarfile.open(tarfilename, mode='r')
fh.extractall(path=path_untar)
fh.close()
# end stage_highfreqobs()



#-----------------------------------------------------------------------
def main():
path_tars = "/gpfs/wolf2/arm/atm124/world-shared/arm-summer-school-2024/lasso_tutorial/ShCu/tars"
path_stage = "/gpfs/wolf2/arm/atm124/world-shared/arm-summer-school-2024/lasso_tutorial/ShCu/untar"

# Get the list of tar files to process and loop over them...
tar_list = glob(f"{path_tars}/*.tar")
for tarfilename in tar_list:
# Parse the datastream type and call the appropriate subroutine to untar it...
# File type options:
# sgplassocogsdiagobsmodC1.m1.20190404.120000.tar
# sgplassodiagconfobsmod4C1.m1.20190404.000000.tar
# sgplassodiagraw5C1.m1.20190404.000000.tar
# sgplassohighfreqobsC1.c1.20190404.000000.tar
# 0 1 2 3
# 0123456789012345678901234567890
# 0123456789012345678901234567890
aa = os.path.basename(tarfilename)
if not aa[0:8] == "sgplasso":
# not a tar we care about, so skip it
continue
# end if
aa = aa[8:] # strip off "sgplasso", should now start with the keys we are looking for

# Call the appropriate staging routine for each tar type to
# account for the folder structure in each type...
if aa[:14] == "diagconfobsmod":
stage_confobsmod(tarfilename, path_stage)

elif aa[:7] == "diagraw":
stage_diagraw(tarfilename, path_stage)

elif aa[:14] == "cogsdiagobsmod":
stage_cogsdiagobsmod(tarfilename, path_stage)

elif aa[:11] == "highfreqobs":
stage_highfreqobs(tarfilename, path_stage)
# end if
# end main()


#-----------------------------------------------------------------------
if __name__ == "__main__":
main()
Binary file not shown.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added _preview/41/_images/Juliano-headshot.jpg
1 change: 1 addition & 0 deletions _preview/41/_images/ProjectPythia_Logo_Final-01-Blue.svg
Binary file added _preview/41/_images/aerosol_sizing.png
Binary file added _preview/41/_images/arm_logo.png
Binary file added _preview/41/_images/campusmap-page-1.png
Binary file added _preview/41/_images/campusmap-page-2.png
Binary file added _preview/41/_images/collis-headshot.jpg
Binary file added _preview/41/_images/feldman-headshot.jpeg
Binary file added _preview/41/_images/feng-headshot.png
Binary file added _preview/41/_images/fridlind-headshot.jpg
Binary file added _preview/41/_images/giansiracusa-headshot.jpg
Binary file added _preview/41/_images/goswami-headshot.jpg
Binary file added _preview/41/_images/grover-headshot.jpeg
Binary file added _preview/41/_images/gustafson-headshot.jpg
Binary file added _preview/41/_images/jackson-headshot.png
Binary file added _preview/41/_images/li-headshot.JPG
Binary file added _preview/41/_images/obrien-headshot.png
Binary file added _preview/41/_images/zhang-headshot.jpg
86 changes: 86 additions & 0 deletions _preview/41/_sources/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# ARM Summer School 2024

***ARM Summer School 2024: Open Science in the Department of Energy's Atmospheric Radiation Measurement (ARM) User Facility: Connecting State-of-the-Art Models with Diverse Field Campaign Observations***


[![badge](https://img.shields.io/static/v1.svg?logo=Jupyter&label=ARM+JupyterHub&message=ACE+Environment&color=blue)](https://jupyterhub.arm.gov/hub/user-redirect/git-pull?repo=https%3A//github.com/ARM-Development/arm-ams-short-course-2024&urlpath=lab/tree/arm-ams-short-course-2024/notebooks&branch=main)
[![nightly-build](https://github.com/ARM-Development/arm-ams-short-course-2024/actions/workflows/nightly-build.yaml/badge.svg)](https://github.com/ARM-Development/arm-ams-short-course-2024/actions/workflows/nightly-build.yaml)

## Motivation

ARM Mobile Facilities (AMF) have traveled to locations all over the world, including South America for the Cloud, Aerosol, and Complex Terrain Interactions (CACTI) field campaign, as well as Norway for the Cold-Air Outbreaks in the Marine Boundary Layer Experiment (COMBLE). One of the key goals of these field deployments is to integrate measurements with a spectrum of model datasets, ranging from high-resolution large-eddy simulation to limited-domain nested weather and climate model datasets, furthering the understanding of Earth’s climate system. Within this tutorial, participants will gain a broad understanding of the ARM User Facility, the open data available to the community, the data workbench that allows data-proximate-computing, and science overviews of two ongoing model-observation intercomparison projects. This will be suitable for a broad audience of atmospheric scientists, bringing together both observations and simulations, as well as deep and shallow convection.

This summer school, aimed at a broad audience, will:
1. Introduce participants to ARM’s observational facilities and data products and the community of atmospheric scientists that use and produce ARM data.
2. Educate attendees on ARM’s measurement suite and data archive.
3. Educate attendees on ARM’s (and collaborators') model data.
4. Highlight the underlying science behind CACTI and COMBLE.
5. Demonstrate how to find and access ARM data.
6. Using open source tools, guide attendees in analyzing ARM’s open data in the Python programming language.
7. Highlight several techniques to compare ARM observations and high-resolution model output.


## Authors

ARM Summer School 2024 Instructors

### Contributors

<a href="https://github.com/ProjectPythia/cookbook-template/graphs/contributors">
<img src="https://contrib.rocks/image?repo=ARM-Development/arm-ams-short-course-2024" />
</a>

## Structure

To familiarize attendees with ARM, its measurements and data discovery systems.
1. To highlight the breadth of measurements and science possible through the AMF deployments for CACTI and COMBLE.
2. To demonstrate a series of analytical methods using open science cookbooks and ARM data, with a focus on robustly comparing observational data with high-resolution simulations.
3. To provide an onramp to open science using ARM data and to remove barriers to using ARM data.
4. To train attendees on the latest features of ARM tools and open HPC platforms.

### Lectures

### Small Group Projects

## Running the Notebooks

You can either run the notebook using [Binder](https://binder.projectpythia.org/) or on your local machine.

### Running on Jupyter

The simplest way to interact with a Jupyter Notebook is through the
[ARM Jupyter](https://jupyterhub.arm.gov), which enables the execution of a
[Jupyter Book](https://jupyterbook.org) on ARM infrastructure. The details of how this works are not
important for now. Navigate your mouse to
the top right corner of the book chapter you are viewing and click
on the rocket ship icon, (see figure below), and be sure to select
“launch Jupyterhub”. After a moment you should be presented with a
notebook that you can interact with. I.e. you’ll be able to execute
and even change the example programs. You’ll see that the code cells
have no output at first, until you execute them by pressing
{kbd}`Shift`\+{kbd}`Enter`. Complete details on how to interact with
a live Jupyter notebook are described in [Getting Started with
Jupyter](https://foundations.projectpythia.org/foundations/getting-started-jupyter.html).

### Running on Your Own Machine
If you are interested in running this material locally on your computer, you will need to follow this workflow:

1. Clone the `https://github.com/ARM-Development/arm-summer-school-2024` repository:

```bash
git clone https://github.com/ARM-Development/arm-summer-school-2024
```
1. Move into the `arm-summer-school-2024` directory
```bash
cd arm-summer-school-2024
```
1. Create and activate your conda environment from the `environment.yml` file
```bash
conda env create -f environment.yml
conda activate arm-summer-school-2024-dev
```
1. Move into the `notebooks` directory and start up Jupyterlab
```bash
cd notebooks/
jupyter lab
```
Loading

0 comments on commit 0306cbc

Please sign in to comment.