feat(weather): Update AMY components to parse new NOAA format

This commit updates the AMY component to parse the new NOAA files from https://www.ncei.noaa.gov/access/search/data-search/global-hourly

dragonfly_grasshopper/json/DF_Import_NOAA_File.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.1.1", 
+  "version": "0.2.0", 
   "nickname": "ImportNOAA", 
   "outputs": [
     [
@@ -72,14 +72,14 @@
     {
       "access": "item", 
       "name": "_noaa_file", 
-      "description": "The path to a .txt file of annual data obtained from the NOAA\ndatabase on your system as a string.", 
+      "description": "The path to a .csv file of annual data obtained from the NOAA\ndatabase on your system as a string.", 
       "type": "string", 
       "default": null
     }, 
     {
       "access": "item", 
       "name": "_timestep_", 
-      "description": "The timestep at which the data collections should be output.\nDefault is 1 but this can be set as high as 60 to ensure that all data\nfrom the .txt file is imported.", 
+      "description": "The timestep at which the data collections should be output.\nDefault is 1 but this can be set as high as 60 to ensure that all data\nfrom the .csv file is imported.", 
       "type": "int", 
       "default": null
     }, 
@@ -92,8 +92,8 @@
     }
   ], 
   "subcategory": "4 :: AlternativeWeather", 
-  "code": "\nimport os\nimport csv\n\ntry:\n    from ladybug.dt import DateTime\n    from ladybug.analysisperiod import AnalysisPeriod\n    from ladybug.header import Header\n    from ladybug.datacollection import HourlyDiscontinuousCollection, HourlyContinuousCollection\n    from ladybug.datatype.temperature import DryBulbTemperature, DewPointTemperature\n    from ladybug.datatype.speed import WindSpeed\n    from ladybug.datatype.angle import WindDirection\n    from ladybug.datatype.fraction import TotalSkyCover\n    from ladybug.datatype.pressure import AtmosphericStationPressure\n    from ladybug.datatype.distance import Visibility, CeilingHeight\n    from ladybug.datatype.generic import GenericType\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug:\\n\\t{}'.format(e))\n\ntry:\n    from ladybug_{{cad}}.{{plugin}} import all_required_inputs\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug_{{cad}}:\\n\\t{}'.format(e))\n\n\n# dictionary that converts from sky cover codes to tenths of sky cover.\nsky_codes_dict = {\n    'CLR': 0,\n    'SCT': 3,\n    'BKN': 7,\n    'OVC': 10,\n    'OBS': 10,\n    'POB': 5\n    }\n\n\ndef build_collection(values, dates, data_type, unit):\n    \"\"\"Build a data collection from raw noaa data and process it to the timestep.\"\"\"\n    \n    if values == []:\n        return None\n    \n    # convert date codes into datetimes.\n    datetimes = [DateTime(int(dat[4:6]), int(dat[6:8]), int(dat[8:10]),\n                 int(dat[10:12])) for dat in dates]\n    \n    # make a discontinuous cata collection\n    data_header = Header(data_type, unit, AnalysisPeriod())\n    data_init = HourlyDiscontinuousCollection(data_header, values, datetimes)\n    data_final = data_init.validate_analysis_period()\n    \n    # cull out unwanted timesteps.\n    if _timestep_:\n        data_final.convert_to_culled_timestep(_timestep_)\n    else:\n        data_final.convert_to_culled_timestep(1)\n    \n    return data_final\n\n\nif all_required_inputs(ghenv.Component) and _run:\n    # check that the file exists.\n    assert os.path.isfile(_noaa_file), 'Cannot find file at {}.'.format(_noaa_file)\n    \n    # empty lists to be filled with data\n    all_years = []\n    header_txt = []\n    db_t = []\n    db_t_dates = []\n    dp_t = []\n    dp_t_dates = []\n    ws = []\n    ws_dates = []\n    wd = []\n    wd_dates = []\n    sc = []\n    sc_dates = []\n    ap = []\n    ap_dates = []\n    slp = []\n    slp_dates = []\n    vis = []\n    vis_dates = []\n    ceil = []\n    ceil_dates = []\n    \n    # pull relevant data out of the file\n    with open(_noaa_file) as csv_file:\n        csv_reader = csv.reader(csv_file, delimiter=' ', skipinitialspace=True)\n        next(csv_reader)  # Skip header row.\n        for row in csv_reader:\n            all_years.append(int(row[2][:4]))\n            if row[3] != '***':\n                wd.append(float(row[3]))\n                wd_dates.append(row[2])\n            if row[4] != '***':\n                ws.append(float(row[4]))\n                ws_dates.append(row[2])\n            if row[6] != '***':\n                ceil.append(float(row[6]))\n                ceil_dates.append(row[2])\n            if row[7] != '***':\n                sc.append(row[7])\n                sc_dates.append(row[2])\n            if row[11] != '****':\n                vis.append(float(row[11]))\n                vis_dates.append(row[2])\n            if row[21] != '****':\n                db_t.append(float(row[21]))\n                db_t_dates.append(row[2])\n            if row[22] != '****':\n                dp_t.append(float(row[22]))\n                dp_t_dates.append(row[2])\n            if row[25] != '******':\n                ap.append(float(row[25]))\n                ap_dates.append(row[2])\n            if row[23] != '******':\n                slp.append(float(row[23]))\n                slp_dates.append(row[2])\n    \n    # check that all years in the file are the same.\n    yr1 = all_years[0]\n    for yr in all_years:\n        assert yr == yr1, 'Not all of the data in the file is from the same ' \\\n            'year. {} != {}'.format(yr1, yr)\n    data_header = Header(GenericType('Years', 'yr'), 'yr', AnalysisPeriod())\n    model_year = HourlyContinuousCollection(data_header, [yr1] * 8760)\n    \n    # perform conversions\n    sc = [sky_codes_dict[cov] for cov in sc]  # sky cover codes to values\n    ceil = [c * 100 for c in ceil]  # hundreds of feet to feet\n    \n    # build data collections from the imported values\n    dry_bulb_temp = build_collection(db_t, db_t_dates, DryBulbTemperature(), 'F')\n    dew_point_temp = build_collection(dp_t, dp_t_dates, DewPointTemperature(), 'F')\n    wind_speed = build_collection(ws, ws_dates, WindSpeed(), 'mph')\n    wind_direction = build_collection(wd, wd_dates, WindDirection(), 'degrees')\n    total_sky_cover = build_collection(sc, sc_dates, TotalSkyCover(), 'tenths')\n    visibility = build_collection(vis, vis_dates, Visibility(), 'mi')\n    ceiling_height = build_collection(ceil, ceil_dates, CeilingHeight(), 'ft')\n    \n    # deal with available atmospheric pressure data\n    if ap != []:\n        ap = [press / 1000 for press in ap]  # pressure mbar to bar\n        atmos_pressure = build_collection(ap, ap_dates, AtmosphericStationPressure(), 'bar')\n    else:\n        slp = [press / 1000 for press in slp]  # pressure mbar to bar\n        atmos_pressure = build_collection(slp, slp_dates, AtmosphericStationPressure(), 'bar')\n    \n    # convert all units to SI.\n    if dry_bulb_temp is not None:\n        dry_bulb_temp.convert_to_unit('C')\n    if dew_point_temp is not None:\n        dew_point_temp.convert_to_unit('C')\n    if wind_speed is not None:\n        wind_speed.convert_to_unit('m/s')\n    if visibility is not None:\n        visibility.convert_to_unit('km')\n    if ceiling_height is not None:\n        ceiling_height.convert_to_unit('m')\n    if atmos_pressure is not None:\n        atmos_pressure.convert_to_unit('Pa')", 
+  "code": "\nimport os\nimport csv\n\ntry:\n    from ladybug.dt import DateTime\n    from ladybug.analysisperiod import AnalysisPeriod\n    from ladybug.header import Header\n    from ladybug.datacollection import HourlyDiscontinuousCollection, HourlyContinuousCollection\n    from ladybug.datatype.temperature import DryBulbTemperature, DewPointTemperature\n    from ladybug.datatype.speed import WindSpeed\n    from ladybug.datatype.angle import WindDirection\n    from ladybug.datatype.fraction import TotalSkyCover\n    from ladybug.datatype.pressure import AtmosphericStationPressure\n    from ladybug.datatype.distance import Visibility, CeilingHeight\n    from ladybug.datatype.generic import GenericType\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug:\\n\\t{}'.format(e))\n\ntry:\n    from ladybug_{{cad}}.{{plugin}} import all_required_inputs\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug_{{cad}}:\\n\\t{}'.format(e))\n\n\ndef build_collection(values, dates, data_type, unit):\n    \"\"\"Build a data collection from raw noaa data and process it to the timestep.\"\"\"\n    if values == []:\n        return None\n    \n    # convert date codes into datetimes.\n    datetimes = [DateTime(int(dat[5:7]), int(dat[8:10]), int(dat[11:13]),\n                 int(dat[14:16])) for dat in dates]\n    \n    # make a discontinuous cata collection\n    data_header = Header(data_type, unit, AnalysisPeriod())\n    data_init = HourlyDiscontinuousCollection(data_header, values, datetimes)\n    data_final = data_init.validate_analysis_period()\n    \n    # cull out unwanted timesteps.\n    if _timestep_:\n        data_final.convert_to_culled_timestep(_timestep_)\n    else:\n        data_final.convert_to_culled_timestep(1)\n    \n    return data_final\n\n\nif all_required_inputs(ghenv.Component) and _run:\n    # check that the file exists.\n    assert os.path.isfile(_noaa_file), 'Cannot find file at {}.'.format(_noaa_file)\n\n    # empty lists to be filled with data\n    all_years = []\n    header_txt = []\n    db_t = []\n    db_t_dates = []\n    dp_t = []\n    dp_t_dates = []\n    ws = []\n    ws_dates = []\n    wd = []\n    wd_dates = []\n    sc = []\n    sc_dates = []\n    slp = []\n    slp_dates = []\n    vis = []\n    vis_dates = []\n    ceil = []\n    ceil_dates = []\n\n    # pull relevant data out of the file\n    with open(_noaa_file) as csv_file:\n        csv_reader = csv.reader(csv_file, delimiter=',', skipinitialspace=True)\n\n        # find the column with total sky cover if it exists\n        header = csv_reader.next()  # get header row\n        sc_col = None\n        for i, colname in enumerate(header):\n            if colname == 'GF1':\n                sc_col = i\n\n        for row in csv_reader:\n            # parse the dates and the years\n            date_row = row[1]\n            all_years.append(int(date_row[:4]))\n\n            # parse the wind information\n            wind_info = row[10].split(',')\n            if wind_info[0] != '999':\n                wd.append(float(wind_info[0]))\n                wd_dates.append(date_row)\n            if wind_info[3] != '9999':\n                ws.append(float(wind_info[3]) / 10)\n                ws_dates.append(date_row)\n\n            # parse the ceiling height information\n            ceil_info = row[11].split(',')\n            if ceil_info[0] != '99999':\n                ceil.append(float(ceil_info[0]))\n                ceil_dates.append(date_row)\n\n            # parse the visibility information\n            vis_info = row[12].split(',')\n            if vis_info[0] != '999999':\n                vis.append(float(vis_info[0]) / 1000)\n                vis_dates.append(date_row)\n\n            # parse the dry bulb and dew point information\n            temp_info = row[13].split(',')\n            if temp_info[0] != '+9999':\n                db_t.append(float(temp_info[0]) / 10)\n                db_t_dates.append(date_row)\n            dwpt_info = row[14].split(',')\n            if dwpt_info[0] != '+9999':\n                dp_t.append(float(dwpt_info[0]) / 10)\n                dp_t_dates.append(date_row)\n\n            # parse the pressure information\n            slp_info = row[15].split(',')\n            if slp_info[0] != '99999':\n                slp.append(float(slp_info[0]) * 10)\n                slp_dates.append(date_row)\n\n            # parse the sky cover info if it exists\n            if sc_col is not None and row[sc_col] != '':\n                sc_info = row[sc_col].split(',')\n                sc_oktas = int(sc_info[0])\n                sc_tenths = sc_oktas * (10 / 8) if sc_oktas != 9 else 10\n                sc.append(sc_tenths)\n                sc_dates.append(date_row)\n\n    # check that all years in the file are the same.\n    yr1 = all_years[0]\n    for yr in all_years:\n        assert yr == yr1, 'Not all of the data in the file is from the same ' \\\n            'year. {} != {}'.format(yr1, yr)\n    data_header = Header(GenericType('Years', 'yr'), 'yr', AnalysisPeriod())\n    model_year = HourlyContinuousCollection(data_header, [yr1] * 8760)\n\n    # build data collections from the imported values\n    dry_bulb_temp = build_collection(db_t, db_t_dates, DryBulbTemperature(), 'C')\n    dew_point_temp = build_collection(dp_t, dp_t_dates, DewPointTemperature(), 'C')\n    wind_speed = build_collection(ws, ws_dates, WindSpeed(), 'm/s')\n    wind_direction = build_collection(wd, wd_dates, WindDirection(), 'degrees')\n    ceiling_height = build_collection(ceil, ceil_dates, CeilingHeight(), 'm')\n    visibility = build_collection(vis, vis_dates, Visibility(), 'km')\n    atmos_pressure = build_collection(slp, slp_dates, AtmosphericStationPressure(), 'bar')\n    total_sky_cover = build_collection(sc, sc_dates, TotalSkyCover(), 'tenths')\n", 
   "category": "Dragonfly", 
   "name": "DF Import NOAA File", 
-  "description": "Import climate data from a .txt file of annual data obtained from the National\nOceanic and Atmospheric Administration (NOAA) database.  The database can be\naccessed here:\nhttps://gis.ncdc.noaa.gov/maps/ncei/cdo/hourly\n-"
+  "description": "Import climate data from a .csv file of annual data obtained from the National\nOceanic and Atmospheric Administration (NOAA) database.  The database can be\naccessed here:\nhttps://gis.ncdc.noaa.gov/maps/ncei/cdo/hourly\n-"
 }

dragonfly_grasshopper/json/DF_Import_NOAA_Staion_Location.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.1.1", 
+  "version": "0.2.0", 
   "nickname": "ImportStaion", 
   "outputs": [
     [
@@ -16,7 +16,7 @@
     {
       "access": "item", 
       "name": "_station_file", 
-      "description": "The path to a .txt file of NOAA station location data\non your system as a string.", 
+      "description": "The path to a .csv file of NOAA station location data\non your system as a string.", 
       "type": "string", 
       "default": null
     }, 
@@ -29,8 +29,8 @@
     }
   ], 
   "subcategory": "4 :: AlternativeWeather", 
-  "code": "\nimport os\n\ntry:\n    from ladybug.datatype.distance import Distance\n    from ladybug.location import Location\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug:\\n\\t{}'.format(e))\n\ntry:\n    from ladybug_{{cad}}.{{plugin}} import all_required_inputs\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug_{{cad}}:\\n\\t{}'.format(e))\n\n\nif all_required_inputs(ghenv.Component):\n    # check that the file exists.\n    assert os.path.isfile(_station_file), 'Cannot find file at {}.'.format(_station_file)\n    \n    with open(_station_file) as station_file:\n        station_file.readline()  # Skip header row\n        \n        # get the pattern of data within the file\n        char_pattern = station_file.readline().strip().split(' ')\n        data_line = station_file.readline()\n        data_list = []\n        total_char = 0\n        for pattern in char_pattern:\n            data_list.append(data_line[total_char:total_char + len(pattern)])\n            total_char += len(pattern) + 1\n        \n        # parse all of the info from the file\n        station_id, wban_id = data_list[0].split(' ')\n        station_name = data_list[1].strip()\n        country = data_list[2].strip()\n        state = data_list[3].strip()\n        latitude = float(data_list[4])\n        longitude = float(data_list[5])\n        elevation = float(data_list[6])\n        \n        elevation = Distance().to_unit([elevation], 'm', 'ft')[0]  # convert to meters\n        \n        # estimate or parse time zone.\n        if time_zone_:\n            assert -12 <= time_zone_ <= 12, ' time_zone_ must be between -12 and '\\\n                ' 12. Got {}.'.format(time_zone_)\n            time_zone = time_zone_\n        else:\n            time_zone = int((longitude / 180) * 12)\n        \n        # build the location object\n        location = Location(city=station_name,\n                            state=state,\n                            country=country,\n                            latitude=latitude,\n                            longitude=longitude,\n                            time_zone=time_zone,\n                            elevation=elevation,\n                            station_id=station_id,\n                            source='NCDC')", 
+  "code": "\nimport os\n\ntry:\n    from ladybug.datatype.distance import Distance\n    from ladybug.location import Location\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug:\\n\\t{}'.format(e))\n\ntry:\n    from ladybug_{{cad}}.{{plugin}} import all_required_inputs\nexcept ImportError as e:\n    raise ImportError('\\nFailed to import ladybug_{{cad}}:\\n\\t{}'.format(e))\n\n\nif all_required_inputs(ghenv.Component):\n    # check that the file exists.\n    assert os.path.isfile(_station_file), 'Cannot find file at {}.'.format(_station_file)\n    \n    with open(_station_file) as station_file:\n        station_file.readline()  # Skip header row\n\n        # get the pattern of data within the file\n        dat_line = station_file.readline().strip().split(',')\n\n        # parse all of the info from the file\n        station_id = dat_line[0].replace('\"', '')\n        city = dat_line[6].replace('\"', '')\n        latitude = float(dat_line[3].replace('\"', ''))\n        longitude = float(dat_line[4].replace('\"', ''))\n        elevation = float(dat_line[5].replace('\"', ''))\n        elevation = Distance().to_unit([elevation], 'm', 'ft')[0]  # convert to meters\n\n        # estimate or parse time zone.\n        if time_zone_:\n            assert -12 <= time_zone_ <= 14, ' time_zone_ must be between -12 and '\\\n                ' 14. Got {}.'.format(time_zone_)\n            time_zone = time_zone_\n        else:\n            time_zone = int((longitude / 180) * 12)\n\n        # build the location object\n        location = Location(\n            city=city, latitude=latitude, longitude=longitude,\n            time_zone=time_zone, elevation=elevation,\n            station_id=station_id, source='NCDC')\n", 
   "category": "Dragonfly", 
   "name": "DF Import NOAA Staion Location", 
-  "description": "Import station location from a .txt file of station information obtained from the\nNational Oceanic and Atmospheric Administration (NOAA) database.  The database can\nbe accessed here:\nhttps://gis.ncdc.noaa.gov/maps/ncei/cdo/hourly\n-"
+  "description": "Import station location from a .csv file of station information obtained from the\nNational Oceanic and Atmospheric Administration (NOAA) database.  The database can\nbe accessed here:\nhttps://gis.ncdc.noaa.gov/maps/ncei/cdo/hourly\n-"
 }