diff --git a/ph5/core/experiment_pn3.py b/ph5/core/experiment_pn3.py new file mode 100755 index 00000000..d866b572 --- /dev/null +++ b/ph5/core/experiment_pn3.py @@ -0,0 +1,1586 @@ +#!/usr/bin/env pnpython3 +# +# Core functionality to manipulate ph5 files +# +# Steve Azevedo, August 2006 +# + +import tables +import numpy +import os +import os.path +import time +import sys +import logging +import string +import re +from ph5.core import columns +try: + import importlib.reload as reload +except ImportError: + pass + +PROG_VERSION = '2021.160' +LOGGER = logging.getLogger(__name__) +ZLIBCOMP = 6 + +os.environ['TZ'] = 'UTM' +time.tzset() + +externalLinkRE = re.compile(".*ExternalLink.*") + + +class HDF5InteractionError (Exception): + def __init__(self, errno, msg): + self.args = (errno, msg) + self.errno = errno + self.msg = msg + + +class MapsGroup: + ''' /Experiment_g/Maps_g + /Das_g_[nnnn] + /Hdr_a_[nnnn] + /Sta_g_[nnnn] + /Evt_g_[nnnn] + */Guides_g_[nnnn] + */Guide_t + */Fn_a_[nnnn] + */Blush_t + *out_s + *sub_s + *n_i + * -> Not implemented. + + Un-tested as of December 4 2013. + See JSON format: + { + "name":"Hdr_a_" { + "properties": { + "FileType":"SAC"|"SEG-Y"|"MSEED"|"SEG-D"|"SEG-2" {, + "type":"string", + "description":"Type of originating file", + "required": True, + }, + "HeaderType":"reel"|"trace"{, + "type":"string", + "description":"The header type", + "required":True, + }, + "HeaderSubType":"iNova"|"Menlo"|"SEG"|"PASSCAL" {, + "type":"string", + "description":"Currently extended header type", + "required":False + }, + }, + "properties": {, + "key:value": {, + "type": "any", + "description": "varies", + "required": True + }, + } + } + } + ''' + + def __init__(self, ph5): + self.ph5 = ph5 + self.current_g_das = None + self.arrayRE = re.compile(r"([H]\w+_a_)(\d+)") + self.groupRE = re.compile(r"([DSE]\w+_g_)(\d+)") + self.ph5_t_index = None + + def initgroup(self): + try: + # Create Maps group + self.ph5_g_maps = initialize_group(self.ph5, + '/Experiment_g', + 'Maps_g') + # Create Index table + self.ph5_t_index = initialize_table(self.ph5, + '/Experiment_g/Maps_g', + 'Index_t', + columns.Index) + columns.add_reference( + '/Experiment_g/Maps_g/Index_t', self.ph5_t_index) + except tables.FileModeError: + pass + + def nuke_index_t(self): + self.ph5_t_index.remove() + self.initgroup() + + def setcurrent(self, g): + # If this is an external link it needs to be redirected. + if externalLinkRE.match(g.__str__()): + g = g() + + self.current_g_das = g + + def getdas_g(self, sn): + ''' Return group for a given serial number ''' + sn = 'Das_g_' + sn + self.current_g_das = None + for g in self.ph5.iter_nodes('/Experiment_g/Maps_g'): + if g._v_name == sn: + self.setcurrent(g) + return g + + return None + + def newdas(self, what, sn): + # Create a new group for a DAS, Station, or Event + choices = ['Das_g_', 'Stn_g_', 'Evt_g_'] + if what in choices: + sn = what + sn + else: + return None + # Create the das group + d = initialize_group(self.ph5, '/Experiment_g/Maps_g', sn) + + self.current_g_das = d + + return d + + def get_array_nodes(self, name): + ''' Find array nodes based on name prefix ''' + return get_nodes_by_name(self.ph5, + '/Experiment_g/Maps_g', + re.compile(name + r'(\d+)'), None) + + def writeheader(self, hdr_json_list, desc=None): + nxt = self.nextarray('Hdr_a_') + a = self.newearray(nxt, desc) + a.append(hdr_json_list) + + a.flush() + + def nextarray(self, prefix): + ns = 0 + name = self.current_g_das._v_name + if name in columns.LAST_ARRAY_NODE_MAPS and\ + prefix in columns.LAST_ARRAY_NODE_MAPS[ + name]: + mo = self.arrayRE.match(columns.LAST_ARRAY_NODE_MAPS[name][prefix]) + cprefix, an = mo.groups() + nombre = "%s%04d" % (prefix, int(an) + 1) + else: + for n in self.ph5.iter_nodes( + self.current_g_das, classname='Array'): + mo = self.arrayRE.match(n._v_name) + if not mo: + continue + cprefix, an = mo.groups() + if cprefix == prefix: + if int(an) > ns: + ns = int(an) + + nombre = "%s%04d" % (prefix, ns + 1) + + columns.add_last_array_node_maps(self.current_g_das, prefix, nombre) + + return nombre + + def newearray(self, name, description=None): + batom = tables.StringAtom(itemsize=40) + a = create_empty_earray(self.ph5, + self.current_g_das, + name, + batom=batom, + expectedrows=2000) + + if description is not None: + a.attrs.description = description + + return a + + def read_index(self): + ''' Read Index table ''' + ret, keys = read_table(self.ph5_t_index) + + return ret, keys + + def read_hdr(self): + ''' Read Hdr text arrays ''' + ret = {} + arrays = self.get_array_nodes('Hdr_a_') + keys = arrays.keys() + for k in keys: + name = arrays[k]._v_name + ret[name] = arrays[k].read() + + return ret + + def populateIndex_t(self, p, key=None): + required_keys = ['serial_number_s', 'external_file_name_s'] + populate_table(self.ph5_t_index, + p, + key, + required_keys) + + self.ph5.flush() +# +# Mixins refactor here Dec 11 +# + + +class SortsGroup: + ''' /Experiment_g/Sorts_g + /Sort_t #groups data by time and location + /Array_t_[array] + # columns.Array, groups data by location + /Offset_t(_[array]_[shotline]) + # columns.Offset, source to receiver offsets + /Event_t(_[shotline]) + # columns.Event, list of events + ''' + + def __init__(self, ph5): + self.ph5 = ph5 + self.ph5_g_sorts = None + self.ph5_t_sort = None + self.ph5_t_array = {} # Local cached nodes keyed on Array_t_xxx + self.ph5_t_offset = {} # Local cached nodes keyed on Offset_t_aaa_sss + self.ph5_t_event = {} # Local cached nodes keyed on Event_t_xxx + self.Array_tRE = re.compile(r"Array_t_(\d+)") + self.Event_tRE = re.compile(r"Event_t(_(\d+))?") + self.Offset_tRE = re.compile(r"Offset_t(_(\d+)_(\d+))?") + + def update_local_table_nodes(self): + ''' + Cache node references to Array_t_xxx, Event_t_xxx, + Offset_t_aaa_sss, tables in this group + ''' + names = columns.TABLES.keys() + for n in names: + name = os.path.basename(n) + mo = self.Array_tRE.match(name) + if mo: + self.ph5_t_array[name] = columns.TABLES[n] + continue + mo = self.Event_tRE.match(name) + if mo: + self.ph5_t_event[name] = columns.TABLES[n] + continue + mo = self.Offset_tRE.match(name) + if mo: + self.ph5_t_offset[name] = columns.TABLES[n] + + def read_offset(self, offset_name=None): + if offset_name is None and 'Offset_t' in self.ph5_t_offset: + # Legacy naming + ret, keys = read_table(self.ph5_t_offset['Offset_t']) + else: + try: + node = self.ph5_t_offset[offset_name] + except KeyError: + node = self.ph5.get_node( + '/Experiment_g/Sorts_g', + name=offset_name, + classname='Table') + self.ph5_t_offset[offset_name] = node + + ret, keys = read_table(node) + + return ret, keys + + def read_events(self, event_name=None): + if event_name is None and 'Event_t' in self.ph5_t_event: + # Legacy + ret, keys = read_table(self.ph5_t_event['Event_t']) + else: + try: + node = self.ph5_t_event[event_name] + except KeyError: + node = self.ph5.get_node( + '/Experiment_g/Sorts_g', + name=event_name, + classname='Table') + self.ph5_t_event[event_name] = node + + ret, keys = read_table(node) + + return ret, keys + + def read_sorts(self): + ret, keys = read_table(self.ph5_t_sort) + + return ret, keys + + def read_arrays(self, array_name, ignore_srm=False): + try: + node = self.ph5_t_array[array_name] + except KeyError: + node = self.ph5.get_node( + '/Experiment_g/Sorts_g', + name=array_name, + classname='Table') + self.ph5_t_array[array_name] = node + except IndexError: + node = self.ph5.get_node( + '/Experiment_g/Sorts_g', + name=array_name, + classname='Table') + self.ph5_t_array = {} + self.ph5_t_array[array_name] = node + + ret, keys = read_table(node) + return ret, keys + + def index_offset_table(self, name='Offset_t', level=9, weight='full'): + ''' Index offset table on event and station id_s + Inputs: + name -> offset table name, Offset_t_002_003 + level -> level of optimization, 0-9 + weight -> kind of index, + 'ultralight', 'light', 'medium', 'full'. + ''' + try: + self.ph5_t_offset[name].cols.event_id_s.create_index( + optlevel=level, kind=weight) + except (ValueError, + tables.exceptions.NodeError, + tables.exceptions.FileModeError): + pass + + try: + self.ph5_t_offset[name].cols.receiver_id_s.create_index( + optlevel=level, kind=weight) + except (ValueError, + tables.exceptions.NodeError, + tables.exceptions.FileModeError): + pass + + def read_offset_fast(self, shot, station, name=None): + if name is None and 'Offset_t' in self.ph5_t_offset: + # Legacy + name = 'Offset_t' + + ret = {} + query = "(event_id_s == b'{0}') & (receiver_id_s == b'{1}')".format( + shot, station) + for row in self.ph5_t_offset[name].where(query): + ret['offset/value_d'] = row['offset/value_d'] + ret['offset/units_s'] = row['offset/units_s'] + ret['azimuth/value_f'] = row['azimuth/value_f'] + ret['azimuth/units_s'] = row['azimuth/units_s'] + ret['event_id_s'] = str(shot) + ret['receiver_id_s'] = str(station) + + return ret + + def read_offsets(self, shotrange=None, stations=None, name='Offset_t'): + offsets = [] + + if stations is not None: + stations = map(int, stations) + stations = map(str, stations) + + keys, names = columns.keys(self.ph5_t_offset[name]) + + for row in self.ph5_t_offset[name].iterrows(): + if shotrange is not None: + try: + shot = int(row['event_id_s']) + except BaseException: + LOGGER.warning("Non-numeric event in Offset_t." + "Event: {0} Station: {1}." + .format(row['event_id_s'], + row['receiver_id_s'])) + continue + + if shot > shotrange[1]: + continue + + if not (shot >= shotrange[0] and shot <= shotrange[1]): + continue + + if stations is not None: + try: + station = str(int(row['receiver_id_s'])) + except BaseException: + LOGGER.warning("Non-numeric station in Offset_t " + "Event_t : {0} Station: {1}." + .format(row['event_id_s'], + row['receiver_id_s'])) + continue + + if station not in stations: + continue + + r = {} + for k in keys: + r[k] = row[k] + + offsets.append(r) + + return offsets, keys + + def newOffsetSort(self, name): + o = initialize_table(self.ph5, + '/Experiment_g/Sorts_g', + name, + columns.Offset) + + self.ph5_t_offset[name] = o + + columns.add_reference('/Experiment_g/Sorts_g/' + + name, self.ph5_t_offset[name]) + + return o + + def newEventSort(self, name): + e = initialize_table(self.ph5, + '/Experiment_g/Sorts_g', + name, + columns.Event) + + self.ph5_t_event = e + + columns.add_reference('/Experiment_g/Sorts_g/' + + name, self.ph5_t_event) + + return e + + def newArraySort(self, name): + ''' Names should be 000 - 999 + Array_t_xxx + ''' + # Create Array table + a = initialize_table(self.ph5, + '/Experiment_g/Sorts_g', + name, + columns.Array) + + self.ph5_t_array = a + + columns.add_reference('/Experiment_g/Sorts_g/' + + name, self.ph5_t_array) + + return a + + def NewSort(self, name): + return self.newArraySort(name) + + def nextName(self): + names = [] + names.append(0) + for n in self.ph5.walk_nodes( + '/Experiment_g/Sorts_g', classname='Table'): + mo = self.Array_tRE.match(n._v_name) + if not mo: + continue + + name = int(mo.groups()[0]) + names.append(name) + + names.sort() + s = "Array_t_%03d" % (names[-1] + 1) + + return s + + def namesRE(self, re): + ''' Sorts_g table names by RE ''' + names = [] + for n in self.ph5.walk_nodes( + '/Experiment_g/Sorts_g', classname='Table'): + if re.match(n._v_name): + names.append(n._v_name) + + return names + + def namesArray_t(self): + names = self.ph5_t_array.keys() + if len(names) == 0: + names = self.namesRE(self.Array_tRE) + + return names + + def names(self): + return self.namesArray_t() + + def namesEvent_t(self): + names = self.ph5_t_event.keys() + if len(names) == 0: + names = self.namesRE(self.Event_tRE) + + return names + + def namesOffset_t(self): + names = self.ph5_t_offset.keys() + if len(names) == 0: + names = self.namesRE(self.Offset_tRE) + + return names + + def populate(self, ref, p, key=[]): + populate_table(ref, p, key) + + ref.flush() + + def populateSort_t(self, p, pkey=[]): + self.populate(self.ph5_t_sort, p, pkey) + + def populateArray_t(self, p, pkey=[], name=None): + self.populate(self.ph5_t_array[name], p, pkey) + + def populateEvent_t(self, p, pkey=[], name='Event_t'): + self.populate(self.ph5_t_event[name], p, pkey) + + def populateOffset_t(self, p, pkey=[], name='Offset_t'): + self.populate(self.ph5_t_offset[name], p, pkey) + + def initgroup(self): + # Create Sorts group + self.ph5_g_sorts = initialize_group( + self.ph5, '/Experiment_g', 'Sorts_g') + + # Create Sort table + self.ph5_t_sort = initialize_table(self.ph5, + '/Experiment_g/Sorts_g', + 'Sort_t', + columns.Sort) + + columns.add_reference('/Experiment_g/Sorts_g/Sort_t', self.ph5_t_sort) + + # Reference Offset table(s) *** See Offset_tRE *** + offsets = get_nodes_by_name(self.ph5, + '/Experiment_g/Sorts_g', + self.Offset_tRE, + 'Table') # Should this be Table or Array? + keys = offsets.keys() + for k in keys: + nombre = offsets[k]._v_name + columns.add_reference( + '/Experiment_g/Sorts_g/' + nombre, offsets[k]) + + # Reference Event table *** See Event_tRE *** + events = get_nodes_by_name(self.ph5, + '/Experiment_g/Sorts_g', + self.Event_tRE, + 'Table') # Should this be Table or Array? + keys = events.keys() + for k in keys: + nombre = events[k]._v_name + columns.add_reference('/Experiment_g/Sorts_g/' + nombre, events[k]) + + # Find any Attay_t_[nnn] + arrays = get_nodes_by_name(self.ph5, + '/Experiment_g/Sorts_g', + self.Array_tRE, + 'Table') # Should this be Table or Array? + keys = arrays.keys() + for k in keys: + nombre = arrays[k]._v_name + columns.add_reference('/Experiment_g/Sorts_g/' + nombre, arrays[k]) + + self.update_local_table_nodes() + + def nuke_array_t(self, n): + nombre = "Array_t_{0:03d}".format(n) + allArray_t = self.names() + + if nombre in allArray_t: + n = self.ph5.get_node('/Experiment_g/Sorts_g', + name=nombre, + classname='Table') + n.remove() + return True + else: + return False + + def nuke_event_t(self, name='Event_t'): + try: + self.ph5_t_event[name].remove() + self.initgroup() + return True + except Exception: + return False + + def nuke_sort_t(self): + try: + self.ph5_t_sort.remove() + self.initgroup() + return True + except Exception: + return False + + def nuke_offset_t(self, name='Offset_t'): + # Remove the indexes before removing the table + try: + self.ph5_t_offset[name].cols.event_id_s.remove_index() + except Exception: + pass + + try: + self.ph5_t_offset[name].cols.receiver_id_s.remove_index() + except Exception: + pass + # Remove cruft left from failed indexing + cruftRE = re.compile(".*value_d") + nodes = get_nodes_by_name( + self.ph5, '/Experiment_g/Sorts_g/', cruftRE, None) + for k in nodes.keys(): + try: + nodes[k].remove() + except Exception: + pass + # Remove the offset table + try: + self.ph5_t_offset[name].remove() + self.initgroup() + return True + except Exception: + return False + + +class Data_Trace (object): + __slots__ = ("das", "epoch", "length", "channel", + "data_trace", "receiver", "keys") + + +class ReceiversGroup: + ''' /Experiment_g/Receivers_g/Das_g_[sn]#Data for this DAS in this group + /Das_t # columns.Data + /Data_a_[n] + #A 1 x n array containing + the data for an event + /SOH_a_[n] + # State of health data, + usually a list of strings + /Event_a_[n] + # Event table, usually a + list of strings + /Log_a_[n] + # Log channel + /Experiment_g/Receivers_g + /Receiver_t # columns.Receiver + /Time_t # columns.Time + /Index_t # columns.Index + ''' + + def __init__(self, ph5): + self.ph5 = ph5 + self.ph5_g_receivers = None # Receivers group + self.current_g_das = None # Current das group + self.current_t_das = None # Current das table + self.ph5_t_receiver = None # Current receiver table + self.ph5_t_time = None # Current time table + self.ph5_t_index = None # + # Match arrays under Das_g_[sn] + self.arrayRE = re.compile(r"([DSEL]\w+_a_)(\d+)") + self.dasRE = re.compile(r"Das_g_(.+)") # Match Das_g groups + self.byteorder = None # Trace atom byte order + self.elementtype = None # atom type"int","float",or"undetermined" + + def get_das_name(self): + ''' Return the current das name ''' + if self.current_g_das is None: + return None + + name = self.current_g_das._v_name.split('_') + + return name[-1] + + def read_das(self, ignore_srm=False): + ''' Read DAS table ''' + def cmp_epoch(a, b): + return int(a['time/epoch_l']) - int(b['time/epoch_l']) + + ret, keys = read_table(self.current_t_das) + if ret is not None: + ret.sort(cmp=cmp_epoch) + else: + return [], keys + return ret, keys + + def read_receiver(self): + ''' Read Receiver table ''' + ret, keys = read_table(self.ph5_t_receiver) + + return ret, keys + + def read_time(self): + ''' Read Time table ''' + ret, keys = read_table(self.ph5_t_time) + + return ret, keys + + def read_index(self): + ''' Read Index table ''' + ret = None + keys = None + if not self.ph5.__contains__('/Experiment_g/Receivers_g/Index_t'): + return ret, keys + + ret, keys = read_table(self.ph5_t_index) + + return ret, keys + + def get_array_nodes(self, name): + ''' Find array nodes based on name prefix ''' + arrays = get_nodes_by_name(self.ph5, + '/Experiment_g/Receivers_g', + re.compile(name + r"(\d+)"), + 'Array') + + return arrays + + def read_soh(self): + ''' Read SOH text arrays ''' + ret = {} + try: + self.current_g_das._v_name + except AttributeError: + return ret + + arrays = get_nodes_by_name(self.ph5, + self.current_g_das, + re.compile('SOH_a_' + r"(\d+)"), + 'Array') + keys = arrays.keys() + for k in keys: + name = arrays[k]._v_name + ret[name] = arrays[k].read() + + return ret + + def read_event(self): + ''' Read Event text arrays ''' + ret = {} + arrays = self.get_array_nodes('Event_a_') + keys = arrays.keys() + for k in keys: + name = arrays[k]._v_name + ret[name] = arrays[k].read() + + return ret + + def read_log(self): + ''' Read Log text arrays ''' + ret = {} + arrays = self.get_array_nodes('Log_a_') + keys = arrays.keys() + for k in keys: + name = arrays[k]._v_name + ret[name] = arrays[k].read() + + return ret + + def trace_info(self, trace_ref): + try: + s = repr(trace_ref.atom) + s = s.split("(")[0] + if s == "Int32Atom": + t = "int" + elif s == "Float32Atom": + t = "float" + else: + t = "undetermined" + except Exception as e: + LOGGER.debug("Unable to get trace element type.\n{0}" + .format(e.message)) + t = "undetermined" + + return t, trace_ref.byteorder + + def read_trace(self, trace_ref, start=None, stop=None): + ''' Read data trace ''' + if start is None and stop is None: + data = trace_ref.read() + else: + data = trace_ref.read(start=start, stop=stop) + return data + + def find_trace_ref(self, name): + try: + node = self.ph5.get_node( + self.current_g_das, name=name, classname='Array') + except Exception: + node = None + + return node + + def find_traces(self, epoch=None): + traces = [] # List of data_trace (above) + das_dict = {} # Keyed on DAS points to traces list + receiver_t = [] + + x = 0 + # Get all Das_g + for g in self.ph5.iter_nodes('/Experiment_g/Receivers_g'): + traces = [] + # Not sure why it returns its own name??? + if g._v_name == 'Receivers_g': + continue + + # Get Das_t + t = g.Das_t + + # Get Receiver_t + tr = g.Receiver_t + tkeys, names = columns.keys(tr) + # XXX This is a kludge! XXX + for receiver in tr: + receiver_t.append(receiver) + + for r in t.iterrows(): + i = r['receiver_table_n_i'] + e = float(r['time/epoch_l']) + \ + float(r['time/micro_seconds_i']) / 1000000.0 + sps = r['sample_rate_i'] + a = r['array_name_data_a'] + an = self.ph5.get_node( + '/Experiment_g/Receivers_g/' + g._v_name + '/' + a) + n = an.nrows + # Length of trace in seconds + ll = float(n) / float(sps) + # Epoch of last sample + s = e + ll + # Does epoch fall in trace? epoch == None flag to get all + if (epoch >= e and epoch <= s) or epoch is None: + # Get an instance of our Data_Trace (structure?) + dt = Data_Trace() + dt.das = g._v_name[6:] + dt.epoch = e + dt.length = ll + dt.channel = r['channel_number_i'] + dt.data_trace = an + # Just return receiver table row for this das + dt.receiver = receiver_t[i] + dt.keys = tkeys + traces.append(dt) + + x = x + 1 + # No epoch so key on x + if epoch is None: + das_dict[str(x)] = traces + # We have an epoch so key on das + else: + das_dict[dt.das] = traces + + return das_dict + + def setcurrent(self, g): + ''' + ''' + # If this is an external link it needs to be redirected. + if externalLinkRE.match(g.__str__()): + try: + if self.ph5.mode == 'r': + g = g() + else: + g = g(mode='a') + except tables.exceptions.NoSuchNodeError: + self.current_g_das = None + self.current_t_das = None + return + + self.current_g_das = g + self.current_t_das = g.Das_t + + def getdas_g(self, sn): + ''' Return group for a given serial number ''' + sn = 'Das_g_' + sn + self.current_g_das = None + try: + g = self.ph5.get_node(self.ph5_g_receivers, name=sn) + self.current_g_das = g + except Exception: + return None + + return self.current_g_das + + def alldas_g(self): + dasGroups = get_nodes_by_name(self.ph5, + '/Experiment_g/Receivers_g', + re.compile('Das_g_' + r'(\w+)'), + None) + + return dasGroups + + def init_Das_t(self, sn): + sn = 'Das_g_' + sn + t = initialize_table(self.ph5, + '/Experiment_g/Receivers_g/' + sn, + 'Das_t', + columns.Data, expectedrows=1000) + return t + + # New das group and tables + + def newdas(self, sn): + t = None + sn = 'Das_g_' + sn + # Create the das group + d = initialize_group(self.ph5, + '/Experiment_g/Receivers_g', + sn) + + t = initialize_table(self.ph5, + '/Experiment_g/Receivers_g/' + sn, + 'Das_t', + columns.Data, expectedrows=1000) + + self.current_g_das = d + self.current_t_das = t + columns.add_reference('/Experiment_g/Receivers_g/' + + sn + '/Das_t', self.current_t_das) + + return d, t, self.ph5_t_receiver, self.ph5_t_time + + def nextarray(self, prefix): + ns = 0 + name = self.current_g_das._v_name + if name in columns.LAST_ARRAY_NODE_DAS and\ + prefix in columns.LAST_ARRAY_NODE_DAS[ + name]: + mo = self.arrayRE.match(columns.LAST_ARRAY_NODE_DAS[name][prefix]) + cprefix, an = mo.groups() + nombre = "%s%04d" % (prefix, int(an) + 1) + else: + for n in self.ph5.iter_nodes( + self.current_g_das, classname='Array'): + mo = self.arrayRE.match(n._v_name) + if not mo: + continue + cprefix, an = mo.groups() + if cprefix == prefix: + if int(an) > ns: + ns = int(an) + + nombre = "%s%04d" % (prefix, ns + 1) + + columns.add_last_array_node_das(self.current_g_das, prefix, nombre) + + return nombre + + def newearray(self, name, description=None, expectedrows=None): + batom = tables.StringAtom(itemsize=80) + a = create_empty_earray(self.ph5, + self.current_g_das, + name, + batom=batom, + expectedrows=expectedrows) + + if description is not None: + a.attrs.description = description + + return a + + def newdataearray(self, name, data, batom=None, rows=None): + a = create_data_earray(self.ph5, + self.current_g_das, + name, + data, + batom, + rows=rows) + + return a + + def newarray(self, name, data, dtype=None, description=None): + ''' + name is name of array as follows: + Data_a_[event_number] --- Numarray array + SOH_a_[n] --- State of health (python list) + Event_a_[n] --- Event table (python list) + Log_a_[n] --- Generic log channel (python list) + + inputs: name --- name of array + data --- data to place in array + description --- description of array + + returns: tables array descriptor + ''' + # If this is a data array convert it to a numarray.array + prefix, body1, suffix = string.split(name, '_') + "_".join([prefix, body1]) + '_' + if prefix == 'Data': + if dtype is None: + dtype = 'i' + + if type(data) != numpy.ndarray: + data = numpy.fromiter(data, dtype=dtype) + + if self.current_g_das is not None: + try: + self.ph5.remove_node(self.current_g_das, name=name) + LOGGER.warning("Node {0} exists. Overwritten." + .format(name)) + except Exception: + pass + + if dtype == 'int32': + a = self.newdataearray(name, data, batom=tables.Int32Atom()) + elif dtype == 'float32': + a = self.newdataearray(name, data, batom=tables.Float32Atom()) + else: + a = self.ph5.create_array(self.current_g_das, name, data) + + if description is not None: + a.attrs.description = description + + return a + + def populateDas_t(self, p, key=None): + required_keys = ['time/epoch_l', + 'channel_number_i', 'array_name_data_a'] + + populate_table(self.current_t_das, p, key, required_keys) + + self.ph5.flush() + + def populateReceiver_t(self, p, key=None): + required_keys = [] + + populate_table(self.current_t_receiver, p, key, required_keys) + + self.ph5.flush() + + def populateTime_t(self, p, key=None): + required_keys = ['das/serial_number_s', 'start_time/epoch_l', + 'end_time/epoch_l', 'offset_l', 'slope_d'] + + populate_table(self.current_t_time, p, key, required_keys) + + self.ph5.flush() + + def populateTime_t_(self, p, key=None): + required_keys = ['das/serial_number_s', 'start_time/epoch_l', + 'end_time/epoch_l', 'offset_d', 'slope_d'] + + populate_table(self.ph5_t_time, p, key, required_keys) + + self.ph5.flush() + + def populateIndex_t(self, p, key=None): + required_keys = ['serial_number_s', 'external_file_name_s'] + + populate_table(self.ph5_t_index, p, key, required_keys) + + self.ph5.flush() + + def indexIndex_t(self): + ''' Set up indexing on DAS SN and external mini filename ''' + try: + self.ph5_t_index.cols.serial_number_s.create_csindex() + except (ValueError, + tables.exceptions.NodeError, + tables.exceptions.FileModeError): + pass + try: + self.ph5_t_index.cols.external_file_name_s.create_csindex() + except (ValueError, + tables.exceptions.NodeError, + tables.exceptions.FileModeError): + pass + + def initgroup(self): + # Create receivers group + self.ph5_g_receivers = initialize_group(self.ph5, + '/Experiment_g', + 'Receivers_g') + # Create receivers table + self.ph5_t_receiver = initialize_table(self.ph5, + '/Experiment_g/Receivers_g', + 'Receiver_t', + columns.Receiver, + expectedrows=1) + # Create time table + self.ph5_t_time = initialize_table(self.ph5, + '/Experiment_g/Receivers_g', + 'Time_t', + columns.Time) + # Create index table + self.ph5_t_index = initialize_table(self.ph5, + '/Experiment_g/Receivers_g', + 'Index_t', + columns.Index) + self.ph5_t_index.expectedrows = 1000000 + + columns.add_reference( + '/Experiment_g/Receivers_g/Receiver_t', self.ph5_t_receiver) + columns.add_reference( + '/Experiment_g/Receivers_g/Time_t', self.ph5_t_time) + columns.add_reference( + '/Experiment_g/Receivers_g/Index_t', self.ph5_t_index) + + def nuke_index_t(self): + self.ph5_t_index.remove() + self.initgroup() + + def nuke_receiver_t(self): + self.ph5_t_receiver.remove() + self.initgroup() + + def nuke_time_t(self): + self.ph5_t_time.remove() + self.initgroup() + + def nuke_das_t(self, das): + g = self.getdas_g(das) + if not g: + return False + self.setcurrent(g) + self.current_t_das.truncate(0) + return True + + +class ReportsGroup: + ''' /Experiment_g/Reports_g # Group to hold experiment reports + /Report_t # Report table, columns.Report + /Report_a_[title]# The report in pdf format + ''' + + def __init__(self, ph5): + self.ph5 = ph5 + self.ph5_g_reports = None + self.ph5_t_report = None + self.Report_aRE = re.compile(r"Report_a_(\d\d\d)") + + def read_reports(self): + ret, keys = read_table(self.ph5_t_report) + + return ret, keys + + def nextName(self): + report_array_nodes = get_nodes_by_name(self.ph5, + '/Experiment_g/Reports_g', + self.Report_aRE, + 'Array') + keys = sorted(report_array_nodes.keys()) + try: + n = int(keys[-1]) + except BaseException: + n = 0 + + s = "Report_a_%03d" % (n + 1) + + return s + + def get_report(self, name): + buf = None + try: + node = self.ph5.get_node( + self.ph5_g_reports, name=name, classname='Array') + buf = node.read() + except Exception: + LOGGER.error("Failed to read report {0}".format(name)) + + return buf + + def newarray(self, title, data, description=None): + name = title + try: + self.ph5.remove_node(self.current_g_reports, name=name) + LOGGER.warning("Node {0} exists. Overwritten." + .format(name)) + except Exception: + pass + + a = self.ph5.create_array(self.ph5_g_reports, name, data) + if description is not None: + a.attrs.description = description + + def populate(self, p, pkey=None): + populate_table(self.ph5_t_report, + p, + key=pkey) + + self.ph5.flush() + + def initgroup(self): + # Create reports group + self.ph5_g_reports = initialize_group(self.ph5, + '/Experiment_g', + 'Reports_g') + # Create reports table + self.ph5_t_report = initialize_table(self.ph5, + '/Experiment_g/Reports_g', + 'Report_t', + columns.Report, + expectedrows=1) + + columns.add_reference( + '/Experiment_g/Reports_g/Report_t', self.ph5_t_report) + + def nuke_report_t(self): + self.ph5_t_report.remove() + self.initgroup() + + +class ResponsesGroup: + def __init__(self, ph5): + self.ph5 = ph5 + self.ph5_g_responses = None + self.ph5_t_response = None + + def populateResponse_t(self, p, pkey=None): + required_keys = [] + populate_table(self.ph5_t_response, p, pkey, required_keys) + self.ph5.flush() + + def read_responses(self): + ret, keys = read_table(self.ph5_t_response) + + return ret, keys + + def get_response(self, name): + try: + node = self.ph5.get_node(name) + out = "" + for i in node: + out = out + i + except Exception: + LOGGER.error("Failed to read response {0}".format(name)) + + return out + + def newearray(self, name, description=None): + batom = tables.StringAtom(itemsize=40) + a = create_empty_earray(self.ph5, + self.current_g_das, + name, + batom=batom, + expectedrows=2000) + + if description is not None: + a.attrs.description = description + + return a + + def initgroup(self): + # Create response group + self.ph5_g_responses = initialize_group(self.ph5, + '/Experiment_g', + 'Responses_g') + # Create response table + self.ph5_t_response = initialize_table(self.ph5, + '/Experiment_g/Responses_g', + 'Response_t', + columns.Response, + expectedrows=1) + + columns.add_reference( + '/Experiment_g/Responses_g/Response_t', self.ph5_t_response) + + def nuke_response_t(self): + self.ph5_t_response.remove() + self.initgroup() + + +class ExperimentGroup: + def __init__(self, currentpath='.', nickname="untitled-experiment"): + self.nickname = nickname # Experiment official nickname + self.currentpath = currentpath # Path to directory holding ph5 file + self.filename = self.ph5buildFilename() # Make filename + self.ph5 = None # PyTables file reference + self.ph5_g_experiment = None # Experiment group + self.ph5_t_experiment = None # Experiment table + self.ph5_g_sorts = None # Sorts group + self.ph5_g_receivers = None # Receivers group + self.ph5_g_reports = None # Reports group + self.ph5_g_responses = None + self.ph5_g_maps = None # Maps group + + def version(self): + return columns.PH5VERSION + + def __version__(self): + self.version() + + def read_experiment(self): + ret, keys = read_table(self.ph5_t_experiment) + + return ret, keys + + def ph5exists(self): + ''' Check to see if file exists + self.h5 -- Reference to hdf5 file ''' + if os.path.exists(self.filename): + if tables.is_pytables_file(self.filename): + # XXX Needs to be modified to return version of ph5 XXX + return True + else: + return False + else: + return False + + def ph5buildFilename(self): + ''' Build filename from path and experiment nickname ''' + postfix = '.ph5' + if self.nickname[-4:] == postfix: + f = os.path.join(self.currentpath, self.nickname) + else: + f = os.path.join(self.currentpath, self.nickname + postfix) + + return f + + def ph5flush(self): + self.ph5.flush() + + def ph5open(self, editmode=False, + ph5title='PIC KITCHEN HDF5 file, Version = ' + + columns.PH5VERSION): + ''' Open ph5 file, create it if it doesn't exist ''' + if self.ph5exists(): + # XXX Needs try:except XXX + if editmode is True: + LOGGER.debug("Opened ph5 file {0} in append edit mode." + .format(self.filename)) + self.ph5 = tables.open_file(self.filename, mode='a') + else: + LOGGER.debug("Opened ph5 file {0} in read only mode." + .format(self.filename)) + self.ph5 = tables.open_file(self.filename, mode='r') + elif editmode is True: + LOGGER.debug("No PH5 file exists at '{0}'! Creating new ph5 file." + .format(self.filename)) + self.ph5 = tables.open_file( + self.filename, mode='w', title=ph5title) + else: + raise OSError('Unable to open "{0}". Does this file exist?' + .format(self.filename)) + + def ph5close(self): + if self.ph5 is not None and self.ph5.isopen: + self.ph5.close() + self.ph5 = None + # This will not work with version 3 + reload(columns) + + def populateExperiment_t(self, p): + ''' Keys: 'time_stamp/type, time_stamp/epoch, + time_stamp/ascii,time_stamp/micro_seconds + nickname, longname, PIs, institutions, + north_west_corner/coordinate_system, + north_west_corner/projection + north_west_corner/ellipsoid, + north_west_corner/[XYZ]/[units,value] + north_west_corner/description, (same for south_east_corner) + summary_paragraph' ''' + + populate_table(self.ph5_t_experiment, p) + + self.ph5.flush() + + def initgroup( + self, ph5_g_title='PIC KITCHEN HDF5 file, Version = ' + + columns.PH5VERSION): + ''' If group Experiment_g does not exist create it, + otherwise get a reference to it + If table Experiment_t does not exist create it, + otherwise get a reference to it ''' + # Create experiment group + self.ph5_g_experiment = initialize_group(self.ph5, + '/', + 'Experiment_g') + # Create experiment table + self.ph5_t_experiment = initialize_table(self.ph5, + '/Experiment_g', + 'Experiment_t', + columns.Experiment, + expectedrows=1) + + # Put handle in lookup table columns.TABLE + columns.add_reference( + '/Experiment_g/Experiment_t', self.ph5_t_experiment) + + # XXX This stuff should be in own methods? XXX + self.ph5_g_sorts = SortsGroup(self.ph5) + self.ph5_g_sorts.initgroup() + + self.ph5_g_receivers = ReceiversGroup(self.ph5) + self.ph5_g_receivers.initgroup() + + self.ph5_g_reports = ReportsGroup(self.ph5) + self.ph5_g_reports.initgroup() + + self.ph5_g_responses = ResponsesGroup(self.ph5) + self.ph5_g_responses.initgroup() + + self.ph5_g_maps = MapsGroup(self.ph5) + self.ph5_g_maps.initgroup() + + def nuke_experiment_t(self): + self.ph5_t_experiment.remove() + # Create experiment group + self.ph5_g_experiment = initialize_group(self.ph5, + '/', + 'Experiment_g') + # Create experiment table + self.ph5_t_experiment = initialize_table(self.ph5, + '/Experiment_g', + 'Experiment_t', + columns.Experiment, + expectedrows=1) + + # Put handle in lookup table columns.TABLE + columns.add_reference( + '/Experiment_g/Experiment_t', self.ph5_t_experiment) + +# +# Mixins +# + + +def initialize_group(filenode, where, group, title=''): + returnnode = None + path = '/'.join([where, group]) + if filenode.__contains__(path): + returnnode = filenode.get_node(where, name=group, classname='Group') + else: + returnnode = filenode.create_group(where, group, title=title) + + return returnnode + + +def initialize_table(filenode, where, table, description, expectedrows=None): + returnnode = None + path = '/'.join([where, table]) + if filenode.__contains__(path): + returnnode = filenode.get_node(where, name=table, classname='Table') + else: + if expectedrows is None: + returnnode = filenode.create_table(where, table, description) + else: + returnnode = filenode.create_table( + where, table, description, expectedrows=expectedrows) + + return returnnode + + +def populate_table(tablenode, key_value, key=None, required_keys=[]): + err_keys, err_required = columns.validate( + tablenode, key_value, required_keys) + + if err_keys: + raise HDF5InteractionError(1, err_keys) + + if err_required: + raise HDF5InteractionError(2, err_required) + + try: + columns.populate(tablenode, key_value, key) + tablenode.flush() + except Exception as e: + raise HDF5InteractionError(3, e.message) + + +def read_table(tablenode): + ret = [] + keys = None + if not tablenode: + return ret, keys + + LOGGER.debug("Read {0}".format(tablenode)) + try: + tableiterator = tablenode.iterrows() + keys, names = columns.keys(tablenode) + ret = columns.rowstolist(tableiterator, keys) + except Exception as e: + raise HDF5InteractionError(4, e.message) + + return ret, keys + + +def create_empty_earray(filenode, groupnode, name, + batom=None, expectedrows=None): + try: + bfilter = tables.Filters(complevel=ZLIBCOMP, complib='zlib') + if expectedrows is None: + a = filenode.create_earray(groupnode, + name, + atom=batom, + shape=(0,), + filters=bfilter) + else: + a = filenode.create_earray(groupnode, + name, + atom=batom, + shape=(0,), + filters=bfilter, + expectedrows=expectedrows) + + except Exception as e: + raise HDF5InteractionError(5, e.message) + + return a + + +def create_data_earray(filenode, groupnode, name, data, batom, rows=None): + try: + if rows is None: + rows = len(data) / 4 + + a = create_empty_earray(filenode, + groupnode, + name, + batom=batom, + expectedrows=rows) + + a.append(data) + except Exception as e: + raise HDF5InteractionError(6, e.message) + + return a + + +def get_nodes_by_name(filenode, where, RE, classname): + nodes = {} + for n in filenode.iter_nodes(where, classname=classname): + mo = RE.match(n._v_name) + if mo: + # key = mo.groups ()[-1] + key = n._v_name + nodes[key] = n + + return nodes + + +if __name__ == '__main__': + ex = ExperimentGroup('.', 'GEO_DESIRE') + EDITMODE = False + ex.ph5open(EDITMODE) + ex.initgroup() + reports, keys = ex.ph5_g_reports.read_reports() + for r in reports: + for k in keys: + print k, r[k] + + ex.ph5close() + sys.exit() + sorts = ex.ph5_g_sorts.read_sorts() + first_sort = True + i = 0 + for s in sorts: + # Get a dictionary of DASs in this array + a_name = s['array_t_name_s'] + array = ex.ph5_g_sorts.read_arrays(a_name) + # The dictionary + dass = {} + for a in array: + dass[a['das/serial_number_s']] = True + + # Loop through events and populate the Sort_t table + events = ex.ph5_g_sorts.read_events() + for e in events: + ep = float(e['time/epoch_l']) + \ + (float(e['time/micro_seconds_i']) / 1000000.0) + dict = ex.ph5_g_receivers.find_traces(ep) + ks = dict.keys() + for k in ks: + + # Is this das in this array? + if dict[k].das not in dass: + print "Not found in any array: ", dict[k].das + continue + lat = dict[k].receiver['location/X/value_d'] + lon = dict[k].receiver['location/Y/value_d'] + print dict[k].das, dict[k].epoch, dict[k].length, lat, lon + i = i + 1 + + print "Matched %d traces." % i + + ex.ph5close() diff --git a/ph5/core/ph5api.py b/ph5/core/ph5api.py index ad11bef8..78a30f4c 100755 --- a/ph5/core/ph5api.py +++ b/ph5/core/ph5api.py @@ -1372,7 +1372,11 @@ def get_extent(self, das, component, sample_rate, start=None, end=None): stop_epoch=end, sample_rate=sample_rate) if not das_t_t: - LOGGER.warning("No Das table found for " + das) + msg = ("No Das table found for %s chan %s sr %s" + % (das, component, sample_rate)) + if start is not None: + msg += " in time [%s, %s]" % (start, end) + LOGGER.warning(msg) return None, None if not das_t_t: diff --git a/ph5/core/ph5api_pn3.py b/ph5/core/ph5api_pn3.py new file mode 100755 index 00000000..147fe5b1 --- /dev/null +++ b/ph5/core/ph5api_pn3.py @@ -0,0 +1,1871 @@ +#!/usr/bin/env pnpython4 +# +# Basic API for reading a family of ph5 files +# +# Steve Azevedo, March 2015 +# modified to work with pn3 +# + +import logging +import os +import time +import re +import numpy as np +import math +from pyproj import Geod +from ph5.core import columns, experiment_pn3 as experiment, timedoy +from tables.exceptions import NoSuchNodeError + +PROG_VERSION = '2021.47' + +LOGGER = logging.getLogger(__name__) +PH5VERSION = columns.PH5VERSION + +# No time corrections applied if slope exceeds this value, normally 0.001 +# (.1%) +MAX_DRIFT_RATE = 0.001 + +__version__ = PROG_VERSION + +# Conversion factors to meters +FACTS_M = {'km': 1000., 'm': 1., 'dm': 1. / 10., 'cm': 1. / 100., + 'mm': 1. / 1000., 'kmi': 1852.0, 'in': 0.0254, + 'ft': 0.3048, 'yd': 0.9144, 'mi': 1609.344, + 'fath': 1.8288, 'ch': 20.1168, 'link': 0.201168, + 'us-in': 1. / 39.37, 'us-ft': 0.304800609601219, + 'us-yd': 0.914401828803658, 'us-ch': 20.11684023368047, + 'us-mi': 1609.347218694437, 'ind-yd': 0.91439523, + 'ind-ft': 0.30479841, 'ind-ch': 20.11669506} + + +class APIError(Exception): + def __init__(self, errno, msg): + self.args = (errno, msg) + self.errno = errno + self.msg = msg + + +class CutHeader(object): + ''' PH5 cut header object + array -> The receiver array number or None if receiver gather + shot_line -> The shot line number or None if shot gather + length -> Number of samples in each line + order -> The order of the station or shot id's + si_us -> Sample interval in micro-seconds + ''' + __slots__ = 'array', 'shot_line', 'length', 'order', 'si_us' + + def __init__(self, array=None, shot_line=None, + length=0, order=[], si_us=0): + self.array = array + self.shot_line = shot_line + self.length = length + self.order = order + self.si_us = si_us + + def __repr__(self): + if self.array: + gather_type = "Shot" + elif self.shot_line: + gather_type = "Receiver" + else: + gather_type = "Unknown" + + return "Gather type: {0}, Trace length: {1} Sample interval: {2} us,\ + Number of traces {3}".format(gather_type, + self.length, + self.si_us, + len(self.order)) + + +class Cut(object): + ''' PH5 cut object + das_sn -> The DAS to cut data from + start_fepoch -> The starting time of the cut as a float + stop_fepoch -> The ending time of the cut as a float + sample_rate -> The sample rate in samples per second as a float + channels -> A list of chanels to cut + das_t_times -> A list of Das_t times found for + the start_fepoch, stop_fepoch, + ((start, stop), (start, stop), (start, stop), ...) + msg -> List of error or warning messages + id_s -> The shot id for receiver gathers or station id + for shot gathers + ''' + __slots__ = 'id_s', 'das_sn', 'das_t_times', 'start_fepoch', + 'stop_fepoch', 'sample_rate', 'channels', 'msg' + + def __init__(self, das_sn, start_fepoch, stop_fepoch, sample_rate, + channels={}, das_t_times=[], msg=None, id_s=None): + self.das_sn = das_sn + self.das_t_times = das_t_times + self.start_fepoch = start_fepoch + self.stop_fepoch = stop_fepoch + self.sample_rate = sample_rate + self.channels = channels + self.msg = [] + if msg is not None: + self.msg.append(msg) + self.id_s = id_s + + def __repr__(self): + ret = '' + ret = "ID: {1} DAS: {0} SR: {2} samp/sec SI: {3:G} us\n" \ + .format(self.das_sn, + self.id_s, + self.sample_rate, + (1. / self.sample_rate) * 1000000) + ret += "Start: {0} Stop: {1}\n" \ + .format(timedoy.epoch2passcal(self.start_fepoch), + timedoy.epoch2passcal(self.stop_fepoch)) + for m in self.msg: + ret += m + '\n' + ret += "DAS windows:\n" + for w in self.das_t_times: + ret += "\t{0} - {1}\n".format(timedoy.epoch2passcal(w[0]), + timedoy.epoch2passcal(w[1])) + + return ret + + +class Clock(object): + ''' Clock performance + slope -> Drift rate in seconds/second + offset_secs -> The offset of the clock at offload + max_drift_rate_allowed -> The maximum allowed drift rate + for time corrections + comment -> Comment on clock performance + ''' + __slots__ = ('slope', 'offset_secs', 'max_drift_rate_allowed', 'comment') + + def __init__(self, slope=0., offset_secs=0., max_drift_rate_allowed=1.): + self.slope = slope + self.offset_secs = offset_secs + self.max_drift_rate_allowed = max_drift_rate_allowed + self.comment = [] + + def __repr__(self): + return "Slope: {0}\nMaximum slope: {1}\nOffload offset:\ + {2}\nComment: {3}\n".format(self.slope, + self.max_drift_rate_allowed, + self.offset_secs, + self.comment) + + +class Trace(object): + ''' PH5 trace object: + data -> Numpy array of trace data points + start_time -> timedoy time object + time_correction_ms -> The correction to account for ocillator drift + clock -> Clock performance object + nsamples -> Number of data samples, ie. length of data + padding -> Number of samples padding as a result of gaps + sample_rate -> Number of samples per second as a float + ttype -> Data sample point type, at this point 'int' or 'float' + byteorder -> Data byteorder + das_t -> A list of Das_t dictionaries + receiver_t -> Orientation + response_t -> Gain and bit weight fo now. + Methods: + time_correct -> Apply any time correction and return a timedoy object + ''' + __slots__ = ('data', 'start_time', 'time_correction_ms', 'clock', + 'nsamples', 'padding', 'sample_rate', 'ttype', 'byteorder', + 'das_t', 'receiver_t', 'response_t', 'time_correct') + + def __init__(self, data, fepoch, time_correction_ms, nsamples, sample_rate, + ttype, byteorder, das_t, receiver_t, + response_t, clock=Clock()): + self.data = data + self.start_time = timedoy.TimeDOY(epoch=fepoch) + self.time_correction_ms = time_correction_ms + self.clock = clock + self.nsamples = nsamples + self.sample_rate = sample_rate + self.ttype = ttype + self.byteorder = byteorder + self.das_t = das_t + self.receiver_t = receiver_t + self.response_t = response_t + + def __repr__(self): + end_time = self.get_endtime() + return "start_time: {0}\nend_time: {7}\nnsamples: {1}/{6}\nsample_rate:\ + {2}\ntime_correction_ms: {3}\nttype: {4}\nchannel_number: {5}" \ + .format(self.start_time, + self.nsamples, + self.sample_rate, + self.time_correction_ms, + self.ttype, + self.das_t[0]['channel_number_i'], + len(self.data), + end_time) + + def get_endtime(self): + if self.sample_rate > 0: + delta = 1. / float(self.sample_rate) + time_diff = float(self.nsamples - 1) * delta + end_time = timedoy.TimeDOY(epoch=(self.start_time.epoch( + fepoch=True) + time_diff)) + else: + end_time = timedoy.TimeDOY(epoch=(self.start_time.epoch( + fepoch=True))) + return end_time.getFdsnTime() + + def time_correct(self): + return timedoy.timecorrect(self.start_time, self.time_correction_ms) + + +class PH5(experiment.ExperimentGroup): + das_gRE = re.compile("Das_g_(.*)") + + def __init__(self, path=None, nickname=None, editmode=False): + ''' path -> Path to ph5 file + nickname -> The master ph5 file name, ie. master.ph5 + editmode -> Always False + ''' + if not os.path.exists(os.path.join(path, nickname)): + raise APIError(0, "PH5 file does not exist: {0}".format( + os.path.join(path, nickname))) + experiment.ExperimentGroup.__init__( + self, currentpath=path, nickname=nickname) + if self.currentpath is not None and self.nickname is not None: + self.ph5open(editmode) + self.initgroup() + + self.clear() + + def clear(self): + ''' Clears key variables ''' + self.Array_t = { + } # Array_t[array_name] = { 'byid':byid, 'order':order, 'keys':keys } + # Event_t[event_name] = { 'byid':byid, 'order':order, 'keys':keys } + self.Event_t = {} + self.Sort_t = {} # Sort_t[array_name] = { 'rows':rows, 'keys':keys } + self.Das_t = {} # Das_t[das] = { 'rows':rows, 'keys':keys } + # Das_t_full[das], internal complete copy of Das_t + self.Das_t_full = {} + # Offset_t[offset_name] = { 'byid':byid, 'order':order, 'keys':keys } + self.Offset_t = {} + self.Index_t = None + self.Time_t = None + self.Receiver_t = None + self.Experiment_t = None + self.Response_t = None + self.Offset_t_names = [] + self.Array_t_names = [] + self.Event_t_names = [] + self.Das_g_names = [] + self.num_found_das = 0 + + def close(self): + self.clear() + self.ph5close() + + def channels(self, array, station): + ''' + Inputs: + array -> The Array_t name example: Array_t_001 + station -> The station id_s + Output: + returns a list of channels for this station + ''' + try: + self.read_array_t(array) + chans = sorted(self.Array_t[array]['byid'][station].keys()) + return chans + except Exception: + return [] + + def channels_Array_t(self, array): + ''' + Inputs: + array -> The Array_t name example: Array_t_001 + Output: + returns a list of channels + ''' + try: + if array in self.Array_t: + order = self.Array_t[array]['order'] + else: + self.read_array_t(array) + order = self.Array_t[array]['order'] + except Exception: + return [] + + ret = {} + for o in order: + chans = self.Array_t[array]['byid'][o].keys() + for c in chans: + ret[c] = True + + ret = sorted(ret.keys()) + + return ret + + def get_offset(self, sta_line, sta_id, evt_line, evt_id): + ''' Calculate offset distance in meters from a shot to a station + Inputs: + sta_line -> the array or line + sta_id -> the station id + evt_line -> the shot line + evt_id -> the event or shot id + Returns: + A dictionary with the following keys: + { 'event_id_s': The event or shot id, + 'receiver_id_s': The station or receiver id, + 'azimuth/value_f:The azimuth from the station to the shot, + 'azimuth/units_s': The units of the azimuth, + 'offset/value_d': The offset distance, + 'offset/units_s': The units of the offset + } + ''' + az = 0.0 + baz = 0.0 + dist = 0.0 + chans = self.channels(sta_line, sta_id) + if chans: + c = chans[0] + else: + LOGGER.warning("Couldn't get offset.") + return {} + try: + if sta_line in self.Array_t and evt_line in self.Event_t: + array_t = self.Array_t[sta_line]['byid'][sta_id][c] + event_t = self.Event_t[evt_line]['byid'][evt_id] + lon0 = array_t[0]['location/X/value_d'] + lat0 = array_t[0]['location/Y/value_d'] + lon1 = event_t['location/X/value_d'] + lat1 = event_t['location/Y/value_d'] + az, baz, dist = run_geod(lat0, lon0, lat1, lon1) + except Exception as e: + LOGGER.warning("Couldn't get offset. {0}".format(repr(e))) + return {} + + return {'event_id_s': evt_id, 'receiver_id_s': sta_id, + 'azimuth/value_f': az, 'azimuth/units_s': 'degrees', + 'offset/value_d': dist, 'offset/units_s': 'm'} + + def calc_offsets(self, array, shot_id, shot_line="Event_t"): + ''' + Calculate offset with sign from a shot point to each station in an + array. + Inputs: + array -> the array or line as named in the ph5 file,'Array_t_001' + shot_id -> the event or shot id, '101'. + shot_line -> the shot line, 'Event_t' (old style), 'Event_t_001' + Returns: + A list of dictionaries in the same format as ph5 Offset_t. + ''' + Offset_t = [] + if not self.Array_t_names: + self.read_array_t_names() + if array not in self.Array_t_names: + return Offset_t + if array not in self.Array_t: + self.read_array_t(array) + if not self.Event_t_names: + self.read_event_t_names() + if shot_line not in self.Event_t: + self.read_event_t(shot_line) + + Array_t = self.Array_t[array] + order = Array_t['order'] + + Event_t = self.Event_t[shot_line] + if shot_id in Event_t['byid']: + Event_t['byid'][shot_id] + else: + return Offset_t + + for o in order: + array_t = Array_t['byid'][o] + chans = self.channels(array, o) + c = chans[0] + offset_t = self.get_offset( + array, array_t[c][0]['id_s'], shot_line, shot_id) + Offset_t.append(offset_t) + + rows = calc_offset_sign(Offset_t) + + byid, order = by_id(rows, key='receiver_id_s') + + return {'byid': byid, 'order': order, 'keys': rows[0].keys()} + + def read_offsets_shot_order( + self, array_table_name, shot_id, shot_line="Event_t"): + ''' Reads shot to station distances from Offset_t_aaa_sss + Inputs: + array_table_name -> The array table name such as Array_t_001 + shot_id -> The shot id, id_s from the event table + shot_line -> The event table name such as Event_t_002 + Returns: + A dictionary keyed on array table id_s that points to a + row from the offset table. + ''' + if shot_line == "Event_t": + # Legacy Offset table name + offset_table_name = "Offset_t" + else: + offset_table_name = "Offset_t_{0}_{1}" \ + .format(array_table_name[-3:], + shot_line[-3:]) + + Offset_t = {} + if not self.Array_t_names: + self.read_array_t_names() + if array_table_name not in self.Array_t_names: + return Offset_t + if array_table_name not in self.Array_t: + self.read_array_t(array_table_name) + if not self.Event_t_names: + self.read_event_t_names() + if shot_line not in self.Event_t: + self.read_event_t(shot_line) + if not self.Offset_t_names: + self.read_offset_t_names() + if offset_table_name not in self.Offset_t_names: + return Offset_t + + Array_t = self.Array_t[array_table_name] + order = Array_t['order'] + for o in order: + c = self.channels(array_table_name, o)[0] + array_t = Array_t['byid'][o] + offset_t = self.ph5_g_sorts. \ + read_offset_fast(shot_id, + array_t[c][0]['id_s'], + name=offset_table_name) + + Offset_t[array_t[c][0]['id_s']] = offset_t + + return Offset_t + + def read_offsets_receiver_order( + self, array_table_name, station_id, shot_line="Event_t"): + ''' Reads shot to station distances from Offset_t_aaa_sss + Inputs: + array_table_name -> The array table name such as Array_t_001 + station_id -> The station id, id_s from the array table + shot_line -> The event table name such as Event_t_002 + Returns: + A dictionary keyed on event table id_s that points to a row + from the offset table. + ''' + if shot_line == "Event_t": + offset_table_name = "Offset_t" + else: + offset_table_name = "Offset_t_{0}_{1}" \ + .format(array_table_name[-3:], + shot_line[-3:]) + + Offset_t = {} + if not self.Array_t_names: + self.read_array_t_names() + if array_table_name not in self.Array_t_names: + return Offset_t + + if not self.Event_t_names: + self.read_event_t_names() + if shot_line not in self.Event_t: + self.read_event_t(shot_line) + if not self.Offset_t_names: + self.read_offset_t_names() + if offset_table_name not in self.Offset_t_names: + return Offset_t + + Event_t = self.Event_t[shot_line] + order = Event_t['order'] + for o in order: + event_t = Event_t['byid'][o] + offset_t = self.ph5_g_sorts. \ + read_offset_fast(event_t['id_s'], + station_id, + name=offset_table_name) + Offset_t[event_t['id_s']] = offset_t + + return Offset_t + + def read_experiment_t(self): + ''' Read Experiment_t + Sets: + Experiment_t['rows'] (a list of dictionaries) + Experiment_t['keys'] (a list of dictionary keys) + ''' + rows, keys = self.read_experiment() + self.Experiment_t = {'rows': rows, 'keys': keys} + + def read_offset_t_names(self): + ''' Read Offset_t names + Sets: + Offset_t_names + ''' + self.Offset_t_names = self.ph5_g_sorts.namesOffset_t() + + def read_offset_t(self, name, id_order='event_id_s'): + ''' + Read Offset_t + Inputs: + name -> Offset_t_aaa_sss name + id_order -> 'event_id_s', or 'receiver_id_s' + Sets: + Offset_t[name]['byid'] + Offset_t[name]['order'] + Offset_t[name]['keys'] + ''' + if not self.Offset_t_names: + self.read_offset_t_names() + if name in self.Offset_t_names: + rows, keys = self.ph5_g_sorts.read_offset(name) + byid, order = by_id(rows, key=id_order) + self.Offset_t[name] = {'byid': byid, 'order': order, 'keys': keys} + + def read_event_t_names(self): + ''' Read Event_t names + Sets: + Event_t_names + ''' + self.Event_t_names = self.ph5_g_sorts.namesEvent_t() + + def read_event_t(self, name): + ''' Read Event_t + Inputs: + name -> the Event_t_xxx name + Sets: + Event_t[name]['byid'] Keyed by shot id(a list of dictionaries) + Event_t[name]['order'] Keyed by order in the PH5 file + (a list of dictionaries) + Event_t[name]['keys'] (a list of dictionary keys) + ''' + if not self.Event_t_names: + self.read_event_t_names() + if name in self.Event_t_names: + rows, keys = self.ph5_g_sorts.read_events(name) + byid, order = by_id(rows) + self.Event_t[name] = {'byid': byid, 'order': order, 'keys': keys} + + def read_array_t_names(self): + ''' Read Array_t names + Sets: + Array_t_names + ''' + self.Array_t_names = self.ph5_g_sorts.namesArray_t() + + def read_array_t(self, name): + ''' Read Array_t n + Inputs: + name -> the name of the array as a string 'Array_t_xxx' + Sets: + Array_t[name]['byid'] Keyed by station id as a list of dict of + array_t lines by channel + Array_t[name]['order'] Keyed in order as in PH5 file + (a list of dictionaries) + Array_t[name]['keys'] (a list of dictionary keys) + ''' + if not self.Array_t_names: + self.read_array_t_names() + if name in self.Array_t_names: + rows, keys = self.ph5_g_sorts.read_arrays(name) + byid, order = by_id( + rows, secondary_key='channel_number_i', unique_key=False) + self.Array_t[name] = {'byid': byid, 'order': order, 'keys': keys} + + def get_sort_t(self, start_epoch, array_name): + ''' + Get list of sort_t lines based on a time and array + Returns: + A list of sort_t lines + ''' + if not self.Sort_t: + self.read_sort_t() + + ret = [] + if array_name not in self.Sort_t: + return ret + + for sort_t in self.Sort_t[array_name]['rows']: + start = sort_t['start_time/epoch_l'] + \ + (sort_t['start_time/micro_seconds_i'] / 1000000.) + if not start_epoch >= start: + continue + stop = sort_t['end_time/epoch_l'] + \ + (sort_t['end_time/micro_seconds_i'] / 1000000.) + if not start_epoch <= stop: + continue + ret.append(sort_t) + + return ret + + def read_sort_t(self): + ''' Read Sort_t + Sets: + Sort_t[array_name]['rows'] (a list sort_t of dictionaries) + Sort_t[array_name]['keys'] + ''' + tmp = {} + rows, keys = self.ph5_g_sorts.read_sorts() + for r in rows: + if r['array_t_name_s'] not in tmp: + tmp[r['array_t_name_s']] = [] + + tmp[r['array_t_name_s']].append(r) + + arrays = tmp.keys() + for a in arrays: + self.Sort_t[a] = {'rows': tmp[a], 'keys': keys} + + def read_index_t(self): + ''' Read Index_t + Sets: + Index_t['rows'] (a list of dictionaries) + Index_t['keys'] (a list of dictionary keys) + ''' + rows, keys = self.ph5_g_receivers.read_index() + self.Index_t = {'rows': rows, 'keys': keys} + + def read_time_t(self): + ''' Read Time_t + Sets: + Time_t['rows'] (a list of dictionaries) + Time_t['keys'] (a list of dictionary keys) + ''' + rows, keys = self.ph5_g_receivers.read_time() + self.Time_t = {'rows': rows, 'keys': keys} + + def get_time_t(self, das): + ''' Return Time_t as a list of dictionaries + Returns: + time_t (a list of dictionaries) + ''' + if not self.Time_t: + self.read_time_t() + + time_t = [] + for t in self.Time_t['rows']: + if t['das/serial_number_s'] == das: + time_t.append(t) + + return time_t + + def read_receiver_t(self): + ''' Read Receiver_t + Sets: + Receiver_t['rows] (a list of dictionaries) + Receiver_t['keys'] (a list of dictionary keys) + ''' + rows, keys = self.ph5_g_receivers.read_receiver() + self.Receiver_t = {'rows': rows, 'keys': keys} + + def get_receiver_t(self, das_t, by_n_i=True): + ''' + Read Receiver_t to match n_i as set in das_t else use channel + Returns: + receiver_t + ''' + + if not self.Receiver_t: + self.read_receiver_t() + + if by_n_i: + try: + + n_i = das_t['receiver_table_n_i'] + + receiver_t = self.Receiver_t['rows'][n_i] + except KeyError: + receiver_t = None + else: + try: + chan = das_t['channel_number_i'] + for receiver_t in self.Receiver_t['rows']: + if receiver_t['orientation/channel_number_i'] == chan: + break + except BaseException: + receiver_t = None + + return receiver_t + + def get_receiver_t_by_n_i(self, n_i): + ''' + Read Receiver_t to match n_i + Returns: + receiver_t + ''' + + if not self.Receiver_t: + self.read_receiver_t() + + try: + receiver_t = self.Receiver_t['rows'][n_i] + except KeyError: + receiver_t = None + + return receiver_t + + def read_response_t(self): + ''' Read Response_t + Sets: + Response_t['rows'] (a list of dictionaries) + Response_t['keys] (a list of dictionary keys) + ''' + rows, keys = self.ph5_g_responses.read_responses() + self.Response_t = {'rows': rows, 'keys': keys} + + def get_response_t(self, das_t): + ''' + Read Response_t to match n_i as set in das_t + Returns: + response_t + ''' + if not self.Response_t: + self.read_response_t() + + try: + try: + n_i = das_t[0]['response_table_n_i'] + except BaseException: + n_i = das_t['response_table_n_i'] + + response_t = self.Response_t['rows'][n_i] + if response_t['n_i'] != n_i: + for response_t in self.Response_t['rows']: + if response_t['n_i'] == n_i: + break + except (KeyError, IndexError): + response_t = None + + return response_t + + def get_response_t_by_n_i(self, n_i): + ''' + Read Response_t to match n_i + Returns: + response_t + ''' + if not self.Response_t: + self.read_response_t() + + try: + for response_t in self.Response_t['rows']: + if response_t['n_i'] == n_i: + return response_t + except BaseException: + return None + + return None + + def read_das_g_names(self): + ''' Read Das_g names + Sets: + Das_g_names (a list of dictionary keys) + ''' + self.Das_g_names = self.ph5_g_receivers.alldas_g() + + def query_das_t(self, + das, + chan=None, + start_epoch=None, + stop_epoch=None, + sample_rate=None, + sample_rate_multiplier=1, + check_samplerate=True): + ''' Uses queries to get data from specific das table''' + das_g = "Das_g_{0}".format(das) + try: + node = self.ph5_g_receivers.getdas_g(das) + except experiment.HDF5InteractionError as e: + raise e + if not node: + return [] + self.ph5_g_receivers.setcurrent(node) + try: + tbl = self.ph5.get_node('/Experiment_g/Receivers_g/' + das_g, + 'Das_t') + except NoSuchNodeError: + return [] + try: + sample_rate_multiplier_i = tbl.cols.sample_rate_multiplier_i # noqa + sample_rate_multiplier_i = sample_rate_multiplier_i + except AttributeError: + errmsg = ("%s has sample_rate_multiplier_i " + "missing. Please run fix_srm to fix " + "sample_rate_multiplier_i for PH5 data." + % tbl._v_parent._v_name.replace('Das_g', 'Das_t')) + raise APIError(-1, errmsg) + + if len(list(tbl.where('sample_rate_multiplier_i==0'))) > 0: + errmsg = ("%s has sample_rate_multiplier_i " + "with value 0. Please run fix_srm to fix " + "sample_rate_multiplier_i for PH5 data." + % tbl._v_parent._v_name.replace('Das_g', 'Das_t')) + raise APIError(-1, errmsg) + + epoch_i = tbl.cols.time.epoch_l # noqa + micro_seconds_i = tbl.cols.time.micro_seconds_i # noqa + sample_count_i = tbl.cols.sample_count_i # noqa + sample_rate_i = tbl.cols.sample_rate_i # noqa + epoch_i = epoch_i + micro_seconds_i = micro_seconds_i + sample_count_i = sample_count_i + sample_rate_i = sample_rate_i + das = [] + if not start_epoch: + start_epoch = 0 + if not stop_epoch: + stop_epoch = 32509613590 + + if sample_rate == 0 or sample_rate is None: + numexprstr = ( + '(channel_number_i == ' + + str(chan) + ' )&(epoch_i+micro_seconds_i/1000000>=' + + str(start_epoch) + + ')&(epoch_i+micro_seconds_i/1000000<=' + + str(stop_epoch) + ')' + ) + elif check_samplerate is False: + numexprstr = ( + '(channel_number_i == ' + + str(chan) + ' )&(epoch_i+micro_seconds_i/1000000 >= ' + + str(start_epoch) + + '-sample_count_i/sample_rate_i/sample_rate_multiplier_i)' + '&(epoch_i+micro_seconds_i/1000000 <= ' + + str(stop_epoch) + ')' + ) + else: + numexprstr = ( + '(channel_number_i == ' + + str(chan) + ' )&(epoch_i+micro_seconds_i/1000000>=' + + str(start_epoch) + + '-sample_count_i/sample_rate_i/sample_rate_multiplier_i)' + '&(epoch_i+micro_seconds_i/1000000<=' + + str(stop_epoch) + ')&(sample_rate_i==' + + str(sample_rate) + + ')&(sample_rate_multiplier_i==' + + str(sample_rate_multiplier) + ')' + ) + + for row in tbl.where(numexprstr): + row_dict = {'array_name_SOH_a': row['array_name_SOH_a'], + 'array_name_data_a': row['array_name_data_a'], + 'array_name_event_a': row['array_name_event_a'], + 'array_name_log_a': row['array_name_log_a'], + 'channel_number_i': row['channel_number_i'], + 'event_number_i': row['event_number_i'], + 'raw_file_name_s': row['raw_file_name_s'], + 'receiver_table_n_i': row['receiver_table_n_i'], + 'response_table_n_i': row['response_table_n_i'], + 'sample_count_i': row['sample_count_i'], + 'sample_rate_i': row['sample_rate_i'], + 'sample_rate_multiplier_i': + row['sample_rate_multiplier_i'], + 'stream_number_i': row['stream_number_i'], + 'time/ascii_s': row['time/ascii_s'], + 'time/epoch_l': row['time/epoch_l'], + 'time/micro_seconds_i': + row['time/micro_seconds_i'], + 'time/type_s': row['time/type_s'], + 'time_table_n_i': row['time_table_n_i'] + } + das.append(row_dict) + return das + + def read_das_t(self, das, start_epoch=None, stop_epoch=None, reread=True): + ''' Read Das_t, return Das_t keyed on DAS serial number + Inputs: + das -> DAS serial number as string or name of das group + start_epoch -> epoch time in seconds + stop_epoch -> epoch time in seconds + reread -> Re-read table even if Das_t[das] exists + Sets: + Das_t[das]['rows'] (a list of dictionaries) + Das_t[das]['keys'] (a list of dictionary keys) + + ''' + dass = self.Das_t.keys() + mo = self.das_gRE.match(das) + if mo: + das_g = das + das = mo.groups()[0] + else: + das_g = "Das_g_{0}".format(das) + + if das in dass and not reread and not start_epoch: + if das in self.Das_t_full: + self.Das_t[das] = self.Das_t_full[das] + return das + if self.Das_g_names == []: + self.read_das_g_names() + node = None + + if das_g in self.Das_g_names: + node = self.ph5_g_receivers.getdas_g(das) + self.ph5_g_receivers.setcurrent(node) + if node is None: + return None + rows_keep = [] + rows = [] + rk = {} + rows, keys = self.ph5_g_receivers.read_das() + self.Das_t_full[das] = {'rows': rows, 'keys': keys} + if stop_epoch is not None and start_epoch is not None: + for r in self.Das_t_full[das]['rows']: + # Start and stop for this das event window + start = float(r['time/epoch_l']) + \ + float(r['time/micro_seconds_i']) / 1000000. + + if r['sample_rate_i'] > 0: + stop = start + (float(r['sample_count_i']) / ( + float(r['sample_rate_i']) / + float(r['sample_rate_multiplier_i']))) + else: + stop = start + if r['sample_rate_i'] > 0: + sr = float(r['sample_rate_i']) / \ + float(r['sample_rate_multiplier_i']) + else: + sr = 0 + # We need to keep this + if is_in(start, stop, start_epoch, stop_epoch): + if sr not in rk: + rk[sr] = [] + rk[sr].append(r) + rkk = rk.keys() + # Sort so higher sample rates are first + rkk.sort(reverse=True) + for s in rkk: + rows_keep.extend(rk[s]) + else: + rows_keep = rows + + if len(rows_keep) > 0: + self.Das_t[das] = {'rows': rows_keep, + 'keys': self.Das_t_full[das]['keys']} + self.num_found_das += 1 + else: + das = None + + return das + + def forget_das_t(self, das): + node = self.ph5_g_receivers.getdas_g(das) + try: + node.umount() + except NoSuchNodeError: + # when no minixxx.ph5 is used + pass + if das in self.Das_t: + del self.Das_t[das] + + def read_t(self, table, n=None): + ''' Read table and return kef + Inputs: + table -> Experiment_t, Sort_t, Offset_t, Event_t, + Array_t requires n, Response_t, Receiver_t, Index_t, + Das_t requires n, Time_t + n -> the number of the table + -> or a tuple n containing n of Array_t and n of Event_t + -> or a DAS serial number + ''' + if table == "Experiment_t": + self.read_experiment_t() + return build_kef("/Experiment_g/Experiment_t", + self.Experiment_t['rows']) + elif table == "Sort_t": + self.read_sort_t() + keys = self.Sort_t.keys() + rows = [] + for k in keys: + rows += self.Sort_t[k]['rows'] + return build_kef("/Experiment_t/Sorts_g/Sort_t", rows) + elif table == "Offset_t": + rows = [] + kef = '' + self.read_offset_t_names() + if n and len(n) == 2: + a = n[0] + s = n[1] + off = ["Offset_t_{0:03d}_{1:03d}".format(a, s)] + else: + off = sorted(self.Offset_t_names) + + for o in off: + self.read_offset_t(o) + bi = self.Offset_t[o]['byid'] + order = self.Offset_t[o]['order'] + for r in order: + rows.append(bi[r]) + kef += build_kef("Experiment_g/Sorts_g/{0}".format(o), rows) + return kef + # This will change once shot lines are implemented + elif table == "Event_t": + rows = [] + en = [] + self.read_event_t_names() + kef = '' + if n: + en = ["Event_t_{0:03d}".format(int(n))] + else: + en = sorted(self.Event_t_names) + + for n in en: + self.read_event_t(n) + bi = self.Event_t[n]['byid'] + order = self.Event_t[n]['order'] + for o in order: + rows.append(bi[o]) + kef += build_kef("/Experiment_g/Sorts_g/{0}".format(n), rows) + return kef + elif table == "Array_t": + n = int(n) + self.read_array_t_names() + self.read_array_t("Array_t_{0:03d}".format(n)) + rows = [] + bi = self.Array_t["Array_t_{0:03d}".format(n)]['byid'] + order = self.Array_t["Array_t_{0:03d}".format(n)]['order'] + for o in order: + rows.append(bi[o]) + return build_kef( + "/Experiment_g/Sorts_g/Array_t_{0:03d}".format(n), rows) + elif table == "Response_t": + self.read_response_t() + return build_kef( + "/Experiment_g/Responses_g/Response_t", + self.Response_t['rows']) + elif table == "Report_t": + raise APIError(-1, "Return of Report_t not implemented.") + elif table == "Receiver_t": + self.read_receiver_t() + return build_kef( + "/Experiment_g/Receivers_g/Receiver_t", + self.Receiver_t['rows']) + elif table == "Index_t": + self.read_index_t() + return build_kef( + "/Experiment_g/Receivers_g/Index_t", self.Index_t['rows']) + elif table == "Das_t": + self.read_das_g_names() + self.read_das_t(n) + return build_kef( + "/Experiment_g/Receivers_g/Das_t_{0}/Das_t".format(n), + self.Das_t[n]['rows']) + elif table == "Time_t": + self.read_time_t() + return build_kef( + "/Experiment_g/Receivers_g/Time_t", self.Time_t['rows']) + else: + return None + + def textural_cut(self, das, + start_fepoch, stop_fepoch, + chan, + das_t=None): + """ + Cuts a text based trace such as LOG file + :param das: + :param start_fepoch: + :param stop_fepoch: + :param chan: + :param das_t: + :return: + """ + if not das_t: + self.read_das_t(das, start_epoch=start_fepoch, + stop_epoch=stop_fepoch, reread=False) + if das not in self.Das_t: + return [] + Das_t = filter_das_t(self.Das_t[das]['rows'], chan) + else: + Das_t = das_t + + traces = list() + for entry in Das_t: + if entry['sample_rate_i'] > 0: + continue + ref = self.ph5_g_receivers.find_trace_ref( + entry['array_name_data_a'].strip()) + stime = (entry['time/epoch_l'] + + entry['time/micro_seconds_i']/1000000) + + data = self.ph5_g_receivers.read_trace(ref) + trace = Trace(data, + stime, + 0, # time_correction + len(data), # samples_read + 0, + '|S1', + None, + Das_t, + None, # receiver_t + None, # response_t + clock=None) + traces.append(trace) + return traces + + def cut(self, das, start_fepoch, stop_fepoch, chan=1, + sample_rate=None, apply_time_correction=True, das_t=None): + ''' Cut trace data and return a Trace object + Inputs: + das -> data logger serial number + start_fepoch -> time to cut start of trace as a + floating point epoch + stop_fepoch -> time to cut end of trace as a + floating point epoch + chan -> channel to cut + sample_rate -> sample rate in samples per second + apply_time_correction -> iff True, slide traces to + correct for clock drift + Returns: + A list of PH5 trace objects split on gaps + ''' + if not das_t: + self.read_das_t(das, start_epoch=start_fepoch, + stop_epoch=stop_fepoch, reread=False) + if das not in self.Das_t: + return [Trace(np.array([]), start_fepoch, 0., 0, + sample_rate, None, None, [], None, None)] + Das_t = filter_das_t(self.Das_t[das]['rows'], chan) + else: + Das_t = das_t + if sample_rate == 0 or chan == -2: + LOGGER.info("calling textural_cut") + cuts = self.textural_cut( + das, + start_fepoch, + stop_fepoch, + chan, + Das_t) + return cuts + + # We shift the samples to match the requested start + # time to apply the time correction + + clock = Clock() + if apply_time_correction: + Time_t = self.get_time_t(das) + time_cor_guess_ms, clock = _cor(start_fepoch, stop_fepoch, Time_t) + if das in self.Das_t: + sr = sample_rate + si = 1. / float(sr) + else: + sr = 0. + si = 0. + time_cor_guess_secs = abs(time_cor_guess_ms / 1000.) + if time_cor_guess_secs > si: + time_cor_guess_samples = int( + (sr * (time_cor_guess_ms / 1000.)) + 0.5) + else: + time_cor_guess_samples = 0 + else: + clock.comment.append("No time correction applied.") + time_cor_guess_samples = 0 + + samples_read = 0 + first = True + new_trace = False + traces = [] + das_t = [] + + window_start_fepoch0 = None + window_stop_fepoch = None + trace_start_fepoch = None + data = None + for d in Das_t: + sr = float(d['sample_rate_i']) / \ + float(d['sample_rate_multiplier_i']) + window_start_fepoch = fepoch( + d['time/epoch_l'], d['time/micro_seconds_i']) + if (d['channel_number_i'] != chan) or ( + sr != sample_rate) or (window_start_fepoch > stop_fepoch): + continue + if window_start_fepoch0 is None: + window_start_fepoch0 = window_start_fepoch + # Number of samples in window + window_samples = d['sample_count_i'] + # Window stop epoch + window_stop_fepoch = window_start_fepoch + (window_samples / sr) + + # Requested start before start of window, we must need to + # start cutting at start of window + if start_fepoch < window_start_fepoch: + cut_start_fepoch = window_start_fepoch + cut_start_sample = 0 + else: + # Cut start is somewhere in window + cut_start_fepoch = start_fepoch + cut_start_sample = int(math.ceil(((cut_start_fepoch - + window_start_fepoch) * + sr))) + # Requested stop is after end of window so we need rest of window + if stop_fepoch > window_stop_fepoch: + cut_stop_fepoch = window_stop_fepoch + cut_stop_sample = window_samples + else: + # Requested stop is somewhere in window + cut_stop_fepoch = round(stop_fepoch, 6) + cut_stop_sample = int(round( + math.ceil((cut_stop_fepoch - + window_start_fepoch) * sr), + 6)) + # Get trace reference and cut data available in this window + trace_reference = self.ph5_g_receivers.find_trace_ref( + d['array_name_data_a'].strip()) + + if trace_reference is None: + continue + + if not trace_reference: + continue + + data_tmp = self.ph5_g_receivers.read_trace( + trace_reference, + start=int(round(cut_start_sample - time_cor_guess_samples)), + stop=int(round(cut_stop_sample - time_cor_guess_samples))) + current_trace_type, current_trace_byteorder = ( + self.ph5_g_receivers.trace_info(trace_reference)) + if first: + # Correct start time to 'actual' time of first sample + if trace_start_fepoch is None: + trace_start_fepoch = \ + window_start_fepoch + cut_start_sample / sr + first = False + dt = 'int32' + if current_trace_type == 'float': + dt = 'float32' + + data = np.array([], dtype=dt) + else: + # Time difference between the end of last window and the start + # of this one + time_diff = abs(window_start_fepoch) + # Overlaps are positive + d['gap_overlap'] = time_diff - (1. / sr) + # Data gap + if abs(time_diff) > (1. / sr): + new_trace = True + if len(data_tmp) > 0: + # Gap!!! + if new_trace: + # Save trace before gap + trace = Trace(data, + trace_start_fepoch, + 0, # time_correction + len(data), # samples_read + sr, + current_trace_type, + current_trace_byteorder, + das_t, + None, # receiver_t + None, # response_t + clock=clock) + traces.append(trace) + # + # Start of trace after gap + # + start_fepoch = trace_start_fepoch + trace_start_fepoch = window_start_fepoch + samples_read = len(data_tmp) + + dt = 'int32' + if current_trace_type == 'float': + dt = 'float32' + data = np.array([], dtype=dt) + + data = np.append(data, data_tmp) + das_t = [d] + new_trace = False + else: + data = np.append(data, data_tmp) + samples_read += len(data_tmp) + das_t.append(d) + # adjust the number of data samples as to not over extend the + # cut_stop_fepoch + if data is None: + return [Trace(np.array([]), start_fepoch, 0., + 0, sample_rate, None, None, das_t, None, + None, clock=clock)] + calc_stop_fepoch = trace_start_fepoch + (len(data) / sr) + + # calculate number of overextending samples + # num_overextend_samples is specific to the data per das table + # needs to be embedded in for loop to work properly. + num_overextend_samples = int(math.floor(calc_stop_fepoch - + cut_stop_fepoch) * sr) + samples_to_cut = int(len(data) - num_overextend_samples) + if num_overextend_samples > 0: + # trim the data array to exclude the over extending samples + data = data[0:samples_to_cut] + # Done reading all the traces catch the last bit + trace = Trace(data, + trace_start_fepoch, + 0, # time_correction_ms + len(data), # nsamples + sample_rate, + current_trace_type, + current_trace_byteorder, + das_t, + None, # receiver_t + None, # response_t + clock=clock) + traces.append(trace) + if das_t: + receiver_t = self.get_receiver_t(das_t[0]) + response_t = self.get_response_t(das_t[0]) + else: + receiver_t = None + response_t = None + ret = [] + + for t in traces: + if apply_time_correction: + window_start_fepoch0 = t.start_time + window_stop_fepoch = window_start_fepoch0 + (t.nsamples / sr) + time_correction, clock = \ + _cor(window_start_fepoch0.epoch(fepoch=True), + window_stop_fepoch.epoch(fepoch=True), + Time_t) + if time_correction != time_cor_guess_ms: + t.clock.comment.append( + "Time correction mismatch. {0}ms/{1}ms" + .format(time_correction, time_cor_guess_ms)) + else: + time_correction = 0. + # Set time correction + t.time_correction_ms = time_correction + # Set receiver_t and response_t + t.receiver_t = receiver_t + t.response_t = response_t + ret.append(t) + + if 'PH5API_DEBUG' in os.environ and os.environ['PH5API_DEBUG']: + for t in ret: + print('-=' * 40) + print(t) + + return ret + + def get_extent(self, das, component, sample_rate, start=None, end=None): + ''' + Takes a das serial number, and option start and end time + and returns the time of the earliest and latest samples + fot a given channel + Required: das serial and component + Optional: Start time, End time + :param das: das serial number + :param component: component channel number + :param start: start time epoch + :param end: end time epoch + :param sample_rate: sample rate + :return: earliest epoch and latest epoch + ''' + das_t_t = None + if component is None: + raise ValueError("Component required for get_extent") + if start or end: + if not (start and end): + raise ValueError("if start or end, both are required") + # self.read_das_t(das, start, end, reread=True) + + if das not in self.Das_t: + das_t_t = self.query_das_t( + das, + chan=component, + start_epoch=start, + stop_epoch=end, + sample_rate=sample_rate) + if not das_t_t: + LOGGER.warning("No Das table found for " + das) + return None, None + + if not das_t_t: + Das_t = filter_das_t(self.Das_t[das]['rows'], component) + else: + Das_t = filter_das_t(das_t_t, component) + new_das_t = sorted(Das_t, key=lambda k: k['time/epoch_l']) + + if not new_das_t: + LOGGER.warning("No Das table found for " + das) + return None, None + earliest_epoch = (float(new_das_t[0]['time/epoch_l']) + + float(new_das_t[0] + ['time/micro_seconds_i']) / 1000000) + + latest_epoch_start = (float(new_das_t[-1]['time/epoch_l']) + + float(new_das_t[-1] + ['time/micro_seconds_i']) / 1000000) + if new_das_t[-1]['sample_rate_i'] > 0: + true_sample_rate = (float(new_das_t[-1]['sample_rate_i']) / + float(new_das_t[-1] + ['sample_rate_multiplier_i'])) + latest_epoch = (latest_epoch_start + + (float(new_das_t[-1]['sample_count_i']) + / true_sample_rate)) + else: + latest_epoch = earliest_epoch + + self.forget_das_t(das) + + return earliest_epoch, latest_epoch + + def get_availability(self, das, sample_rate, component, + start=None, end=None): + ''' + Required: das, sample_rate and component + Optional: Start time, End time + :param das: das serial number + :param sample_rate: sample rate + :param component: component channel number + :param start: start time epoch + :param end: end time epoch + :return: list of tuples (sample_rate, start, end) + ''' + das_t_t = None + gaps = 0 + if component is None: + raise ValueError("Component required for get_availability") + if sample_rate is None: + raise ValueError("Sample rate required for get_availability") + + self.read_das_t(das, start, end, reread=True) + + if das not in self.Das_t: + das_t_t = self.query_das_t( + das, + chan=component, + start_epoch=start, + stop_epoch=end, + sample_rate=sample_rate) + if not das_t_t: + LOGGER.warning("No Das table found for " + das) + return None + if not das_t_t: + Das_t = filter_das_t(self.Das_t[das]['rows'], component) + else: + Das_t = filter_das_t(das_t_t, component) + if sample_rate > 0: + Das_t = [das_t for das_t in Das_t if + das_t['sample_rate_i'] / + das_t['sample_rate_multiplier_i'] == sample_rate] + else: + Das_t = [das_t for das_t in Das_t if + das_t['sample_rate_i'] == sample_rate] + + new_das_t = sorted(Das_t, key=lambda k: k['time/epoch_l']) + + if not new_das_t: + LOGGER.warning("No Das table found for " + das) + return None + + gaps = 0 + prev_start = None + prev_end = None + prev_len = None + prev_sr = None + times = [] + for entry in new_das_t: + # set the values for this entry + cur_time = (float(entry['time/epoch_l']) + + float(entry['time/micro_seconds_i']) / + 1000000) + if entry['sample_rate_i'] > 0: + cur_len = (float(entry['sample_count_i']) / + float(entry['sample_rate_i']) / + float(entry['sample_rate_multiplier_i'])) + cur_sr = (float(entry['sample_rate_i']) / + float(entry['sample_rate_multiplier_i'])) + else: + cur_len = 0 + cur_sr = 0 + cur_end = cur_time + cur_len + + if (prev_start is None and prev_end is None and + prev_len is None and prev_sr is None): + prev_start = cur_time + prev_end = cur_end + prev_len = cur_len + prev_sr = cur_sr + else: + if (cur_time == prev_start and + cur_len == prev_len and + cur_sr == prev_sr): + # duplicate entry - skip + continue + elif (cur_time > prev_end or + cur_sr != prev_sr): + # there is a gap so add a new entry + times.append((prev_sr, + prev_start, + prev_end)) + # increment the number of gaps and reset previous + gaps = gaps + 1 + prev_start = cur_time + prev_end = cur_end + prev_len = cur_len + prev_sr = cur_sr + elif (cur_time == prev_end and + cur_sr == prev_sr): + # extend the end time since this was a continuous segment + prev_end = cur_end + prev_len = cur_len + prev_sr = cur_sr + + # add the last continuous segment + times.append((prev_sr, + prev_start, + prev_end)) + + self.forget_das_t(das) + + return times + +# +# Mix-ins +# + + +def pad_traces(traces): + ''' + Input: + A list of ph5 Trace objects + Return: + A trace object with gaps padded with the mean + ''' + + def pad(data, n, dtype): + m = np.mean(data, dtype=dtype) + + return np.append(data, [m] * n) + + ret = Trace(traces[0].data, # Gets extended (np.append) + 0., # Gets set at begining + 0, # ??? + 0., # Gets set at end + traces[0].sample_rate, # Should not change + traces[0].ttype, # Should not change + traces[0].byteorder, # Should not change + traces[0].das_t, # Gets appended to + traces[0].receiver_t, # Should not change + traces[0].response_t, # Should not change + clock=traces[0].clock) + ret.start_time = traces[0].start_time + + end_time0 = None + end_time1 = None + x = 0 + tcor_sum = 0 + N = 0 + for t in traces: + tcor_sum += t.time_correction_ms + x += 1. + end_time0 = t.start_time.epoch( + fepoch=True) + (t.nsamples / t.sample_rate) + if end_time1 is not None: + if end_time0 != end_time1: + n = int(((end_time1 - end_time0) * ret.sample_rate) + 0.5) + # Pad + d = pad(t.data, n, dtype=ret.ttype) + ret.data = np.append(ret.data, d) + N += n + + end_time1 = end_time0 + (1. / t.sample_rate) + + ret.padding = N + ret.nsamples = len(ret.data) + ret.time_correction_ms = int((tcor_sum / x) + 0.5) + + return ret + + +def seed_channel_code(array_t): + try: + if len(array_t['seed_band_code_s']) == 1 and len( + array_t['seed_instrument_code_s']) == 1 and \ + len(array_t['seed_orientation_code_s']) == 1: + return array_t['seed_band_code_s'] + \ + array_t['seed_instrument_code_s'] + \ + array_t['seed_orientation_code_s'] + else: + return "---" + except KeyError: + return "---" + + +def by_id(rows, key='id_s', secondary_key=None, unique_key=True): + ''' Order table info by id_s (usually) then if required a secondary key. + ''' + order = [] + byid = {} + for r in rows: + if key in r: + Id = r[key] + if unique_key: + byid[Id] = r + order.append(Id) + elif secondary_key and secondary_key in r: + if Id not in byid: + byid[Id] = {} + order.append(Id) + if r[secondary_key] not in byid[Id]: + byid[Id][r[secondary_key]] = [r] + else: + byid[Id][r[secondary_key]].append(r) + else: + if Id not in byid: + byid[Id] = [] + order.append(Id) + byid[Id].append(r) + + return byid, order + + +def run_geod(lat0, lon0, lat1, lon1): + UNITS = 'm' + ELLIPSOID = 'WGS84' + + config = "+ellps={0}".format(ELLIPSOID) + + g = Geod(config) + + az, baz, dist = g.inv(lon0, lat0, lon1, lat1) + + if dist: + dist /= FACTS_M[UNITS] + + # Return list containing azimuth, back azimuth, distance + return az, baz, dist + + +def rect(r, w, deg=0): + # Convert from polar to rectangular coordinates + # radian if deg=0; degree if deg=1 + from math import cos, sin, pi + if deg: + w = pi * w / 180.0 + return r * cos(w), r * sin(w) + + +def linreg(X, Y): + if len(X) != len(Y): + raise ValueError( + 'Unequal length, X and Y. Can\'t do linear regression.') + + N = len(X) + Sx = Sy = Sxx = Syy = Sxy = 0.0 + for x, y in map(None, X, Y): + Sx = Sx + x + Sy = Sy + y + Sxx = Sxx + x * x + Syy = Syy + y * y + Sxy = Sxy + x * y + + det = Sxx * N - Sx * Sx + if det == 0: + return 0.0, 0.0, None + + a, b = (Sxy * N - Sy * Sx) / det, (Sxx * Sy - Sx * Sxy) / det + + meanerror = residual = 0.0 + for x, y in map(None, X, Y): + meanerror = meanerror + (y - Sy / N) ** 2 + residual = residual + (y - a * x - b) ** 2 + + RR = 1 - residual / meanerror + if N > 2: + ss = residual / (N - 2) + else: + ss = 1. + + return a, b, (RR, ss) + + +def calc_offset_sign(offsets): + ''' offsets is a list of offset_t ''' + if not offsets: + return [] + from math import atan, degrees + X = [] + Y = [] + OO = [] + offsetmin = 21 ** 63 - 1 + for offset_t in offsets: + try: + w = offset_t['azimuth/value_f'] + r = offset_t['offset/value_d'] + if abs(r) < abs(offsetmin): + offsetmin = r + + x, y = rect(r, w, deg=True) + X.append(x) + Y.append(y) + except Exception as e: + LOGGER.error(e) + + # The seismic line is abx + c (ab => w) + ab, c, err = linreg(X, Y) + + if abs(ab) > 1: + regangle = degrees(atan(1. / ab)) + else: + regangle = degrees(atan(ab)) + + sig = 0 + flop = False + for offset_t in offsets: + try: + # Rotate line to have zero slope + a = offset_t['azimuth/value_f'] + + w = a - regangle + # Pick initial sign + if sig == 0: + if w < 0: + sig = -1 + else: + sig = 1 + + offset_t['offset/value_d'] = sig * \ + float(offset_t['offset/value_d']) + + # Once we pass the minimum offset flip the sign + if abs(offsetmin) == abs(offset_t['offset/value_d']) and not flop: + flop = True + sig *= -1 + + OO.append(offset_t) + except Exception as e: + LOGGER.error(e) + + # Returning Oh not zero + return OO + + +def is_in(start, stop, start_epoch, stop_epoch): + ''' + start is start of window + stop is stop of window + start_epoch is start of desired data + stop_epoch is stop of desired data + ''' + # start_epoch is in between start and stop + if start_epoch >= start and start_epoch <= stop: + return True + # stop_epoch is in between start and stop + elif stop_epoch >= start and stop_epoch <= stop: + return True + # entire recording window is in between start_epoch and stop_epoch + elif start_epoch <= start and stop_epoch >= stop: + return True + else: + return False + + +def build_kef(ts, rs): + ''' + ts -> table string + rs -> rows object + ''' + tdoy = timedoy.TimeDOY(epoch=time.time()) + ret = "#\n### Written by ph5api v{0} at {1}\n#\n".format( + PROG_VERSION, tdoy.getFdsnTime()) + i = 0 + for r in rs: + i += 1 + ret += "# {0}\n".format(i) + ret += ts + '\n' + keys = r.keys() + for k in keys: + line = "\t{0} = {1}\n".format(k, r[k]) + ret += line + + return ret + + +def fepoch(epoch, ms): + ''' + Given ascii epoch and miliseconds return epoch as a float. + ''' + epoch = float(int(epoch)) + secs = float(int(ms)) / 1000000.0 + + return epoch + secs + + +def _cor(start_fepoch, stop_fepoch, Time_t, max_drift_rate=MAX_DRIFT_RATE): + ''' Calculate clock correction in miliseconds ''' + clock = Clock() + if not Time_t: + Time_t = [] + + time_t = None + for t in Time_t: + if hasattr(t, 'corrected_i'): + if t['corrected_i'] != 1: + data_start = fepoch(t['start_time/epoch_l'], + t['start_time/micro_seconds_i']) + data_stop = fepoch(t['end_time/epoch_l'], + t['end_time/micro_seconds_i']) + if is_in(data_start, data_stop, start_fepoch, stop_fepoch): + time_t = t + break + else: + data_start = fepoch(t['start_time/epoch_l'], + t['start_time/micro_seconds_i']) + data_stop = fepoch(t['end_time/epoch_l'], + t['end_time/micro_seconds_i']) + if is_in(data_start, data_stop, start_fepoch, stop_fepoch): + time_t = t + break + + if time_t is None: + clock.comment.append("No clock drift information available.") + return 0., clock + + clock = Clock(slope=time_t['slope_d'], offset_secs=time_t['offset_d'], + max_drift_rate_allowed=max_drift_rate) + # Handle fixed offset correction + if time_t['slope_d'] == 0. and time_t['offset_d'] != 0.: + return 1000. * time_t['offset_d'], clock + + if abs(time_t['slope_d']) > MAX_DRIFT_RATE: + clock.comment.append("Clock drift rate exceeds maximum drift rate.") + + mid_fepoch = start_fepoch + ((stop_fepoch - start_fepoch) / 2.) + delta_fepoch = mid_fepoch - data_start + + time_correction_ms = int(time_t['slope_d'] * (delta_fepoch * 1000.)) * -1 + return time_correction_ms, clock + + +def filter_das_t(Das_t, chan): + def sort_on_epoch(a, b): + a_epoch = a['time/epoch_l'] + \ + (float(a['time/micro_seconds_i']) / 1000000.) + b_epoch = b['time/epoch_l'] + \ + (float(b['time/micro_seconds_i']) / 1000000.) + + if a_epoch > b_epoch: + return 1 + elif a_epoch < b_epoch: + return -1 + else: + return 0 + + ret = [] + Das_t = [das_t for das_t in Das_t if das_t['channel_number_i'] == chan] + + for das_t in Das_t: + if not ret: + ret.append(das_t) + continue + if (ret[-1]['sample_rate_i'] == das_t['sample_rate_i'] and + ret[-1]['sample_rate_multiplier_i'] == + das_t['sample_rate_multiplier_i'] and + ret[-1]['time/micro_seconds_i'] == + das_t['time/micro_seconds_i'] and + ret[-1]['time/epoch_l'] == das_t['time/epoch_l']): + continue + else: + ret.append(das_t) + + ret.sort(cmp=sort_on_epoch) + + return ret diff --git a/ph5/entry_points.py b/ph5/entry_points.py index 0e37b558..c6771acf 100644 --- a/ph5/entry_points.py +++ b/ph5/entry_points.py @@ -255,6 +255,11 @@ def __init__(self): 'Generate data_description.txt ' 'and/or data_request_key.txt.', type=EntryPointTypes.ALL), + EntryPoint('correctwpn3', + 'ph5.utilities.correct_w_pn3:main', + "Correct recreate pn4's array_t, index_t, " + "das data with info from pn3.", + type=EntryPointTypes.ALL), EntryPoint('ph5toevt', 'ph5.clients.ph5toevt:main', 'Extract events from a ph5 archive, ' diff --git a/ph5/test_data/ph5_correct_w_pn3/README.txt b/ph5/test_data/ph5_correct_w_pn3/README.txt new file mode 100644 index 00000000..2349aa0b --- /dev/null +++ b/ph5/test_data/ph5_correct_w_pn3/README.txt @@ -0,0 +1,5 @@ +This data tests are created to test correctwpn3 in which, ++ pn3_master.ph5 and miniPH5_00001.ph5 has Index_t, Array_t, Das_t using pn3 format. Other tables aren’t touch the were ignored when these data files were created. ++ arrays_testx.kef, index_test.kef are metadata to create inconsistencies for data in unites ++ addInfo to test if data are corrected using flag -a ++ pn4_master.ph5 is ph5 data with pn4 format. This is the base data that correctwpn3 corrects in unit tests. \ No newline at end of file diff --git a/ph5/test_data/ph5_correct_w_pn3/addInfo.txt b/ph5/test_data/ph5_correct_w_pn3/addInfo.txt new file mode 100644 index 00000000..55468279 --- /dev/null +++ b/ph5/test_data/ph5_correct_w_pn3/addInfo.txt @@ -0,0 +1,3 @@ +sensor/model_s=L28 +sensor/manufacturer_s=Sercel +das/model_s=rt125 \ No newline at end of file diff --git a/ph5/test_data/ph5_correct_w_pn3/arrays_test1.kef b/ph5/test_data/ph5_correct_w_pn3/arrays_test1.kef new file mode 100644 index 00000000..fe9d2951 --- /dev/null +++ b/ph5/test_data/ph5_correct_w_pn3/arrays_test1.kef @@ -0,0 +1,354 @@ +# row 11: deploy=pickup: ids 1111, chan 3 +# delete das 3X500 +# row 10 duplicated row 5: id_s 1111, chan 2 +# Table row 1 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=1 + +# Table row 2 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=2 + +# Table row 3 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=3 + +# Table row 4 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=1 + +# Table row 5 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=2 + +# Table row 6 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=3 + +# Table row 7 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=1 + +# Table row 8 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=2 + +# Table row 9 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=3 + +# Table row 10 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=2 + +# Table row 11 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Wed Dec 31 17:00:00 1969 + deploy_time/epoch_l=25200 + deploy_time/micro_seconds_i=0 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Wed Dec 31 17:00:00 1969 + pickup_time/epoch_l=25200 + pickup_time/micro_seconds_i=0 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=3 \ No newline at end of file diff --git a/ph5/test_data/ph5_correct_w_pn3/arrays_test2.kef b/ph5/test_data/ph5_correct_w_pn3/arrays_test2.kef new file mode 100644 index 00000000..b05ecdb7 --- /dev/null +++ b/ph5/test_data/ph5_correct_w_pn3/arrays_test2.kef @@ -0,0 +1,354 @@ +# +# remove row 12: das 3X500 channel=3 +# row 10: change time from 2017 to 2019 for das 1X1111 channel 1 +# Table row 1 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=1 + +# Table row 2 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=2 + +# Table row 3 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jun 29 18:03:13 2019 + deploy_time/epoch_l=1561831393 + deploy_time/micro_seconds_i=846999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Jul 20 14:40:04 2019 + pickup_time/epoch_l=1563633604 + pickup_time/micro_seconds_i=726999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4892 + channel_number_i=3 + +# Table row 4 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=1 + +# Table row 5 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=2 + +# Table row 6 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Jul 20 14:46:58 2019 + deploy_time/epoch_l=1563634018 + deploy_time/micro_seconds_i=690000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Aug 17 13:56:27 2019 + pickup_time/epoch_l=1566050187 + pickup_time/micro_seconds_i=786999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 5118 + channel_number_i=3 + +# Table row 7 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=1 + +# Table row 8 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=2 + +# Table row 9 +/Experiment_g/Sorts_g/Array_t_001 + id_s=1111 + location/X/value_d=327519.7 + location/X/units_s=degrees + location/Y/value_d=3773952.7 + location/Y/units_s=degrees + location/Z/value_d=1408.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l=1567269236 + deploy_time/micro_seconds_i=857000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i=609999 + pickup_time/type_s=BOTH + das/serial_number_s=1X1111 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 1X1111, Node ID: 4886 + channel_number_i=3 + +# Table row 10 +/Experiment_g/Sorts_g/Array_t_001 + id_s=500 + location/X/value_d=469565.2 + location/X/units_s=degrees + location/Y/value_d=5280709.7 + location/Y/units_s=degrees + location/Z/value_d=0.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Sat Aug 31 16:33:56 2019 + deploy_time/epoch_l= 1567269236 + deploy_time/micro_seconds_i= 609999 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Sat Sep 28 14:30:37 2019 + pickup_time/epoch_l=1569681037 + pickup_time/micro_seconds_i= 609999 + pickup_time/type_s=BOTH + das/serial_number_s=3X500 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 3X500, Node ID: 2240 + channel_number_i=1 + +# Table row 11 +/Experiment_g/Sorts_g/Array_t_001 + id_s=500 + location/X/value_d=469565.2 + location/X/units_s=degrees + location/Y/value_d=5280709.7 + location/Y/units_s=degrees + location/Z/value_d=0.0 + location/Z/units_s=unknown + location/coordinate_system_s=geographic + location/projection_s=WGS84 + location/ellipsoid_s= + location/description_s=Read from SEG-D as is. + deploy_time/ascii_s=Wed Aug 9 15:46:32 2017 + deploy_time/epoch_l=1502293592 + deploy_time/micro_seconds_i=230000 + deploy_time/type_s=BOTH + pickup_time/ascii_s=Wed Aug 9 20:06:58 2017 + pickup_time/epoch_l=1502309218 + pickup_time/micro_seconds_i=119999 + pickup_time/type_s=BOTH + das/serial_number_s=3X500 + das/model_s=ZLAND 3C + das/manufacturer_s=FairfieldNodal + das/notes_s=manufacturer and model not read from data file. + sensor/serial_number_s= + sensor/model_s=GS-30CT + sensor/manufacturer_s=Geo Space + sensor/notes_s=manufacturer and model not read from file. + description_s=DAS: 3X500, Node ID: 2240 + channel_number_i=2 diff --git a/ph5/test_data/ph5_correct_w_pn3/index_test.kef b/ph5/test_data/ph5_correct_w_pn3/index_test.kef new file mode 100644 index 00000000..8670923d --- /dev/null +++ b/ph5/test_data/ph5_correct_w_pn3/index_test.kef @@ -0,0 +1,488 @@ +# remove all entries related to das 3X500 +# +# +# Table row 1 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:00:30 2019 + end_time/epoch_l=1562256030 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:00 2019 + start_time/epoch_l=1562256000 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 2 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:00 2019 + end_time/epoch_l=1562256060 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:30 2019 + start_time/epoch_l=1562256030 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 3 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:30 2019 + end_time/epoch_l=1562256090 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:01:00 2019 + start_time/epoch_l=1562256060 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 4 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:00:30 2019 + end_time/epoch_l=1562256030 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:00 2019 + start_time/epoch_l=1562256000 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 5 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:00 2019 + end_time/epoch_l=1562256060 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:30 2019 + start_time/epoch_l=1562256030 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 6 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:30 2019 + end_time/epoch_l=1562256090 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:01:00 2019 + start_time/epoch_l=1562256060 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 7 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:00:30 2019 + end_time/epoch_l=1562256030 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:00 2019 + start_time/epoch_l=1562256000 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 8 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:00 2019 + end_time/epoch_l=1562256060 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:00:30 2019 + start_time/epoch_l=1562256030 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 9 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Jul 4 16:01:30 2019 + end_time/epoch_l=1562256090 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Jul 4 16:01:00 2019 + start_time/epoch_l=1562256060 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 10 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:00:30 2019 + end_time/epoch_l=1564675230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:00 2019 + start_time/epoch_l=1564675200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 11 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:00 2019 + end_time/epoch_l=1564675260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:30 2019 + start_time/epoch_l=1564675230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 12 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:30 2019 + end_time/epoch_l=1564675290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:01:00 2019 + start_time/epoch_l=1564675260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 13 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:00:30 2019 + end_time/epoch_l=1564675230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:00 2019 + start_time/epoch_l=1564675200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 14 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:00 2019 + end_time/epoch_l=1564675260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:30 2019 + start_time/epoch_l=1564675230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 15 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:30 2019 + end_time/epoch_l=1564675290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:01:00 2019 + start_time/epoch_l=1564675260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 16 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:00:30 2019 + end_time/epoch_l=1564675230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:00 2019 + start_time/epoch_l=1564675200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 17 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:00 2019 + end_time/epoch_l=1564675260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:00:30 2019 + start_time/epoch_l=1564675230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 18 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Thu Aug 1 16:01:30 2019 + end_time/epoch_l=1564675290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Thu Aug 1 16:01:00 2019 + start_time/epoch_l=1564675260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 19 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:00:30 2019 + end_time/epoch_l=1568563230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:00 2019 + start_time/epoch_l=1568563200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 20 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:00 2019 + end_time/epoch_l=1568563260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:30 2019 + start_time/epoch_l=1568563230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 21 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:30 2019 + end_time/epoch_l=1568563290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:01:00 2019 + start_time/epoch_l=1568563260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 22 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:00:30 2019 + end_time/epoch_l=1568563230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:00 2019 + start_time/epoch_l=1568563200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 23 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:00 2019 + end_time/epoch_l=1568563260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:30 2019 + start_time/epoch_l=1568563230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 24 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:30 2019 + end_time/epoch_l=1568563290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:01:00 2019 + start_time/epoch_l=1568563260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 25 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:00:30 2019 + end_time/epoch_l=1568563230 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:00 2019 + start_time/epoch_l=1568563200 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 26 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:00 2019 + end_time/epoch_l=1568563260 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:00:30 2019 + start_time/epoch_l=1568563230 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH + +# Table row 27 +/Experiment_g/Receivers_g/Index_t + end_time/ascii_s=Sun Sep 15 16:01:30 2019 + end_time/epoch_l=1568563290 + end_time/micro_seconds_i=329998 + end_time/type_s=BOTH + external_file_name_s=./miniPH5_00001.ph5 + hdf5_path_s=/Experiment_g/Receivers_g/Das_g_1X1111 + serial_number_s=1X1111 + start_time/ascii_s=Sun Sep 15 16:01:00 2019 + start_time/epoch_l=1568563260 + start_time/micro_seconds_i=329998 + start_time/type_s=BOTH + time_stamp/ascii_s=Tue Oct 26 21:18:02 2021 + time_stamp/epoch_l=1635283082 + time_stamp/micro_seconds_i=0 + time_stamp/type_s=BOTH diff --git a/ph5/test_data/ph5_correct_w_pn3/miniPH5_00001.ph5 b/ph5/test_data/ph5_correct_w_pn3/miniPH5_00001.ph5 new file mode 100644 index 00000000..a3902b78 Binary files /dev/null and b/ph5/test_data/ph5_correct_w_pn3/miniPH5_00001.ph5 differ diff --git a/ph5/test_data/ph5_correct_w_pn3/pn3_master.ph5 b/ph5/test_data/ph5_correct_w_pn3/pn3_master.ph5 new file mode 100644 index 00000000..5309f7ca Binary files /dev/null and b/ph5/test_data/ph5_correct_w_pn3/pn3_master.ph5 differ diff --git a/ph5/test_data/ph5_correct_w_pn3/pn4_master.ph5 b/ph5/test_data/ph5_correct_w_pn3/pn4_master.ph5 new file mode 100644 index 00000000..cc4abed6 Binary files /dev/null and b/ph5/test_data/ph5_correct_w_pn3/pn4_master.ph5 differ diff --git a/ph5/utilities/correct_w_pn3.py b/ph5/utilities/correct_w_pn3.py new file mode 100644 index 00000000..f04e77e6 --- /dev/null +++ b/ph5/utilities/correct_w_pn3.py @@ -0,0 +1,631 @@ +# +# Lan Dam, September 2021 +# + + +import argparse +import os +import sys +import logging +from shutil import copy +import time + +from ph5.core import ph5api, experiment, columns, ph5api_pn3, timedoy +from ph5.utilities import tabletokef as T2K +from ph5 import LOGGING_FORMAT + + +PROG_VERSION = "2021.301" +LOGGER = logging.getLogger(__name__) + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + + parser.usage = ("correctwpn3 --pn4 pn4path --pn3 pn3path") + + parser.description = ( + "Require: full data set of pn3 in pn3path" + " and master.ph5 of pn4 in pn4path" + "Exclude none-matched entries in array_t, index_t, das_t.\n" + "Remove external links and all pn4's minifiles.\n" + "Recreate Array_t, Index_t with info from pn3.\n" + "Copy pn3's minifiles to pn4.\n" + "Reformat das_t table.\n" + "Create new external link based on info from Index_t.\n" + "Version: {0}".format(PROG_VERSION)) + + parser.add_argument( + "--pn4", dest="pn4path", metavar="pn4path", required=True, + help="Path to pn4 master file (not include master.ph5)") + + parser.add_argument( + "--pn3", dest="pn3path", metavar="pn3path", required=True, + help="Path to pn3 master file (not include master.ph5)") + + parser.add_argument( + "-a", "--add_info", dest="addInfoFile", metavar="addInfoFile", + help=("Path to file containing user's values for array_t. " + "Ex: -a path/addInfo.txt \n" + "Format for each line: key=value\n" + "Ex:" + "\n\tsensor/model_s=L28" + "\n\tsensor/manufacturer_s=Sercel")) + parser.add_argument( + "-s", dest="skip_missing_minifile", action='store_true', default=False, + help="Flag to skip reporting about minifile") + + parser.add_argument( + "-S", dest="skip_questioning", action='store_true', default=False, + help="Flag to skip asking for response from user") + + args = parser.parse_args() + skip_missing_minifile = args.skip_missing_minifile + skip_ask_response = args.skip_questioning + pn4path = args.pn4path + pn3path = args.pn3path + addInfoFile = args.addInfoFile + return (pn4path, pn3path, addInfoFile, + skip_missing_minifile, skip_ask_response) + + +def set_logger(): + """ + Setting logger's format and filehandler + """ + + # set filehandler + ch = logging.FileHandler("correct_w_pn3.log") + ch.setLevel(logging.INFO) + # Add formatter + formatter = logging.Formatter(LOGGING_FORMAT) + ch.setFormatter(formatter) + LOGGER.addHandler(ch) + + +def getDupOfField(listOfDict, mainK1, mainK2, mainK3, dupK): + """ + return list of (mainK1, mainK2, mainK3) that have dupK duplicated + """ + c = {} + for d in listOfDict: + c.setdefault((d[mainK1], d[mainK2], d[mainK3]), []).append(d[dupK]) + b = [{'mixK': k, dupK: v} for k, v in c.iteritems()] + d = [e['mixK'] for e in b if len(e[dupK]) > 1] + return d + + +# ########################### CHECK ISSUES ############################## +def get_array_t(pn3object): + """ + + read array_t from pn3 + + remove entry with deploy_t=pickup_time + + return list of entry with array_name to identify array + """ + entryList = [] + sameDepPic = {} + pn3object.read_array_t_names() + count = 0 + remCount = 0 + dasDict = {} + for aname in sorted(pn3object.Array_t_names): + pn3object.read_array_t(aname) + arraybyid = pn3object.Array_t[aname]['byid'] + arrayorder = pn3object.Array_t[aname]['order'] + + for ph5_station in arrayorder: + station_list = arraybyid.get(ph5_station) + for deployment in station_list: + station_len = len(station_list[deployment]) + for st_num in range(0, station_len): + e = station_list[deployment][st_num] + chan = e['channel_number_i'] + if e['deploy_time/epoch_l'] >= e['pickup_time/epoch_l']: + key = (aname, chan, e['deploy_time/ascii_s']) + if key not in sameDepPic.keys(): + sameDepPic[key] = {} + if e['das/serial_number_s'] not in sameDepPic[key]: + sameDepPic[key][e['das/serial_number_s']] = 0 + sameDepPic[key][e['das/serial_number_s']] += 1 + remCount += 1 + else: + e['array_name'] = aname + entryList.append(e) + if e['das/serial_number_s'] not in dasDict.keys(): + dasDict[e['das/serial_number_s']] = [] + dasDict[e['das/serial_number_s']].append(e) + count += 1 + + for t in sameDepPic.keys(): + for das in sameDepPic[t].keys(): + remNo = sameDepPic[t][das] + total = sameDepPic[t][das] + if das in dasDict: + total += len(dasDict[das]) + sameDepPic[t][das] = "%s/%s" % (remNo, total) + + if len(sameDepPic) != 0: + msg = ("Due to coincided deploy and pickup times, %s/%s " + "entries will be removed from array_t:\n" + "[array, chan, deptime]: {[das_serial]: [rem/total], " + "[das_serial]: [rem/total], ...},\n" % (remCount, count)) + for key, dasncount in sameDepPic.items(): + msg += "%s: %s\n" % (key, dasncount) + LOGGER.warning(msg) + + for d in dasDict: + res = getDupOfField(dasDict[d], 'array_name', 'channel_number_i', + 'deploy_time/ascii_s', 'pickup_time/ascii_s') + for r in res: + array, channel, dep = r + LOGGER.warning('Das %s channel %s deploy time %s duplicated in %s.' + ' User need to handle this manually' + % (d, channel, dep, array)) + + return entryList, dasDict.keys() + + +def get_index_t(pn3object, pn3_array_t, das_of_array_t): + """ + + read index_t from pn3object + + remove entries that have mismatch das + + return new pn3_array_t, pn3_index_t + """ + pn3object.read_index_t() + pn3_index_t = [] + index_t_remDas = set() # set of das to be removed from index_t + totalOrgIndexE = len(pn3object.Index_t['rows']) + for e in pn3object.Index_t['rows']: + if e['serial_number_s'] not in das_of_array_t: + index_t_remDas.add(e['serial_number_s']) + else: + pn3_index_t.append(e) + + if len(index_t_remDas) != 0: + msg = "Due to nonexistency in array_t %s/%s entries will be removed " \ + "from index_t for the following das:\n%s" + LOGGER.warning(msg % (len(index_t_remDas), totalOrgIndexE, + sorted(index_t_remDas))) + + array_t_remDas = {} # dict of das to be removed from each array_t + totalOrgArrayE = len(pn3_array_t) + das_of_index_t = [e['serial_number_s'] for e in pn3_index_t] + new_pn3_array_t = [] + count = 0 + for e in pn3_array_t: + if e['das/serial_number_s'] not in das_of_index_t: + if e['array_name'] not in array_t_remDas: + array_t_remDas[e['array_name']] = [0, set()] + array_t_remDas[e['array_name']][1].add(e['das/serial_number_s']) + array_t_remDas[e['array_name']][0] += 1 + count += 1 + else: + new_pn3_array_t.append(e) + + if len(array_t_remDas) != 0: + msg = ("Due to nonexistency in index_t %s/%s entries will be removed" + " from array_t:\n" + % (count, totalOrgArrayE)) + for a, e in array_t_remDas.items(): + msg += "%s: remove %s entries of das: %s\n" % ( + a, e[0], sorted(e[1])) + LOGGER.warning(msg) + + return pn3_index_t, new_pn3_array_t + + +def get_das_t(pn3object, pn3_array_t, pn3_index_t): + """ + + read das_g from pn3 + + compare das with filtered pn3_index_t and remove ones that are mismatch + + build pn3_das_t for the ones that's in existing minifile + """ + pn3object.read_das_g_names() + all_das_g_name = pn3object.Das_g_names + # print("all_das_g_name: ", all_das_g_name) + all_das = [name.replace("Das_g_", "") for name in all_das_g_name.keys()] + # print(all_das) + # + index_t_das = [e['serial_number_s'] for e in pn3_index_t] + in_das_no_index = [d for d in all_das if d not in index_t_das] + if len(in_das_no_index) != 0: + msg = ("Compare Das_g against the filtered list of index_t and " + "array_t, the following Das will be removed from Das data:\n" + " %s" % sorted(in_das_no_index)) + LOGGER.warning(msg) + all_das = [d for d in all_das if d in index_t_das] + + in_index_no_das = [d for d in index_t_das if d not in all_das] + if len(in_index_no_das) != 0: + msg = ("Compare filtered list of index_t and array_t agains Das_g, " + "the following Das's entries in array_t and index_t will be " + "removed:\n %s" % sorted(in_index_no_das)) + LOGGER.warning(msg) + pn3_array_t = [e for e in pn3_array_t + if e['das/serial_number_s'] in all_das] + pn3_index_t = [e for e in pn3_index_t + if e['serial_number_s'] in all_das] + + # print(all_das_g_name['Das_g_11736']._v_pathname) + pn3dir = os.path.join(os.getcwd(), pn3object.currentpath) + # p = all_das_g_name['Das_g_11736']._get_filename_node()[0] + + existing_minifile_dict = {} + missing_minifiles = set() + empty_das_t_list = [] + pn3_das_t = {} + + for das_g_name in all_das_g_name.keys(): + das = das_g_name.replace("Das_g_", "") + das_g = all_das_g_name[das_g_name] + minifile = das_g._get_filename_node()[0] + minifile_fullpath = os.path.join(pn3dir, minifile) + if not os.path.isfile(minifile_fullpath): + missing_minifiles.add(minifile) + continue + else: + if minifile not in existing_minifile_dict: + existing_minifile_dict[minifile] = [] + existing_minifile_dict[minifile].append(das) + das = das_g_name.replace("Das_g_", "") + pn3object.read_das_t(das) + if pn3object.Das_t[das]['rows'] == []: + empty_das_t_list.append(das) + pn3_das_t[das] = pn3object.Das_t[das]['rows'] + if len(empty_das_t_list) != 0: + msg = "The following Das have empty das_t: %s" % empty_das_t_list + LOGGER.warning(msg) + + if len(missing_minifiles) != 0: + msg = "The following minifiles are missing:\n%s" % sorted( + missing_minifiles) + LOGGER.warning(msg) + return (pn3_das_t, pn3_index_t, pn3_array_t, + in_das_no_index, existing_minifile_dict) + + +def check_pn3_issues(pn3object): + """ + compare between array_t, index_t, das_t, remove mismatch das, + remove station with deploy=pickup + """ + pn3_array_t, das_of_array_t = get_array_t(pn3object) + pn3_index_t, pn3_array_t = get_index_t( + pn3object, pn3_array_t, das_of_array_t) + return get_das_t(pn3object, pn3_array_t, pn3_index_t) +# ######################### END CHECK ISSUES ############################ + + +def create_index_t_backup(pn3object, path): + """ + create index_t backup in pn4path + """ + T2K.init_local() + T2K.EX = pn3object + T2K.read_index_table() + tdoy = timedoy.TimeDOY(epoch=time.time()) + tt = "{0:04d}{1:03d}".format(tdoy.dtobject.year, tdoy.dtobject.day) + outfile = "Index_t_{0}_backup_from_pn3.kef".format(tt) + # Exit if we can't write backup kef + if os.access(os.getcwd(), os.W_OK): + LOGGER.info("Writing table backup: {0}." + .format(os.path.join(os.getcwd(), outfile))) + else: + LOGGER.error( + "Can't write: {0}.\nExiting!" + .format(os.path.join(os.getcwd(), outfile))) + sys.exit(-3) + try: + fh = open(os.path.join(path, outfile), 'w') + fh.write("# Created by correctwpn3 from pn3's index_t to recover " + "removed das") + T2K.table_print('/Experiment_g/Receivers_g/Index_t', T2K.INDEX_T, + fh=fh) + fh.close() + except Exception as e: + LOGGER.error( + "Failed to save {0}.\n{1}\nExiting!" + .format(os.path.join(os.getcwd(), outfile), e.message)) + sys.exit(-4) + return outfile + + +# ########################### CLEAN UP PN4 ############################# +def cleanup_pn4(pn4object, skip_ask_response): + """ + + truncate array_t, index_t + + remove ext_link for das_g + """ + if not skip_ask_response: + ret = raw_input("\n=========================================\n" + "All existing external links to minifiles are going " + "to be removed from : %s.\n" + "Do you want to continue?(y/n)" % pn4object.filename) + if ret == 'n': + return False + # #### REMOVE Array_t ##### + pn4object.read_array_t_names() + rem_arrays = [] + failed_rem_arrays = [] + for aname in sorted(pn4object.Array_t_names): + ARRAY_TABLE = int(aname[-3:]) + if pn4object.ph5_g_sorts.nuke_array_t(ARRAY_TABLE): + rem_arrays.append(aname) + else: + failed_rem_arrays.append(aname) + LOGGER.info("Remove the following array_t from pn4: %s" % rem_arrays) + + # #### REMOVE INDEX_T ##### + pn4object.ph5_g_receivers.nuke_index_t() + LOGGER.info("Remove Index_t from pn4") + + # ### REMOVE EXT_LINK FOR Das_g #### + pn4object.read_das_g_names() + all_das_g_name = pn4object.Das_g_names + rem_das = [] + for das_g_name in all_das_g_name.keys(): + external_path = all_das_g_name[das_g_name]._v_pathname + try: + group_node = pn4object.ph5.get_node(external_path) + group_node.remove() + rem_das.append(external_path.split('/')[3].replace("Das_g_", "")) + except Exception: + pass + LOGGER.info("Remove Das_g external links from pn4: %s" % rem_das) + return pn4object +# ########################## END CLEAN UP PN4 ########################### + + +# ############################ RECREATE PN4 ############################# +def get_band_code(sample_rate): + if sample_rate >= 1000: + band_code = 'G' + elif sample_rate >= 250 and sample_rate < 1000: + band_code = 'D' + elif sample_rate >= 80 and sample_rate < 250: + band_code = 'E' + elif sample_rate >= 10 and sample_rate < 80: + band_code = 'S' + else: + band_code = 'X' + return band_code + + +def convert_to_pn4_array(entry, pn3_das_t, addInfo, skip_missing_minifile): + """ + param pn3e: array entry form pn3 + param addInfoFile + """ + del entry['array_name'] + das = entry['das/serial_number_s'] + chan = entry['channel_number_i'] + dep = entry['deploy_time/epoch_l'] + pic = entry['pickup_time/epoch_l'] + if das in pn3_das_t.keys(): + rel_das_chan_rows = [e for e in pn3_das_t[das] + if e['channel_number_i'] == chan] + if len(rel_das_chan_rows) == 0: + LOGGER.error("Cannot fill in station's sample_rate_i, " + "seed_band_code_s and receiver_table_n_i because " + "chan %s doesn't exist for %s in das_t " + % (chan, das)) + else: + rel_das_time_rows = [e for e in rel_das_chan_rows + if dep <= e['time/epoch_l'] <= pic] + sample_rates = list({e['sample_rate_i'] + for e in rel_das_time_rows}) + if len(sample_rates) > 1: + msg = ("There are more than one sample rate for das %s at " + "channel %s. User need to correct sample rate in " + "array_t by themselves." % (das, chan)) + LOGGER.warning(msg) + if len(sample_rates) == 0: + msg = ("Das %s at channel %s has no trace between time " + "[%s, %s]." % (das, chan, dep, pic)) + LOGGER.error(msg) + else: + entry['sample_rate_i'] = sample_rates[0] + entry['seed_band_code_s'] = get_band_code(sample_rates[0]) + entry['receiver_table_n_i'] = rel_das_time_rows[0][ + 'receiver_table_n_i'] + else: + if not skip_missing_minifile: + LOGGER.error("Cannot fill in station's sample_rate_i, " + "seed_band_code_s and receiver_table_n_i because das " + "%s belongs to a missing minifile." % das) + + entry['sample_rate_multiplier_i'] = 1 + entry['seed_instrument_code_s'] = 'P' + entry['seed_orientation_code_s'] = 'Z' + entry['seed_station_name_s'] = entry['id_s'] + + incorrect_keys = set() + for k in addInfo: + if k in entry.keys(): + entry[k] = addInfo[k] + else: + incorrect_keys.add(k) + if len(incorrect_keys) != 0: + LOGGER.warning("AddInfo file contents incorrect key(s): %s" % + incorrect_keys) + + return entry + + +def recreate_array_t(pn4object, pn3_array_t, pn3_das_t, addInfo, + skip_missing_minifile): + pn3_array_dict = {} + for e in pn3_array_t: + if e['array_name'] not in pn3_array_dict.keys(): + pn3_array_dict[e['array_name']] = [] + pn3_array_dict[e['array_name']].append(e) + for array_name in pn3_array_dict.keys(): + a = pn4object.ph5_g_sorts.newArraySort(array_name) + for e in pn3_array_dict[array_name]: + e = convert_to_pn4_array(e, pn3_das_t, addInfo, + skip_missing_minifile) + try: + columns.populate(a, e) + except Exception as err: + LOGGER.error(err.message) + + +def recreate_index_t(pn4object, pn3_index_t): + for e in pn3_index_t: + pn4object.ph5_g_receivers.populateIndex_t(e) + + +def prepare_minifiles(pn4path, pn3path, pn3_das_t, + existing_minifiles, skip_ask_response): + if not skip_ask_response: + ret = raw_input("\n=========================================\n" + "All existing minifiles are going to be deleted in %s." + "\nDo you want to continue?(y/n)" % pn4path) + if ret == 'n': + return False + + for f in os.listdir((pn4path)): + if f.endswith(".ph5") and f.startswith("miniPH5"): + LOGGER.info("Deleting pn4's existing minifile: %s", f) + os.remove(os.path.join(pn4path, f)) + + for f in existing_minifiles: + LOGGER.info("Preparing minifile: %s" % f) + minipn3path = os.path.join(pn3path, f) + minipn4path = os.path.join(pn4path, f) + copy(minipn3path, minipn4path) + miniobj = ph5api.PH5(path=pn4path, nickname=f, editmode=True) + miniobj.read_das_g_names() + all_das_g_name = miniobj.Das_g_names + for group in all_das_g_name: + das = group.replace("Das_g_", "") + groupnode = miniobj.ph5_g_receivers.getdas_g(das) + # remove das_t + das_t = miniobj.ph5_g_receivers.ph5.get_node( + '/Experiment_g/Receivers_g/Das_g_%s' % das, + name='Das_t', + classname='Table') + das_t.remove() + # initialize das_t with pn4 format + experiment.initialize_table( + miniobj.ph5_g_receivers.ph5, + '/Experiment_g/Receivers_g/Das_g_%s' % das, + 'Das_t', + columns.Data, expectedrows=1000) + miniobj.ph5_g_receivers.setcurrent(groupnode) + for e in pn3_das_t[das]: + # add pn3's das_t entry with sample_rate_multiplier_i=1 + e['sample_rate_multiplier_i'] = 1 + miniobj.ph5_g_receivers.populateDas_t(e) + miniobj.close() + return True + + +def create_external_links(pn4object, pn3_index_t, rem_das, + existing_minifile_dict): + ext_links = {} + for entry in pn3_index_t: + das = entry['serial_number_s'] + ext_file = entry['external_file_name_s'][2:] + ext_path = entry['hdf5_path_s'] + target = ext_file + ':' + ext_path + ext_group = ext_path.split('/')[3] + try: + pn4object.ph5.create_external_link( + '/Experiment_g/Receivers_g', ext_group, target) + if ext_file not in ext_links: + ext_links[ext_file] = [] + ext_links[ext_file].append(das) + except Exception: + pass + + for ext_file in ext_links: + LOGGER.info("External link to %s is created for the following das: %s" + % (ext_file, ext_links[ext_file])) + + das_in_existing_mini = [] + for das_list in existing_minifile_dict.values(): + das_in_existing_mini += das_list + no_ext_link_das = [] + for das in rem_das: + if das in das_in_existing_mini: + no_ext_link_das.append(das) + if len(no_ext_link_das) != 0: + LOGGER.warning("External link is not created for the following das, " + "Use tool 'create_ext' when metadata is found: %s" + % no_ext_link_das) + + +# ########################## END RECREATE PN4 ########################### + +def getAddInfo(filename): + """ + Use flag -a to add addInfo file to change info + """ + with open(filename, 'r') as file: + lines = file.readlines() + addInfo = {} + for line in lines: + ss = [s.strip() for s in line.split('=')] + if len(ss) != 2: + raise Exception("The format of addInfo file is incorrect. " + "It should be fieldname=value for each line.") + addInfo[ss[0]] = ss[1] + if ss[0][-1] == 'i': + addInfo[ss[0]] = int(addInfo[ss[0]]) + elif ss[0][-1] == 'd': + addInfo[ss[0]] = float(addInfo[ss[0]]) + return addInfo + + +def main(): + (pn4path, + pn3path, + addInfoFile, + skip_missing_minifile, + skip_ask_response) = get_args() + addInfo = [] + if addInfoFile is not None: + addInfo = getAddInfo(addInfoFile) + + set_logger() + pn3object = ph5api_pn3.PH5( + path=pn3path, nickname='master.ph5', editmode=False) + (pn3_das_t, + pn3_index_t, + pn3_array_t, + rem_das, + existing_minifile_dict) = check_pn3_issues(pn3object) + + create_index_t_backup(pn3object, pn4path) + + pn3object.close() + + os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" + pn4object = ph5api.PH5(path=pn4path, nickname='master.ph5', editmode=True) + + pn4object = cleanup_pn4(pn4object, skip_ask_response) + if not pn4object: + LOGGER.warning("The program was interupted by user.") + sys.exit() + + recreate_array_t(pn4object, pn3_array_t, pn3_das_t, addInfo, + skip_missing_minifile) + + recreate_index_t(pn4object, pn3_index_t) + + ret = prepare_minifiles(pn4path, pn3path, pn3_das_t, + existing_minifile_dict.keys(), skip_ask_response) + if not ret: + LOGGER.warning("The program was interupted by user.") + sys.exit() + create_external_links(pn4object, pn3_index_t, rem_das, + existing_minifile_dict) + pn4object.close() + os.environ["HDF5_USE_FILE_LOCKING"] = "TRUE" + LOGGER.info("FINISH correcting pn4 data using info from pn3.") + + +if __name__ == '__main__': + main() diff --git a/ph5/utilities/ph5validate.py b/ph5/utilities/ph5validate.py index fb1eac55..31e5a6cd 100755 --- a/ph5/utilities/ph5validate.py +++ b/ph5/utilities/ph5validate.py @@ -330,7 +330,7 @@ def check_response_t(self): header = header % len(err) return [ValidationBlock(heading=header, error=err)] - def check_station_completeness(self, station): + def check_station_completeness(self, station, array_name): """ Checks that the following are present in Experiment_t: #### STATION LEVEL CHECKS @@ -471,7 +471,8 @@ def check_station_completeness(self, station): "You may need to reload the raw " "data for this station." .format(str(das_serial))) - dt = self.das_time[(das_serial, channel_id, sample_rate)] + dt = self.das_time[( + das_serial, array_name, channel_id, sample_rate)] # add bound_errors if applicable if deploy_time == dt['min_deploy_time'][0]: try: @@ -479,7 +480,7 @@ def check_station_completeness(self, station): except IndexError: pass - das_time_list = copy.copy(dt['time_windows']) + das_time_list = copy.deepcopy(dt['time_windows']) # check for duplicates: item = (deploy_time, pickup_time, station_id) @@ -492,15 +493,24 @@ def check_station_completeness(self, station): index = das_time_list.index((deploy_time, pickup_time, station_id)) - overlaps = [] + overlap_stations = [] + overlap_entries = [] # check if there is any overlap time for this das for t in das_time_list: if ((t[0] <= deploy_time < t[1]) or (t[0] < pickup_time <= t[1])): - overlaps.append(t[2]) + overlap_stations.append(t[2]) + overlap_entries.append(t) + + # remove overlaped windows times to use current deploy, pickup + if len(overlap_stations) > 1: + for t in overlap_entries: + das_time_list = list(filter((t).__ne__, x)) + das_time_list += [(deploy_time, pickup_time)] + das_time_list.sort() - if len(overlaps) > 1: error.append("Overlap time on station(s): %s" % - ", ".join(overlaps)) + ", ".join(overlap_stations)) + index = das_time_list.index((deploy_time, pickup_time)) try: # don't need to read das_t because it will be read in get_extent @@ -515,11 +525,7 @@ def check_station_completeness(self, station): # -- check data from current deploy time to # next deploy time -1 (-1 to avoid include next deploy time check_end = das_time_list[index+1][0] - 1 - i = 1 - # while loop to avoid using overlaping row - while check_end < check_start: - i += 1 - check_end = das_time_list[index+i][0] - 1 + try: # clear das to make sure get_extent consider channel & sr self.ph5.forget_das_t(das_serial) @@ -576,7 +582,7 @@ def check_station_completeness(self, station): def analyze_time(self): """ Analyze the array table to create dictionary self.das_time with key - is a set of (das, channel, sample_rate) + is a set of (das, array, channel, sample_rate) Each item's value includes * time_windows: a deploy-time-sorted list of all time windows of stations that match its key @@ -600,7 +606,7 @@ def analyze_time(self): d = stat['das/serial_number_s'] c = stat['channel_number_i'] spr = stat['sample_rate_i'] - key = (d, c, spr) + key = (d, array_name, c, spr) if key not in self.das_time.keys(): self.das_time[key] = {'time_windows': []} self.das_time[key]['time_windows'].append( @@ -611,7 +617,7 @@ def analyze_time(self): for key in self.das_time.keys(): dt = self.das_time[key] dt['time_windows'].sort() - d, c, spr = key + d, a, c, spr = key dt['min_deploy_time'] = [dt['time_windows'][0][0]] dt['max_pickup_time'] = [max([t[1] for t in dt['time_windows']])] # look for data outside time border of each set @@ -683,7 +689,8 @@ def check_array_t(self): .format(str(station_id), str(channel_id))) info, warning, error = \ - self.check_station_completeness(station) + self.check_station_completeness( + station, array_name) check_info = validation.check_response_info( resp_info, @@ -916,29 +923,29 @@ def get_args(): def main(): - try: - args = get_args() - ph5API_object = ph5api.PH5(path=args.ph5path, nickname=args.nickname) - ph5validate = PH5Validate(ph5API_object, - args.ph5path) - validation_blocks = [] - validation_blocks.extend(ph5validate.check_experiment_t()) - validation_blocks.extend(ph5validate.check_array_t()) - validation_blocks.extend(ph5validate.check_response_t()) - validation_blocks.extend(ph5validate.check_event_t()) - with open(args.outfile, "w") as log_file: - for vb in validation_blocks: - vb.write_to_log(log_file, - args.level) - ph5API_object.close() - sys.stdout.write("\nWarnings, Errors and suggestions " - "written to logfile: %s\n" % args.outfile) - except ph5api.APIError as err: - LOGGER.error(err) - except PH5ValidateException as err: - LOGGER.error(err) - except Exception as e: - LOGGER.error(e) + # try: + args = get_args() + ph5API_object = ph5api.PH5(path=args.ph5path, nickname=args.nickname) + ph5validate = PH5Validate(ph5API_object, + args.ph5path) + validation_blocks = [] + validation_blocks.extend(ph5validate.check_experiment_t()) + validation_blocks.extend(ph5validate.check_array_t()) + validation_blocks.extend(ph5validate.check_response_t()) + validation_blocks.extend(ph5validate.check_event_t()) + with open(args.outfile, "w") as log_file: + for vb in validation_blocks: + vb.write_to_log(log_file, + args.level) + ph5API_object.close() + sys.stdout.write("\nWarnings, Errors and suggestions " + "written to logfile: %s\n" % args.outfile) + # except ph5api.APIError as err: + # LOGGER.error(err) + # except PH5ValidateException as err: + # LOGGER.error(err) + # except Exception as e: + # LOGGER.error(e) if __name__ == '__main__': diff --git a/ph5/utilities/tests/test_correct_w_pn3.py b/ph5/utilities/tests/test_correct_w_pn3.py new file mode 100755 index 00000000..dbc3a917 --- /dev/null +++ b/ph5/utilities/tests/test_correct_w_pn3.py @@ -0,0 +1,374 @@ +''' +Tests for correct_w_pn3 +''' +import os +import sys +import unittest +import logging +import time +import shutil + +from mock import patch +from testfixtures import LogCapture, OutputCapture + +from ph5.utilities import correct_w_pn3, nuke_table, kef2ph5 +from ph5.core import ph5api, timedoy, ph5api_pn3 +from ph5.core.tests.test_base import LogTestCase, TempDirTestCase + + +def change_table(path, type, repl_datafile): + """ + remove table according to type in path/master.ph5 with data from + repl_datafile + """ + if type == 'array': + addargs = ['-A', '001'] + elif type == 'index': + addargs = ['-I'] + testargs = ['nuke_table', '-n', 'master.ph5', '-p', path] + addargs + with patch.object(sys, 'argv', testargs): + nuke_table.main() + testargs = ['kef2ph5', '-n', 'master.ph5', '-p', path, '-k', repl_datafile] + with patch.object(sys, 'argv', testargs): + kef2ph5.main() + + +def count_arrays_entries(ph5obj): + """ + count all entries of array tables in ph5obj + """ + count = 0 + for aname in ph5obj.Array_t_names: + ph5obj.read_array_t(aname) + arraybyid = ph5obj.Array_t[aname]['byid'] + for station in arraybyid.values(): + for deployment in station.values(): + count += len(deployment) + return count + + +class TestCheckPn3Issues(TempDirTestCase, LogTestCase): + def setUp(self): + super(TestCheckPn3Issues, self).setUp() + os.mkdir('pn3') + os.mkdir('pn4') + self.datapath = os.path.join(self.home, + "ph5/test_data/ph5_correct_w_pn3") + self.pn3path = os.path.join(self.tmpdir, 'pn3') + shutil.copy( + os.path.join(self.datapath, "pn3_master.ph5"), + os.path.join(self.pn3path, "master.ph5") + ) + + def tearDown(self): + self.pn3object.close() + super(TestCheckPn3Issues, self).tearDown() + + def test_consistent_data(self): + pn3path = os.path.join(self.home, + "ph5/test_data/ph5_correct_w_pn3") + self.pn3object = ph5api_pn3.PH5( + path=pn3path, nickname='pn3_master.ph5', editmode=False) + with LogCapture() as log: + log.setLevel(logging.WARNING) + (pn3_das_t, + pn3_index_t, + pn3_array_t, + rem_das, + existing_minifile_dict) = correct_w_pn3.check_pn3_issues( + self.pn3object) + self.assertEqual(len(log.records), 0) + self.assertEqual(len(pn3_das_t), 2) + self.assertEqual(len(pn3_index_t), 33) + self.assertEqual(len(pn3_array_t), 12) + self.assertEqual(rem_das, []) + self.assertEqual(existing_minifile_dict, + {'miniPH5_00001.ph5': ['1X1111', '3X500']}) + + def test_inconsistent_data1(self): + """ + array_test1.kef: + deploy=pickup: ids 1111, chan 3 + delete das 3X500 + duplicated rows: id_s 1111, chan 2 + """ + change_table(self.pn3path, 'array', + os.path.join(self.datapath, 'arrays_test1.kef')) + + self.pn3object = ph5api_pn3.PH5( + path=self.pn3path, nickname='master.ph5', editmode=False) + errmsg = [ + "Due to coincided deploy and pickup times, 1/11 entries will be " + "removed from array_t:\n[array, chan, deptime]: {[das_serial]: " + "[rem/total], [das_serial]: [rem/total], ...},\n('Array_t_001', " + "3, 'Wed Dec 31 17:00:00 1969'): {'1X1111': '1/11'}\n", + 'Das 1X1111 channel 2 deploy time Sat Jul 20 14:46:58 2019 ' + 'duplicated in Array_t_001. User need to handle this manually', + "Due to nonexistency in array_t 1/33 entries will be removed " + "from index_t for the following das:\n['3X500']", + "Compare Das_g against the filtered list of index_t and array_t, " + "the following Das will be removed from Das data:\n ['3X500']", + "The following minifiles are missing:\n['miniPH5_00001.ph5']" + ] + with LogCapture() as log: + log.setLevel(logging.WARNING) + (pn3_das_t, + pn3_index_t, + pn3_array_t, + rem_das, + existing_minifile_dict) = correct_w_pn3.check_pn3_issues( + self.pn3object) + self.assertEqual(len(log.records), 5) + for i in range(len(log.records)): + self.assertEqual(log.records[i].msg, errmsg[i]) + self.assertEqual(len(pn3_das_t), 0) # since no minifile + self.assertEqual(len(pn3_index_t), 27) + self.assertEqual(len(pn3_array_t), 10) + self.assertEqual(rem_das, ['3X500']) + self.assertEqual(existing_minifile_dict, {}) + + def test_inconsistent_data3(self): + """ + index_test.kef: + remove row 12: das 3X500 channel=3 + row 10: change time from 2017 to 2019 for das 1X1111 channel 1 + """ + change_table(self.pn3path, 'index', + os.path.join(self.datapath, 'index_test.kef')) + + self.pn3object = ph5api_pn3.PH5( + path=self.pn3path, nickname='master.ph5', editmode=False) + + errmsg = [ + "Due to nonexistency in index_t 3/12 entries will be removed from " + "array_t:\nArray_t_001: remove 3 entries of das: ['3X500']\n", + "Compare Das_g against the filtered list of index_t and array_t, " + "the following Das will be removed from Das data:\n ['3X500']", + "The following minifiles are missing:\n['miniPH5_00001.ph5']" + ] + with LogCapture() as log: + log.setLevel(logging.WARNING) + (pn3_das_t, + pn3_index_t, + pn3_array_t, + rem_das, + existing_minifile_dict) = correct_w_pn3.check_pn3_issues( + self.pn3object) + + self.assertEqual(len(log.records), 3) + for i in range(len(log.records)): + self.assertEqual(log.records[i].msg, errmsg[i]) + self.assertEqual(len(pn3_das_t), 0) # since no minifile + self.assertEqual(len(pn3_index_t), 27) + self.assertEqual(len(pn3_array_t), 9) + self.assertEqual(rem_das, ['3X500']) + self.assertEqual(existing_minifile_dict, {}) + + +class TestCleanupPn4(TempDirTestCase, LogTestCase): + def setUp(self): + super(TestCleanupPn4, self).setUp() + os.mkdir('pn4') + self.datapath = os.path.join(self.home, + "ph5/test_data/ph5_correct_w_pn3") + self.pn4path = os.path.join(self.tmpdir, 'pn4') + shutil.copy( + os.path.join(self.datapath, "pn4_master.ph5"), + os.path.join(self.pn4path, "master.ph5") + ) + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5', + editmode=True) + + def tearDown(self): + self.pn4object.close() + super(TestCleanupPn4, self).tearDown() + + def test_cleanup_pn4(self): + with OutputCapture(): + with LogCapture(): + correct_w_pn3.cleanup_pn4(self.pn4object, True) + # close and reopen to update changes + self.pn4object.close() + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5') + self.pn4object.read_array_t_names() + self.assertEqual(self.pn4object.Array_t_names, []) + self.pn4object.read_index_t() + self.assertEqual(len(self.pn4object.Index_t['rows']), 0) + self.pn4object.read_das_g_names() + self.assertEqual(len(self.pn4object.Das_g_names), 0) + + +class TestMain(TempDirTestCase, LogTestCase): + def setUp(self): + super(TestMain, self).setUp() + os.mkdir('pn3') + os.mkdir('pn4') + self.datapath = os.path.join(self.home, + "ph5/test_data/ph5_correct_w_pn3") + self.pn3path = os.path.join(self.tmpdir, 'pn3') + self.pn4path = os.path.join(self.tmpdir, 'pn4') + shutil.copy( + os.path.join(self.datapath, "pn3_master.ph5"), + os.path.join(self.pn3path, "master.ph5") + ) + shutil.copy( + os.path.join(self.datapath, "miniPH5_00001.ph5"), + os.path.join(self.pn3path, "miniPH5_00001.ph5") + ) + shutil.copy( + os.path.join(self.datapath, "pn4_master.ph5"), + os.path.join(self.pn4path, "master.ph5") + ) + + def tearDown(self): + self.pn4object.close() + super(TestMain, self).tearDown() + + def test_consistent_data(self): + addInfoPath = os.path.join(self.datapath, 'addInfo.txt') + testargs = ['correctwpn3', '--pn3', self.pn3path, + '--pn4', self.pn4path, '-a', addInfoPath, '-S'] + with patch.object(sys, 'argv', testargs): + with LogCapture() as log: + correct_w_pn3.main() + tdoy = timedoy.TimeDOY(epoch=time.time()) + tt = "{0:04d}{1:03d}".format(tdoy.dtobject.year, tdoy.dtobject.day) + index_backup = "Index_t_{0}_backup_from_pn3.kef".format(tt) + msg = [ + 'Opened ph5 file %s in read only mode.' + % os.path.join(self.pn3path, 'master.ph5'), + "Read /Experiment_g/Sorts_g/Array_t_001 (Table(12,)) ''", + "Read /Experiment_g/Receivers_g/Index_t (Table(33,)) ''", + "Read /Experiment_g/Receivers_g/Das_g_1X1111/Das_t (Table(27,))" + " ''", + "Read /Experiment_g/Receivers_g/Das_g_3X500/Das_t (Table(6,)) ''", + "Read /Experiment_g/Receivers_g/Index_t (Table(33,)) ''", + 'Writing table backup: %s.' + % os.path.join(self.tmpdir, index_backup), + 'Opened ph5 file %s in append edit mode.' + % os.path.join(self.pn4path, 'master.ph5'), + "Remove the following array_t from pn4: ['Array_t_001']", + 'Remove Index_t from pn4', + "Remove Das_g external links from pn4: ['1X1111', '3X500']", + 'Preparing minifile: miniPH5_00001.ph5', + 'Opened ph5 file %s in append edit mode.' + % os.path.join(self.pn4path, 'miniPH5_00001.ph5'), + "External link to miniPH5_00001.ph5 is created for the following " + "das: ['1X1111', '3X500']", + 'FINISH correcting pn4 data using info from pn3.' + ] + for i in range(len(log.records)): + self.assertEqual(log.records[i].msg, msg[i]) + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5') + self.pn4object.read_array_t_names() + self.assertEqual(self.pn4object.Array_t_names, ['Array_t_001']) + for aname in self.pn4object.Array_t_names: + self.pn4object.read_array_t(aname) + arraybyid = self.pn4object.Array_t[aname]['byid'] + for station in arraybyid.values(): + for deployment in station.values(): + for e in deployment: + self.assertEqual(e['sensor/model_s'], 'L28') + self.assertEqual(e['sensor/manufacturer_s'], 'Sercel') + self.assertEqual(e['das/model_s'], 'rt125') + self.pn4object.read_index_t() + self.assertEqual(len(self.pn4object.Index_t['rows']), 33) + self.pn4object.read_das_g_names() + self.assertEqual(len(self.pn4object.Das_g_names), 2) + + def test_inconsistent_data1(self): + """ + array_test1.kef: + deploy=pickup: ids 1111, chan 3 + delete das 3X500 + duplicated rows: id_s 1111, chan 2 + """ + change_table(self.pn3path, 'array', + os.path.join(self.datapath, 'arrays_test1.kef')) + + testargs = ['correctwpn3', '--pn3', self.pn3path, + '--pn4', self.pn4path, '-S'] + with patch.object(sys, 'argv', testargs): + with LogCapture() as log: + correct_w_pn3.main() + + self.assertEqual(len(log.records), 20) + self.assertEqual( + log.records[-2].msg, + "External link is not created for the following das, " + "Use tool 'create_ext' when metadata is found: ['3X500']" + ) + # not test other log mesg because they are similar to + # TestCheckPn3Issues.test_inconsistent_data1()'s + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5') + self.pn4object.read_array_t_names() + self.assertEqual(self.pn4object.Array_t_names, ['Array_t_001']) + self.assertEqual(count_arrays_entries(self.pn4object), 10) + self.pn4object.read_index_t() + self.assertEqual(len(self.pn4object.Index_t['rows']), 27) + self.pn4object.read_das_g_names() + self.assertEqual(len(self.pn4object.Das_g_names), 1) + + def test_inconsistent_data2(self): + """ + array_test2.kef: + remove row 12: das 3X500 channel=3 + row 10: change time from 2017 to 2019 for das 1X1111 channel 1 + """ + change_table(self.pn3path, 'array', + os.path.join(self.datapath, 'arrays_test2.kef')) + + testargs = ['correctwpn3', '--pn3', self.pn3path, + '--pn4', self.pn4path, '-S'] + with patch.object(sys, 'argv', testargs): + with LogCapture() as log: + log.setLevel(logging.WARNING) + correct_w_pn3.main() + + self.assertEqual(len(log.records), 1) + self.assertEqual( + log.records[0].msg, + 'Das 3X500 at channel 1 has no trace between time ' + '[1567269236, 1569681037].') + + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5') + self.pn4object.read_array_t_names() + self.assertEqual(self.pn4object.Array_t_names, ['Array_t_001']) + self.assertEqual(count_arrays_entries(self.pn4object), 11) + self.pn4object.read_index_t() + self.assertEqual(len(self.pn4object.Index_t['rows']), 33) + self.pn4object.read_das_g_names() + self.assertEqual(len(self.pn4object.Das_g_names), 2) + + def test_inconsistent_data3(self): + """ + index_test.kef: + remove row 12: das 3X500 channel=3 + row 10: change time from 2017 to 2019 for das 1X1111 channel 1 + """ + change_table(self.pn3path, 'index', + os.path.join(self.datapath, 'index_test.kef')) + + testargs = ['correctwpn3', '--pn3', self.pn3path, + '--pn4', self.pn4path, '-S'] + with patch.object(sys, 'argv', testargs): + with LogCapture() as log: + log.setLevel(logging.WARNING) + correct_w_pn3.main() + + # not test other log mesg because they are similar to + # TestCheckPn3Issues.test_inconsistent_data3()'s + self.assertEqual(len(log.records), 3) + + self.pn4object = ph5api.PH5(path=self.pn4path, nickname='master.ph5') + self.pn4object.read_array_t_names() + self.assertEqual(self.pn4object.Array_t_names, ['Array_t_001']) + self.assertEqual(count_arrays_entries(self.pn4object), 9) + self.pn4object.read_index_t() + self.assertEqual(len(self.pn4object.Index_t['rows']), 27) + self.pn4object.read_das_g_names() + self.assertEqual(len(self.pn4object.Das_g_names), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/ph5/utilities/tests/test_ph5validate.py b/ph5/utilities/tests/test_ph5validate.py index 9b7c010f..673260b8 100755 --- a/ph5/utilities/tests/test_ph5validate.py +++ b/ph5/utilities/tests/test_ph5validate.py @@ -252,7 +252,7 @@ def test_check_array_t(self): self.assertEqual( self.ph5validate.das_time, - {('12183', 1, 500): + {('12183', 'Array_t_009', 1, 500): {'max_pickup_time': [1550850187], 'time_windows': [(1550849950, 1550850034, '9001'), (1550849950, 1550850034, '9001'), @@ -313,8 +313,9 @@ def test_analyze_time(self): + check if it catch the case data exists before the whole time range """ self.ph5validate.analyze_time() - self.assertEqual(self.ph5validate.das_time.keys(), [('12183', 1, 500)]) - Dtime = self.ph5validate.das_time[('12183', 1, 500)] + self.assertEqual(self.ph5validate.das_time.keys(), + [('12183', 'Array_t_009', 1, 500)]) + Dtime = self.ph5validate.das_time[('12183', 'Array_t_009', 1, 500)] # 3 different deploy time self.assertEqual(len(Dtime['time_windows']), 5) @@ -339,7 +340,7 @@ def test_analyze_time(self): def test_check_station_completeness(self): self.ph5validate.das_time = { - ('12183', 1, 500): + ('12183', 'Array_t_009', 1, 500): {'time_windows': [(1550849950, 1550850034, '9001'), (1550849950, 1550850034, '9001'), (1550849950, 1550850034, '9001'), @@ -353,7 +354,7 @@ def test_check_station_completeness(self): self.ph5validate.read_arrays('Array_t_009') arraybyid = self.ph5validate.ph5.Array_t['Array_t_009']['byid'] - DT = self.ph5validate.das_time[('12183', 1, 500)] + DT = self.ph5validate.das_time[('12183', 'Array_t_009', 1, 500)] # check lon/lat not in range # check warning data exist before min_deploy_time @@ -364,7 +365,8 @@ def test_check_station_completeness(self): station['location/Y/units_s'] = 'degrees' station['location/Z/value_d'] = 1403 station['location/Z/units_s'] = 'm' - ret = self.ph5validate.check_station_completeness(station) + ret = self.ph5validate.check_station_completeness( + station, 'Array_t_009') warnings = ret[1] self.assertEqual( warnings, @@ -386,7 +388,8 @@ def test_check_station_completeness(self): station['location/Y/units_s'] = None station['location/Z/value_d'] = None station['location/Z/units_s'] = '' - ret = self.ph5validate.check_station_completeness(station) + ret = self.ph5validate.check_station_completeness( + station, 'Array_t_009') warnings = ret[1] self.assertEqual( warnings, @@ -405,7 +408,8 @@ def test_check_station_completeness(self): # check error overlaping # => change deploy time of the 3rd station DT['time_windows'][5] = (1550850090, 1550850187, '9003') - ret = self.ph5validate.check_station_completeness(station) + ret = self.ph5validate.check_station_completeness( + station, 'Array_t_009') errors = ret[2] self.assertIn('Overlap time on station(s): 9002, 9003', errors) @@ -415,7 +419,8 @@ def test_check_station_completeness(self): station['deploy_time/epoch_l'] = 1550850190 station['pickup_time/epoch_l'] = 1550850191 DT['time_windows'][5] = (1550850190, 1550850191, '9003') - ret = self.ph5validate.check_station_completeness(station) + ret = self.ph5validate.check_station_completeness( + station, 'Array_t_009') errors = ret[2] self.assertIn("No data found for das serial number 12183 during this " "station's time. You may need to reload the raw data " @@ -425,8 +430,10 @@ def test_check_station_completeness(self): station = arraybyid.get('9002')[1][0] station['das/serial_number_s'] = '1218' self.ph5validate.das_time[ - ('1218', 1, 500)] = self.ph5validate.das_time[('12183', 1, 500)] - ret = self.ph5validate.check_station_completeness(station) + ('1218', 'Array_t_009', 1, 500)] = self.ph5validate.das_time[ + ('12183', 'Array_t_009', 1, 500)] + ret = self.ph5validate.check_station_completeness( + station, 'Array_t_009') errors = ret[2] self.assertIn("No data found for das serial number 1218. " "You may need to reload the raw data for this station.", @@ -464,53 +471,62 @@ def test_check_station_completeness(self): station = self.ph5_object.Array_t['Array_t_008']['byid']['8001'][1][0] # id_s isn't a whole number => error - das_time[('9EEF', 1, 100)]['time_windows'][0] = \ + das_time[('9EEF', 'Array_t_008', 1, 100)]['time_windows'][0] = \ (1463568480, 1463568540, '33a33') station['id_s'] = '33a33' - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_008') self.assertIn("Station ID '33a33' not a whole number " "between 0 and 65535.", err) # id_s not in range [0,65535] => error - das_time[('9EEF', 1, 100)]['time_windows'][0] = \ + das_time[('9EEF', 'Array_t_008', 1, 100)]['time_windows'][0] = \ (1463568480, 1463568540, '65536') station['id_s'] = '65536' - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_008') self.assertIn("Station ID '65536' not between 0 and 65535.", err) # id_s in range [32768, 65534] => warning - das_time[('9EEF', 1, 100)]['time_windows'][0] = \ + das_time[('9EEF', 'Array_t_008', 1, 100)]['time_windows'][0] = \ (1463568480, 1463568540, '33333') station['id_s'] = '33333' - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_008') self.assertIn("Station ID '33333' is more than 32767. " "Not compatible with SEGY revision 1.", warn) # sample_rate=0 => warning - das_time[('12183', 1, 0)] = das_time[('12183', 1, 500)] + das_time[('12183', 'Array_t_009', 1, 0)] = das_time[ + ('12183', 'Array_t_009', 1, 500)] station = self.ph5_object.Array_t['Array_t_009']['byid']['9001'][1][0] station['sample_rate_i'] = 0 - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_009') self.assertIn("Sample rate seems to be 0. Is this correct???", warn) # sample_rate<0 => error - das_time[('12183', 1, -1)] = das_time[('12183', 1, 500)] + das_time[('12183', 'Array_t_009', 1, -1)] = das_time[ + ('12183', 'Array_t_009', 1, 500)] station['sample_rate_i'] = -1 - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_009') self.assertIn("Sample rate = -1 not positive.", err) # sample_rate_multiplier_i isn't a integer => error station['sample_rate_i'] = 500 station['sample_rate_multiplier_i'] = 1.1 - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_009') self.assertIn("Sample rate multiplier = 1.1 is not an" " integer greater than 1.", err) # sample_rate_multiplier_i<1 => error station['sample_rate_multiplier_i'] = 0 - inf, warn, err = self.ph5validate.check_station_completeness(station) + inf, warn, err = self.ph5validate.check_station_completeness( + station, 'Array_t_009') self.assertIn("Sample rate multiplier = 0 is not an integer " "greater than 1.", err)