Nortek Bottom Track Update

examples/data/dolfyn/test_data/BenchFile01.repr.txt

@@ -17,10 +17,9 @@
   - pressure ('time',)
   - pressure_alt ('time',)
   - pressure_altraw ('time_altraw',)
+  - pressure_b5 ('time_b5',)
   - amp ('beam', 'range', 'time')
   - amp_b5 ('range_b5', 'time_b5')
   - corr ('beam', 'range', 'time')
   - corr_b5 ('range_b5', 'time_b5')
-  - accel ('dirIMU', 'time')
-  - mag ('dirIMU', 'time')
   ... and others (see `<obj>.variables`)

examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log

@@ -1,30 +1,101 @@
 root - INFO -       4464: A5, 10, 18, 188, 0, 1, 1
-
 root - INFO -       4662: A5, 10, 15, 524, 0, 1, 1
-
 root - INFO -       5196: A5, 10, 1C, 214, 0, 1, 1
-
-root - INFO -       5420: A5, 10, 18, 188, 1, 2, 2
-
-root - INFO -       5618: A5, 10, 15, 524, 1, 2, 2
-
-root - INFO -       6152: A5, 10, 1C, 214, 1, 2, 2
-
-root - INFO -       6376: A5, 10, 18, 188, 2, 3, 3
-
-root - INFO -       6574: A5, 10, 15, 524, 2, 3, 3
-
-root - INFO -       7108: A5, 10, 1C, 214, 2, 3, 3
-
-root - INFO -       7332: A5, 10, 18, 188, 3, 4, 4
-
-root - INFO -       7530: A5, 10, 15, 524, 3, 4, 4
-
-root - INFO -       8064: A5, 10, 1C, 214, 3, 4, 4
-
-root - INFO -       8288: A5, 10, 18, 188, 4, 5, 5
-
-root - INFO -       8486: A5, 10, 15, 524, 4, 5, 5
-
-root - INFO -       9020: A5, 10, 1C, 214, 4, 5, 5
-
+root - INFO -       5420: A5, 10, 18, 188, 1, 1, 1
+root - INFO -       5618: A5, 10, 15, 524, 1, 1, 1
+root - INFO -       6152: A5, 10, 1C, 214, 1, 1, 1
+root - INFO -       6376: A5, 10, 18, 188, 2, 1, 1
+root - INFO -       6574: A5, 10, 15, 524, 2, 1, 1
+root - INFO -       7108: A5, 10, 1C, 214, 2, 1, 1
+root - INFO -       7332: A5, 10, 18, 188, 3, 1, 1
+root - INFO -       7530: A5, 10, 15, 524, 3, 1, 1
+root - INFO -       8064: A5, 10, 1C, 214, 3, 1, 1
+root - INFO -       8288: A5, 10, 18, 188, 4, 1, 1
+root - INFO -       8486: A5, 10, 15, 524, 4, 1, 1
+root - INFO -       9020: A5, 10, 1C, 214, 4, 1, 1
+root - INFO -       9244: A5, 10, 18, 188, 5, 1, 1
+root - INFO -       9442: A5, 10, 15, 524, 5, 1, 1
+root - INFO -       9976: A5, 10, 1C, 214, 5, 1, 1
+root - INFO -      10200: A5, 10, 18, 188, 6, 1, 1
+root - INFO -      10398: A5, 10, 15, 524, 6, 1, 1
+root - INFO -      10932: A5, 10, 1C, 214, 6, 1, 1
+root - INFO -      11156: A5, 10, 18, 188, 7, 1, 1
+root - INFO -      11354: A5, 10, 15, 524, 7, 1, 1
+root - INFO -      11888: A5, 10, 1C, 214, 7, 1, 1
+root - INFO -      12112: A5, 10, 18, 188, 8, 1, 1
+root - INFO -      12310: A5, 10, 15, 524, 8, 1, 1
+root - INFO -      12844: A5, 10, 1C, 214, 8, 1, 1
+root - INFO -      13068: A5, 10, 18, 188, 9, 1, 1
+root - INFO -      13266: A5, 10, 15, 524, 9, 1, 1
+root - INFO -      13800: A5, 10, 1C, 214, 9, 1, 1
+root - INFO -      14024: A5, 10, 18, 188, 10, 1, 1
+root - INFO -      14222: A5, 10, 15, 524, 10, 1, 1
+root - INFO -      14756: A5, 10, 1C, 214, 10, 1, 1
+root - INFO -      14980: A5, 10, 18, 188, 11, 1, 1
+root - INFO -      15178: A5, 10, 15, 524, 11, 1, 1
+root - INFO -      15712: A5, 10, 1C, 214, 11, 1, 1
+root - INFO -      15936: A5, 10, 18, 188, 12, 1, 1
+root - INFO -      16134: A5, 10, 15, 524, 12, 1, 1
+root - INFO -      16668: A5, 10, 1C, 214, 12, 1, 1
+root - INFO -      16892: A5, 10, 18, 188, 13, 1, 1
+root - INFO -      17090: A5, 10, 15, 524, 13, 1, 1
+root - INFO -      17624: A5, 10, 1C, 214, 13, 1, 1
+root - INFO -      17848: A5, 10, 18, 188, 14, 1, 1
+root - INFO -      18046: A5, 10, 15, 524, 14, 1, 1
+root - INFO -      18580: A5, 10, 1C, 214, 14, 1, 1
+root - INFO -      18804: A5, 10, 18, 188, 15, 1, 1
+root - INFO -      19002: A5, 10, 15, 524, 15, 1, 1
+root - INFO -      19536: A5, 10, 1C, 214, 15, 1, 1
+root - INFO -      19760: A5, 10, 18, 188, 16, 1, 1
+root - INFO -      19958: A5, 10, 15, 524, 16, 1, 1
+root - INFO -      20492: A5, 10, 1C, 214, 16, 1, 1
+root - INFO -      20716: A5, 10, 18, 188, 17, 1, 1
+root - INFO -      20914: A5, 10, 15, 524, 17, 1, 1
+root - INFO -      21448: A5, 10, 1C, 214, 17, 1, 1
+root - INFO -      21672: A5, 10, 18, 188, 18, 1, 1
+root - INFO -      21870: A5, 10, 15, 524, 18, 1, 1
+root - INFO -      22404: A5, 10, 1C, 214, 18, 1, 1
+root - INFO -      22628: A5, 10, 18, 188, 19, 1, 1
+root - INFO -      22826: A5, 10, 15, 524, 19, 1, 1
+root - INFO -      23360: A5, 10, 1C, 214, 19, 1, 1
+root - INFO -      23584: A5, 10, 18, 188, 20, 1, 1
+root - INFO -      23782: A5, 10, 15, 524, 20, 1, 1
+root - INFO -      24316: A5, 10, 1C, 214, 20, 1, 1
+root - INFO -      24540: A5, 10, 18, 188, 21, 1, 1
+root - INFO -      24738: A5, 10, 15, 524, 21, 1, 1
+root - INFO -      25272: A5, 10, 1C, 214, 21, 1, 1
+root - INFO -      25496: A5, 10, 18, 188, 22, 1, 1
+root - INFO -      25694: A5, 10, 15, 524, 22, 1, 1
+root - INFO -      26228: A5, 10, 1C, 214, 22, 1, 1
+root - INFO -      26452: A5, 10, 18, 188, 23, 1, 1
+root - INFO -      26650: A5, 10, 15, 524, 23, 1, 1
+root - INFO -      27184: A5, 10, 1C, 214, 23, 1, 1
+root - INFO -      27408: A5, 10, 18, 188, 24, 1, 1
+root - INFO -      27606: A5, 10, 15, 524, 24, 1, 1
+root - INFO -      28140: A5, 10, 1C, 214, 24, 1, 1
+root - INFO -      28364: A5, 10, 18, 188, 25, 1, 1
+root - INFO -      28562: A5, 10, 15, 524, 25, 1, 1
+root - INFO -      29096: A5, 10, 1C, 214, 25, 1, 1
+root - INFO -      29320: A5, 10, 18, 188, 26, 1, 1
+root - INFO -      29518: A5, 10, 15, 524, 26, 1, 1
+root - INFO -      30052: A5, 10, 1C, 214, 26, 1, 1
+root - INFO -      30276: A5, 10, 18, 188, 27, 1, 1
+root - INFO -      30474: A5, 10, 15, 524, 27, 1, 1
+root - INFO -      31008: A5, 10, 1C, 214, 27, 1, 1
+root - INFO -      31232: A5, 10, 18, 188, 28, 1, 1
+root - INFO -      31430: A5, 10, 15, 524, 28, 1, 1
+root - INFO -      31964: A5, 10, 1C, 214, 28, 1, 1
+root - INFO -      32188: A5, 10, 18, 188, 29, 1, 1
+root - INFO -      32386: A5, 10, 15, 524, 29, 1, 1
+root - INFO -      32920: A5, 10, 1C, 214, 29, 1, 1
+root - INFO -      33144: A5, 10, 18, 188, 30, 1, 1
+root - INFO -      33342: A5, 10, 15, 524, 30, 1, 1
+root - INFO -      33876: A5, 10, 1C, 214, 30, 1, 1
+root - INFO -      34100: A5, 10, 18, 188, 31, 1, 1
+root - INFO -      34298: A5, 10, 15, 524, 31, 1, 1
+root - INFO -      34832: A5, 10, 1C, 214, 31, 1, 1
+root - INFO -      35056: A5, 10, 18, 188, 32, 1, 1
+root - INFO -      35254: A5, 10, 15, 524, 32, 1, 1
+root - INFO -      35788: A5, 10, 1C, 214, 32, 1, 1
+root - INFO -      36012: A5, 10, 18, 188, 33, 1, 1
+root - INFO -      36210: A5, 10, 15, 524, 33, 1, 1

mhkit/dolfyn/io/base.py

@@ -323,11 +323,8 @@ def _create_dataset(data):
                 else:  # amp, corr, prcnt_gd, status
                     dim0 = "beam"
 
-                if not any(val in key for val in tag) or ("_avg" in key):
-                    if "_avg" in key:
-                        tg = "_avg"
-                    else:
-                        tg = ""
+                if not any(val in key for val in tag) or (tg in ["_avg", "_bt"]):
+                    tg = "" if tg not in ["_avg", "_bt"] else tg
                     ds_dict[key] = {
                         "dims": (dim0, "range" + tg, "time" + tg),
                         "data": data["data_vars"][key],

mhkit/dolfyn/io/nortek2.py

@@ -81,10 +81,8 @@ def read_signature(
     )
     d = rdr.readfile(nens[0], nens[1])
     rdr.sci_data(d)
-    if rdr._dp:
-        _clean_dp_skips(d)
-    out = _reorg(d)
-    _reduce(out)
+    _clean_skips(d)
+    out = _reduce(_reorg(d))
 
     # Convert time to dt64 and fill gaps
     coords = out["coords"]
@@ -246,7 +244,7 @@ def _init_burst_readers(
                 )
             elif rdr_id == 23:
                 self._burst_readers[rdr_id] = defs._calc_bt_struct(
-                    cfg["_config"], cfg["n_beams"]
+                    cfg["_config"], cfg["n_beams"], cfg["n_cells"], cfg["_d_ver"]
                 )
             else:
                 self._burst_readers[rdr_id] = defs._calc_burst_struct(
@@ -301,9 +299,11 @@ def _read_str(self, size):
         string = string[1:-1]
         return id, string
 
-    def _read_burst(self, id, dat, c, echo=False):
+    def _read_burst(self, id, dat, c, sz, echo=False):
         rdr = self._burst_readers[id]
-        rdr.read_into(self.f, dat, c)
+        to_skip = rdr.read_into(self.f, dat, c, sz)
+        if to_skip:
+            self.f.seek(to_skip, 1)
 
     def readfile(self, ens_start=0, ens_stop=None):
         # If the lastblock is not whole, we don't read it.
@@ -326,10 +326,13 @@ def readfile(self, ens_start=0, ens_stop=None):
             except IOError:
                 return outdat
             id = hdr["id"]
-            if id in [21, 22, 23, 24, 28]:  # "burst data record" (vel + ast),
+            if id not in outdat:
+                # If the ID was skipped in the indexer, it won't be in outdat
+                self.f.seek(hdr["sz"], 1)
+            elif id in [21, 22, 23, 24, 28]:  # "burst data record" (vel + ast),
                 # "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt),
                 # "interleaved burst data record" (vel_b5), "echosounder record" (echo)
-                self._read_burst(id, outdat[id], c)
+                self._read_burst(id, outdat[id], c, hdr["sz"])
             elif id in [26, 31]:
                 # "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg)
                 rdr = self._burst_readers[id]
@@ -367,7 +370,7 @@ def readfile(self, ens_start=0, ens_stop=None):
                             "The number of samples in this 'Altimeter Raw' "
                             "burst is different from prior bursts."
                         )
-                self._read_burst(id, outdat[id], c_altraw[id])
+                self._read_burst(id, outdat[id], c_altraw[id], hdr["sz"])
                 outdat[id]["ensemble"][c_altraw[id]] = c
                 c_altraw[id] += 1
 
@@ -506,10 +509,6 @@ def _reorg(dat):
         (28, "_echo"),
         (31, "raw_avg"),
     ]:
-        if id in [24, 26]:
-            collapse_exclude = [0]
-        else:
-            collapse_exclude = []
         if id not in dat:
             continue
         dnow = dat[id]
@@ -520,7 +519,8 @@ def _reorg(dat):
                 dnow["standard_name"].pop(ky)
         outdat["standard_name"].update(dnow["standard_name"])
         cfg["burst_config" + tag] = lib._headconfig_int2dict(
-            lib._collapse(dnow["config"], exclude=collapse_exclude, name="config")
+            lib._collapse(dnow["config"], exclude=[0], name="config"),
+            mode="bt" if "bt" in tag else "burst",
         )
         outdat["coords"]["time" + tag] = lib._calc_time(
             dnow["year"] + np.uint16(1900),
@@ -531,34 +531,30 @@ def _reorg(dat):
             dnow["second"],
             dnow["usec100"].astype("uint32") * 100,
         )
-        tmp = lib._beams_cy_int2dict(
-            lib._collapse(
-                dnow["beam_config"], exclude=collapse_exclude, name="beam_config"
-            ),
-            21,  # always 21 here
-        )
+
+        d_ver = lib._collapse(dnow["ver"], exclude=[0], name="ver")
+        beams_cy = lib._collapse(dnow["beam_config"], exclude=[0], name="beam_config")
+        # Deal with data version 9 for bottom track
+        if (id == 23) and (d_ver >= 9):
+            n_cells = lib._collapse(dnow["n_cells"], exclude=[0], name="n_cells")
+            beams_cy = lib._repack_bt_beams_cy(beams_cy, n_cells)
+
+        # Unpack cells, coordinate axes and beams
+        tmp = lib._beams_cy_int2dict(beams_cy, id, d_ver)
         cfg["n_cells" + tag] = int(tmp["n_cells"])
         cfg["coord_sys_axes" + tag] = tmp["cy"]
         cfg["n_beams" + tag] = int(tmp["n_beams"])
-        cfg["ambig_vel" + tag] = round(
-            float(lib._collapse(dnow["ambig_vel"], name="ambig_vel")), 3
-        )
 
-        for ky in [
-            "SerialNum",
-            "nominal_corr",
-        ]:
-            cfg[ky + tag] = int(
-                lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky)
+        cfg["SerialNum" + tag] = int(
+            lib._collapse(dnow["SerialNum"], exclude=[0], name="SerialNum")
+        )
+        for ky in ["cell_size", "power_level_dB", "ambig_vel"]:
+            cfg[ky + tag] = round(
+                float(lib._collapse(dnow[ky], exclude=[0], name=ky)), 3
             )
-
-        for ky in [
-            "cell_size",
-            "blank_dist",
-            "power_level_dB",
-        ]:
-            cfg[ky + tag] = float(
-                lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky)
+        if id != 23:
+            cfg["blank_dist" + tag] = float(
+                lib._collapse(dnow["blank_dist"], exclude=[0], name="blank_dist")
             )
 
         for ky in [
@@ -674,9 +670,7 @@ def _reorg(dat):
         cfg["rotate_vars"] = []
         cy = cfg["coord_sys_axes_avg"]
     else:
-        cfg["rotate_vars"] = [
-            "vel",
-        ]
+        cfg["rotate_vars"] = ["vel"]
         cy = cfg["coord_sys_axes"]
     outdat["attrs"]["coord_sys"] = {"XYZ": "inst", "ENU": "earth", "beam": "beam"}[cy]
 
@@ -693,10 +687,9 @@ def _reorg(dat):
     return outdat
 
 
-def _clean_dp_skips(data):
+def _clean_skips(data):
     """
-    Removes zeros from interwoven measurements taken in a dual profile
-    configuration.
+    Removes zeros from interwoven measurements
     """
 
     for id in data:
@@ -724,14 +717,6 @@ def _reduce(data):
     dc = data["coords"]
     da = data["attrs"]
 
-    # Average these fields
-    for ky in ["c_sound", "temp", "pressure", "temp_press", "temp_clock", "batt"]:
-        lib._reduce_by_average(dv, ky, ky + "_b5")
-
-    # Angle-averaging is treated separately
-    for ky in ["heading", "pitch", "roll"]:
-        lib._reduce_by_average_angle(dv, ky, ky + "_b5")
-
     if "vel" in dv:
         dc["range"] = (np.arange(dv["vel"].shape[1]) + 1) * da["cell_size"] + da[
             "blank_dist"
@@ -784,6 +769,8 @@ def _reduce(data):
             tm[irow, icol] = tmat["M" + str(irow + 1) + str(icol + 1)]
     dv["beam2inst_orientmat"] = tm
 
+    return data
+
 
 def split_dp_datasets(ds):
     """