Skip to content
GitLab
Commit 6a76f1dd authored by Calmel Blaise's avatar Calmel Blaise
Browse files

Update open_functions and DepthData

parent 96762081
main
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 102 additions and 83 deletions
+102 -83
Classes/DepthData.py
+ 69
− 70
View file @ 6a76f1dd
......@@ -61,7 +61,7 @@ class DepthData(object):
valid_beams: np.array
Logical array, 1 row for each beam identifying valid data.
"""
def __init__(self):
"""Initialize attributes.
"""
......@@ -88,7 +88,7 @@ class DepthData(object):
self.valid_data_method = None # QRev or TRDI
self.valid_data = None # Logical array of valid mean depth for each ensemble
self.valid_beams = None # Logical array, 1 row for each beam identifying valid data
def populate_data(self, depth_in, source_in, freq_in, draft_in, cell_depth_in, cell_size_in):
"""Stores data in DepthData.
......@@ -120,7 +120,7 @@ class DepthData(object):
self.filter_type = 'None'
self.interp_type = 'None'
self.valid_data_method = 'QRev'
# For BT data set method to average multiple beam depths
if source_in == 'BT':
self.avg_method = 'IDW'
......@@ -223,19 +223,19 @@ class DepthData(object):
def change_draft(self, draft):
"""Changes the draft for object
draft: new draft for object
"""
# Compute draft change
draft_change = draft - self.draft_use_m
self.draft_use_m = draft
# Apply draft to ensemble depths if BT or VB
if self.depth_source != 'DS':
self.depth_beams_m = self.depth_beams_m + draft_change
self.depth_processed_m = self.depth_processed_m + draft_change
# Apply draft to depth cell locations
self.depth_processed_m = self.depth_processed_m + draft_change
# Apply draft to depth cell locations
if len(self.depth_cell_depth_m) > 0:
self.depth_cell_depth_m = self.depth_cell_depth_m + draft_change
......@@ -249,7 +249,7 @@ class DepthData(object):
bt_depths: DepthData
Object of DepthData with bottom track depths
"""
self.depth_cell_depth_orig_m = bt_depths.depth_cell_depth_orig_m
self.depth_cell_size_m = bt_depths.depth_cell_size_m
self.depth_cell_depth_m = bt_depths.depth_cell_depth_m
......@@ -301,7 +301,7 @@ class DepthData(object):
# TRDI filter for multiple returns
self.filter_trdi()
self.filter_type = 'TRDI'
self.valid_mean_data()
# Update processed depth with filtered results
......@@ -312,7 +312,7 @@ class DepthData(object):
# Single beam (VB or DS) save to 1-D array
self.depth_processed_m = np.array(self.depth_beams_m[0, :])
self.depth_processed_m[np.squeeze(np.equal(self.valid_data, 0))] = np.nan
def apply_interpolation(self, transect, method=None):
"""Coordinates application of interpolations
......@@ -323,11 +323,10 @@ class DepthData(object):
method: str
Type of interpolation to apply (None, HoldLast, Smooth, Linear)
"""
# Determine interpolation to apply
if method is None:
method = self.interp_type
# Apply selected interpolation
self.interp_type = method
# No filtering
......@@ -345,7 +344,7 @@ class DepthData(object):
# Linear interpolation
else:
self.interpolate_linear(transect=transect)
# Identify ensembles with interpolated depths
idx = np.where(np.logical_not(self.valid_data[:]))
if len(idx[0]) > 0:
......@@ -354,7 +353,7 @@ class DepthData(object):
if len(idx2) > 0:
idx2 = idx2[0]
self.depth_source_ens[idx[idx2]] = 'IN'
def apply_composite(self, comp_depth, comp_source):
"""Applies the data from CompDepth computed in DepthStructure
to DepthData object
......@@ -366,17 +365,17 @@ class DepthData(object):
comp_source: str
Source of composite depth (BT, VB, DS)
"""
# Assign composite depth to property
self.depth_processed_m = comp_depth
# Assign appropriate composite source for each ensemble
self.depth_source_ens[comp_source == 1] = 'BT'
self.depth_source_ens[comp_source == 2] = 'VB'
self.depth_source_ens[comp_source == 3] = 'DS'
self.depth_source_ens[comp_source == 4] = 'IN'
self.depth_source_ens[comp_source == 0] = 'NA'
def sos_correction(self, ratio):
"""Correct depth for new speed of sound setting
......@@ -385,48 +384,48 @@ class DepthData(object):
ratio: float
Ratio of new to old speed of sound value
"""
# Correct unprocessed depths
self.depth_beams_m = self.draft_use_m+np.multiply(self.depth_beams_m-self.draft_use_m, ratio)
self.depth_beams_m = self.draft_use_m + np.multiply(self.depth_beams_m - self.draft_use_m, ratio)
# Correct processed depths
self.depth_processed_m = self.draft_use_m+np.multiply(self.depth_processed_m-self.draft_use_m, ratio)
self.depth_processed_m = self.draft_use_m + np.multiply(self.depth_processed_m - self.draft_use_m, ratio)
# Correct cell size and location
self.depth_cell_size_m = np.multiply(self.depth_cell_size_m, ratio)
self.depth_cell_depth_m = self.draft_use_m + np.multiply(self.depth_cell_depth_m - self.draft_use_m, ratio)
def valid_mean_data(self):
"""Determines if raw data are sufficient to compute a valid depth without interpolation.
"""
if self.depth_source == 'BT':
self.valid_data = np.tile(True, self.valid_beams.shape[1])
nvalid = np.sum(self.valid_beams, axis=0)
if self.valid_data_method == 'TRDI':
self.valid_data[nvalid < 3] = False
else:
self.valid_data[nvalid < 2] = False
else:
self.valid_data = self.valid_beams[0, :]
def filter_none(self):
"""Applies no filter to depth data. Removes filter if one was applied.
"""
# Set all ensembles to have valid data
if len(self.depth_beams_m.shape) > 1:
self.valid_beams = np.tile(True, self.depth_beams_m.shape)
else:
self.valid_beams = np.tile(True, (1, self.depth_beams_m.shape[0]))
# Set ensembles with no depth data to invalid
self.valid_beams[self.depth_beams_m == 0] = False
self.valid_beams[np.isnan(self.depth_beams_m)] = False
self.filter_type = 'None'
def filter_smooth(self, transect):
"""This filter uses a moving InterQuartile Range filter on residuals from a
robust Loess smooth of the depths in each beam to identify unnatural spikes in the depth
......@@ -444,18 +443,18 @@ class DepthData(object):
This is the raw number of points actual points used may be less if some are bad.
multiplier - number multiplied times the IQR to determine the filter criteria
"""
# If the smoothed depth has not been computed
if self.smooth_depth is None or len(self.smooth_depth) == 0:
# Set filter characteristics
self.filter_type = 'Smooth'
# cycles = 3
# half_width = 10
# multiplier = 15
# Determine number of beams
if len(self.depth_orig_m.shape) > 1:
n_beams, n_ensembles = self.depth_orig_m.shape[0], self.depth_orig_m.shape[1]
......@@ -487,7 +486,7 @@ class DepthData(object):
idx = np.where(np.isnan(track_x))
if len(idx[0]) < 2:
x = np.nancumsum(np.sqrt(track_x**2+track_y**2))
x = np.nancumsum(np.sqrt(track_x ** 2 + track_y ** 2))
else:
x = np.nancumsum(transect.date_time.ens_duration_sec)
......@@ -515,10 +514,10 @@ class DepthData(object):
# Reset valid data
self.filter_none()
# Set filter type
self.filter_type = 'Smooth'
# Determine number of beams
if len(self.depth_orig_m.shape) > 1:
n_beams, n_ensembles = self.depth_orig_m.shape[0], self.depth_orig_m.shape[1]
......@@ -533,7 +532,7 @@ class DepthData(object):
# Set bad depths to nan
depth = repmat(np.nan, depth_raw.shape[0], depth_raw.shape[1])
depth[nan_greater(depth_raw, 0)] = depth_raw[nan_greater(depth_raw, 0)]
# Apply filter
for j in range(n_beams):
if np.nansum(self.smooth_upper_limit[j, :]) > 0:
......@@ -746,7 +745,7 @@ class DepthData(object):
def interpolate_none(self):
"""Applies no interpolation.
"""
# Compute processed depth without interpolation
if self.depth_source == 'BT':
# Bottom track methods
......@@ -754,25 +753,25 @@ class DepthData(object):
else:
# Vertical beam or depth sounder depths
self.depth_processed_m = self.depth_beams_m[0, :]
self.depth_processed_m[np.squeeze(np.equal(self.valid_data, False))] = np.nan
# Set interpolation type
self.interp_type = 'None'
def interpolate_hold_last(self):
"""This function holds the last valid value until the next valid data point.
"""
# Get number of ensembles
n_ensembles = len(self.depth_processed_m)
# Process data by ensemble
for n in range(1, n_ensembles):
# If current ensemble's depth is invalid assign depth from previous example
if not self.valid_data[n]:
self.depth_processed_m[n] = self.depth_processed_m[n-1]
self.depth_processed_m[n] = self.depth_processed_m[n - 1]
def interpolate_next(self):
"""This function back fills with the next valid value.
......@@ -782,7 +781,7 @@ class DepthData(object):
n_ens = len(self.depth_processed_m)
# Process data by ensemble
for n in np.arange(0, n_ens-1)[::-1]:
for n in np.arange(0, n_ens - 1)[::-1]:
# If current ensemble's depth is invalid assign depth from previous example
if not self.valid_data[n]:
......@@ -791,15 +790,15 @@ class DepthData(object):
def interpolate_smooth(self):
"""Apply interpolation based on the robust loess smooth
"""
self.interp_type = 'Smooth'
# Get depth data from object
depth_new = self.depth_beams_m
# Update depth data with interpolated depths
depth_new[not self.valid_beams] = self.smooth_depth[not self.valid_beams]
# Compute processed depths with interpolated values
if self.depth_source == 'BT':
# Temporarily change self.depth_beams_m to compute average
......@@ -812,11 +811,10 @@ class DepthData(object):
else:
# Assignment for VB or DS
self.depth_processed_m = depth_new[0, :]
def interpolate_linear(self, transect):
"""Apply linear interpolation
"""
# Set interpolation type
self.interp_type = 'Linear'
......@@ -831,31 +829,31 @@ class DepthData(object):
select = getattr(transect.boat_vel, 'bt_vel')
track_x = np.tile(np.nan, select.u_processed_mps.shape)
track_y = np.tile(np.nan, select.v_processed_mps.shape)
idx = np.where(np.isnan(track_x[1:]))
# If the navigation reference has no gaps use it for interpolation, if not use time
if len(idx[0]) < 1:
x = np.nancumsum(np.sqrt(track_x**2 + track_y**2))
x = np.nancumsum(np.sqrt(track_x ** 2 + track_y ** 2))
else:
# Compute accumulated time
x = np.nancumsum(transect.date_time.ens_duration_sec)
# Determine number of beams
n_beams = self.depth_beams_m.shape[0]
depth_mono = copy.deepcopy(self.depth_beams_m)
depth_new = copy.deepcopy(self.depth_beams_m)
# Create strict monotonic arrays for depth and track by identifying duplicate
# track values. The first track value is used and the remaining duplicates
# are set to nan. The depth assigned to that first track value is the average
# of all duplicates. The depths for the duplicates are then set to nan. Only
# valid strictly monotonic track and depth data are used for the input in to linear
# interpolation. Only the interpolated data for invalid depths are added
# to the valid depth data to create depth_new
# Create strict monotonic arrays for depth and track by identifying duplicate
# track values. The first track value is used and the remaining duplicates
# are set to nan. The depth assigned to that first track value is the average
# of all duplicates. The depths for the duplicates are then set to nan. Only
# valid strictly monotonic track and depth data are used for the input in to linear
# interpolation. Only the interpolated data for invalid depths are added
# to the valid depth data to create depth_new
x_mono = x
idx0 = np.where(np.diff(x) == 0)[0]
if len(idx0) > 0:
if len(idx0) > 1:
......@@ -876,7 +874,7 @@ class DepthData(object):
depth_mono[:, indices[0]] = depth_avg
depth_mono[:, indices[1:]] = np.nan
x[indices[1:]] = np.nan
# Interpolate each beam
for n in range(n_beams):
......@@ -912,12 +910,12 @@ class DepthData(object):
Draft of ADCP
method: str
Averaging method (Simple, IDW)
Returns
-------
avg_depth: np.array(float)
Average depth for each ensemble
"""
if method == 'Simple':
avg_depth = np.nanmean(depth, 0)
......@@ -925,7 +923,8 @@ class DepthData(object):
# Compute inverse weighted mean depth
rng = depth - draft
w = 1 - np.divide(rng, np.nansum(rng, 0))
avg_depth = draft+np.nansum(np.divide((rng * w), np.nansum(w, 0), where=np.nansum(w, 0) != 0), 0)
avg_depth = draft + np.nansum(np.divide((rng * w), np.nansum(w, 0), out=np.zeros_like(rng),
where=np.nansum(w, 0) != 0), 0)
avg_depth[avg_depth == draft] = np.nan
return avg_depth
......
open_functions.py
+ 33
− 13
View file @ 6a76f1dd
......@@ -46,18 +46,21 @@ def select_directory():
if path_folder:
# ADCP folders path
path_folder = '\\'.join(path_folder.split('/'))
path_folders = np.array(glob.glob(path_folder + "/*"))
# path_folders = np.array(glob.glob(path_folder + "/*"))
path_folders = [f.path for f in os.scandir(path_folder) if f.is_dir()]
# Load their name
name_folders = np.array([os.path.basename((x)) for x in path_folders])
# Exclude files
excluded_folders = [s.find('.') == -1 for s in name_folders]
path_folders = path_folders[excluded_folders]
name_folders = name_folders[excluded_folders]
# excluded_folders = [s.find('.') == -1 for s in name_folders]
# path_folders = path_folders[excluded_folders]
# name_folders = name_folders[excluded_folders]
# Open measurement
type_meas = list()
path_meas = list()
name_meas = list()
no_adcp = list()
for id_meas in range(len(path_folders)):
list_files = os.listdir(path_folders[id_meas])
exte_files = list([i.split('.', 1)[-1] for i in list_files])
......@@ -66,17 +69,30 @@ def select_directory():
loca_meas = [i for i, s in enumerate(exte_files) if "mat" in s] # transects index
fileNameRaw = [(list_files[x]) for x in loca_meas]
qrev_data = False
path_qrev_meas = []
for name in fileNameRaw:
path = os.path.join(path_folders[id_meas], name)
mat_data = sio.loadmat(path, struct_as_record=False, squeeze_me=True)
if 'version' in mat_data:
type_meas.append('QRev')
name_meas.append(name_folders[id_meas])
path_meas.append(mat_data)
path_qrev_meas.append(os.path.join(path_folders[id_meas], name))
qrev_data = True
print('QRev detected')
break
if not qrev_data:
if qrev_data:
recent_time = None
recent_id = 0
for i in range(len(path_qrev_meas)):
qrev_time = os.path.getmtime(path_qrev_meas[i])
if recent_time is None or qrev_time < recent_time:
recent_time = qrev_time
recent_id = i
path = path_qrev_meas[recent_id]
mat_data = sio.loadmat(path, struct_as_record=False, squeeze_me=True)
print('QRev file')
type_meas.append('QRev')
name_meas.append(name_folders[id_meas])
path_meas.append(mat_data)
else:
print('SonTek file')
type_meas.append('SonTek')
fileName = [s for i, s in enumerate(fileNameRaw)
if "QRev.mat" not in s]
......@@ -85,16 +101,20 @@ def select_directory():
[os.path.join(path_folders[id_meas], fileName[x]) for x in
range(len(fileName))])
elif 'mmt' in exte_files:
else:
print('TRDI file')
type_meas.append('TRDI')
loca_meas = exte_files.index("mmt") # index of the measurement file
fileName = list_files[loca_meas]
path_meas.append(os.path.join(path_folders[id_meas], fileName))
name_meas.append(name_folders[id_meas])
return path_folder, path_meas, type_meas, name_meas
else:
no_adcp.append(name_folders[id_meas])
print(f"No ADCP : {name_folders[id_meas]}")
return path_folder, path_meas, type_meas, name_meas, no_adcp
else:
warnings.warn('No folder selected - end')
return None, None, None, None
return None, None, None, None, None
def open_measurement(path_meas, type_meas, apply_settings=False, navigation_reference=None,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment