Compare commits
No commits in common. "5895f39874f970c7b2c52b7fe12a7361a7d06274" and "99ba4e390ea4a446f359da611a92761d73db0e7f" have entirely different histories.
5895f39874
...
99ba4e390e
@ -1,263 +0,0 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import scipy.signal as signal
|
||||
import matplotlib as mpl
|
||||
mpl.use("TkAgg") # fixes focus issues for me
|
||||
import matplotlib.pyplot as plt
|
||||
from time import sleep
|
||||
from random import choice as r_choice
|
||||
from sys import exit
|
||||
import os
|
||||
import re
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __package__ is None:
|
||||
# make relative imports work as described here: https://peps.python.org/pep-0366/#proposed-change
|
||||
__package__ = "teng_ml"
|
||||
import sys
|
||||
from os import path
|
||||
filepath = path.realpath(path.abspath(__file__))
|
||||
sys.path.insert(0, path.dirname(path.dirname(filepath)))
|
||||
|
||||
from .util.transform import Normalize
|
||||
from .util.data_loader import get_datafiles
|
||||
from .util.file_io import get_next_digits
|
||||
|
||||
file = "/home/matth/Uni/TENG/teng_2/data/2023-06-28_foam_black_1_188mm_06V001.csv"
|
||||
|
||||
|
||||
class InteractiveDataSelector:
|
||||
re_file = r'\d{4}-\d{2}-\d{2}_([a-zA-Z0-9_]+)_([a-zA-Z0-9]+)_(\d+(?:\.\d+)?mm)_(\d+V)(\d+)\.csv'
|
||||
re_index_group_nr = 5 # group number of the index part of the filename
|
||||
"""
|
||||
Go through all .csv files in a directory, split the data and exclude sections with the mouse, then write the sections as single files into a new directory
|
||||
"""
|
||||
def __init__(self, in_dir, out_dir, keep_index=True, split_at_exclude=True):
|
||||
"""
|
||||
@param keep_index:
|
||||
If True: append the split number as triple digits to the existing filename (file001.csv -> file001001.csv, file001002.csv ...)
|
||||
Else: remove the indices from the filename before adding the split number (file001.csv -> file001.csv, file002.csv ...)
|
||||
@param split_at_exclude:
|
||||
If True: When excluding an area, split the data before and after the excluded zone
|
||||
Else: remove the excluded zone and join the previous and later part
|
||||
"""
|
||||
if os.path.isdir(out_dir):
|
||||
if os.listdir(out_dir):
|
||||
raise ValueError(f"'out_dir' = '{out_dir}' is not empty")
|
||||
else:
|
||||
os.makedirs(out_dir)
|
||||
self._out_dir = out_dir
|
||||
|
||||
self._in_dir = in_dir
|
||||
self._in_files = os.listdir(in_dir)
|
||||
self._in_files.sort()
|
||||
for i in reversed(range(len(self._in_files))):
|
||||
if not re.fullmatch(InteractiveDataSelector.re_file, self._in_files[i]):
|
||||
print(f"Dropping non-matching file '{self._in_files[i]}'")
|
||||
self._in_files.pop(i)
|
||||
if not self._in_files:
|
||||
raise ValueError(f"No matching files in 'in_dir' = '{in_dir}'")
|
||||
|
||||
self._history: list[tuple[str, list]] = [] # (in_file, [out_files...])
|
||||
|
||||
self._keep_index = keep_index
|
||||
self.split_at_exclude = split_at_exclude
|
||||
|
||||
plt.ion()
|
||||
self._fig, self._ax = plt.subplots()
|
||||
|
||||
mpl.rcParams['keymap.save'].remove('s') # s is used for split
|
||||
mpl.rcParams['keymap.quit'].remove('q')
|
||||
self._fig.canvas.mpl_connect("button_press_event", lambda ev: self._fig_on_button_press(ev))
|
||||
self._fig.canvas.mpl_connect("key_press_event", lambda ev: self._fig_on_key_press(ev))
|
||||
|
||||
def run(self):
|
||||
self._next_file()
|
||||
while plt.fignum_exists(self._fig.number):
|
||||
plt.pause(0.01)
|
||||
|
||||
def _set_titles(self):
|
||||
help_str = "[(e)xclude, (s)plit, (w)rite+next, (U)ndo last file, (Q)uit]"
|
||||
self._fig.suptitle(f"{help_str}\nuse left click to select, right click to undo\ncurret mode: {self._mode}")
|
||||
|
||||
def _undo_file(self):
|
||||
if len(self._history) == 0:
|
||||
print("Nothing to undo")
|
||||
return
|
||||
# delete written files
|
||||
for outfile in self._history[-1][1]:
|
||||
print(f"Deleting '{outfile}'")
|
||||
os.remove(outfile)
|
||||
self._in_files.insert(0, self._history[-1][0])
|
||||
self._history.pop()
|
||||
self._next_file()
|
||||
|
||||
def _next_file(self):
|
||||
# runtime stuff
|
||||
if len(self._in_files) == 0:
|
||||
raise IndexError("No more files to process")
|
||||
self._current_file = self._in_files.pop(0)
|
||||
self._current_dataframe = pd.read_csv(os.path.join(self._in_dir, self._current_file))
|
||||
self._current_array = self._current_dataframe.to_numpy()
|
||||
self._current_array = np.loadtxt(os.path.join(self._in_dir, self._current_file), skiprows=1, delimiter=",")
|
||||
|
||||
|
||||
# plot stuff
|
||||
self._splits_lines = None # vlines
|
||||
self._excludes_lines = None
|
||||
self._excludes_areas = [] # list of areas
|
||||
self._fig.clear()
|
||||
self._ax = self._fig.subplots()
|
||||
self._ax.plot(self._current_array[:,0], self._current_array[:,2])
|
||||
self._ax.set_xlabel(self._current_file)
|
||||
|
||||
self._splits: list[int] = []
|
||||
self._excludes: list[int] = []
|
||||
self._mode = "exclude" # split or exclude
|
||||
self._set_titles()
|
||||
|
||||
def _fig_on_button_press(self, event):
|
||||
"""
|
||||
left click: set split / exclude section (depends on mode)
|
||||
right click: undo last action of selected mode
|
||||
"""
|
||||
if event.xdata is None: return
|
||||
if event.xdata in self._excludes or event.xdata in self._splits: return
|
||||
if event.button == 1: # left click, add position
|
||||
if self._mode == "split":
|
||||
self._splits.append(event.xdata)
|
||||
else:
|
||||
self._excludes.append(event.xdata)
|
||||
elif event.button == 3: # right click, undo
|
||||
if self._mode == "split":
|
||||
if len(self._splits) > 0:
|
||||
self._splits.pop()
|
||||
else:
|
||||
if len(self._excludes) > 0:
|
||||
self._excludes.pop()
|
||||
self._update_lines()
|
||||
|
||||
def _fig_on_key_press(self, event):
|
||||
"""
|
||||
s: set split mode
|
||||
e: set exclude mode
|
||||
w: write and got to next file
|
||||
Q: quit all
|
||||
"""
|
||||
if event.key == 's':
|
||||
self._mode = "split"
|
||||
elif event.key == 'e':
|
||||
self._mode = "exclude"
|
||||
elif event.key == 'w':
|
||||
self._save_as_new_files()
|
||||
try:
|
||||
self._next_file()
|
||||
except IndexError:
|
||||
print(f"All files processed.")
|
||||
exit(0)
|
||||
elif event.key == 'U':
|
||||
self._undo_file()
|
||||
elif event.key == 'Q':
|
||||
print(f"Quitting before all files have been processed!")
|
||||
exit(1)
|
||||
self._set_titles()
|
||||
|
||||
|
||||
def _update_lines(self):
|
||||
# print(self._splits, self._excludes)
|
||||
ymin, ymax = self._ax.get_ylim()
|
||||
|
||||
if self._splits_lines is not None: self._splits_lines.remove()
|
||||
self._splits_lines = self._ax.vlines(self._splits, ymin, ymax, color="b")
|
||||
|
||||
if self._excludes_lines is not None: self._excludes_lines.remove()
|
||||
self._excludes_lines = self._ax.vlines(self._excludes, ymin, ymax, color="r")
|
||||
|
||||
for area in self._excludes_areas:
|
||||
area.remove()
|
||||
self._excludes_areas.clear()
|
||||
excludes = self._excludes.copy()
|
||||
if len(excludes) % 2 == 1: excludes.pop() # only draw pairs
|
||||
excludes.sort()
|
||||
for i in range(1, len(excludes), 2):
|
||||
self._excludes_areas.append(self._ax.axvspan(excludes[i-1], excludes[i], facecolor='r', alpha=0.3))
|
||||
|
||||
self._ax.set_ylim(ymin, ymax) # reset, since margins are added to lines
|
||||
self._fig.canvas.draw()
|
||||
|
||||
def _get_next_filename(self):
|
||||
if self._keep_index:
|
||||
# 5th group is index
|
||||
match = re.fullmatch(InteractiveDataSelector.re_file, self._current_file)
|
||||
assert(type(match) is not None)
|
||||
basename = self._current_file[:match.start(InteractiveDataSelector.re_index_group_nr)]
|
||||
else:
|
||||
basename = self._current_file[:-4] # extension
|
||||
index = get_next_digits(basename, self._out_dir, digits=3)
|
||||
return f"{basename}{index}.csv"
|
||||
|
||||
def _save_as_new_files(self):
|
||||
# convert timestamps to their closest index
|
||||
excludes_idx = [np.abs(self._current_array[:,0] - t).argmin() for t in self._excludes]
|
||||
splits_idx = [np.abs(self._current_array[:,0] - t).argmin() for t in self._splits]
|
||||
if self.split_at_exclude:
|
||||
# split before the start of the exclucded range
|
||||
splits_idx += [ excludes_idx[i]-1 for i in range(0, len(excludes_idx), 2) ]
|
||||
# split after the end of the exclucded range
|
||||
splits_idx += [ excludes_idx[i]+1 for i in range(1, len(excludes_idx), 2) ]
|
||||
splits_idx = list(set(splits_idx)) # remove duplicates
|
||||
splits_idx.sort()
|
||||
|
||||
df = self._current_dataframe.copy()
|
||||
|
||||
# 1) remove excluded parts
|
||||
for i in range(1, len(excludes_idx), 2):
|
||||
df = df.drop(index=range(excludes_idx[i-1], excludes_idx[i]+1))
|
||||
|
||||
# 2) splits
|
||||
new_frames = []
|
||||
start_i = df.index[0]
|
||||
for i in range(0, len(splits_idx)):
|
||||
end_i = splits_idx[i]
|
||||
# print(start_i, end_i)
|
||||
# check if valid start and end index
|
||||
if start_i in df.index and end_i in df.index:
|
||||
new_frames.append(df.loc[start_i:end_i])
|
||||
start_i = end_i + 1
|
||||
# append rest
|
||||
if start_i in df.index:
|
||||
new_frames.append(df.loc[start_i:])
|
||||
|
||||
# 3) remove empty
|
||||
for i in reversed(range(len(new_frames))):
|
||||
if len(new_frames[i]) == 0:
|
||||
new_frames.pop(i)
|
||||
|
||||
self._history.append((self._current_file, []))
|
||||
for frame in new_frames:
|
||||
filename = self._get_next_filename()
|
||||
pathname = os.path.join(self._out_dir, filename)
|
||||
# until now, frame is a copy of a slice
|
||||
frame = frame.copy()
|
||||
# transform timestamps so that first value is 0
|
||||
t_column_name = frame.columns[0]
|
||||
frame[t_column_name] -= frame.iloc[0][t_column_name]
|
||||
frame.to_csv(pathname, index=False)
|
||||
self._history[-1][1].append(pathname)
|
||||
print(f"Saved range of length {len(frame.index):04} to {pathname}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser("data_preprocess")
|
||||
parser.add_argument("in_dir")
|
||||
parser.add_argument("out_dir")
|
||||
parser.add_argument("-i", "--keep_index", action="store_true")
|
||||
parser.add_argument("-e", "--split_at_exclude", action="store_true")
|
||||
ns = parser.parse_args()
|
||||
|
||||
selector = InteractiveDataSelector(ns.in_dir, ns.out_dir, keep_index=ns.keep_index, split_at_exclude=ns.split_at_exclude)
|
||||
selector.run()
|
||||
exit(2) # selector should exit in _fig_on_key_press
|
||||
|
||||
@ -18,9 +18,8 @@ import time
|
||||
from os import makedirs, path
|
||||
|
||||
from .util.transform import ConstantInterval, Normalize
|
||||
from .util.data_loader import load_datasets, LabelConverter, count_data
|
||||
from .util.data_loader import load_datasets, LabelConverter
|
||||
from .util.split import DataSplitter
|
||||
from .util.pad import PadSequences
|
||||
from .util.settings import MLSettings
|
||||
from .rnn.rnn import RNN
|
||||
from .rnn.training import train_validate_save, select_device
|
||||
@ -42,30 +41,34 @@ def test_interpol():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
labels = LabelConverter(["foam_PDMS_white", "foam_PDMS_black", "foam_PDMS_TX100", "foam_PE", "antistatic_foil", "cardboard", "glass", "kapton", "bubble_wrap_PE", "fabric_PP", ])
|
||||
# labels = LabelConverter(["foam_PDMS_white", "foam_PDMS_black", "foam_PDMS_TX100", "foam_PE", "kapton", "bubble_wrap_PE", "fabric_PP", ])
|
||||
models_dir = "/home/matth/Uni/TENG/teng_2/models_gen_12" # where to save models, settings and results
|
||||
labels = LabelConverter(["white_foam", "glass", "Kapton", "bubble_wrap", "cloth", "black_foam"])
|
||||
models_dir = "/home/matth/Uni/TENG/models" # where to save models, settings and results
|
||||
if not path.isdir(models_dir):
|
||||
makedirs(models_dir)
|
||||
data_dir = "/home/matth/Uni/TENG/teng_2/sorted_data"
|
||||
data_dir = "/home/matth/Uni/TENG/data"
|
||||
|
||||
# gen_5 best options: datasplitter, not bidirectional, lr=0.001, no scheduler
|
||||
# gen_6 best options: no glass, cardboard and antistatic_foil, not bidirectional, lr=0.0007, no datasplitter, 2 layers n_hidden = 10
|
||||
|
||||
# Test with
|
||||
num_layers = [ 2, 3 ]
|
||||
hidden_size = [ 21, 28 ]
|
||||
bidirectional = [ False, True ]
|
||||
t_const_int = ConstantInterval(0.01) # TODO check if needed: data was taken at equal rate, but it isnt perfect -> maybe just ignore?
|
||||
t_norm = Normalize(-1, 1)
|
||||
transforms = [[ t_norm ]] #, [ t_norm, t_const_int ]]
|
||||
batch_sizes = [ 4 ]
|
||||
splitters = [ DataSplitter(50, drop_if_smaller_than=30) ] # smallest file has length 68 TODO: try with 0.5-1second snippets
|
||||
num_layers = [ 3 ]
|
||||
hidden_size = [ 8 ]
|
||||
bidirectional = [ True ]
|
||||
t_const_int = ConstantInterval(0.01)
|
||||
t_norm = Normalize(0, 1)
|
||||
transforms = [[ t_const_int ]] #, [ t_const_int, t_norm ]]
|
||||
batch_sizes = [ 64 ] # , 16]
|
||||
splitters = [ DataSplitter(100) ]
|
||||
num_epochs = [ 80 ]
|
||||
# (epoch, min_accuracy)
|
||||
training_cancel_points = [(15, 20), (40, 25)]
|
||||
# training_cancel_points = []
|
||||
|
||||
# num_layers=1,
|
||||
# hidden_size=1,
|
||||
# bidirectional=True,
|
||||
# optimizer=None,
|
||||
# scheduler=None,
|
||||
# loss_func=None,
|
||||
# transforms=[],
|
||||
# splitter=None,
|
||||
# num_epochs=10,
|
||||
# batch_size=5,
|
||||
args = [num_layers, hidden_size, bidirectional, [None], [None], [None], transforms, splitters, num_epochs, batch_sizes]
|
||||
|
||||
# create settings for every possible combination
|
||||
@ -75,26 +78,23 @@ if __name__ == "__main__":
|
||||
|
||||
loss_func = nn.CrossEntropyLoss()
|
||||
optimizers = [
|
||||
lambda model: torch.optim.Adam(model.parameters(), lr=0.0007),
|
||||
lambda model: torch.optim.Adam(model.parameters(), lr=0.03),
|
||||
# lambda model: torch.optim.Adam(model.parameters(), lr=0.25),
|
||||
# lambda model: torch.optim.Adam(model.parameters(), lr=0.50),
|
||||
]
|
||||
schedulers = [
|
||||
None,
|
||||
# lambda optimizer, st: torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9),
|
||||
# lambda optimizer, st: torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.5),
|
||||
lambda optimizer, st: torch.optim.lr_scheduler.StepLR(optimizer, step_size=st.num_epochs // 8, gamma=0.60, verbose=False),
|
||||
lambda optimizer, st: torch.optim.lr_scheduler.StepLR(optimizer, step_size=st.num_epochs // 10, gamma=0.40, verbose=False),
|
||||
# lambda optimizer, st: torch.optim.lr_scheduler.StepLR(optimizer, step_size=st.num_epochs // 10, gamma=0.75, verbose=False),
|
||||
]
|
||||
|
||||
device = select_device(force_device="cpu") # TODO cuda is not supported because something throws NotImplementedError with my gpu
|
||||
n_total = len(settings) * len(optimizers) * len(schedulers)
|
||||
print(f"Testing {n_total} possible configurations, device='{device}'")
|
||||
print(f"Testing {n_total} possible configurations")
|
||||
# scheduler2 =
|
||||
def create_model(st, optimizer_f, scheduler_f):
|
||||
model=RNN(input_size=st.num_features, hidden_size=st.hidden_size, num_layers=st.num_layers, num_classes=len(labels), bidirectional=st.bidirectional)
|
||||
optimizer = optimizer_f(model)
|
||||
if scheduler_f is not None:
|
||||
scheduler = scheduler_f(optimizer, st)
|
||||
else: scheduler = None
|
||||
scheduler = scheduler_f(optimizer, st)
|
||||
return model, optimizer, scheduler
|
||||
|
||||
t_begin = time.time()
|
||||
@ -103,21 +103,19 @@ if __name__ == "__main__":
|
||||
for s in range(len(schedulers)):
|
||||
for i in range(len(settings)):
|
||||
st = settings[i]
|
||||
train_set, test_set = load_datasets(data_dir, labels, exclude_n_object=None, voltage=None, transforms=st.transforms, split_function=st.splitter, train_to_test_ratio=0.7, random_state=123, num_workers=4)
|
||||
# print(st.get_name())
|
||||
train_set, test_set = load_datasets(data_dir, labels, voltage=8.2, transforms=st.transforms, split_function=st.splitter, train_to_test_ratio=0.7, random_state=42, num_workers=4)
|
||||
|
||||
generator = torch.manual_seed(42)
|
||||
train_loader = DataLoader(train_set, batch_size=st.batch_size, shuffle=True, generator=generator, collate_fn=PadSequences())
|
||||
test_loader = DataLoader(test_set, batch_size=None, shuffle=True, generator=generator)
|
||||
|
||||
# set batch_size to None and remove collate_fn for this to work
|
||||
# count_data(train_loader, st.labels, print_summary="training data")
|
||||
# count_data(test_loader, st.labels, print_summary="validation data")
|
||||
|
||||
|
||||
model, optimizer, scheduler = create_model(st, optimizers[o], schedulers[s])
|
||||
# train_loader = iter(DataLoader(train_set))
|
||||
# test_loader = iter(DataLoader(test_set))
|
||||
train_loader = DataLoader(train_set, batch_size=st.batch_size, shuffle=True, generator=generator)
|
||||
test_loader = DataLoader(test_set, batch_size=st.batch_size, shuffle=True, generator=generator)
|
||||
print(f"Testing {n}/{n_total}: (o={o}, s={s}, i={i})")
|
||||
model, optimizer, scheduler = create_model(st, optimizers[o], schedulers[s])
|
||||
device = select_device(force_device="cpu")
|
||||
try:
|
||||
train_validate_save(model, optimizer, scheduler, loss_func, train_loader, test_loader, st, models_dir, print_interval=1, print_continuous=True, training_cancel_points=training_cancel_points)
|
||||
train_validate_save(model, optimizer, scheduler, loss_func, train_loader, test_loader, st, models_dir, print_interval=1)
|
||||
except KeyboardInterrupt:
|
||||
if input("Cancelled current training. Quit? (q/*): ") == "q":
|
||||
t_end = time.time()
|
||||
|
||||
@ -10,7 +10,7 @@ from sys import exit
|
||||
if __name__ == "__main__":
|
||||
if __package__ is None:
|
||||
# make relative imports work as described here: https://peps.python.org/pep-0366/#proposed-change
|
||||
__package__ = "teng_ml"
|
||||
__package__ = "teng-ml"
|
||||
import sys
|
||||
from os import path
|
||||
filepath = path.realpath(path.abspath(__file__))
|
||||
|
||||
@ -19,59 +19,52 @@ class RNN(nn.Module):
|
||||
self.softmax = nn.Softmax(dim=1)
|
||||
self.D = 2 if self.is_bidirectional == True else 1
|
||||
|
||||
def forward(self, x, unpadded_lengths=None):
|
||||
"""
|
||||
@param x:
|
||||
Tensor (seq_length, features) for unbatched inputs
|
||||
Tensor (batch_size, seq_length, features) for batch inputs
|
||||
PackedSequence for padded batched inputs
|
||||
@param unpadded_lengths: Tensor(batch_size) with lengths of the unpadded sequences, when using padding but without PackedSequence
|
||||
@returns (batch_size, num_classes) with batch_size == 1 for unbatched inputs
|
||||
"""
|
||||
# if type(x) == torch.Tensor:
|
||||
# device = x.device
|
||||
# # h0: initial hidden states
|
||||
# # c0: initial cell states
|
||||
# if len(x.shape) == 2: # x: (seq_length, features)
|
||||
# h0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
# c0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
# elif len(x.shape) == 3: # x: (batch, seq_length, features)
|
||||
# batch_size = x.shape[0]
|
||||
# h0 = torch.zeros(self.D * self.num_layers, batch_size, self.hidden_size).to(device)
|
||||
# c0 = torch.zeros(self.D * self.num_layers, batch_size, self.hidden_size).to(device)
|
||||
# else:
|
||||
# raise ValueError(f"RNN.forward: invalid input shape: {x.shape}. Must be (batch, seq_length, features) or (seq_length, features)")
|
||||
# elif type(x) == nn.utils.rnn.PackedSequence:
|
||||
# device = x.data.device
|
||||
# h0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
# c0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
# else:
|
||||
# raise ValueError(f"RNN.forward: invalid input type: {type(x)}. Must be Tensor or PackedSequence")
|
||||
def forward(self, x):
|
||||
device = x.device
|
||||
|
||||
# h0: initial hidden states
|
||||
# c0: initial cell states
|
||||
if len(x.shape) == 2: # x: (seq_length, features)
|
||||
h0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
c0 = torch.zeros(self.D * self.num_layers, self.hidden_size).to(device)
|
||||
elif len(x.shape) == 3: # x: (batch, seq_length, features)
|
||||
batch_size = x.shape[0]
|
||||
h0 = torch.zeros(self.D * self.num_layers, batch_size, self.hidden_size).to(device)
|
||||
c0 = torch.zeros(self.D * self.num_layers, batch_size, self.hidden_size).to(device)
|
||||
else:
|
||||
raise ValueError(f"RNN.forward: invalid iput shape: {x.shape}. Must be (batch, seq_length, features) or (seq_length, features)")
|
||||
|
||||
# lstm: (batch_size, seq_length, features) -> (batch_size, hidden_size)
|
||||
# or: packed_sequence -> packed_sequence
|
||||
# out, (h_n, c_n) = self.lstm(x, (h0, c0))
|
||||
out, (h_n, c_n) = self.lstm(x) # (h0, c0) defaults to zeros
|
||||
out, (h_n, c_n) = self.lstm(x, (h0, c0))
|
||||
print(f"forward: out.shape={out.shape} TODO verify comment")
|
||||
# out: (N, L, D * hidden_size)
|
||||
# h_n: (D * num_layers, hidden_size)
|
||||
# c_n: (D * num_layers, hidden_size)
|
||||
# print(f"out({out.shape})={out}")
|
||||
# print(f"h_n({h_n.shape})={h_n}")
|
||||
# print(f"c_n({c_n.shape})={c_n}")
|
||||
# print(f"out({out.shape})=...")
|
||||
# print(f"h_n({h_n.shape})=...")
|
||||
# print(f"c_n({c_n.shape})=...")
|
||||
|
||||
# select the last state of lstm's neurons
|
||||
if type(out) == nn.utils.rnn.PackedSequence:
|
||||
# padding has to be considered
|
||||
out, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
|
||||
# the unpadded length of batch i is lengths[i], so that is the last non-zero state
|
||||
out = torch.stack([out[i,lengths[i].item()-1,:] for i in range(len(lengths))])
|
||||
elif unpadded_lengths is not None:
|
||||
out = torch.stack([out[i,unpadded_lengths[i].item()-1,:] for i in range(len(unpadded_lengths))])
|
||||
"""
|
||||
# select only last layer [-1] -> last layer,
|
||||
last_layer_state = h_n.view(self.num_layers, D, batch_size, self.hidden_size)[-1]
|
||||
if D == 1:
|
||||
# [1, batch_size, hidden_size] -> [batch_size, hidden_size]
|
||||
X = last_layer_state.squeeze() # TODO what if batch_size == 1
|
||||
elif D == 2:
|
||||
h_1, h_2 = last_layer_state[0], last_layer_state[1] # states of both directions
|
||||
# concatenate both states, X-size: (Batch, hidden_size * 2）
|
||||
X = torch.cat((h_1, h_2), dim=1)
|
||||
else:
|
||||
if out.shape[0] == 3: # batched
|
||||
out = out[:,-1,:]
|
||||
else: # unbatched
|
||||
# softmax requires (batch_size, *)
|
||||
out = torch.stack([out[-1,:]])
|
||||
raise ValueError("D must be 1 or 2")
|
||||
""" # all this is quivalent to line below
|
||||
out = out[:,-1,:] # select last time step
|
||||
|
||||
# fc fully connected layer: (*, hidden_size) -> (*, num_classes)
|
||||
out = self.fc(out)
|
||||
|
||||
# softmax: (batch_size, *) -> (batch_size, *)
|
||||
# softmax: (*) -> (*)
|
||||
out = self.softmax(out)
|
||||
return out
|
||||
|
||||
@ -7,7 +7,7 @@ from torch.utils.data import DataLoader
|
||||
from ..util.settings import MLSettings
|
||||
from ..tracker.epoch_tracker import EpochTracker
|
||||
from ..util.file_io import get_next_digits
|
||||
from ..util.string import class_str, optimizer_str
|
||||
from ..util.string import class_str
|
||||
|
||||
from ..util import model_io as mio
|
||||
|
||||
@ -30,20 +30,29 @@ def select_device(force_device=None):
|
||||
return device
|
||||
|
||||
|
||||
def train(model, optimizer, scheduler, loss_func, train_loader: DataLoader, st: MLSettings, print_interval=1, print_continuous=False, training_cancel_points=[]) -> EpochTracker:
|
||||
def train(model, optimizer, scheduler, loss_func, train_loader: DataLoader, st: MLSettings, print_interval=1) -> EpochTracker:
|
||||
epoch_tracker = EpochTracker(st.labels)
|
||||
epoch_tracker.begin()
|
||||
for ep in range(st.num_epochs):
|
||||
loss = -1
|
||||
for i, (data, lengths, y) in enumerate(train_loader):
|
||||
for i, (data, y) in enumerate(train_loader):
|
||||
# print(data, y)
|
||||
# data = batch, seq, features
|
||||
# print(f"data({data.shape})={data}")
|
||||
x = data[:,:,[2]].float() # select voltage data
|
||||
# print(f"x({x.shape}, {x.dtype})=...")
|
||||
# print(f"y({y.shape}, {y.dtype})=...")
|
||||
# pack = torch.nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True)
|
||||
# out = model(pack) # really slow
|
||||
out = model(x, lengths)
|
||||
# length = torch.tensor([x.shape[1] for _ in range(x.shape[0])], dtype=torch.int64)
|
||||
# print(f"length({length.shape})={length}")
|
||||
# batch_size = x.shape[0]
|
||||
# print(f"batch_size={batch_size}")
|
||||
# v = x.view(batch_size, -1, feature_count)
|
||||
# data = rnn_utils.pack_padded_sequence(v.type(torch.FloatTensor), length, batch_first=True).to(device)[0]
|
||||
# print(f"data({data.shape})={data}")
|
||||
out = model(x)
|
||||
|
||||
# print(f"out({out.shape}={out})")
|
||||
# print(f" y({y.shape}={y})")
|
||||
with torch.no_grad():
|
||||
predicted = torch.argmax(out, dim=1, keepdim=False) # -> [ label_indices ]
|
||||
correct = torch.argmax(y, dim=1, keepdim=False) # -> [ label_indices ]
|
||||
@ -63,19 +72,9 @@ def train(model, optimizer, scheduler, loss_func, train_loader: DataLoader, st:
|
||||
|
||||
# predicted = torch.max(torch.nn.functional.softmax(out), 1)[1]
|
||||
epoch_tracker.end_epoch(loss, optimizer.param_groups[0]["lr"])
|
||||
if (ep+1) % print_interval == 0:
|
||||
if print_continuous: end='\r'
|
||||
else: end='\n'
|
||||
print(f"Training:", epoch_tracker.get_epoch_summary_str(), end=end)
|
||||
# cancel training if model is not good enough
|
||||
if len(training_cancel_points) > 0 and ep+1 == training_cancel_points[0][0]:
|
||||
if epoch_tracker.accuracies[-1] < training_cancel_points[0][1]:
|
||||
print(f"Training cancelled because the models accuracy={epoch_tracker.accuracies[-1]:.2f} < {training_cancel_points[0][1]} after {ep+1} epochs.")
|
||||
break;
|
||||
training_cancel_points.pop(0)
|
||||
|
||||
if scheduler is not None:
|
||||
scheduler.step()
|
||||
if ep+1 % print_interval == 0:
|
||||
print(f"Training:", epoch_tracker.get_epoch_summary_str())
|
||||
scheduler.step()
|
||||
print("Training:", epoch_tracker.end())
|
||||
return epoch_tracker
|
||||
|
||||
@ -86,27 +85,24 @@ def validate(model, test_loader: DataLoader, st: MLSettings) -> EpochTracker:
|
||||
with torch.no_grad():
|
||||
for i, (data, y) in enumerate(test_loader):
|
||||
# print(ep, "Test")
|
||||
x = data[:,[2]].float()
|
||||
x = data[:,:,[2]].float()
|
||||
out = model(x)
|
||||
|
||||
predicted = torch.argmax(out, dim=1, keepdim=False) # -> [ label_indices ]
|
||||
if y.shape[0] == 2: # batched
|
||||
correct = torch.argmax(y, dim=1, keepdim=False) # -> [ label_indices ]
|
||||
else: # unbatched
|
||||
correct = torch.argmax(y, dim=0, keepdim=True) # -> [ label_indices ]
|
||||
correct = torch.argmax(y, dim=1, keepdim=False) # -> [ label_indices ]
|
||||
|
||||
epoch_tracker.add_prediction(correct, predicted)
|
||||
print("Validation:", epoch_tracker.end())
|
||||
return epoch_tracker
|
||||
|
||||
|
||||
def train_validate_save(model, optimizer, scheduler, loss_func, train_loader: DataLoader, test_loader: DataLoader, st: MLSettings, models_dir, print_interval=1, print_continuous=False, show_plots=False, training_cancel_points=[]):
|
||||
def train_validate_save(model, optimizer, scheduler, loss_func, train_loader: DataLoader, test_loader: DataLoader, st: MLSettings, models_dir, print_interval=1, show_plots=False):
|
||||
# assumes model and data is already on correct device
|
||||
# train_loader.to(device)
|
||||
# test_loader.to(device)
|
||||
|
||||
# store optimizer, scheduler and loss_func in settings
|
||||
st.optimizer = optimizer_str(optimizer)
|
||||
st.optimizer = class_str(optimizer)
|
||||
st.scheduler = class_str(scheduler)
|
||||
st.loss_func = class_str(loss_func)
|
||||
|
||||
@ -115,15 +111,15 @@ def train_validate_save(model, optimizer, scheduler, loss_func, train_loader: Da
|
||||
def add_tab(s):
|
||||
return "\t" + str(s).replace("\n", "\n\t")
|
||||
print(100 * '=')
|
||||
print("model name:", model_name)
|
||||
print("Model Name:", model_name)
|
||||
print(f"model:\n", add_tab(model))
|
||||
print(f"loss_func: {st.loss_func}")
|
||||
print(f"optimizer: {st.optimizer}")
|
||||
print(f"scheduler: {st.scheduler}")
|
||||
# print(f"loss_func:\n", add_tab(class_str(loss_func)))
|
||||
# print(f"optimizer:\n", add_tab(class_str(optimizer)))
|
||||
# print(f"scheduler:\n", add_tab(class_str(scheduler)))
|
||||
|
||||
|
||||
print(100 * '-')
|
||||
training_tracker = train(model, optimizer, scheduler, loss_func, train_loader, st, print_interval=print_interval, print_continuous=print_continuous, training_cancel_points=training_cancel_points)
|
||||
training_tracker = train(model, optimizer, scheduler, loss_func, train_loader, st, print_interval=print_interval)
|
||||
# print("Training: Count per label:", training_tracker.get_count_per_label())
|
||||
# print("Training: Predictions per label:", training_tracker.get_predictions_per_label())
|
||||
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
from os import stat
|
||||
from ..util.data_loader import LabelConverter
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.colors as colors
|
||||
import time
|
||||
import torch
|
||||
import numpy as np
|
||||
@ -143,7 +141,8 @@ class EpochTracker:
|
||||
label_names = self.labels.get_labels()
|
||||
|
||||
fig, ax = plt.subplots(layout="tight")
|
||||
im = ax.imshow(normalized_predictions, cmap='Blues') # cmap='BuPu', , norm=colors.PowerNorm(1./2.)
|
||||
|
||||
im = ax.imshow(normalized_predictions, cmap='Blues') # cmap='BuPu'
|
||||
ax.set_xticks(np.arange(N))
|
||||
ax.set_yticks(np.arange(N))
|
||||
ax.set_xticklabels(label_names)
|
||||
|
||||
@ -4,15 +4,13 @@ import re
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.sparse import data
|
||||
import torch
|
||||
|
||||
import threading
|
||||
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
# groups: date, name, n_object, voltage, distance, index
|
||||
# re_filename = r"(\d{4}-\d{2}-\d{2})_([a-zA-Z_]+)_(\d{1,2}(?:\.\d*)?)V_(\d+(?:\.\d*)?)mm(\d+).csv"
|
||||
re_filename = r"(\d{4}-\d{2}-\d{2})_([a-zA-Z0-9_]+)_(\d+)_(\d{1,2}(?:\.\d*)?)V_(\d+(?:\.\d*)?)mm(\d+).csv"
|
||||
# groups: date, name, voltage, distance, index
|
||||
re_filename = r"(\d{4}-\d{2}-\d{2})_([a-zA-Z_]+)_(\d{1,2}(?:\.\d*)?)V_(\d+(?:\.\d*)?)mm(\d+).csv"
|
||||
|
||||
class LabelConverter:
|
||||
def __init__(self, class_labels: list[str]):
|
||||
@ -25,13 +23,7 @@ class LabelConverter:
|
||||
vec[self.class_labels.index(label)] = 1.0
|
||||
return vec
|
||||
|
||||
def get_label_index(self, one_hot: torch.Tensor):
|
||||
"""return one hot vector for given label"""
|
||||
return int(torch.argmax(one_hot).item())
|
||||
|
||||
def __getitem__(self, index):
|
||||
if type(index) == torch.Tensor:
|
||||
return self.class_labels[self.get_label_index(index)]
|
||||
return self.class_labels[index]
|
||||
|
||||
def __contains__(self, value):
|
||||
@ -48,10 +40,9 @@ class LabelConverter:
|
||||
|
||||
|
||||
class Datasample:
|
||||
def __init__(self, date: str, label: str, n_object: str, voltage: str, distance: str, index: str, label_vec, datapath: str, init_data=False):
|
||||
def __init__(self, date: str, label: str, voltage: str, distance: str, index: str, label_vec, datapath: str, init_data=False):
|
||||
self.date = date
|
||||
self.label = label
|
||||
self.n_object = int(n_object)
|
||||
self.voltage = float(voltage)
|
||||
self.distance = float(distance)
|
||||
self.index = int(index)
|
||||
@ -91,11 +82,8 @@ class Dataset:
|
||||
if split_function is None:
|
||||
self.data.append((data, sample.label_vec))
|
||||
else:
|
||||
try:
|
||||
for data_split in split_function(data):
|
||||
self.data.append((data_split, sample.label_vec))
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Exception occured during splitting of sample '{sample.datapath}': {e}")
|
||||
for data_split in split_function(data):
|
||||
self.data.append((data_split, sample.label_vec))
|
||||
|
||||
def apply_transforms(self, data):
|
||||
if type(self.transforms) == list:
|
||||
@ -112,41 +100,36 @@ class Dataset:
|
||||
return len(self.data)
|
||||
|
||||
|
||||
def get_datafiles(datadir, labels: LabelConverter, exclude_n_object=None, filter_voltage=None):
|
||||
def get_datafiles(datadir, labels: LabelConverter, voltage=None):
|
||||
"""
|
||||
get a list of all matching datafiles from datadir that are in the format: yyyy-mm-dd_label__n_object_x.xV_xxxmm.csv
|
||||
get a list of all matching datafiles from datadir that are in the format: yyyy-mm-dd_label_x.xV_xxxmm.csv
|
||||
"""
|
||||
datafiles = []
|
||||
files = listdir(datadir)
|
||||
files.sort()
|
||||
for file in files:
|
||||
match = re.fullmatch(re_filename, file)
|
||||
if not match:
|
||||
print(f"get_datafiles: dropping non matching file '{file}'")
|
||||
continue
|
||||
if not match: continue
|
||||
|
||||
label = match.groups()[1]
|
||||
if label not in labels: continue
|
||||
|
||||
sample_n_object = float(match.groups()[2])
|
||||
if exclude_n_object and exclude_n_object == sample_n_object: continue
|
||||
sample_voltage = float(match.groups()[3])
|
||||
if filter_voltage and filter_voltage != sample_voltage: continue
|
||||
|
||||
sample_voltage = float(match.groups()[2])
|
||||
if voltage and voltage != sample_voltage: continue
|
||||
datafiles.append((datadir + "/" + file, match, label))
|
||||
return datafiles
|
||||
|
||||
|
||||
def load_datasets(datadir, labels: LabelConverter, transforms=None, split_function=None, exclude_n_object=None, voltage=None, train_to_test_ratio=0.7, random_state=None, num_workers=None):
|
||||
def load_datasets(datadir, labels: LabelConverter, transforms=None, split_function=None, voltage=None, train_to_test_ratio=0.7, random_state=None, num_workers=None):
|
||||
"""
|
||||
load all data from datadir that are in the format: yyyy-mm-dd_label_x.xV_xxxmm.csv
|
||||
"""
|
||||
datasamples = []
|
||||
if num_workers == None:
|
||||
for file, match, label in get_datafiles(datadir, labels, exclude_n_object=exclude_n_object, filter_voltage=voltage):
|
||||
for file, match, label in get_datafiles(datadir, labels, voltage):
|
||||
datasamples.append(Datasample(*match.groups(), labels.get_one_hot(label), file))
|
||||
else:
|
||||
files = get_datafiles(datadir, labels, exclude_n_object=exclude_n_object, filter_voltage=voltage)
|
||||
files = get_datafiles(datadir, labels, voltage)
|
||||
def worker():
|
||||
while True:
|
||||
try:
|
||||
@ -161,36 +144,10 @@ def load_datasets(datadir, labels: LabelConverter, transforms=None, split_functi
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
|
||||
# TODO do the train_test_split after the Dataset split
|
||||
# problem: needs to be after transforms
|
||||
train_samples, test_samples = train_test_split(datasamples, train_size=train_to_test_ratio, shuffle=True, random_state=random_state)
|
||||
train_dataset = Dataset(train_samples, transforms=transforms, split_function=split_function)
|
||||
test_dataset = Dataset(test_samples, transforms=transforms, split_function=split_function)
|
||||
return train_dataset, test_dataset
|
||||
|
||||
|
||||
def count_data(data_loader, label_converter: LabelConverter, print_summary=False):
|
||||
"""
|
||||
@param data_loader: unbatched data loader
|
||||
"""
|
||||
n_sequences = 0 # count number of sequences
|
||||
labels = [ 0 for _ in range(len(label_converter)) ] # count number of sequences per label
|
||||
len_data = [ 0 for _ in range(len(label_converter)) ] # count number of datapoints per label
|
||||
for i, (data, y) in enumerate(data_loader):
|
||||
n_sequences = i
|
||||
label_i = label_converter.get_label_index(y)
|
||||
len_data[label_i] += data.shape[0]
|
||||
labels[label_i] += 1
|
||||
if print_summary:
|
||||
print("=" * 50)
|
||||
print("Dataset summary" + f" for {print_summary}:" if type(print_summary) == str else ":")
|
||||
print(f"Number of sequences: {n_sequences}")
|
||||
for i in range(len(label_converter)):
|
||||
print(f"- {label_converter[i]:15}: {labels[i]:3} sequences, {len_data[i]:5} datapoints")
|
||||
|
||||
return n_sequences, labels, len_data
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
import torch
|
||||
import torch.nn.utils.rnn as rnn
|
||||
import numpy as np
|
||||
|
||||
class PadSequences:
|
||||
def __call__(self, batch):
|
||||
# batch = [(data, label)]
|
||||
# sort by length
|
||||
sorted_batch = sorted(batch, key=lambda sample: sample[0].shape[0], reverse=True)
|
||||
sequences = [torch.Tensor(sample[0]) for sample in sorted_batch]
|
||||
labels = torch.Tensor(np.array([sample[1] for sample in sorted_batch]))
|
||||
lengths = torch.IntTensor(np.array([seq.shape[0] for seq in sequences]))
|
||||
sequences_padded = rnn.pad_sequence(sequences, batch_first=True)
|
||||
return sequences_padded, lengths, labels
|
||||
@ -5,12 +5,8 @@ class DataSplitter:
|
||||
Split a numpy array into smaller arrays of size datapoints_per_split
|
||||
If data.shape(0) % datapoints_per_split != 0, the remaining datapoints are dropped
|
||||
"""
|
||||
def __init__(self, datapoints_per_split, drop_if_smaller_than=-1):
|
||||
"""
|
||||
@param drop_if_smaller_than: drop the remaining datapoints if the sequence would be smaller than this value. -1 means drop_if_smaller_than=datapoints_per_split
|
||||
"""
|
||||
def __init__(self, datapoints_per_split):
|
||||
self.split_size = datapoints_per_split
|
||||
self.drop_threshhold = datapoints_per_split if drop_if_smaller_than == -1 else drop_if_smaller_than
|
||||
|
||||
def __call__(self, data: np.ndarray):
|
||||
"""
|
||||
@ -19,11 +15,6 @@ class DataSplitter:
|
||||
ret_data = []
|
||||
for i in range(self.split_size, data.shape[0], self.split_size):
|
||||
ret_data.append(data[i-self.split_size:i, :])
|
||||
|
||||
rest_start = len(ret_data) * self.split_size
|
||||
if len(data) - rest_start >= self.drop_threshhold:
|
||||
ret_data.append(data[rest_start:,:])
|
||||
|
||||
if len(ret_data) == 0:
|
||||
raise ValueError(f"data has only {data.shape[0]}, but datapoints_per_split is set to {self.split_size}")
|
||||
return ret_data
|
||||
|
||||
@ -13,50 +13,25 @@ def fill_and_center(s: str, fill_char="=", length=100):
|
||||
else:
|
||||
return s
|
||||
|
||||
|
||||
def class_str(x):
|
||||
"""
|
||||
Return the constructor of the class of x with arguemnts
|
||||
"""
|
||||
name = type(x).__name__
|
||||
signature = inspect.signature(type(x))
|
||||
params = []
|
||||
try:
|
||||
signature = inspect.signature(type(x))
|
||||
for param_name, param_value in x.__dict__.items():
|
||||
if param_name not in signature.parameters:
|
||||
continue
|
||||
default_value = signature.parameters[param_name].default
|
||||
if param_value != default_value:
|
||||
params.append(f"{param_name}={param_value!r}")
|
||||
except ValueError:
|
||||
pass
|
||||
for param_name, param_value in x.__dict__.items():
|
||||
if param_name not in signature.parameters:
|
||||
continue
|
||||
default_value = signature.parameters[param_name].default
|
||||
if param_value != default_value:
|
||||
params.append(f"{param_name}={param_value!r}")
|
||||
if params:
|
||||
return f"{name}({', '.join(params)})"
|
||||
else:
|
||||
return name
|
||||
|
||||
|
||||
def optimizer_str(x):
|
||||
# optimizer stores everything in 'defaults' dict and is thus not compatible with class_str
|
||||
name = type(x).__name__
|
||||
params = []
|
||||
try:
|
||||
signature = inspect.signature(type(x))
|
||||
for param_name, param_value in x.__dict__["defaults"].items():
|
||||
if param_name not in signature.parameters:
|
||||
continue
|
||||
default_value = signature.parameters[param_name].default
|
||||
if param_value != default_value:
|
||||
params.append(f"{param_name}={param_value!r}")
|
||||
except ValueError:
|
||||
pass
|
||||
if params:
|
||||
return f"{name}({', '.join(params)})"
|
||||
else:
|
||||
return name
|
||||
|
||||
|
||||
|
||||
def cleanup_str(s):
|
||||
"""
|
||||
convert to string if necessary and
|
||||
|
||||
Loading…
Reference in New Issue
Block a user