11 changed files with 573 additions and 504 deletions
@ -1,2 +0,0 @@
@@ -1,2 +0,0 @@
|
||||
from . import gui |
||||
from . import main |
||||
@ -1,127 +0,0 @@
@@ -1,127 +0,0 @@
|
||||
import numpy as np |
||||
import matplotlib.pyplot as plt |
||||
import matplotlib.ticker as ticker |
||||
import pandas as pd |
||||
import seaborn as sns |
||||
import pathlib |
||||
|
||||
from pandas.plotting import register_matplotlib_converters |
||||
|
||||
register_matplotlib_converters() |
||||
|
||||
from .logparser import CheckWhen, CheckResult |
||||
|
||||
# set plotting styles |
||||
sns.set_style("darkgrid") |
||||
sns.set_style( |
||||
"ticks", |
||||
{ |
||||
"legend.frameon": True, |
||||
"xtick.direction": "in", |
||||
"ytick.direction": "in", |
||||
"axes.linewidth": 2, |
||||
}, |
||||
) |
||||
sns.set(rc={"figure.figsize": (12, 6)}) |
||||
sns.set_context("paper") |
||||
|
||||
|
||||
def generate_point_plot(df, figure_index, what, y_limit, y_label, colors, hue_order): |
||||
ax = sns.pointplot( |
||||
data=df, |
||||
x="well", |
||||
y=what, |
||||
hue="when", |
||||
hue_order=hue_order, |
||||
style="when", |
||||
markers=list("X."), |
||||
ax=figure_index, |
||||
style_order=hue_order[::-1], |
||||
palette=colors, |
||||
join=False, |
||||
) |
||||
ax.set_ylim(y_limit) |
||||
ax.set_ylabel(y_label) |
||||
return ax |
||||
|
||||
|
||||
def adjust_common_figure_style(ax): |
||||
show_ticks = [] |
||||
selected_tick_labels = [] |
||||
source_columns = set() |
||||
for i, tick_label in enumerate(ax.get_xticklabels()): |
||||
well = tick_label.get_text() |
||||
column = well[0] |
||||
if column not in source_columns: |
||||
show_ticks.append(i) |
||||
selected_tick_labels.append(well) |
||||
source_columns.add(column) |
||||
|
||||
ax.set_xticks(show_ticks) |
||||
ax.set_xticklabels(selected_tick_labels) |
||||
ax.xaxis.grid(True) |
||||
ax.set_xlabel("") |
||||
|
||||
|
||||
def generate_figure(*args): |
||||
ax = generate_point_plot(*args) |
||||
adjust_common_figure_style(ax) |
||||
|
||||
|
||||
def generate_drop_check_chart(df): |
||||
df_sorted = df.sort_values(by="path", ascending=True) |
||||
colors = sns.palettes.color_palette() |
||||
hue_order = [CheckWhen.PRE_RUN.value, CheckWhen.POST_RUN.value] |
||||
palette = {CheckWhen.PRE_RUN.value: colors[1], CheckWhen.POST_RUN.value: colors[0]} |
||||
|
||||
plt.clf() |
||||
# figsize looks strange, but is fittet for pdf report |
||||
fig, axs = plt.subplots(nrows=3, sharex=True, figsize=(8.75, 8.75)) |
||||
|
||||
generate_figure( |
||||
df_sorted, axs[0], "distance", (0, 400), "Distance [pixels]", palette, hue_order |
||||
) |
||||
generate_figure( |
||||
df_sorted, |
||||
axs[1], |
||||
"traverse", |
||||
(-100, 100), |
||||
"Traverse [pixels]", |
||||
palette, |
||||
hue_order, |
||||
) |
||||
generate_figure( |
||||
df_sorted, axs[2], "volume", (0, 600), "Drop Volume [pl]", palette, hue_order |
||||
) |
||||
axs[-1].set_xlabel("Print Solution Well") |
||||
|
||||
|
||||
def generate_environment_graph(df): |
||||
plt.clf() |
||||
fig, axs = plt.subplots(nrows=2, sharex=True, figsize=(8.5, 5.8)) |
||||
|
||||
ax = sns.lineplot(data=df["temperature"], ax=axs[0]) |
||||
ax.set_ylabel("Temperature [°C]") |
||||
ax.set_ylim((10, 40)) |
||||
ax.set_xlabel("") |
||||
|
||||
ax = sns.lineplot(data=df["humidity"], ax=axs[1]) |
||||
ax.set_ylabel("Humidity [%rH]") |
||||
ax.set_ylim((10, 90)) |
||||
ax.set_xlabel("Date and Time") |
||||
|
||||
|
||||
def find_missing_drops(df): |
||||
mask = df["result"] != "ok" |
||||
missing = df.loc[mask].copy() |
||||
pivot = missing.pivot(index="well", columns="when", values="well") |
||||
if "pre run" not in pivot.columns: |
||||
pivot["pre run"] = np.nan |
||||
if "post run" not in pivot.columns: |
||||
pivot["post run"] = np.nan |
||||
pivot = pivot.fillna("") |
||||
# remove labels for post run fails if there are pre run fails |
||||
pivot["nodups"] = pivot["post run"] |
||||
mask = pivot["pre run"] == pivot["post run"] |
||||
pivot["nodups"][mask] = "" |
||||
return pivot.drop(columns=["post run"]).rename(columns={"nodups": "post run"}) |
||||
@ -0,0 +1,153 @@
@@ -0,0 +1,153 @@
|
||||
import matplotlib.pyplot as plt |
||||
import matplotlib.ticker as ticker |
||||
import pandas |
||||
import pathlib |
||||
import seaborn |
||||
|
||||
from pandas.plotting import register_matplotlib_converters |
||||
from collections import namedtuple |
||||
|
||||
register_matplotlib_converters() |
||||
|
||||
# set plotting styles |
||||
seaborn.set_style("darkgrid") |
||||
seaborn.set_style( |
||||
"ticks", |
||||
{ |
||||
"legend.frameon": True, |
||||
"xtick.direction": "in", |
||||
"ytick.direction": "in", |
||||
"axes.linewidth": 2, |
||||
}, |
||||
) |
||||
seaborn.set(rc={"figure.figsize": (12, 6)}) |
||||
seaborn.set_context("paper") |
||||
|
||||
|
||||
GraphPaths = namedtuple("GraphPaths", ["environment", "drops"]) |
||||
|
||||
|
||||
def save_plot(data, label, suffix=".png"): |
||||
if not suffix.startswith("."): |
||||
suffix = f".{suffix}" |
||||
folder = data.files.folder |
||||
path = folder / f"{folder.name}_{label}{suffix}" |
||||
plt.savefig(path) |
||||
return path |
||||
|
||||
|
||||
def generate_environment_graph(data): |
||||
dataframe = data.print.environment |
||||
plt.clf() |
||||
fig, axs = plt.subplots(nrows=2, sharex=True, figsize=(8.5, 5.8)) |
||||
|
||||
ax = seaborn.lineplot(data=dataframe["temperature"], ax=axs[0]) |
||||
ax.set_ylabel("Temperature [°C]") |
||||
ax.set_ylim((10, 40)) |
||||
ax.set_xlabel("") |
||||
|
||||
ax = seaborn.lineplot(data=dataframe["humidity"], ax=axs[1]) |
||||
ax.set_ylabel("Humidity [%rH]") |
||||
ax.set_ylim((10, 90)) |
||||
ax.set_xlabel("Date / Time") |
||||
|
||||
return save_plot(data, "environment") |
||||
|
||||
|
||||
def _drop_point_plot(df, figure_index, what, y_limit, y_label, colors, hue_order): |
||||
ax = seaborn.pointplot( |
||||
data=df, |
||||
x="well", |
||||
y=what, |
||||
hue="measurement", |
||||
hue_order=hue_order, |
||||
style="measurement", |
||||
markers=list("X."), |
||||
ax=figure_index, |
||||
style_order=hue_order[::-1], |
||||
palette=colors, |
||||
join=False, |
||||
) |
||||
ax.set_ylim(y_limit) |
||||
ax.set_ylabel(y_label) |
||||
return ax |
||||
|
||||
|
||||
def _drop_figure_styles(ax): |
||||
show_ticks = [] |
||||
selected_tick_labels = [] |
||||
source_columns = set() |
||||
for i, tick_label in enumerate(ax.get_xticklabels()): |
||||
well = tick_label.get_text() |
||||
column = well[0] |
||||
if column not in source_columns: |
||||
show_ticks.append(i) |
||||
selected_tick_labels.append(well) |
||||
source_columns.add(column) |
||||
|
||||
ax.set_xticks(show_ticks) |
||||
ax.set_xticklabels(selected_tick_labels) |
||||
ax.xaxis.grid(True) |
||||
ax.set_xlabel("") |
||||
|
||||
|
||||
def _make_drop_figure(*args): |
||||
ax = _drop_point_plot(*args) |
||||
_drop_figure_styles(ax) |
||||
|
||||
|
||||
def generate_drop_graph(data, nozzle): |
||||
# select the data of the nozlle |
||||
selection = data.drops["nozzle"] == nozzle |
||||
nozzle_df = data.drops[selection] |
||||
sorted_df = nozzle_df.sort_values(by="when", ascending=True) |
||||
|
||||
# setup some parameters |
||||
colors = seaborn.palettes.color_palette() |
||||
hue_order = ["pre run", "post run"] |
||||
palette = {"pre run": colors[1], "post run": colors[0]} |
||||
settings = data.print.graph_settings |
||||
|
||||
plt.clf() |
||||
# figsize looks strange, but is fittet for pdf report |
||||
fig, axs = plt.subplots(nrows=3, sharex=True, figsize=(8.75, 8.75)) |
||||
|
||||
_make_drop_figure( |
||||
sorted_df, |
||||
axs[0], |
||||
"distance", |
||||
(settings.distance.min, settings.distance.max), |
||||
settings.distance.label, |
||||
palette, |
||||
hue_order, |
||||
) |
||||
_make_drop_figure( |
||||
sorted_df, |
||||
axs[1], |
||||
"offset", |
||||
(settings.offset.min, settings.offset.max), |
||||
settings.offset.label, |
||||
palette, |
||||
hue_order, |
||||
) |
||||
_make_drop_figure( |
||||
sorted_df, |
||||
axs[2], |
||||
"volume", |
||||
(settings.volume.min, settings.volume.max), |
||||
settings.volume.label, |
||||
palette, |
||||
hue_order, |
||||
) |
||||
axs[-1].set_xlabel("Print Solution Well") |
||||
|
||||
return save_plot(data, f"nozzle_{nozzle}") |
||||
|
||||
|
||||
def generate_all_graphs(data): |
||||
env_graph = generate_environment_graph(data) |
||||
|
||||
nozzles = data.drops["nozzle"].unique() |
||||
drop_graphs = {n: generate_drop_graph(data, n) for n in nozzles} |
||||
|
||||
return GraphPaths(env_graph, drop_graphs) |
||||
@ -1,201 +0,0 @@
@@ -1,201 +0,0 @@
|
||||
# basic imports |
||||
import numpy as np |
||||
import matplotlib.pyplot as plt |
||||
import matplotlib.ticker as ticker |
||||
import pandas as pd |
||||
import seaborn as sns |
||||
import pathlib |
||||
|
||||
from collections import namedtuple |
||||
from enum import Enum |
||||
from io import StringIO |
||||
|
||||
|
||||
PrintLogResult = namedtuple("PrintLogResult", ["environment", "info"]) |
||||
|
||||
|
||||
class CheckWhen(Enum): |
||||
PRE_RUN = "pre run" |
||||
POST_RUN = "post run" |
||||
|
||||
|
||||
class CheckResult(Enum): |
||||
OK = "ok" |
||||
FAIL = "fail" |
||||
SKIPPED = "skipped" |
||||
|
||||
|
||||
class DropCheckResult: |
||||
def __init__( |
||||
self, |
||||
path, |
||||
well, |
||||
result, |
||||
distance=np.nan, |
||||
traverse=np.nan, |
||||
volume=np.nan, |
||||
when=None, |
||||
): |
||||
self.well = well |
||||
self.path = path |
||||
self.result = result |
||||
self.distance = distance |
||||
self.traverse = traverse |
||||
self.volume = volume |
||||
self.when = when |
||||
|
||||
def as_dict(self): |
||||
return { |
||||
"well": self.well, |
||||
"path": self.path, |
||||
"result": self.result.value, |
||||
"distance": self.distance, |
||||
"traverse": self.traverse, |
||||
"volume": self.volume, |
||||
"when": self.when.value, |
||||
} |
||||
|
||||
@classmethod |
||||
def from_file(cls, path, encoding="iso-8859-1"): |
||||
with open(path, "r", encoding=encoding) as file_handle: |
||||
lines = file_handle.readlines() |
||||
|
||||
# get x and y values, will be distance and traverse |
||||
xy_line = lines[1] |
||||
x_part, y_part = xy_line.split("\t") |
||||
x = parse_str_value(x_part, float) |
||||
y = parse_str_value(y_part, float) |
||||
|
||||
# get other data values |
||||
for line in lines: |
||||
if line.startswith("Well"): |
||||
well = parse_log_line(line, str) |
||||
if well.startswith("1"): |
||||
# the source plate number is encoded, we remove it, |
||||
# our printers have only one source plate |
||||
well = well[1:] |
||||
elif line.startswith("Drop Volume"): |
||||
volume = parse_log_line(line, float) |
||||
|
||||
# check for status |
||||
if path.stem.lower().endswith("ok"): |
||||
return cls( |
||||
path, well, CheckResult.OK, distance=x, traverse=y, volume=volume |
||||
) |
||||
else: |
||||
return cls(path, well, CheckResult.FAIL) |
||||
|
||||
|
||||
# helper functions |
||||
|
||||
|
||||
def parse_str_value(str_data, cast_to, default_value=np.nan): |
||||
try: |
||||
return cast_to(str_data.strip()) |
||||
except Exception: |
||||
return default_value |
||||
|
||||
|
||||
def parse_log_line(line, cast_to, default_value=np.nan, separator="="): |
||||
_, str_data = line.rsplit(separator, 1) |
||||
return parse_str_value(str_data, cast_to, default_value) |
||||
|
||||
|
||||
def parse_log_files(log_list): |
||||
pre_run = dict() |
||||
post_run = dict() |
||||
well_list = list() |
||||
# use the files sorted by date and time |
||||
for path in sorted(log_list): |
||||
log_result = DropCheckResult.from_file(path) |
||||
if log_result.well not in pre_run: |
||||
log_result.when = CheckWhen.PRE_RUN |
||||
pre_run[log_result.well] = log_result |
||||
# we keep a separate list of wells in the order they appear |
||||
# there might be skipped wells after the pre run check |
||||
well_list.append(log_result.well) |
||||
else: |
||||
log_result.when = CheckWhen.POST_RUN |
||||
post_run[log_result.well] = log_result |
||||
|
||||
skipped_runs = {well for well in pre_run if well not in post_run} |
||||
for well in skipped_runs: |
||||
post_result = DropCheckResult( |
||||
"", well, CheckResult.SKIPPED, when=CheckWhen.POST_RUN |
||||
) |
||||
post_run[well] = post_result |
||||
|
||||
parsed_files = [] |
||||
for well in well_list: |
||||
parsed_files.append(pre_run[well]) |
||||
parsed_files.append(post_run[well]) |
||||
|
||||
return pd.DataFrame([pf.as_dict() for pf in parsed_files]) |
||||
|
||||
|
||||
def split_print_log_line(line): |
||||
_, value = line.split(":", 1) |
||||
return value.strip() |
||||
|
||||
|
||||
def count_solutions(file_handle): |
||||
solutions = set() |
||||
for line in file_handle: |
||||
line = line.strip() |
||||
if not line or line[0] in ("X", "Y", "F", "["): |
||||
# empty line or uninteresting one, pick next one |
||||
continue |
||||
elif line.startswith("Drops/Field"): |
||||
# finished with all field definition, leave loop |
||||
break |
||||
entries = (item.strip() for item in line.split("\t")) |
||||
wells = (well for well in entries if well) |
||||
solutions.update(wells) |
||||
return len(solutions) |
||||
|
||||
|
||||
def parse_print_log(log_files): |
||||
env_lines = [] |
||||
print_info = {} |
||||
with open(log_files, "r", encoding="iso-8859-1") as file_handle: |
||||
for line in file_handle: |
||||
if "\tHumidity=\t" in line: |
||||
env_lines.append(line) |
||||
elif line.startswith("Probe:"): |
||||
print_info["source"] = split_print_log_line(line) |
||||
elif line.startswith("Target:"): |
||||
target_and_fields = split_print_log_line(line) |
||||
target, fields = target_and_fields.rsplit(":", 1) |
||||
print_info["target"] = target.strip() |
||||
print_info["fields"] = len(fields.split(",")) |
||||
elif line.startswith("Humidity:"): |
||||
print_info["humidity"] = split_print_log_line(line) |
||||
elif line.startswith("Run Name:"): |
||||
print_info["run"] = split_print_log_line(line) |
||||
elif line.startswith("Dot Pitch:"): |
||||
# important to pass the filehandle iterator here |
||||
print_info["solutions"] = count_solutions(file_handle) |
||||
|
||||
buff = StringIO("".join(env_lines)) |
||||
columns = ["datetime", "garbage 1", "humidity", "garbage 2", "temperature"] |
||||
tmp_df = pd.read_csv( |
||||
buff, sep="\t", header=None, names=columns, index_col=0, parse_dates=True |
||||
) |
||||
environment_df = tmp_df.drop(columns=["garbage 1", "garbage 2"]) |
||||
return PrintLogResult(environment_df, print_info) |
||||
|
||||
|
||||
def augment_print_info(print_log_result, drop_log_list, encoding="iso-8859-1"): |
||||
""" gets voltage and pulse from a drop log file |
||||
|
||||
Since the voltage and pulse should not change during a print run, |
||||
we add this information to the print log info |
||||
""" |
||||
one_log_file = drop_log_list[0] |
||||
with open(one_log_file, "r", encoding=encoding) as file_handle: |
||||
for line in file_handle: |
||||
if line.startswith("Nozzle Voltage"): |
||||
print_log_result.info["voltage"] = parse_log_line(line, str) |
||||
elif line.startswith("Nozzle Pulse"): |
||||
print_log_result.info["pulse"] = parse_log_line(line, str) |
||||
return print_log_result |
||||
@ -0,0 +1,243 @@
@@ -0,0 +1,243 @@
|
||||
import io |
||||
import numpy |
||||
import pandas |
||||
import datetime |
||||
|
||||
from collections import namedtuple |
||||
|
||||
from . import utils |
||||
|
||||
|
||||
DropStatusInfo = namedtuple("DropStatusInfo", ["when", "status"]) |
||||
GraphProperties = namedtuple("GraphProperties", ["min", "max", "label"]) |
||||
GraphSettings = namedtuple("GraphSettings", ["distance", "offset", "volume"]) |
||||
LogResult = namedtuple("LogResult", ["files", "print", "drops", "statistics"]) |
||||
Nozzle = namedtuple("Nozzle", ["number", "voltage", "pulse", "drops_failed"]) |
||||
SoftwareVersion = namedtuple("Version", ["major", "minor", "patch"]) |
||||
Statistics = namedtuple("Statistics", ["nozzles", "failed_pre_run", "failed_post_run"]) |
||||
|
||||
GRAPH_SETTINGS = { |
||||
3: GraphSettings( |
||||
distance=GraphProperties(min=0, max=400, label="Distance [pixels]"), |
||||
offset=GraphProperties(min=-100, max=100, label="Traverse [pixels]"), |
||||
volume=GraphProperties(min=0, max=600, label="Volume [pl]"), |
||||
), |
||||
10: GraphSettings( |
||||
distance=GraphProperties(min=0, max=3, label="Speed [m/s]"), |
||||
offset=GraphProperties(min=-140, max=140, label="Deviaton [µm]"), |
||||
volume=GraphProperties(min=0, max=600, label="Volume [pl]"), |
||||
), |
||||
} |
||||
|
||||
|
||||
class PrintLog: |
||||
def __init__(self, log_file, printer, version): |
||||
# construction parameters |
||||
self.log_file = log_file |
||||
self.printer = printer |
||||
self.software_version = version |
||||
|
||||
# runid is derived from the filename |
||||
run_id, _ = log_file.stem.rsplit("_", 1) |
||||
self.run_id = run_id |
||||
|
||||
try: |
||||
self.graph_settings = GRAPH_SETTINGS[version.major] |
||||
except KeyError: |
||||
raise ValueError(f"Unknown Scienion Software Version {version.major}") |
||||
|
||||
# common parameters of the print log |
||||
self.humidity_setting = None |
||||
self.pattern_file = None |
||||
self.print_solutions = None |
||||
self.run_method = None |
||||
self.source_plate = None |
||||
self.target_substrate = None |
||||
self.target_count = None |
||||
|
||||
# dataframe for humidity and temperature |
||||
self.environment = None |
||||
|
||||
def parse(self, filehandle): |
||||
self.parse_header(filehandle) |
||||
self.parse_source_wells(filehandle) |
||||
self.parse_environment(filehandle) |
||||
|
||||
def parse_header(self, iterator): |
||||
for line in iterator: |
||||
if line.startswith("Field(s):"): |
||||
break |
||||
|
||||
parts = line.split(":", 1) |
||||
if len(parts) != 2: |
||||
continue |
||||
|
||||
key, value = parts[0].strip(), parts[1].strip() |
||||
if key == "Probe": |
||||
self.source_plate = value |
||||
elif key == "Target": |
||||
substrate, targets_str = value.split(":") |
||||
self.target_substrate = substrate.strip() |
||||
self.target_count = len(targets_str.split(",")) |
||||
elif key.startswith("Pattern File"): |
||||
self.pattern_file = value |
||||
elif key == "Humidity": |
||||
self.humidity_setting = value |
||||
elif key == "Run Name": |
||||
self.run_method = value |
||||
|
||||
def parse_source_wells(self, iterator): |
||||
# first we need to move ahead a little bit |
||||
for line in iterator: |
||||
if line.startswith("Field "): |
||||
break |
||||
raw_wells = [] |
||||
|
||||
for line in iterator: |
||||
if line.startswith("Drops"): |
||||
break |
||||
line = line.strip() |
||||
if line == "" or line[0] in ("F", "["): |
||||
continue |
||||
else: |
||||
raw_wells.extend(line.split("\t")) |
||||
|
||||
stripped = (entry.strip() for entry in raw_wells) |
||||
wells = (entry for entry in stripped if entry) |
||||
self.print_solutions = len(set(wells)) |
||||
|
||||
def parse_environment(self, iterator): |
||||
buff = io.StringIO() |
||||
for line in iterator: |
||||
if "\tHumidity=\t" in line: |
||||
buff.write(line) |
||||
buff.seek(0) |
||||
|
||||
f = lambda s: datetime.datetime.strptime(s, "%d.%m.%y-%H:%M:%S.%f") |
||||
tmp_df = pandas.read_csv( |
||||
buff, sep="\t", header=None, index_col=0, parse_dates=True, date_parser=f |
||||
) |
||||
self.environment = pandas.DataFrame( |
||||
{"humidity": tmp_df.iloc[:, 1], "temperature": tmp_df.iloc[:, 3]} |
||||
) |
||||
|
||||
|
||||
def parse_print_log(log_files): |
||||
with open(log_files.print, "r", encoding="iso-8859-1") as filehandle: |
||||
# parse the printer name |
||||
printer_line = next(filehandle) |
||||
printer = printer_line.split()[0] |
||||
|
||||
# get the software version info |
||||
version_line = next(filehandle) |
||||
_, version_info = version_line.split(":", 1) |
||||
major, minor, patch, _ = version_info.strip().split(".", 3) |
||||
version = SoftwareVersion(int(major), int(minor), int(patch)) |
||||
|
||||
log_parser = PrintLog(log_files.print, printer, version) |
||||
log_parser.parse(filehandle) |
||||
return log_parser |
||||
|
||||
|
||||
def cast(original, to, default=numpy.nan): |
||||
if hasattr(original, "strip"): |
||||
original = original.strip() |
||||
try: |
||||
return to(original) |
||||
except: |
||||
return default |
||||
|
||||
|
||||
def parse_value(log_line, to, default=numpy.nan): |
||||
_, value = log_line.split("=", 1) |
||||
return cast(value, to, default) |
||||
|
||||
|
||||
def parse_file_name(file_path): |
||||
name_parts = [p for p in file_path.stem.split("_") if p] |
||||
*_, date, unknown, autodrop, time, info = name_parts |
||||
when = date + time # parsing datetime is done in the pandas dataframe |
||||
if info.lower().endswith("ok"): |
||||
status = utils.DropState.OK |
||||
else: |
||||
status = utils.DropState.FAULT |
||||
return DropStatusInfo(when, status) |
||||
|
||||
|
||||
def parse_drop_file(file_path): |
||||
status_info = parse_file_name(file_path) |
||||
data = { |
||||
"path": file_path, |
||||
"when": status_info.when, |
||||
"status": status_info.status.value, |
||||
"distance": numpy.nan, # as default value |
||||
"offset": numpy.nan, # as default value |
||||
"volume": numpy.nan, # as default value |
||||
} |
||||
|
||||
with open(file_path, "r", encoding="iso-8859-1") as filehandle: |
||||
if status_info.status == utils.DropState.OK: |
||||
# only parse distance and offset if it is not a failed check |
||||
next(filehandle) # ignore first line |
||||
flight_info = next(filehandle) |
||||
distance, offset = flight_info.split() |
||||
data["distance"] = cast(distance, float) |
||||
data["offset"] = cast(offset, float) |
||||
|
||||
for line in filehandle: |
||||
if line.startswith("Well"): |
||||
well_id = parse_value(line, str) |
||||
data["plate"] = cast(well_id[0], int) |
||||
data["well"] = well_id[1:] |
||||
elif line.startswith("Nozzle No"): |
||||
data["nozzle"] = parse_value(line, int) |
||||
elif line.startswith("Nozzle Voltage"): |
||||
data["voltage"] = parse_value(line, int) |
||||
elif line.startswith("Nozzle Pulse"): |
||||
data["pulse"] = parse_value(line, int) |
||||
elif ( |
||||
line.startswith("Drop Volume") |
||||
and status_info.status == utils.DropState.OK |
||||
): |
||||
data["volume"] = parse_value(line, int) |
||||
|
||||
data["well_id"] = f"{data['nozzle']}.{well_id}" # nozzle is added for a complete id |
||||
return data |
||||
|
||||
|
||||
def parse_drop_logs(log_files): |
||||
collection = (parse_drop_file(f) for f in log_files.drops) |
||||
df = pandas.DataFrame(collection) |
||||
df["when"] = pandas.to_datetime(df["when"], format="%Y%m%d%H%M%S") |
||||
|
||||
# find the pre run values |
||||
grouped = df.groupby("well_id") |
||||
pre_run_df = grouped["when"].min().reset_index() |
||||
pre_run_df["measurement"] = "pre run" |
||||
|
||||
# merge them back into the dataframe |
||||
df = df.merge(pre_run_df, on=["well_id", "when"], how="outer") |
||||
|
||||
# the ones with not set values are post runs |
||||
df = df.fillna({"measurement": "post run"}) |
||||
return df |
||||
|
||||
|
||||
def collect_statistics(drop_log): |
||||
nozzle_df = drop_log.groupby("nozzle").first() |
||||
nozzles = [] |
||||
for nozzle_nr, row in nozzle_df.iterrows(): |
||||
failures = utils.find_failed_drops(drop_log, nozzle_nr) |
||||
nozzles.append(Nozzle(nozzle_nr, row["voltage"], row["pulse"], failures)) |
||||
|
||||
total_failures = utils.find_failed_drops(drop_log, nozzle=None) |
||||
return Statistics( |
||||
nozzles, len(total_failures.pre_run), len(total_failures.post_run) |
||||
) |
||||
|
||||
|
||||
def parse_logs(log_files): |
||||
print_log = parse_print_log(log_files) |
||||
drop_log = parse_drop_logs(log_files) |
||||
stats = collect_statistics(drop_log) |
||||
return LogResult(log_files, print_log, drop_log, stats) |
||||
@ -0,0 +1,72 @@
@@ -0,0 +1,72 @@
|
||||
import enum |
||||
import os |
||||
import pathlib |
||||
import subprocess |
||||
import sys |
||||
|
||||
from collections import namedtuple |
||||
|
||||
FailedWells = namedtuple("FailedWells", ["pre_run", "post_run"]) |
||||
|
||||
|
||||
class DropState(enum.Enum): |
||||
OK = "ok" |
||||
FAULT = "fault" |
||||
|
||||
|
||||
class LogFiles(namedtuple("_LogFiles", ["folder", "print", "drops"])): |
||||
__slots__ = () |
||||
|
||||
def __bool__(self): |
||||
if self.print and self.drops: |
||||
return True |
||||
else: |
||||
return False |
||||
|
||||
|
||||
def _find_files(dir_path, endswith): |
||||
visible = (i for i in dir_path.iterdir() if not i.name.startswith(".")) |
||||
return [item for item in visible if item.name.endswith(endswith)] |
||||
|
||||
|
||||
def find_log_files(folder): |
||||
dir_path = pathlib.Path(folder) |
||||
tmp_print_log = _find_files(dir_path, "_Logfile.log") |
||||
if len(tmp_print_log) == 1: |
||||
print_log = tmp_print_log[0] |
||||
else: |
||||
print_log = None |
||||
drop_logs = _find_files(dir_path, ".cor") |
||||
return LogFiles(dir_path, print_log, drop_logs) |
||||
|
||||
|
||||
def get_failed_drop_checks(dataframe, measurement, nozzle=None): |
||||
if nozzle is not None: |
||||
selection = dataframe["nozzle"] == nozzle |
||||
nozzle_df = dataframe[selection] |
||||
else: |
||||
nozzle_df = dataframe |
||||
# select first only the failed rows |
||||
selection = nozzle_df["status"] == DropState.FAULT.value |
||||
failure_df = nozzle_df[selection] |
||||
# selection based on measurement type |
||||
selection = failure_df["measurement"] == measurement |
||||
return failure_df[selection] |
||||
|
||||
|
||||
def find_failed_drops(dataframe, nozzle=None): |
||||
pre_run_df = get_failed_drop_checks(dataframe, "pre run", nozzle) |
||||
all_post_run_df = get_failed_drop_checks(dataframe, "post run", nozzle) |
||||
# if a check already failed in the pre run, we exclude it from the post run |
||||
selection = all_post_run_df["well_id"].isin(pre_run_df["well_id"]) |
||||
post_run_df = all_post_run_df[~selection] |
||||
return FailedWells(pre_run_df, post_run_df) |
||||
|
||||
|
||||
def open_with_default_app(some_path): |
||||
if sys.platform.startswith("linux"): |
||||
subprocess.call(["xdg-open", some_path]) |
||||
elif sys.platform.startswith("darwin"): |
||||
subprocess.call(["open", some_path]) |
||||
elif sys.platform.startswith("win"): |
||||
os.startfile(some_path) |
||||
Loading…
Reference in new issue