From a887e4c9ad6aad1b14be52cce6c54f14c39224d7 Mon Sep 17 00:00:00 2001 From: Holger Frey Date: Mon, 29 Apr 2019 13:21:02 +0200 Subject: [PATCH] added report generation --- mtor/dataproc.py | 125 ++++++++++++++++++++++++++++++++++++++++++++++ mtor/report.py | 0 mtor/workflows.py | 16 +++--- test_mtor.py | 4 +- 4 files changed, 137 insertions(+), 8 deletions(-) delete mode 100644 mtor/report.py diff --git a/mtor/dataproc.py b/mtor/dataproc.py index a80f67d..aad23cf 100644 --- a/mtor/dataproc.py +++ b/mtor/dataproc.py @@ -6,8 +6,20 @@ import pickle import seaborn import matplotlib.pyplot as pyplot +from reportlab.platypus import ( + SimpleDocTemplate, + Paragraph, + PageBreak, + Spacer, + KeepTogether, + Image, +) +from reportlab.lib.pagesizes import A4 +from reportlab.lib.styles import getSampleStyleSheet +from reportlab.lib.units import mm from scipy.signal import savgol_filter + from .commons import ROI_STATISTIC_FUNCTIONS @@ -394,6 +406,119 @@ def save_data(data_frame, selected_df, extremas_df, parameters): writer.save() +def create_report(data_frame, selected_df, extremas_df, parameters): + styles = getSampleStyleSheet() + style_headline = styles["Heading1"] + style_section = styles["Heading2"] + style_text = styles["Normal"] + + data_dir = parameters.data_dir + path = data_dir / "report.pdf" + doc = SimpleDocTemplate(str(path), pagesize=A4) + + img_width = doc.width * 0.9 + img_height = (900 * img_width / 1200) * 0.9 + + num_images = len(data_frame) + num_selected = len(selected_df) + num_discarded = num_images - num_selected + + def text_and_graph(text, name): + flowable = KeepTogether( + [ + Paragraph(text, style_text), + Image( + str(data_dir / name), width=img_width, height=img_height + ), + Spacer(1, 7 * mm), + ] + ) + return flowable + + story = [ + Paragraph(f"Analysis of {num_images} Tif Images", style_headline), + Spacer(1, 10 * mm), + Paragraph("Estimating Guard Threshold", style_section), + text_and_graph( + ( + "In a first step, the histogram of the combined left and " + "right guard values is calculated." + ), + "1-histogram-of-guard-avarages-not-filtered.png", + ), + text_and_graph( + ( + "A Savitzky-Golay filter is applied to the histogram to " + "smooth the curve." + ), + "2-histogram-of-guard-avarages-filtered.png", + ), + text_and_graph( + ( + "The first minima after the first peak is used as the guard " + f"threshold value: {int(parameters.guard_max_value)} au" + ), + "3-histogram-of-guard-avarages-filtered-with-first-minima.png", + ), + text_and_graph( + ( + "The images with one of the guard values above the threshold " + "are discarded." + ), + "4-image-selection-based-on-guard-values.png", + ), + Image( + str(data_dir / "5-selected-values-based-on-guard-values.png"), + width=img_width, + height=img_height, + ), + PageBreak(), + Paragraph("Removing Outliers", style_section), + text_and_graph( + "From the remaining values, outliers are removed.", + "6-boxplot-of-guarded-values.png", + ), + text_and_graph( + ( + f"From {num_images} images {num_discarded} images were " + f"discarded, leaving {num_selected} selected. The finally " + "selected values are listed in the excel sheet 'selection' " + "in the data file." + ), + "7-selected-images-outliers-removed.png", + ), + PageBreak(), + Paragraph( + "Experimental: Applying a rolling min calculation", style_section + ), + text_and_graph( + ( + "Due to the nature of the experiment, unusable images tend " + "to have a higher value as the desiered ones. Therfore a " + "rolling min filter is applied" + ), + "8-selected-images-outliers-removed-rolling-min-applied.png", + ), + Paragraph("Experimental: Finding Maxima and Minima", style_section), + text_and_graph( + "To smooth the resulting curve, a Savitzky-Golay filter is used.", + ( + "9-selected-images-outliers-" + "removed-rolling-min-savgol-filtered.png" + ), + ), + text_and_graph( + ( + "The most interesting data points should be the maxima and " + "minima of this curve. These are listed in the sheet " + "'extremas' in the data file" + ), + "11-finding-minimas.png", + ), + ] + doc.build(story) + + def save_temp(data_frame, parameters): csv_path = parameters.tif_dir / "_data.csv" data_frame.to_csv(csv_path, sep="\t") diff --git a/mtor/report.py b/mtor/report.py deleted file mode 100644 index e69de29..0000000 diff --git a/mtor/workflows.py b/mtor/workflows.py index 6da53a9..fedd86d 100644 --- a/mtor/workflows.py +++ b/mtor/workflows.py @@ -18,6 +18,7 @@ from .dataproc import ( smooth_savgol_filter, find_extremas, save_data, + create_report, ) from .postproc import ( stem_file_list, @@ -28,7 +29,7 @@ from .postproc import ( def prescan_workflow(folder, top, right, bottom, left, **kargs): - print("1/4: scanning tifs for common autocontrast values") + print("1/5: scanning tifs for common autocontrast values") parameters = Parameters(folder, top, right, bottom, left) parameters = scan_tifs(parameters) @@ -43,7 +44,7 @@ def prescan_workflow(folder, top, right, bottom, left, **kargs): def image_workflow(parameters): - print("2/4: Image analysis and conversion") + print("2/5: Image analysis and conversion") func = functools.partial(process_one_tif, parameters=parameters) tif_list = parameters.tif_list num_items = len(tif_list) @@ -56,7 +57,7 @@ def image_workflow(parameters): def data_workflow(stats_results, parameters): from .dataproc import save_temp - print("3/4: Data analysis") + print("3/5: Data analysis") set_plotting_styles() data_frame = construct_data_frame(stats_results, parameters) save_temp(data_frame, parameters) @@ -72,22 +73,24 @@ def data_workflow(stats_results, parameters): extremas_df = find_extremas(selected_df, parameters) save_data(data_frame, selected_df, extremas_df, parameters) + create_report(data_frame, selected_df, extremas_df, parameters) return WorkflowResult(list(selected_df["file"]), parameters) def postprocessing_workflow(selected_files, parameters): - print("4/4: Post processing") + print("4/5: Post processing, sorting cut images") file_stems = stem_file_list(selected_files) - annotate_color_coded_images(file_stems, parameters) sort_cut_images(file_stems, parameters) remove_cuts_dir(parameters) + print("5/5: Post processing, annotating color coded images") + annotate_color_coded_images(file_stems, parameters) def cached_data_workflow(folder): from .dataproc import load_temp - print("3/4: Data analysis") + print("3/5: Data analysis") set_plotting_styles() data_frame, parameters = load_temp(folder) find_guard_threshold(data_frame, parameters) @@ -101,5 +104,6 @@ def cached_data_workflow(folder): extremas_df = find_extremas(selected_df, parameters) save_data(data_frame, selected_df, extremas_df, parameters) + create_report(data_frame, selected_df, extremas_df, parameters) return WorkflowResult(list(selected_df["file"]), parameters) diff --git a/test_mtor.py b/test_mtor.py index 459d471..cef63aa 100644 --- a/test_mtor.py +++ b/test_mtor.py @@ -5,5 +5,5 @@ tif_dir = "original_tifs" #mtor.process_tifs(tif_dir, 50, 910, 300, 660, boost=5) -mtor.run() -#mtor.run_data() +#mtor.run() +mtor.run_data()