added web interface with pyramid

development.ini

@@ -0,0 +1,75 @@
+###
+# app configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
+###
+
+[app:main]
+use = egg:superx_budget
+
+session.secret = "change me in production"
+budgets.dir = %(here)s/test_data
+
+pyramid.reload_templates = true
+pyramid.includes =
+    pyramid_mailer.debug
+
+mail.host = "localhost"        # SMTP host
+mail.port = 2525               # SMTP port
+mail.username = ""             # SMTP username
+mail.password = ""             # SMTP password
+mail.tls = false               # Use TLS
+mail.ssl = false               # Use SSL
+mail.default_sender = ""       # Default from address
+mail.debug = 0                 # SMTP debug level
+mail.debug_include_bcc = true  # Include Bcc headers when Debugging
+
+[pshell]
+setup = ordr3.pshell.setup
+
+###
+# wsgi server configuration
+###
+
+[server:main]
+use = egg:waitress#main
+listen = localhost:6543
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, ordr3, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_ordr3]
+level = DEBUG
+handlers =
+qualname = ordr3
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARN" logs neither.  (Recommended for production systems.)
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s

pyproject.toml

@@ -25,6 +25,10 @@ classifiers = [
 
 requires = [
     "openpyxl >= 3.0.0",
+    "pyramid >= 1.10",
+    "pyramid_jinja2 >= 2.7",
+    "pyramid_mailer >= 0.15.1",
+    "waitress >= 1.4.3",
 ]
 requires-python = ">=3.7"
 
@@ -44,6 +48,9 @@ dev = [
     "pre-commit",
 ]
 
+[tool.flit.entrypoints."paste.app_factory"]
+main = "superx_budget:main"
+
 [tool.black]
 line-length = 79
 py37 = true

superx_budget/__init__.py

@@ -11,6 +11,8 @@ from .helpers import (  # noqa: F401
     find_recipients,
     find_budget_file,
     get_sheet_of_file,
+    list_budget_files,
+    is_budget_file_name,
 )
 from .pyramid import main  # noqa: F401
 from .overview import create_overview  # noqa: F401

superx_budget/budget.py

@@ -34,7 +34,9 @@ BudgetData = namedtuple(
 
 
 def _check_table_header(xl_row):
-    fields = [c.strip() for c in xl_row.data[:7]]
+    fields_ignore_none = (("" if c is None else c) for c in xl_row.data[:7])
+    fields_str = (str(c) for c in fields_ignore_none)
+    fields = [c.strip() for c in fields_str]
     if fields != EXPECTED_TABLE_HEADERS:
         raise BudgetParserError(f"unexpected headers: '{xl_row.data}'")
 

superx_budget/helpers.py

@@ -48,13 +48,30 @@ def strip_excel_value(value):
     return value
 
 
-def find_budget_file(folder, year):
-    """ searches for a file with the name "budget[...]-<year>.xlsx """
+def is_budget_file_name(path_or_name):
+    """ checks if a filename has the format "budget[...]-<year>.xlsx """
+    path = Path(path_or_name)
+    if not path.suffix.lower() == ".xlsx":
+        return False
+    if not path.name.lower().startswith("budget"):
+        return False
+    try:
+        return int(path.stem[-5:]) <= -2019
+    except ValueError:
+        pass
+    return False
+
+
+def list_budget_files(folder):
+    """ lists all files with the name "budget[...]-<year>.xlsx """
     files = (i for i in Path(folder).iterdir() if i.is_file())
     visible = (i for i in files if not i.name.startswith("."))
-    workbooks = (i for i in visible if i.suffix.lower() == ".xlsx")
-    budgets = (i for i in workbooks if i.name.lower().startswith("budget"))
-    for path in budgets:
+    return [i for i in visible if is_budget_file_name(i)]
+
+
+def find_budget_file(folder, year):
+    """ searches for a file with the name "budget[...]-<year>.xlsx """
+    for path in list_budget_files(folder):
         if path.stem.endswith(f"-{year}"):
             return path
 

superx_budget/pyramid/__init__.py

@@ -0,0 +1,47 @@
+""" Superx Budget GUI """
+
+from pathlib import Path
+
+from pyramid.view import notfound_view_config
+from pyramid.config import Configurator
+from pyramid.session import JSONSerializer, SignedCookieSessionFactory
+from pyramid.httpexceptions import HTTPFound
+
+from ..overview import create_overview  # noqa: F401
+from ..exceptions import BudgetParserError, SuperXParserError  # noqa: F401
+
+XLSX_CONTENT_TYPE = "application/vnd.ms-excel"
+
+
+def root_factory(request):
+    return {}
+
+
+def main(global_config, **settings):
+    """ This function returns a Pyramid WSGI application. """
+    with Configurator(settings=settings) as config:
+
+        config.include("pyramid_jinja2")
+
+        session_factory = SignedCookieSessionFactory(
+            settings["session.secret"], serializer=JSONSerializer()
+        )
+        config.set_session_factory(session_factory)
+
+        config.set_root_factory(root_factory)
+
+        config.add_request_method(
+            lambda r: Path(settings["budgets.dir"]), "budgets_dir", reify=True,
+        )
+
+        age = int(settings.get("static_views.cache_max_age", 0))
+        config.add_static_view("static", "static", cache_max_age=age)
+
+        config.scan()
+
+        return config.make_wsgi_app()
+
+
+@notfound_view_config()
+def not_found(context, request):
+    return HTTPFound("/")

superx_budget/pyramid/overview.py

@@ -0,0 +1,145 @@
+""" Views for the create overview part """
+
+from tempfile import NamedTemporaryFile
+
+from pyramid.view import view_config
+from pyramid.httpexceptions import HTTPFound
+from pyramid_mailer.message import Message, Attachment
+
+from . import XLSX_CONTENT_TYPE
+from ..budget import parse_budget_file
+from ..superx import parse_exported_file
+from ..helpers import find_recipients, find_budget_file, get_sheet_of_file
+from ..overview import create_overview  # noqa: F401
+from ..exceptions import BudgetParserError, SuperXParserError  # noqa: F401
+
+
+@view_config(
+    context=dict,
+    request_method="GET",
+    renderer="superx_budget:pyramid/templates/start.jinja2",
+)
+def index(context, request):
+    return {}
+
+
+@view_config(
+    context=dict,
+    request_method="POST",
+    renderer="superx_budget:pyramid/templates/overview.jinja2",
+)
+def superx_upload(context, request):
+    upload = request.POST.get("superx")
+
+    if upload == b"" or not upload.filename.endswith(".xlsx"):
+        request.session.flash("No Excel file uploaded.", "error")
+        return HTTPFound("/")
+
+    try:
+        superx_export = parse_exported_file(upload.file)
+    except SuperXParserError:
+        request.session.flash(
+            "File does not appear to be the required SuperX export.", "error"
+        )
+        return HTTPFound("/")
+
+    budget_file = find_budget_file(
+        request.budgets_dir, superx_export.account_year
+    )
+    if budget_file is None:
+        request.session.flash(
+            f"No budget file for year {superx_export.account_year} found.",
+            "error",
+        )
+        return HTTPFound("/")
+
+    try:
+        budget_data = parse_budget_file(budget_file)
+    except BudgetParserError:
+        request.session.flash(
+            "Budget File does not appear to be in the required format.",
+            "error",
+        )
+        return HTTPFound("/")
+
+    overview_map = create_overview(budget_data, superx_export)
+    overview = sorted(overview_map.values(), key=lambda i: i.row)
+
+    if any(not (item.found) for item in overview):
+        request.session.flash(
+            (
+                "Some projects in the budget template were not in the SuperX "
+                "export. Please adjust their expenses manually."
+            ),
+            "info",
+        )
+
+    recipients = find_recipients(request.budgets_dir)
+
+    return {
+        "account_year": superx_export.account_year,
+        "export_date": superx_export.export_date.strftime("%Y-%m-%d"),
+        "overview": overview,
+        "template": budget_file.name,
+        "recipients": recipients,
+    }
+
+
+@view_config(
+    context=dict,
+    name="send",
+    request_method="POST",
+    renderer="superx_budget:pyramid/templates/sent.jinja2",
+)
+def send_overview(context, request):
+    export_date = request.POST.get("export_date").strip()
+    tmp_recipients = request.POST.get("recipients").strip()
+    recipients = tmp_recipients.splitlines()
+    budget_template = request.POST.get("template")
+    budget_file = request.budgets_dir / budget_template
+    expenses = {}
+    for key, value in request.POST.items():
+        if key.startswith("expense-"):
+            row_str = key.split("-")[-1]
+            row = int(row_str)
+            try:
+                value = float(value)
+            except ValueError:
+                value = 0
+            expenses[row] = value
+
+    # sanity check
+    if (
+        not export_date
+        or not recipients
+        or not expenses
+        or not budget_file.is_file()
+    ):
+        request.session.flash(
+            f"There was an error with your submisssion, please try again.",
+            "error",
+        )
+        return HTTPFound("/")
+
+    sheet = get_sheet_of_file(budget_file)
+    for row, value in expenses.items():
+        cell = f"F{row}"
+        sheet[cell] = value
+
+    message = Message(
+        subject=f"Budget Übersicht, Stand {export_date}",
+        sender="cpiserver@imtek.uni-freiburg.de",
+        recipients=recipients,
+        body="hello from ford",
+    )
+
+    budget_year = budget_file.stem.split("-")[-1]
+    xls_name = f"{export_date}-Budget-Overview-{budget_year}.xlsx"
+    with NamedTemporaryFile() as tmp:
+        sheet._parent.save(tmp.name)
+        tmp.seek(0)
+        attachment = Attachment(xls_name, XLSX_CONTENT_TYPE, tmp)
+        message.attach(attachment)
+        request.mailer.send(message)
+
+    return {"recipients": recipients, "xls_name": xls_name}

superx_budget/pyramid/static/style.css

@@ -0,0 +1,20 @@
+.form input.form-control.currency {
+    display: inline-block;
+    vertical-align:middle;
+    width: 7em;
+    border-color:#007bff!important;
+    }
+
+
+/* Remove Spinner from number input fields */
+/* For Firefox */
+input[type='number'] {
+    -moz-appearance:textfield;
+    }
+
+/* Webkit browsers like Safari and Chrome */
+input[type=number]::-webkit-inner-spin-button,
+input[type=number]::-webkit-outer-spin-button {
+    -webkit-appearance: none;
+    margin: 0;
+    }

superx_budget/pyramid/templates.py

@@ -0,0 +1,82 @@
+""" Views for the templates part """
+
+from pathlib import Path
+
+from pyramid.view import view_config
+from pyramid.response import FileResponse
+from pyramid.httpexceptions import HTTPFound
+
+from . import XLSX_CONTENT_TYPE
+from ..budget import parse_budget_file
+from ..helpers import list_budget_files, is_budget_file_name
+from ..overview import create_overview  # noqa: F401
+from ..exceptions import BudgetParserError, SuperXParserError  # noqa: F401
+
+
+@view_config(
+    context=dict,
+    name="templates",
+    request_method="GET",
+    renderer="superx_budget:pyramid/templates/templates.jinja2",
+)
+def templates(context, request):
+    if "f" in request.GET:
+        file_name = request.GET["f"]
+        file_path = request.budgets_dir / file_name
+        if not file_path.is_file():
+            return HTTPFound("/templates")
+        response = FileResponse(
+            file_path,
+            request=request,
+            cache_max_age=0,
+            content_type=XLSX_CONTENT_TYPE,
+        )
+        response.headers[
+            "Content-Disposition"
+        ] = f"attachment;filename={file_name}"
+        return response
+
+    tmp = list_budget_files(request.budgets_dir)
+    tmp = sorted(tmp, key=lambda p: p.stem[-4:])
+    budget_files = [path.name for path in tmp]
+    return {"budget_files": budget_files}
+
+
+@view_config(
+    context=dict,
+    name="templates",
+    request_method="POST",
+    renderer="superx_budget:pyramid/templates/templates.jinja2",
+)
+def templates_update(context, request):
+    upload = request.POST.get("budget")
+
+    if upload == b"" or not is_budget_file_name(upload.filename):
+        request.session.flash("No Excel file uploaded.", "error")
+        return HTTPFound("/templates")
+
+    try:
+        parse_budget_file(upload.file)
+    except BudgetParserError:
+        request.session.flash(
+            "File does not appear to be in the required format.", "error"
+        )
+        return HTTPFound("/templates")
+
+    # sanitizing upload filename:
+    file_name = Path(upload.filename).name
+
+    tmp = list_budget_files(request.budgets_dir)
+    available_files = {p.name.lower(): p for p in tmp}
+    old_path = available_files.get(file_name.lower())
+    if old_path:
+        old_path.unlink()
+
+    new_path = request.budgets_dir / file_name
+    with new_path.open("wb") as fh:
+        upload.file.seek(0)
+        fh.write(upload.file.read())
+
+    request.session.flash("File upload successful.", "info")
+
+    return HTTPFound("/templates")

superx_budget/pyramid/templates/layout.jinja2

@@ -0,0 +1,72 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+
+  	<title>SuperX -> Budget Overview</title>
+
+	<link href="{{request.static_url('superx_budget.pyramid:static/img/favicon.ico')}}" type="image/x-icon" rel="shortcut icon">
+
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">
+    <link rel="stylesheet" href="{{request.static_url('superx_budget.pyramid:static/style.css')}}" type="text/css" media="screen" />
+</head>
+<body>
+<div class="container">
+
+    <div class="row mb-4">
+        <div class="col">
+            <header>
+                <nav class="navbar navbar-expand navbar-light bg-light">
+                    <span class="navbar-brand bg-info p-3 rounded">Budget Overview From SuperX</span>
+
+                    <div class="collapse navbar-collapse" id="navbarNav">
+                        <ul class="navbar-nav na">
+                            <li class="nav-item {% if request.view_name !='templates' %}active{% endif %}">
+                                <a class="nav-link" href="/">Create Budget Overview <span class="sr-only">(current)</span></a>
+                            </li>
+
+                            <li class="nav-item {% if request.view_name =='templates' %}active{% endif %}">
+                                <a class="nav-link" href="/templates">Budget Templates</a>
+                            </li>
+                        </ul>
+                    </div>
+                </nav>
+            </header>
+        </div>
+    </div>
+
+    {% if request.session.peek_flash('error') %}
+    <div class="row">
+        <div class="col">
+            <div class="alert alert-danger" role="alert">
+                {{ request.session.pop_flash('error')[0] }}
+            </div>
+        </div>
+    </div>
+    {%endif%}
+    {% if request.session.peek_flash('info') %}
+    <div class="row">
+        <div class="col">
+            <div class="alert alert-primary" role="alert">
+                {{ request.session.pop_flash('info')[0] }}
+            </div>
+        </div>
+    </div>
+    {%endif%}
+
+
+{% block content %}
+    <p>No content</p>
+{% endblock content %}
+
+    <div class="row mt-3">
+        <div class="col">
+            <footer>
+                <p class="bg-light p-3">Any problems or questions? Please contact <a href="https://wiki.cpi.imtek.uni-freiburg.de/HolgerFrey">Holgi</a>.</p>
+            </footer>
+        </div>
+    </div>
+
+</div></div></div>
+</body>
+</html>

superx_budget/pyramid/templates/overview.jinja2

@@ -0,0 +1,63 @@
+{% extends "superx_budget:pyramid/templates/layout.jinja2" %}
+
+{% block content %}
+
+    <div class="row">
+        <div class="col">
+            <h2>
+                Budget Overview {{ account_year }}
+                <small class="text-muted">as of {{ export_date }}</small>
+            </h2>
+
+            <form class="form" action="/send" method="post">
+                <input type="hidden" name="export_date" value="{{ export_date }}" >
+                <input type="hidden" name="template" value="{{ template }}" >
+                <table class="table table-striped">
+                    <thead>
+                        <tr>
+                            <th scope="col">Project</th>
+                            <th scope="col">PSP</th>
+                            <th scope="col" class="text-right">Budget</th>
+                            <th scope="col" class="text-right">Expenses</th>
+                            <th scope="col" class="text-right">Rest</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+
+                    {% for budget in overview %}
+
+                        <tr scope="row" class="{% if budget.available<0 %}table-danger{% endif %}">
+                            <td>
+                                {{budget.budget_data.project_name}}
+                            </td>
+                            <td>{{budget.budget_data.project}}</td>
+                            <td class="text-right">{{ "{:,.2f}".format(budget.budget_data.budget) }} €</td>
+                            <td class="text-right">
+                                {% if budget.found %}
+                                    <input type="hidden" name="expense-{{budget.row}}" value="{{budget.expenses}}">
+                                    {{ "{:,.2f}".format(budget.expenses) }} €
+
+                                {% else %}
+                                    <input type="number" name="expense-{{budget.row}}" value="0" min="0" required="required" class="currency text-right form-control"> €
+                                {% endif %}
+                            </td>
+                            <td class="text-right">{{ "{:,.2f}".format(budget.available) }} €</td>
+                        </tr>
+
+                    {% endfor %}
+
+                    </tbody>
+                </table>
+
+                <h4 class="mt-5">Who should receive the list </h4>
+                <label for="recipients">Please separate the recipients by new lines:</label>
+                <textarea class="form-control" name="recipients" id="recipients" rows="{{ recipients|count + 3 }}">{{ recipients|join("\n") }}{{ "\n" }}</textarea>
+                <p class="mt-5">
+                    <button type="submit" name="submit" value="submitted" class="btn btn-primary">Send List</button>
+                </p>
+
+            </form>
+
+        </div>
+    </div>
+{% endblock content %}

superx_budget/pyramid/templates/sent.jinja2

@@ -0,0 +1,16 @@
+{% extends "superx_budget:pyramid/templates/layout.jinja2" %}
+
+{% block content %}
+    <div class="row">
+        <div class="col">
+            <h2>Email sent!</h2>
+            <p>An email with the attatched file "<span class="text-info">{{ xls_name }}</span>" was sent to the following recipients:
+            <ul>
+                {% for r in recipients: %}
+                    <li>{{ r }}</li>
+                {% endfor %}
+            </ul>
+            <h4>You can now close this browser window.</h4>
+        </div>
+    </div>
+{% endblock content %}

superx_budget/pyramid/templates/start.jinja2

@@ -0,0 +1,26 @@
+{% extends "superx_budget:pyramid/templates/layout.jinja2" %}
+
+{% block content %}
+
+    <div class="row">
+        <div class="col">
+            <h2>Procedure to get the right SuperX export</h2>
+            <ol>
+                <li>Log in to SuperX</li>
+                <li>select …</li>
+                <li>select …</li>
+                <li>select …</li>
+                <li>select …</li>
+                <li>select …</li>
+            </ol>
+            <h4>Done?</h4>
+            <form enctype="multipart/form-data" method="post" action="/">
+                <div class="form-group">
+                    <label for="fileupload">Then please upload the just exported Excel file:</label>
+                    <input id="fileupload" name="superx" type="file" class="form-control-file" />
+                </div>
+                <button type="submit" name="submit" value="submitted" class="btn btn-primary">Submit</button>
+            </form>
+        </div>
+    </div>
+{% endblock content %}

superx_budget/pyramid/templates/templates.jinja2

@@ -0,0 +1,25 @@
+{% extends "superx_budget:pyramid/templates/layout.jinja2" %}
+
+{% block content %}
+
+    <div class="row">
+        <div class="col">
+            <h2 class="mb-4">Budget Template Files</h2>
+            <ul>
+                {% for name in budget_files %}
+                    <li><a href="/templates?f={{ name }}">{{ name }}</a></li>
+                {% endfor %}
+            </ul>
+            <h4 class="mt-4">Add a New Template</h4>
+            <p> If you upload a file with an existing name, the currently stored file gets replaced – that's how you do updates.</p>
+            <p> The filename must be in the format "<span class="text-info">budget[...]-[year].xlsx</span>"; character capitalization does not matter.</p>
+            <form enctype="multipart/form-data" method="post" action="/templates">
+                <div class="form-group">
+                    <label for="fileupload">Upload a new file:</label>
+                    <input id="fileupload" name="budget" type="file" class="form-control-file" />
+                </div>
+                <button type="submit" name="submit" value="submitted" class="btn btn-primary">Upload</button>
+            </form>
+        </div>
+    </div>
+{% endblock content %}

tests/test_budget_parser.py

@@ -5,7 +5,7 @@ def test_check_table_header_raises_error():
     from superx_budget.budget import _check_table_header, ExcelRow
     from superx_budget.exceptions import BudgetParserError
 
-    row = ExcelRow(None, ["not", "the", "expected", "row"])
+    row = ExcelRow(None, ["not", "the", "expected", "row", None, 0])
 
     with pytest.raises(BudgetParserError):
         _check_table_header(row)

tests/test_helpers.py

@@ -54,11 +54,11 @@ def test_get_sheet_of_file_first(budget_example_file):
 def test_get_sheet_of_file_named(budget_example_file):
     from superx_budget.helpers import get_sheet_of_file
 
-    sheet = get_sheet_of_file(budget_example_file, sheet="Safeguard I")
+    sheet = get_sheet_of_file(budget_example_file, sheet="Übersicht")
     first_row = next(sheet.values)
     first_cell = first_row[0]
 
-    assert first_cell == 1
+    assert first_cell.strip() == "Nr."
 
 
 @pytest.mark.parametrize(
@@ -81,6 +81,37 @@ def test_excel_value_as_number(input, expected):
     assert result == expected
 
 
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        ("budget-2019.xlsx", True),
+        ("budget-template-2020.xlsx", True),
+        ("budget-2018.xlsx", False),
+        ("xxxxxx-2019.xlsx", False),
+        ("budget-2019.xxxx", False),
+        ("budget-2x19.xlsx", False),
+        ("budget_2019.xlsx", False),
+    ],
+)
+def test_is_budget_file_name(name, expected):
+    from superx_budget.helpers import is_budget_file_name
+
+    result = is_budget_file_name(name)
+
+    assert result == expected
+
+
+def test_list_budget_files(example_root):
+    from superx_budget.helpers import list_budget_files
+
+    result = list_budget_files(example_root)
+
+    assert sorted(r.name for r in result) == [
+        "Budget-Vorlage-2019.xlsx",
+        "Budget-Vorlage-2020.xlsx",
+    ]
+
+
 def test_find_budget_file_found(example_root):
     from superx_budget.helpers import find_budget_file
 
@@ -92,7 +123,7 @@ def test_find_budget_file_found(example_root):
 def test_find_budget_file_not_found(example_root):
     from superx_budget.helpers import find_budget_file
 
-    result = find_budget_file(example_root, 2019)
+    result = find_budget_file(example_root, 1999)
 
     assert result is None