diff --git a/development.ini b/development.ini new file mode 100644 index 0000000..971bcf7 --- /dev/null +++ b/development.ini @@ -0,0 +1,75 @@ +### +# app configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +[app:main] +use = egg:superx_budget + +session.secret = "change me in production" +budgets.dir = %(here)s/test_data + +pyramid.reload_templates = true +pyramid.includes = + pyramid_mailer.debug + +mail.host = "localhost" # SMTP host +mail.port = 2525 # SMTP port +mail.username = "" # SMTP username +mail.password = "" # SMTP password +mail.tls = false # Use TLS +mail.ssl = false # Use SSL +mail.default_sender = "" # Default from address +mail.debug = 0 # SMTP debug level +mail.debug_include_bcc = true # Include Bcc headers when Debugging + +[pshell] +setup = ordr3.pshell.setup + +### +# wsgi server configuration +### + +[server:main] +use = egg:waitress#main +listen = localhost:6543 + +### +# logging configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, ordr3, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_ordr3] +level = DEBUG +handlers = +qualname = ordr3 + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/pyproject.toml b/pyproject.toml index 40b5680..438e29b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,10 @@ classifiers = [ requires = [ "openpyxl >= 3.0.0", + "pyramid >= 1.10", + "pyramid_jinja2 >= 2.7", + "pyramid_mailer >= 0.15.1", + "waitress >= 1.4.3", ] requires-python = ">=3.7" @@ -44,6 +48,9 @@ dev = [ "pre-commit", ] +[tool.flit.entrypoints."paste.app_factory"] +main = "superx_budget:main" + [tool.black] line-length = 79 py37 = true diff --git a/superx_budget/__init__.py b/superx_budget/__init__.py index e695864..dcf75e4 100644 --- a/superx_budget/__init__.py +++ b/superx_budget/__init__.py @@ -11,6 +11,8 @@ from .helpers import ( # noqa: F401 find_recipients, find_budget_file, get_sheet_of_file, + list_budget_files, + is_budget_file_name, ) from .pyramid import main # noqa: F401 from .overview import create_overview # noqa: F401 diff --git a/superx_budget/budget.py b/superx_budget/budget.py index 79442a1..5d1f6e2 100644 --- a/superx_budget/budget.py +++ b/superx_budget/budget.py @@ -34,7 +34,9 @@ BudgetData = namedtuple( def _check_table_header(xl_row): - fields = [c.strip() for c in xl_row.data[:7]] + fields_ignore_none = (("" if c is None else c) for c in xl_row.data[:7]) + fields_str = (str(c) for c in fields_ignore_none) + fields = [c.strip() for c in fields_str] if fields != EXPECTED_TABLE_HEADERS: raise BudgetParserError(f"unexpected headers: '{xl_row.data}'") diff --git a/superx_budget/helpers.py b/superx_budget/helpers.py index 184d952..34c167e 100644 --- a/superx_budget/helpers.py +++ b/superx_budget/helpers.py @@ -48,13 +48,30 @@ def strip_excel_value(value): return value -def find_budget_file(folder, year): - """ searches for a file with the name "budget[...]-.xlsx """ +def is_budget_file_name(path_or_name): + """ checks if a filename has the format "budget[...]-.xlsx """ + path = Path(path_or_name) + if not path.suffix.lower() == ".xlsx": + return False + if not path.name.lower().startswith("budget"): + return False + try: + return int(path.stem[-5:]) <= -2019 + except ValueError: + pass + return False + + +def list_budget_files(folder): + """ lists all files with the name "budget[...]-.xlsx """ files = (i for i in Path(folder).iterdir() if i.is_file()) visible = (i for i in files if not i.name.startswith(".")) - workbooks = (i for i in visible if i.suffix.lower() == ".xlsx") - budgets = (i for i in workbooks if i.name.lower().startswith("budget")) - for path in budgets: + return [i for i in visible if is_budget_file_name(i)] + + +def find_budget_file(folder, year): + """ searches for a file with the name "budget[...]-.xlsx """ + for path in list_budget_files(folder): if path.stem.endswith(f"-{year}"): return path diff --git a/superx_budget/pyramid/__init__.py b/superx_budget/pyramid/__init__.py new file mode 100644 index 0000000..6d398d9 --- /dev/null +++ b/superx_budget/pyramid/__init__.py @@ -0,0 +1,47 @@ +""" Superx Budget GUI """ + +from pathlib import Path + +from pyramid.view import notfound_view_config +from pyramid.config import Configurator +from pyramid.session import JSONSerializer, SignedCookieSessionFactory +from pyramid.httpexceptions import HTTPFound + +from ..overview import create_overview # noqa: F401 +from ..exceptions import BudgetParserError, SuperXParserError # noqa: F401 + +XLSX_CONTENT_TYPE = "application/vnd.ms-excel" + + +def root_factory(request): + return {} + + +def main(global_config, **settings): + """ This function returns a Pyramid WSGI application. """ + with Configurator(settings=settings) as config: + + config.include("pyramid_jinja2") + + session_factory = SignedCookieSessionFactory( + settings["session.secret"], serializer=JSONSerializer() + ) + config.set_session_factory(session_factory) + + config.set_root_factory(root_factory) + + config.add_request_method( + lambda r: Path(settings["budgets.dir"]), "budgets_dir", reify=True, + ) + + age = int(settings.get("static_views.cache_max_age", 0)) + config.add_static_view("static", "static", cache_max_age=age) + + config.scan() + + return config.make_wsgi_app() + + +@notfound_view_config() +def not_found(context, request): + return HTTPFound("/") diff --git a/superx_budget/pyramid/overview.py b/superx_budget/pyramid/overview.py new file mode 100644 index 0000000..d8550e3 --- /dev/null +++ b/superx_budget/pyramid/overview.py @@ -0,0 +1,145 @@ +""" Views for the create overview part """ + +from tempfile import NamedTemporaryFile + +from pyramid.view import view_config +from pyramid.httpexceptions import HTTPFound +from pyramid_mailer.message import Message, Attachment + +from . import XLSX_CONTENT_TYPE +from ..budget import parse_budget_file +from ..superx import parse_exported_file +from ..helpers import find_recipients, find_budget_file, get_sheet_of_file +from ..overview import create_overview # noqa: F401 +from ..exceptions import BudgetParserError, SuperXParserError # noqa: F401 + + +@view_config( + context=dict, + request_method="GET", + renderer="superx_budget:pyramid/templates/start.jinja2", +) +def index(context, request): + return {} + + +@view_config( + context=dict, + request_method="POST", + renderer="superx_budget:pyramid/templates/overview.jinja2", +) +def superx_upload(context, request): + upload = request.POST.get("superx") + + if upload == b"" or not upload.filename.endswith(".xlsx"): + request.session.flash("No Excel file uploaded.", "error") + return HTTPFound("/") + + try: + superx_export = parse_exported_file(upload.file) + except SuperXParserError: + request.session.flash( + "File does not appear to be the required SuperX export.", "error" + ) + return HTTPFound("/") + + budget_file = find_budget_file( + request.budgets_dir, superx_export.account_year + ) + if budget_file is None: + request.session.flash( + f"No budget file for year {superx_export.account_year} found.", + "error", + ) + return HTTPFound("/") + + try: + budget_data = parse_budget_file(budget_file) + except BudgetParserError: + request.session.flash( + "Budget File does not appear to be in the required format.", + "error", + ) + return HTTPFound("/") + + overview_map = create_overview(budget_data, superx_export) + overview = sorted(overview_map.values(), key=lambda i: i.row) + + if any(not (item.found) for item in overview): + request.session.flash( + ( + "Some projects in the budget template were not in the SuperX " + "export. Please adjust their expenses manually." + ), + "info", + ) + + recipients = find_recipients(request.budgets_dir) + + return { + "account_year": superx_export.account_year, + "export_date": superx_export.export_date.strftime("%Y-%m-%d"), + "overview": overview, + "template": budget_file.name, + "recipients": recipients, + } + + +@view_config( + context=dict, + name="send", + request_method="POST", + renderer="superx_budget:pyramid/templates/sent.jinja2", +) +def send_overview(context, request): + export_date = request.POST.get("export_date").strip() + tmp_recipients = request.POST.get("recipients").strip() + recipients = tmp_recipients.splitlines() + budget_template = request.POST.get("template") + budget_file = request.budgets_dir / budget_template + expenses = {} + for key, value in request.POST.items(): + if key.startswith("expense-"): + row_str = key.split("-")[-1] + row = int(row_str) + try: + value = float(value) + except ValueError: + value = 0 + expenses[row] = value + + # sanity check + if ( + not export_date + or not recipients + or not expenses + or not budget_file.is_file() + ): + request.session.flash( + f"There was an error with your submisssion, please try again.", + "error", + ) + return HTTPFound("/") + + sheet = get_sheet_of_file(budget_file) + for row, value in expenses.items(): + cell = f"F{row}" + sheet[cell] = value + + message = Message( + subject=f"Budget Übersicht, Stand {export_date}", + sender="cpiserver@imtek.uni-freiburg.de", + recipients=recipients, + body="hello from ford", + ) + + budget_year = budget_file.stem.split("-")[-1] + xls_name = f"{export_date}-Budget-Overview-{budget_year}.xlsx" + with NamedTemporaryFile() as tmp: + sheet._parent.save(tmp.name) + tmp.seek(0) + attachment = Attachment(xls_name, XLSX_CONTENT_TYPE, tmp) + message.attach(attachment) + request.mailer.send(message) + + return {"recipients": recipients, "xls_name": xls_name} diff --git a/superx_budget/pyramid/static/pyramid-16x16.png b/superx_budget/pyramid/static/pyramid-16x16.png new file mode 100644 index 0000000..9792031 Binary files /dev/null and b/superx_budget/pyramid/static/pyramid-16x16.png differ diff --git a/superx_budget/pyramid/static/pyramid.png b/superx_budget/pyramid/static/pyramid.png new file mode 100644 index 0000000..4ab837b Binary files /dev/null and b/superx_budget/pyramid/static/pyramid.png differ diff --git a/superx_budget/pyramid/static/style.css b/superx_budget/pyramid/static/style.css new file mode 100644 index 0000000..4de011e --- /dev/null +++ b/superx_budget/pyramid/static/style.css @@ -0,0 +1,20 @@ +.form input.form-control.currency { + display: inline-block; + vertical-align:middle; + width: 7em; + border-color:#007bff!important; + } + + +/* Remove Spinner from number input fields */ +/* For Firefox */ +input[type='number'] { + -moz-appearance:textfield; + } + +/* Webkit browsers like Safari and Chrome */ +input[type=number]::-webkit-inner-spin-button, +input[type=number]::-webkit-outer-spin-button { + -webkit-appearance: none; + margin: 0; + } diff --git a/superx_budget/pyramid/templates.py b/superx_budget/pyramid/templates.py new file mode 100644 index 0000000..cb1ca40 --- /dev/null +++ b/superx_budget/pyramid/templates.py @@ -0,0 +1,82 @@ +""" Views for the templates part """ + +from pathlib import Path + +from pyramid.view import view_config +from pyramid.response import FileResponse +from pyramid.httpexceptions import HTTPFound + +from . import XLSX_CONTENT_TYPE +from ..budget import parse_budget_file +from ..helpers import list_budget_files, is_budget_file_name +from ..overview import create_overview # noqa: F401 +from ..exceptions import BudgetParserError, SuperXParserError # noqa: F401 + + +@view_config( + context=dict, + name="templates", + request_method="GET", + renderer="superx_budget:pyramid/templates/templates.jinja2", +) +def templates(context, request): + if "f" in request.GET: + file_name = request.GET["f"] + file_path = request.budgets_dir / file_name + if not file_path.is_file(): + return HTTPFound("/templates") + response = FileResponse( + file_path, + request=request, + cache_max_age=0, + content_type=XLSX_CONTENT_TYPE, + ) + response.headers[ + "Content-Disposition" + ] = f"attachment;filename={file_name}" + return response + + tmp = list_budget_files(request.budgets_dir) + tmp = sorted(tmp, key=lambda p: p.stem[-4:]) + budget_files = [path.name for path in tmp] + return {"budget_files": budget_files} + + +@view_config( + context=dict, + name="templates", + request_method="POST", + renderer="superx_budget:pyramid/templates/templates.jinja2", +) +def templates_update(context, request): + upload = request.POST.get("budget") + + if upload == b"" or not is_budget_file_name(upload.filename): + request.session.flash("No Excel file uploaded.", "error") + return HTTPFound("/templates") + + try: + parse_budget_file(upload.file) + except BudgetParserError: + request.session.flash( + "File does not appear to be in the required format.", "error" + ) + return HTTPFound("/templates") + + # sanitizing upload filename: + file_name = Path(upload.filename).name + + tmp = list_budget_files(request.budgets_dir) + available_files = {p.name.lower(): p for p in tmp} + old_path = available_files.get(file_name.lower()) + if old_path: + old_path.unlink() + + new_path = request.budgets_dir / file_name + with new_path.open("wb") as fh: + upload.file.seek(0) + fh.write(upload.file.read()) + + request.session.flash("File upload successful.", "info") + + return HTTPFound("/templates") diff --git a/superx_budget/pyramid/templates/layout.jinja2 b/superx_budget/pyramid/templates/layout.jinja2 new file mode 100644 index 0000000..db9b1ac --- /dev/null +++ b/superx_budget/pyramid/templates/layout.jinja2 @@ -0,0 +1,72 @@ + + + + + + SuperX -> Budget Overview + + + + + + + +
+ +
+
+
+ +
+
+
+ + {% if request.session.peek_flash('error') %} +
+
+ +
+
+ {%endif%} + {% if request.session.peek_flash('info') %} +
+
+ +
+
+ {%endif%} + + +{% block content %} +

No content

+{% endblock content %} + +
+
+
+

Any problems or questions? Please contact Holgi.

+
+
+
+ +
+ + diff --git a/superx_budget/pyramid/templates/overview.jinja2 b/superx_budget/pyramid/templates/overview.jinja2 new file mode 100644 index 0000000..3032fbc --- /dev/null +++ b/superx_budget/pyramid/templates/overview.jinja2 @@ -0,0 +1,63 @@ +{% extends "superx_budget:pyramid/templates/layout.jinja2" %} + +{% block content %} + +
+
+

+ Budget Overview {{ account_year }} + as of {{ export_date }} +

+ +
+ + + + + + + + + + + + + + + {% for budget in overview %} + + + + + + + + + + {% endfor %} + + +
ProjectPSPBudgetExpensesRest
+ {{budget.budget_data.project_name}} + {{budget.budget_data.project}}{{ "{:,.2f}".format(budget.budget_data.budget) }} € + {% if budget.found %} + + {{ "{:,.2f}".format(budget.expenses) }} € + + {% else %} + € + {% endif %} + {{ "{:,.2f}".format(budget.available) }} €
+ +

Who should receive the list

+ + +

+ +

+ +
+ +
+
+{% endblock content %} diff --git a/superx_budget/pyramid/templates/sent.jinja2 b/superx_budget/pyramid/templates/sent.jinja2 new file mode 100644 index 0000000..9c096d5 --- /dev/null +++ b/superx_budget/pyramid/templates/sent.jinja2 @@ -0,0 +1,16 @@ +{% extends "superx_budget:pyramid/templates/layout.jinja2" %} + +{% block content %} +
+
+

Email sent!

+

An email with the attatched file "{{ xls_name }}" was sent to the following recipients: +

    + {% for r in recipients: %} +
  • {{ r }}
    • + {% endfor %} +
+

You can now close this browser window.

+
+
+{% endblock content %} diff --git a/superx_budget/pyramid/templates/start.jinja2 b/superx_budget/pyramid/templates/start.jinja2 new file mode 100644 index 0000000..88a0484 --- /dev/null +++ b/superx_budget/pyramid/templates/start.jinja2 @@ -0,0 +1,26 @@ +{% extends "superx_budget:pyramid/templates/layout.jinja2" %} + +{% block content %} + +
+
+

Procedure to get the right SuperX export

+
    +
  1. Log in to SuperX
    2. +
  2. select …
    3. +
  3. select …
    4. +
  4. select …
    5. +
  5. select …
    6. +
  6. select …
    7. +
+

Done?

+
+
+ + +
+ +
+
+
+{% endblock content %} diff --git a/superx_budget/pyramid/templates/templates.jinja2 b/superx_budget/pyramid/templates/templates.jinja2 new file mode 100644 index 0000000..deab1be --- /dev/null +++ b/superx_budget/pyramid/templates/templates.jinja2 @@ -0,0 +1,25 @@ +{% extends "superx_budget:pyramid/templates/layout.jinja2" %} + +{% block content %} + +
+
+

Budget Template Files

+
    + {% for name in budget_files %} +
  • {{ name }}
    • + {% endfor %} +
+

Add a New Template

+

If you upload a file with an existing name, the currently stored file gets replaced – that's how you do updates.

+

The filename must be in the format "budget[...]-[year].xlsx"; character capitalization does not matter.

+
+
+ + +
+ +
+
+
+{% endblock content %} diff --git a/test data/Verbrauchsmittel-Toto-2020.xlsx b/test data/Verbrauchsmittel-Toto-2020.xlsx deleted file mode 100755 index a23bb10..0000000 Binary files a/test data/Verbrauchsmittel-Toto-2020.xlsx and /dev/null differ diff --git a/test data/Verwendungsnachweis_und_Kassenstand_SAP.xlsx b/test data/Verwendungsnachweis_und_Kassenstand_SAP.xlsx deleted file mode 100644 index 3f712d6..0000000 Binary files a/test data/Verwendungsnachweis_und_Kassenstand_SAP.xlsx and /dev/null differ diff --git a/test data/Verwendungsnachweis_und_Kassenstand_SAP_Zahlen.xlsx b/test data/Verwendungsnachweis_und_Kassenstand_SAP_Zahlen.xlsx deleted file mode 100644 index 52a877f..0000000 Binary files a/test data/Verwendungsnachweis_und_Kassenstand_SAP_Zahlen.xlsx and /dev/null differ diff --git a/test data/test export data.numbers b/test data/test export data.numbers deleted file mode 100755 index 74e7891..0000000 Binary files a/test data/test export data.numbers and /dev/null differ diff --git a/test_data/Budget-Vorlage-2019.xlsx b/test_data/Budget-Vorlage-2019.xlsx new file mode 100644 index 0000000..adf517a Binary files /dev/null and b/test_data/Budget-Vorlage-2019.xlsx differ diff --git a/test_data/Budget-Vorlage-2020.xlsx b/test_data/Budget-Vorlage-2020.xlsx old mode 100755 new mode 100644 index a23bb10..14d7e8e Binary files a/test_data/Budget-Vorlage-2020.xlsx and b/test_data/Budget-Vorlage-2020.xlsx differ diff --git a/tests/test_budget_parser.py b/tests/test_budget_parser.py index bd80183..892e07d 100644 --- a/tests/test_budget_parser.py +++ b/tests/test_budget_parser.py @@ -5,7 +5,7 @@ def test_check_table_header_raises_error(): from superx_budget.budget import _check_table_header, ExcelRow from superx_budget.exceptions import BudgetParserError - row = ExcelRow(None, ["not", "the", "expected", "row"]) + row = ExcelRow(None, ["not", "the", "expected", "row", None, 0]) with pytest.raises(BudgetParserError): _check_table_header(row) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 8aee264..91c78ed 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -54,11 +54,11 @@ def test_get_sheet_of_file_first(budget_example_file): def test_get_sheet_of_file_named(budget_example_file): from superx_budget.helpers import get_sheet_of_file - sheet = get_sheet_of_file(budget_example_file, sheet="Safeguard I") + sheet = get_sheet_of_file(budget_example_file, sheet="Übersicht") first_row = next(sheet.values) first_cell = first_row[0] - assert first_cell == 1 + assert first_cell.strip() == "Nr." @pytest.mark.parametrize( @@ -81,6 +81,37 @@ def test_excel_value_as_number(input, expected): assert result == expected +@pytest.mark.parametrize( + "name,expected", + [ + ("budget-2019.xlsx", True), + ("budget-template-2020.xlsx", True), + ("budget-2018.xlsx", False), + ("xxxxxx-2019.xlsx", False), + ("budget-2019.xxxx", False), + ("budget-2x19.xlsx", False), + ("budget_2019.xlsx", False), + ], +) +def test_is_budget_file_name(name, expected): + from superx_budget.helpers import is_budget_file_name + + result = is_budget_file_name(name) + + assert result == expected + + +def test_list_budget_files(example_root): + from superx_budget.helpers import list_budget_files + + result = list_budget_files(example_root) + + assert sorted(r.name for r in result) == [ + "Budget-Vorlage-2019.xlsx", + "Budget-Vorlage-2020.xlsx", + ] + + def test_find_budget_file_found(example_root): from superx_budget.helpers import find_budget_file @@ -92,7 +123,7 @@ def test_find_budget_file_found(example_root): def test_find_budget_file_not_found(example_root): from superx_budget.helpers import find_budget_file - result = find_budget_file(example_root, 2019) + result = find_budget_file(example_root, 1999) assert result is None