diff --git a/ordr3.sqlite b/ordr3.sqlite new file mode 100644 index 0000000..8ac8999 Binary files /dev/null and b/ordr3.sqlite differ diff --git a/ordr3/adapters.py b/ordr3/adapters.py index 9bec765..05c9366 100644 --- a/ordr3/adapters.py +++ b/ordr3/adapters.py @@ -60,9 +60,14 @@ log_table = Table( Column("status", Enum(models.OrderStatus), primary_key=True), Column("date", DateTime, primary_key=True, default=datetime.utcnow), Column("by", Text, nullable=False), - Column("user_id", Integer, nullable=False, index=True), ) +vendor_table = Table( + "vendors", + metadata, + Column("term", Text, primary_key=True), + Column("name", Text, index=True), +) user_table = Table( "users", @@ -89,6 +94,7 @@ def start_mappers(): }, ) mapper(models.LogEntry, log_table) + mapper(models.Vendor, vendor_table) mapper(models.User, user_table) diff --git a/ordr3/models.py b/ordr3/models.py index 709b1c4..21b33d3 100644 --- a/ordr3/models.py +++ b/ordr3/models.py @@ -121,16 +121,20 @@ class LogEntry(Model): order_id = None status = None by = None - user_id = None date = None - def __init__(self, order_id, status, by, user_id, date=None): + def __init__(self, order_id, status, by, date=None): self.order_id = order_id self.status = status self.by = by - self.user_id = user_id self.date = date or datetime.utcnow() + def __repr__(self): + return ( + f"" + ) + class ProposedConsumable: """ counting orders to find out if they are consumables """ @@ -140,6 +144,17 @@ class ProposedConsumable: self.times = 0 +class Vendor(Model): + """ a model for finding vendor names and their search terms """ + + term = None + name = None + + def __init__(self, term, name): + self.term = term + self.name = name + + class User(Model): id = None diff --git a/ordr3/repo.py b/ordr3/repo.py index 022d4fe..c8148e9 100644 --- a/ordr3/repo.py +++ b/ordr3/repo.py @@ -2,6 +2,8 @@ import abc +from sqlalchemy import func + from . import models @@ -42,6 +44,10 @@ class AbstractOrderRepository(abc.ABC): def list_users(self): """ list users orderd by username """ + @abc.abstractmethod + def search_vendor(self, reference): + """ search for a vendor by its canonical name """ + class SqlAlchemyRepository(AbstractOrderRepository): """ Repository implementation for SQLAlchemy """ @@ -85,7 +91,11 @@ class SqlAlchemyRepository(AbstractOrderRepository): def get_user_by_email(self, reference): """ get a user from the database by email """ - return self.session.query(models.User).filter_by(email=reference).one() + return ( + self.session.query(models.User) + .filter(func.lower(models.User.email) == func.lower(reference)) + .one() + ) def list_users(self): """ list users orderd by username """ @@ -94,3 +104,14 @@ class SqlAlchemyRepository(AbstractOrderRepository): .order_by(models.User.username) .all() ) + + def search_vendor(self, reference): + """ search for a vendor by its canonical name """ + vendor = ( + self.session.query(models.Vendor) + .filter_by(term=reference) + .one_or_none() + ) + if vendor is None: + return None + return vendor.name diff --git a/ordr3/scripts/__init__.py b/ordr3/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ordr3/scripts/migrate_db.py b/ordr3/scripts/migrate_db.py new file mode 100644 index 0000000..a6c0467 --- /dev/null +++ b/ordr3/scripts/migrate_db.py @@ -0,0 +1,344 @@ +import sys +import sqlite3 +import argparse +from pathlib import Path +from datetime import datetime +from urllib.parse import urlparse + +from sqlalchemy import func +from pyramid.paster import bootstrap, setup_logging, get_appsettings +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm.exc import NoResultFound + +from .. import models, adapters, services + +vendor_map = { + "Accurion GmbH": ["accurion"], + "Acros Organics": ["acros"], + "Agilent Technologies": ["agilent"], + "Alfa Aesar": [ + "alefa aesar", + "alfa", + "aesar", + "alfa aesar - vwr", + "alfa-aesar", + "alfaaesar", + ], + "Amazon": [], + "analytics-shop.com": ["analytics-shop"], + "architekturbedarf.de": ["architekturbedarf"], + "Avanti Polar Lipids": ["avanti"], + "Biesterfeld Spezialchemie GmbH": ["biesterfeld", "biesterfeld gmbh"], + "Bio-Rad": ["bio rad"], + "Biocat": [], + "Bioleague GmbH & co.kg": ["bioleague"], + "Biomol": [], + "Bürkert Indelfingen": ["bürkert"], + "Carl Roth GmbH": [ + "carl roth", + "carl-roth", + "carlroth", + "cr", + "karl roth", + "roth", + ], + "Cellsystems Biotech": ["cellsystems"], + "Chemicell GmbH": ["chemicell"], + "Clickbox e.k.": ["clickbox"], + "Cole Parmer": ["coleparmer"], + "Conrad Electronics": [ + "conrad", + "conrad electronic", + "conrad, fax 09604408937, k-nr 6043142", + ], + "Cytoskeleton inc": ["cytoskeleton"], + "Daniel Maas Dichtstoffhandel & Co.": ["daniel maas"], + "Delta Mask b.v.": ["delta mask", "deltamask"], + "der-rollende-shop.de": ["der-rollende-shop"], + "Dianova": [], + "Distrelec": ["distrilec"], + "Edmund Optics GmbH": ["edmund optics", "edmund"], + "Faust Lab Science GmbH": ["faust", "faust lab"], + "Fisher Scientific": [ + "fischer scientific", + "fisher scientific/acros organic", + ], + "form.in Lasercenter": ["form.in"], + "Uwe Markus (Glasbläser)": [ + "glasbläser markus", + "markus", + "markus, glasbläser", + "uwe markus", + "glasbläser", + ], + "Goodfellow": ["good fellow"], + "Greiner Bio-One": ["greiner bio one"], + "Hach": [], + "Hellma GmbH & Co KG": [ + "hellma analytics", + "hellma optics", + "hellma optik jena", + ], + "Hiss dx": ["hiss"], + "HJ-Bioanalytik GmbH": ["hj bioanalytik gmbh", "hj bioanalytik"], + "laborhandel24.de": ["laborhandel24"], + "locheisen.com": ["locheisen"], + "praezisionsmesstechnik.de": ["praezisionsmesstechnik"], + "siebgewebeshop.de": ["siebgewebeshop"], + "Häberle Labortechnik": ["häberle"], + "High-Tech-Flon": ["hightechflon"], + "hygie.de": ["hygie"], + "Iolitec": [ + "ionic liquids technologie", + "ionic liquids technologies", + "iolitec ionic liquids technologies", + ], + "Iris Biotech GmbH": ["iris biotech"], + "Ismatec": ["ismatec/idex"], + "Jackson Immunoresearch": ["jacksonimmuno", "jackson"], + "Kinesis Abimed": ["kinesis", "kinesisgmbh"], + "Kisker Biotech GmbH & co. kg": ["kisker"], + "Knick Elektronische Messgeräte GmbH": ["knick"], + "Kummer Laborbedarf": ["kummer"], + "Laborhandel Krumpholz": ["krumpholz"], + "leuchtmittelmarkt.com": ["leuchtmittelmarkt"], + "Life Sciences Advanced Technologies inc": [ + "life sciences advanced technologies", + ], + "Life Technologies": ["lifetechnologies", "ife technologies"], + "Macherey Nagel": ["m-n", "mn", "macherey-nagel"], + "magnets4you GmbH": ["magnets4you"], + "magnet-shop.com": ["magnet-shop"], + "Mercateo ag": ["mercateo"], + "Merck": ["merck über vwr"], + "merck berufsbekleidung": [ + "merck berufskleidung", + "merck-berufsbekleidung", + ], + "Merck millipore inc.": ["merck millipore", "merckmillipore"], + "Mettler-Toledo GmbH": ["mettler toledo", "mettler-toledo", "mettler"], + "Micro Particles GmbH": ["micro particles"], + "Microdyn-Nadir GmbH": ["microdyn-nadir"], + "Millipore GmbH": ["millipore"], + "Molecular Devices / Genetix": ["molecular devices", "genetix"], + "Nanoandmore GmbH": ["nano-and-more", "nanoandmore"], + "Nanocyl s.a.": ["nanocyl"], + "Neo Lab": ["neolab"], + "Newport Spectra-Physics": ["newport"], + "OCO Ortenauer Gase GmbH": ["oco ortenauer gase", "oco"], + "Pall Corporation": ["pall"], + "Plano GmbH": ["plano", "plano-eu", "plano-em"], + "Polyan GmbH": ["polyan"], + "Polyscience Europa GmbH": [ + "polysciences", + "polysciences europe gmbh", + "polysciences, inc.", + ], + "ProLiquid GmbH": ["pro-liquid"], + "Qiagen": ["quiagen"], + "R&D Systems": ["r&d system"], + "Reichelt Elektronik": ["reichelt"], + "RS Components": ["rs"], + "S-Polytec": ["s-polytech"], + "Sarstedt AG & Co. KG": ["sarstedt"], + "Science Services GmbH": ["science services"], + "Scienion AG": ["scienion"], + "Sigma Aldrich": [ + "aldrich", + "adrich", + "aldich", + "aldritch", + "aldrích", + "fluka", + "sa", + "sigma - aldrich", + "sigma adrich", + "sigma aldich", + "sigma aldritch", + "sigma aldrích", + "sigma- aldrich", + "sigma-adrich", + "sigma-aldrich", + "sigma-aldrich (fluka)", + "sigmaaldrich", + "roche, sigma-aldrich", + "sigmar", + "sigma", + ], + "Sterlitech": [], + "Supermagnete": ["supermagnet"], + "taq-dna.com": ["taq-dna"], + "TCI Deutschland GmbH": [ + "tci", + "tci chemical", + "tci chemicals", + "tci deutschland", + "tci europe", + "tci europe n.v.", + ], + "Thermo Fisher Scientific": [ + "themofisher", + "thermofischer", + "thermofisher", + "thermo fisher", + "thermofisher scientific", + "thermo scientific", + "thermo scientific - www.perbio.com", + "perbio", + "thermo scientific / pierce", + "thermo ", + "pierce", + ], + "Tib MolBiol": ["tibmolbiol", "tibmolbio"], + "Tse Systems GmbH": ["tse systems"], + "Vilber Lourmat GmbH": ["vilber"], + "VWR International GmbH": [ + "vwr", + "vwr (lenz laborglas gmbh & co.kg)", + "vwr chemicals", + "vwr collection", + "vwt", + ], + "Weigert": ["drweigert"], + "Xantec Bioanalytics": ["xantec", "xantec bioanalyticss"], + "Zitt-Thoma": ["zitt thoma"], +} + + +def _query_table(cursor, table): + cursor.execute(f"SELECT * FROM {table}") + columns = [d[0] for d in cursor.description] + return (dict(zip(columns, values)) for values in cursor) + + +def _case_insensitive_query_user_by_username(repo, reference): + try: + user = ( + repo.session.query(models.User) + .filter(func.lower(models.User.username) == func.lower(reference)) + .one() + ) + return user.username + except NoResultFound: + return reference + + +def migrate(old_db, repo): + inpsql3 = sqlite3.connect(old_db) + cursor = inpsql3.cursor() + migrate_users(cursor, repo) + migrate_vendors(cursor, repo) + migrate_orders(cursor, repo) + + +def migrate_users(cursor, repo): + for old_user in _query_table(cursor, "users"): + user = models.User( + old_user["id"], + old_user["user_name"], + old_user["first_name"], + old_user["last_name"], + old_user["email"], + old_user["password_hash"], + models.UserRole[old_user["role"]], + ) + repo.add_user(user) + + +def migrate_vendors(cursor, repo): + for name, replacements in vendor_map.items(): + vendor = models.Vendor(name.lower(), name) + repo._add_item_to_db(vendor) + for replacement in replacements: + vendor = models.Vendor(replacement.lower(), name) + repo._add_item_to_db(vendor) + + +def migrate_orders(cursor, repo): + status_fields = [ + ("created", models.OrderStatus.OPEN), + ("approval", models.OrderStatus.APPROVAL), + ("ordered", models.OrderStatus.ORDERED), + ("completed", models.OrderStatus.COMPLETED), + ] + + for old_order in _query_table(cursor, "orders"): + + result = services.check_vendor_name(repo, old_order["vendor"]) + if result.found: + vendor = result.name + else: + vendor = old_order["vendor"] + repo._add_item_to_db(models.Vendor(result.name.lower(), vendor)) + + order = models.OrderItem( + old_order["id"], + old_order["cas_description"], + old_order["catalog_nr"], + vendor, + models.OrderCategory[old_order["category"]], + old_order["package_size"], + old_order["unit_price"], + old_order["amount"], + old_order["currency"], + old_order["account"], + old_order["comment"], + ) + repo.add_order(order) + + for old_field, status in status_fields: + field_by = old_order[f"{old_field}_by"] + + if field_by: + field_date = datetime.fromisoformat( + old_order[f"{old_field}_date"] + ) + user = _case_insensitive_query_user_by_username(repo, field_by) + log_entry = models.LogEntry( + order.id, status, user, field_date, + ) + order.add_to_log(log_entry) + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + "config_uri", help="Configuration file, e.g., development.ini", + ) + parser.add_argument( + "old_db", help="ordr2 sqlite database file", + ) + return parser.parse_args(argv[1:]) + + +def main(argv=sys.argv): + args = parse_args(argv) + setup_logging(args.config_uri) + env = bootstrap(args.config_uri) + settings = get_appsettings(args.config_uri) + + # remove an existing sqlite database to issue a restart + database_url = urlparse(settings["sqlalchemy.url"]) + dabase_path = Path(database_url.path) + if database_url.scheme == "sqlite" and dabase_path.is_file(): + dabase_path.unlink() + + try: + with env["request"].tm: + repo = env["request"].repo + adapters.metadata.create_all(repo.session.get_bind()) + migrate(args.old_db, repo) + except OperationalError: + print( + """ +Pyramid is having a problem using your SQL database. The problem +might be caused by one of the following things: + +1. You may need to initialize your database tables with `alembic`. + Check your README.txt for description and try to run it. + +2. Your database server may not be running. Check that the + database server referred to by the "sqlalchemy.url" setting in + your "development.ini" file is running. + """ + ) diff --git a/ordr3/security.py b/ordr3/security.py index 3540cd8..b9f8f7c 100644 --- a/ordr3/security.py +++ b/ordr3/security.py @@ -1,6 +1,6 @@ """ User Authentication and Authorization """ -from passlib import CryptContext +from passlib.context import CryptContext from pyramid.security import Everyone, Authenticated from sqlalchemy.orm.exc import NoResultFound from pyramid.authorization import ACLAuthorizationPolicy diff --git a/ordr3/services.py b/ordr3/services.py index 2942e50..9703f80 100644 --- a/ordr3/services.py +++ b/ordr3/services.py @@ -1,4 +1,6 @@ from datetime import datetime +from collections import namedtuple +from urllib.parse import urlparse from sqlalchemy.orm.exc import NoResultFound @@ -34,7 +36,7 @@ def _find_consumables(repo, repeat=3, days=365 * 2): def create_log_entry(order, status, user): - log_entry = models.LogEntry(order.id, status, user.username, user.id) + log_entry = models.LogEntry(order.id, status, user.username) order.add_to_log(log_entry) @@ -52,3 +54,36 @@ def verify_credentials(repo, pass_ctx, username, password): # we need to update the password hash to a algorithm user.password = new_hash return user + + +def _vendor_from_url(vendor): + parsed = urlparse(vendor) + if parsed.netloc != "": + return parsed.netloc + else: + return vendor + + +def _vendor_with_common_domain(vendor): + for tld in (".eu", ".com", ".de"): + if vendor.endswith(tld): + parts = vendor.split(".") + return parts[-2] + return vendor + + +CheckVendorResult = namedtuple("CheckVendorResult", ["name", "found"]) + + +def check_vendor_name(repo, to_check): + # remove unused whitespace + cleaned = " ".join(to_check.strip().split()) + tmp = _vendor_from_url(cleaned) + canonical_name = _vendor_with_common_domain(tmp) + + vendor = repo.search_vendor(canonical_name.lower()) + + if vendor is None: + return CheckVendorResult(canonical_name, False) + else: + return CheckVendorResult(vendor, True) diff --git a/pyproject.toml b/pyproject.toml index 082bc7b..b9ce827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ dev = [ "pre-commit", ] +[tool.flit.scripts] +migrate = "ordr3.scripts.migrate_db:main" + [tool.flit.entrypoints."paste.app_factory"] main = "ordr3:main" diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 53279d4..3c23c8b 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -71,26 +71,20 @@ def test_log_mapper(session, order_sql): session.execute( """ INSERT INTO logs - (order_id, status, by, user_id, date) + (order_id, status, by, date) VALUES - (1, "OPEN", "ME", 1, "2020-02-05 15:14:13.000000"), - (1, "COMPLETED", "YOU", 2, "2020-02-07 15:14:13.000000"), - (1, "ORDERED", "ME", 1, "2020-02-06 15:14:13.000000") + (1, "OPEN", "ME", "2020-02-05 15:14:13.000000"), + (1, "COMPLETED", "YOU", "2020-02-07 15:14:13.000000"), + (1, "ORDERED", "ME", "2020-02-06 15:14:13.000000") """ ) expected = [ + LogEntry(1, OrderStatus.OPEN, "ME", datetime(2020, 2, 5, 15, 14, 13),), LogEntry( - 1, OrderStatus.OPEN, "ME", 1, datetime(2020, 2, 5, 15, 14, 13), + 1, OrderStatus.COMPLETED, "YOU", datetime(2020, 2, 7, 15, 14, 13), ), LogEntry( - 1, - OrderStatus.COMPLETED, - "YOU", - 2, - datetime(2020, 2, 7, 15, 14, 13), - ), - LogEntry( - 1, OrderStatus.ORDERED, "ME", 1, datetime(2020, 2, 6, 15, 14, 13) + 1, OrderStatus.ORDERED, "ME", datetime(2020, 2, 6, 15, 14, 13) ), ] @@ -140,7 +134,7 @@ def test_adapter_behaviour(session): ) session.add(order) - log_entry = LogEntry(order.id, OrderStatus.OPEN, "ME", 123) + log_entry = LogEntry(order.id, OrderStatus.OPEN, "ME") order.add_to_log(log_entry) from_db = session.query(OrderItem).first() diff --git a/tests/test_models.py b/tests/test_models.py index 67bed0e..c4d7d8a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -42,7 +42,7 @@ def test_orderitem_add_to_log_empty_log(): from ordr3.models import OrderItem, LogEntry order = OrderItem(*list("ABCDEFGHIJK")) - log_entry = LogEntry(order, "critical", "me", "myid", "yesterday") + log_entry = LogEntry(order, "critical", "me", "yesterday") order.add_to_log(log_entry) assert order.created_on == log_entry.date @@ -54,8 +54,8 @@ def test_orderitem_add_to_log_non_empty_log(): from ordr3.models import OrderItem, LogEntry order = OrderItem(*list("ABCDEFGHIJK")) - log_entry_1 = LogEntry(order, "critical", "me", "myid", "yesterday") - log_entry_2 = LogEntry(order, "normal", "you", "yourid", "today") + log_entry_1 = LogEntry(order, "critical", "me", "yesterday") + log_entry_2 = LogEntry(order, "normal", "you", "today") order.add_to_log(log_entry_1) order.add_to_log(log_entry_2) @@ -67,15 +67,32 @@ def test_orderitem_add_to_log_non_empty_log(): def test_LogEntry_init(): from ordr3.models import LogEntry - log_entry = LogEntry(1, "critical", "me", "myid", "yesterday") + log_entry = LogEntry(1, "critical", "me", "yesterday") assert log_entry.order_id == 1 assert log_entry.status == "critical" assert log_entry.by == "me" - assert log_entry.user_id == "myid" assert log_entry.date == "yesterday" +def test_LogEntry_repr(): + from ordr3.models import LogEntry + + log_entry = LogEntry(1, "critical", "me", "yesterday") + result = repr(log_entry) + + assert result == "" + + +def test_vendor_init(): + from ordr3.models import Vendor + + vendor = Vendor("A", "B") + + assert vendor.term == "A" + assert vendor.name == "B" + + def test_user_init(): from ordr3.models import User diff --git a/tests/test_repo.py b/tests/test_repo.py index 1f9b7e3..5a4c483 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -144,3 +144,16 @@ def test_sql_repo_list_users(session, example_users): session.flush() assert repo.list_users() == [earlier, later] + + +def test_sql_search_vendor(session, example_users): + from ordr3.repo import SqlAlchemyRepository + from ordr3.models import Vendor + + repo = SqlAlchemyRepository(session) + rep = Vendor("sa", "Sigma Aldrich") + session.add(rep) + session.flush() + + assert repo.search_vendor("sa") == "Sigma Aldrich" + assert repo.search_vendor("unknown") is None diff --git a/tests/test_services.py b/tests/test_services.py index d5ad74c..6064a58 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -11,6 +11,7 @@ class FakeOrderRepository(AbstractOrderRepository): def __init__(self, session): self._orders = set() self._users = set() + self._vendors = {"sa": "Sigma Aldrich"} def add_order(self, order): """ add an order to the datastore """ @@ -21,6 +22,7 @@ class FakeOrderRepository(AbstractOrderRepository): return next(o for o in self._orders if o.id == reference) def list_orders(self): + """ list orders, sorted by descending creation date """ return sorted(self._orders, reverse=True, key=lambda x: x.created_on) def add_user(self, user): @@ -40,8 +42,13 @@ class FakeOrderRepository(AbstractOrderRepository): return next(o for o in self._users if o.email == reference) def list_users(self): + """ list users, sorted by username """ return sorted(self._users, key=lambda x: x.username) + def search_vendor(self, reference): + """ search for a vendor by a canonical search term """ + return self._vendors.get(reference, None) + class FakePasslibContext: def __init__(self, needs_update): @@ -125,7 +132,6 @@ def test_create_log_entry(prefilled_repo): assert log_entry.order_id == order.id assert log_entry.status == OrderStatus.APPROVAL assert log_entry.by == "B" - assert log_entry.user_id == "A" assert isinstance(log_entry.date, datetime) assert order.status == log_entry.status assert order.created_by == log_entry.by @@ -162,3 +168,51 @@ def test_verify_username_and_password_invalid(name, pwd): pass_ctx = FakePasslibContext(False) assert verify_credentials(repo, pass_ctx, name, pwd) is None + + +@pytest.mark.parametrize( + "input,expected", + [("no url", "no url"), ("http://company.com/path", "company.com")], +) +def test_vendor_from_url(input, expected): + from ordr3.services import _vendor_from_url + + assert _vendor_from_url(input) == expected + + +@pytest.mark.parametrize( + "input,expected", + [ + ("no domain", "no domain"), + ("company.de", "company"), + ("company.com", "company"), + ("company.eu", "company"), + ("company.co.uk", "company.co.uk"), + ], +) +def test_vendor_with_common_domain(input, expected): + from ordr3.services import _vendor_with_common_domain + + assert _vendor_with_common_domain(input) == expected + + +@pytest.mark.parametrize( + "input,name,found", + [ + ("Happy Company", "Happy Company", False), + ("Company Cleanup\n", "Company Cleanup", False), + ("http://url-company.it/", "url-company.it", False), + ("http://url-company.de/", "url-company", False), + ("domain.eu", "domain", False), + ("sa", "Sigma Aldrich", True), + ("http://SA.com/some/path", "Sigma Aldrich", True), + ], +) +def test_check_vendor_name(input, name, found): + from ordr3.services import check_vendor_name + + repo = FakeOrderRepository(None) + result = check_vendor_name(repo, input) + + assert result.name == name + assert result.found == found