Holger Frey
6 years ago
13 changed files with 529 additions and 27 deletions
Binary file not shown.
@ -0,0 +1,344 @@
@@ -0,0 +1,344 @@
|
||||
import sys |
||||
import sqlite3 |
||||
import argparse |
||||
from pathlib import Path |
||||
from datetime import datetime |
||||
from urllib.parse import urlparse |
||||
|
||||
from sqlalchemy import func |
||||
from pyramid.paster import bootstrap, setup_logging, get_appsettings |
||||
from sqlalchemy.exc import OperationalError |
||||
from sqlalchemy.orm.exc import NoResultFound |
||||
|
||||
from .. import models, adapters, services |
||||
|
||||
vendor_map = { |
||||
"Accurion GmbH": ["accurion"], |
||||
"Acros Organics": ["acros"], |
||||
"Agilent Technologies": ["agilent"], |
||||
"Alfa Aesar": [ |
||||
"alefa aesar", |
||||
"alfa", |
||||
"aesar", |
||||
"alfa aesar - vwr", |
||||
"alfa-aesar", |
||||
"alfaaesar", |
||||
], |
||||
"Amazon": [], |
||||
"analytics-shop.com": ["analytics-shop"], |
||||
"architekturbedarf.de": ["architekturbedarf"], |
||||
"Avanti Polar Lipids": ["avanti"], |
||||
"Biesterfeld Spezialchemie GmbH": ["biesterfeld", "biesterfeld gmbh"], |
||||
"Bio-Rad": ["bio rad"], |
||||
"Biocat": [], |
||||
"Bioleague GmbH & co.kg": ["bioleague"], |
||||
"Biomol": [], |
||||
"Bürkert Indelfingen": ["bürkert"], |
||||
"Carl Roth GmbH": [ |
||||
"carl roth", |
||||
"carl-roth", |
||||
"carlroth", |
||||
"cr", |
||||
"karl roth", |
||||
"roth", |
||||
], |
||||
"Cellsystems Biotech": ["cellsystems"], |
||||
"Chemicell GmbH": ["chemicell"], |
||||
"Clickbox e.k.": ["clickbox"], |
||||
"Cole Parmer": ["coleparmer"], |
||||
"Conrad Electronics": [ |
||||
"conrad", |
||||
"conrad electronic", |
||||
"conrad, fax 09604408937, k-nr 6043142", |
||||
], |
||||
"Cytoskeleton inc": ["cytoskeleton"], |
||||
"Daniel Maas Dichtstoffhandel & Co.": ["daniel maas"], |
||||
"Delta Mask b.v.": ["delta mask", "deltamask"], |
||||
"der-rollende-shop.de": ["der-rollende-shop"], |
||||
"Dianova": [], |
||||
"Distrelec": ["distrilec"], |
||||
"Edmund Optics GmbH": ["edmund optics", "edmund"], |
||||
"Faust Lab Science GmbH": ["faust", "faust lab"], |
||||
"Fisher Scientific": [ |
||||
"fischer scientific", |
||||
"fisher scientific/acros organic", |
||||
], |
||||
"form.in Lasercenter": ["form.in"], |
||||
"Uwe Markus (Glasbläser)": [ |
||||
"glasbläser markus", |
||||
"markus", |
||||
"markus, glasbläser", |
||||
"uwe markus", |
||||
"glasbläser", |
||||
], |
||||
"Goodfellow": ["good fellow"], |
||||
"Greiner Bio-One": ["greiner bio one"], |
||||
"Hach": [], |
||||
"Hellma GmbH & Co KG": [ |
||||
"hellma analytics", |
||||
"hellma optics", |
||||
"hellma optik jena", |
||||
], |
||||
"Hiss dx": ["hiss"], |
||||
"HJ-Bioanalytik GmbH": ["hj bioanalytik gmbh", "hj bioanalytik"], |
||||
"laborhandel24.de": ["laborhandel24"], |
||||
"locheisen.com": ["locheisen"], |
||||
"praezisionsmesstechnik.de": ["praezisionsmesstechnik"], |
||||
"siebgewebeshop.de": ["siebgewebeshop"], |
||||
"Häberle Labortechnik": ["häberle"], |
||||
"High-Tech-Flon": ["hightechflon"], |
||||
"hygie.de": ["hygie"], |
||||
"Iolitec": [ |
||||
"ionic liquids technologie", |
||||
"ionic liquids technologies", |
||||
"iolitec ionic liquids technologies", |
||||
], |
||||
"Iris Biotech GmbH": ["iris biotech"], |
||||
"Ismatec": ["ismatec/idex"], |
||||
"Jackson Immunoresearch": ["jacksonimmuno", "jackson"], |
||||
"Kinesis Abimed": ["kinesis", "kinesisgmbh"], |
||||
"Kisker Biotech GmbH & co. kg": ["kisker"], |
||||
"Knick Elektronische Messgeräte GmbH": ["knick"], |
||||
"Kummer Laborbedarf": ["kummer"], |
||||
"Laborhandel Krumpholz": ["krumpholz"], |
||||
"leuchtmittelmarkt.com": ["leuchtmittelmarkt"], |
||||
"Life Sciences Advanced Technologies inc": [ |
||||
"life sciences advanced technologies", |
||||
], |
||||
"Life Technologies": ["lifetechnologies", "ife technologies"], |
||||
"Macherey Nagel": ["m-n", "mn", "macherey-nagel"], |
||||
"magnets4you GmbH": ["magnets4you"], |
||||
"magnet-shop.com": ["magnet-shop"], |
||||
"Mercateo ag": ["mercateo"], |
||||
"Merck": ["merck über vwr"], |
||||
"merck berufsbekleidung": [ |
||||
"merck berufskleidung", |
||||
"merck-berufsbekleidung", |
||||
], |
||||
"Merck millipore inc.": ["merck millipore", "merckmillipore"], |
||||
"Mettler-Toledo GmbH": ["mettler toledo", "mettler-toledo", "mettler"], |
||||
"Micro Particles GmbH": ["micro particles"], |
||||
"Microdyn-Nadir GmbH": ["microdyn-nadir"], |
||||
"Millipore GmbH": ["millipore"], |
||||
"Molecular Devices / Genetix": ["molecular devices", "genetix"], |
||||
"Nanoandmore GmbH": ["nano-and-more", "nanoandmore"], |
||||
"Nanocyl s.a.": ["nanocyl"], |
||||
"Neo Lab": ["neolab"], |
||||
"Newport Spectra-Physics": ["newport"], |
||||
"OCO Ortenauer Gase GmbH": ["oco ortenauer gase", "oco"], |
||||
"Pall Corporation": ["pall"], |
||||
"Plano GmbH": ["plano", "plano-eu", "plano-em"], |
||||
"Polyan GmbH": ["polyan"], |
||||
"Polyscience Europa GmbH": [ |
||||
"polysciences", |
||||
"polysciences europe gmbh", |
||||
"polysciences, inc.", |
||||
], |
||||
"ProLiquid GmbH": ["pro-liquid"], |
||||
"Qiagen": ["quiagen"], |
||||
"R&D Systems": ["r&d system"], |
||||
"Reichelt Elektronik": ["reichelt"], |
||||
"RS Components": ["rs"], |
||||
"S-Polytec": ["s-polytech"], |
||||
"Sarstedt AG & Co. KG": ["sarstedt"], |
||||
"Science Services GmbH": ["science services"], |
||||
"Scienion AG": ["scienion"], |
||||
"Sigma Aldrich": [ |
||||
"aldrich", |
||||
"adrich", |
||||
"aldich", |
||||
"aldritch", |
||||
"aldrích", |
||||
"fluka", |
||||
"sa", |
||||
"sigma - aldrich", |
||||
"sigma adrich", |
||||
"sigma aldich", |
||||
"sigma aldritch", |
||||
"sigma aldrích", |
||||
"sigma- aldrich", |
||||
"sigma-adrich", |
||||
"sigma-aldrich", |
||||
"sigma-aldrich (fluka)", |
||||
"sigmaaldrich", |
||||
"roche, sigma-aldrich", |
||||
"sigmar", |
||||
"sigma", |
||||
], |
||||
"Sterlitech": [], |
||||
"Supermagnete": ["supermagnet"], |
||||
"taq-dna.com": ["taq-dna"], |
||||
"TCI Deutschland GmbH": [ |
||||
"tci", |
||||
"tci chemical", |
||||
"tci chemicals", |
||||
"tci deutschland", |
||||
"tci europe", |
||||
"tci europe n.v.", |
||||
], |
||||
"Thermo Fisher Scientific": [ |
||||
"themofisher", |
||||
"thermofischer", |
||||
"thermofisher", |
||||
"thermo fisher", |
||||
"thermofisher scientific", |
||||
"thermo scientific", |
||||
"thermo scientific - www.perbio.com", |
||||
"perbio", |
||||
"thermo scientific / pierce", |
||||
"thermo ", |
||||
"pierce", |
||||
], |
||||
"Tib MolBiol": ["tibmolbiol", "tibmolbio"], |
||||
"Tse Systems GmbH": ["tse systems"], |
||||
"Vilber Lourmat GmbH": ["vilber"], |
||||
"VWR International GmbH": [ |
||||
"vwr", |
||||
"vwr (lenz laborglas gmbh & co.kg)", |
||||
"vwr chemicals", |
||||
"vwr collection", |
||||
"vwt", |
||||
], |
||||
"Weigert": ["drweigert"], |
||||
"Xantec Bioanalytics": ["xantec", "xantec bioanalyticss"], |
||||
"Zitt-Thoma": ["zitt thoma"], |
||||
} |
||||
|
||||
|
||||
def _query_table(cursor, table): |
||||
cursor.execute(f"SELECT * FROM {table}") |
||||
columns = [d[0] for d in cursor.description] |
||||
return (dict(zip(columns, values)) for values in cursor) |
||||
|
||||
|
||||
def _case_insensitive_query_user_by_username(repo, reference): |
||||
try: |
||||
user = ( |
||||
repo.session.query(models.User) |
||||
.filter(func.lower(models.User.username) == func.lower(reference)) |
||||
.one() |
||||
) |
||||
return user.username |
||||
except NoResultFound: |
||||
return reference |
||||
|
||||
|
||||
def migrate(old_db, repo): |
||||
inpsql3 = sqlite3.connect(old_db) |
||||
cursor = inpsql3.cursor() |
||||
migrate_users(cursor, repo) |
||||
migrate_vendors(cursor, repo) |
||||
migrate_orders(cursor, repo) |
||||
|
||||
|
||||
def migrate_users(cursor, repo): |
||||
for old_user in _query_table(cursor, "users"): |
||||
user = models.User( |
||||
old_user["id"], |
||||
old_user["user_name"], |
||||
old_user["first_name"], |
||||
old_user["last_name"], |
||||
old_user["email"], |
||||
old_user["password_hash"], |
||||
models.UserRole[old_user["role"]], |
||||
) |
||||
repo.add_user(user) |
||||
|
||||
|
||||
def migrate_vendors(cursor, repo): |
||||
for name, replacements in vendor_map.items(): |
||||
vendor = models.Vendor(name.lower(), name) |
||||
repo._add_item_to_db(vendor) |
||||
for replacement in replacements: |
||||
vendor = models.Vendor(replacement.lower(), name) |
||||
repo._add_item_to_db(vendor) |
||||
|
||||
|
||||
def migrate_orders(cursor, repo): |
||||
status_fields = [ |
||||
("created", models.OrderStatus.OPEN), |
||||
("approval", models.OrderStatus.APPROVAL), |
||||
("ordered", models.OrderStatus.ORDERED), |
||||
("completed", models.OrderStatus.COMPLETED), |
||||
] |
||||
|
||||
for old_order in _query_table(cursor, "orders"): |
||||
|
||||
result = services.check_vendor_name(repo, old_order["vendor"]) |
||||
if result.found: |
||||
vendor = result.name |
||||
else: |
||||
vendor = old_order["vendor"] |
||||
repo._add_item_to_db(models.Vendor(result.name.lower(), vendor)) |
||||
|
||||
order = models.OrderItem( |
||||
old_order["id"], |
||||
old_order["cas_description"], |
||||
old_order["catalog_nr"], |
||||
vendor, |
||||
models.OrderCategory[old_order["category"]], |
||||
old_order["package_size"], |
||||
old_order["unit_price"], |
||||
old_order["amount"], |
||||
old_order["currency"], |
||||
old_order["account"], |
||||
old_order["comment"], |
||||
) |
||||
repo.add_order(order) |
||||
|
||||
for old_field, status in status_fields: |
||||
field_by = old_order[f"{old_field}_by"] |
||||
|
||||
if field_by: |
||||
field_date = datetime.fromisoformat( |
||||
old_order[f"{old_field}_date"] |
||||
) |
||||
user = _case_insensitive_query_user_by_username(repo, field_by) |
||||
log_entry = models.LogEntry( |
||||
order.id, status, user, field_date, |
||||
) |
||||
order.add_to_log(log_entry) |
||||
|
||||
|
||||
def parse_args(argv): |
||||
parser = argparse.ArgumentParser() |
||||
parser.add_argument( |
||||
"config_uri", help="Configuration file, e.g., development.ini", |
||||
) |
||||
parser.add_argument( |
||||
"old_db", help="ordr2 sqlite database file", |
||||
) |
||||
return parser.parse_args(argv[1:]) |
||||
|
||||
|
||||
def main(argv=sys.argv): |
||||
args = parse_args(argv) |
||||
setup_logging(args.config_uri) |
||||
env = bootstrap(args.config_uri) |
||||
settings = get_appsettings(args.config_uri) |
||||
|
||||
# remove an existing sqlite database to issue a restart |
||||
database_url = urlparse(settings["sqlalchemy.url"]) |
||||
dabase_path = Path(database_url.path) |
||||
if database_url.scheme == "sqlite" and dabase_path.is_file(): |
||||
dabase_path.unlink() |
||||
|
||||
try: |
||||
with env["request"].tm: |
||||
repo = env["request"].repo |
||||
adapters.metadata.create_all(repo.session.get_bind()) |
||||
migrate(args.old_db, repo) |
||||
except OperationalError: |
||||
print( |
||||
""" |
||||
Pyramid is having a problem using your SQL database. The problem |
||||
might be caused by one of the following things: |
||||
|
||||
1. You may need to initialize your database tables with `alembic`. |
||||
Check your README.txt for description and try to run it. |
||||
|
||||
2. Your database server may not be running. Check that the |
||||
database server referred to by the "sqlalchemy.url" setting in |
||||
your "development.ini" file is running. |
||||
""" |
||||
) |
||||
Loading…
Reference in new issue