from os.path import isdir from stat import S_ISDIR, S_ISREG import re import pathlib import os # from base64 import decodebytes import json import mariadb import logging import schedule import time import pytz CATEGORIES = [ "Prüfungen", "Klausuren", "Übungen", "Labore", "Unterlagen", "Zusammenfassungen", "Multimedia", ] SUBCAT_CATEGORIES = ["Klausuren", "Übungen", "Labore"] unizeug_path = os.environ.get("UNIZEUG_PATH", "./unizeug") log = logging.getLogger(__name__) logging.basicConfig( filename="init.log", level=logging.INFO, format="[%(asctime)s, %(filename)s:%(lineno)s -> %(funcName)10s() ]%(levelname)s: %(message)s", ) debug = log.debug info = log.info error = log.error db = mariadb.connect( host=os.environ.get("DB_HOST", "db"), user=os.environ.get("DB_USER", "user"), password=os.environ.get("DB_PASSWORD", "DBPASSWORD"), database=os.environ.get("DB_DATABASE", "unizeug"), ) c = db.cursor() try: c.execute("DROP TABLE LVAs") except mariadb.OperationalError: pass c.execute( "CREATE TABLE LVAs(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,lvid VARCHAR(6), lvname VARCHAR(256), lvpath VARCHAR(256),PRIMARY KEY(id))" ) try: c.execute("DROP TABLE Profs") except mariadb.OperationalError: pass c.execute( "CREATE TABLE Profs(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,name VARCHAR(256),PRIMARY KEY(id))" ) try: c.execute("DROP TABLE LPLink") except mariadb.OperationalError: pass c.execute( "CREATE TABLE LPLink(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,LId bigint(20),PId bigint(20),PRIMARY KEY(id))" ) try: c.execute("DROP TABLE SubCats") except mariadb.OperationalError: pass c.execute( "CREATE TABLE SubCats(id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,LId BIGINT(20),PId BIGINT(20),cat TINYINT UNSIGNED,name VARCHAR(256), PRIMARY KEY(id))" ) try: c.execute( "CREATE TABLE FIP(id UUID DEFAULT(UUID()), filename VARCHAR(256), filetype VARCHAR(8),initTimeStamp DATETIME, PRIMARY KEY(id))" ) except mariadb.OperationalError: pass db.commit() def get_dirstruct(): # with open("app/pwfile.json", "r") as f: # cred = json.load(f) # ssh = paramiko.SSHClient() # print(cred["sftpurl"]) # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # key=paramiko.RSAKey(data=decodebytes(bytes(cred["key"],"utf-8"))) # ssh.get_host_keys().add(cred["sftpurl"], 'ssh-rsa', key) # ssh.connect(cred["sftpurl"], username=cred["sftpuser"], password=cred["sftpPW"]) # sftp = ssh.open_sftp() # folders = sftp.listdir_attr(unizeug_path) folders = pathlib.Path(unizeug_path) for entry in folders.iterdir(): if entry is None: continue if not entry.is_dir(): continue fname = str(entry.name) regex = re.compile(r"Multimedia_only") if regex.search(fname): continue # print(fname) lvid = re.search(r"[a-zA-Z0-9]{3}\.[a-zA-Z0-9]{3}", fname) # print(lvid) if lvid is None: error(f"Didnt Find LVA ID in Directory {fname}") continue lvid = lvid.group()[:3] + lvid.group()[4:] # name = fname[:-8] name = re.sub(r"[a-zA-Z0-9]{3}\.[a-zA-Z0-9]{3}", "", fname) # print(name) # print(lvid) cur = db.cursor() cur.execute( "INSERT INTO LVAs (lvid, lvname, lvpath) VALUES(?,?,?)", (lvid, name, fname) ) cur.execute("SELECT id FROM LVAs WHERE lvid=?", (lvid,)) lid = cur.fetchone()[0] db.commit() for profsdir in entry.iterdir(): if profsdir is None: continue if not profsdir.is_dir(): continue # print(profsdir.filename) try: lastname, firstname = re.split(r"[_\-\s]", str(profsdir.name)) pid = link_prof(firstname, lastname, lid) except ValueError: error(f"Couldnt get Profs from {fname}") continue for cat in profsdir.iterdir(): if cat is None: continue if not cat.is_dir(): continue if cat.name not in SUBCAT_CATEGORIES: continue idx = CATEGORIES.index(cat.name) for subcat in cat.iterdir(): if subcat is None: continue if not subcat.is_dir(): continue cur = db.cursor() cur.execute( "INSERT INTO SubCats (LId,PId,cat,name) VALUES(?,?,?,?)", (lid, pid, idx, subcat.name), ) db.commit() def link_prof(firstname, lastname, lid): cur = db.cursor() cur.execute("SELECT id from Profs WHERE name=?", (lastname + " " + firstname,)) res = cur.fetchone() if res is not None: cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0])) db.commit() return res[0] cur.execute("SELECT id from Profs WHERE name=?", (firstname + " " + lastname,)) res = cur.fetchone() if res is not None: cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0])) db.commit() return res[0] cur.execute("INSERT INTO Profs (name) VALUES(?)", (lastname + " " + firstname,)) cur.execute("SELECT id FROM Profs WHERE name=?", (lastname + " " + firstname,)) res = cur.fetchone() cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0])) db.commit() return res[0] if __name__ == "__main__": get_dirstruct() info("Database updated") schedule.every().day.at("04:00", "Europe/Vienna").do(get_dirstruct) while True: schedule.run_pending() time.sleep(1)