fet_Unizeug/app/init.py

from os.path import isdir
from stat import S_ISDIR, S_ISREG
import re
import pathlib
import os

# from base64 import decodebytes
import json
import mariadb
import logging
from pathlib import Path
import schedule
import time
import pytz

CATEGORIES = [
    "Prüfungen",
    "Klausuren",
    "Übungen",
    "Labore",
    "Unterlagen",
    "Zusammenfassungen",
    "Multimedia",
]
SUBCAT_CATEGORIES = ["Klausuren", "Übungen", "Labore"]
unizeug_path = os.environ.get("UNIZEUG_PATH", "./unizeug")
APP_ROOT_PATH = Path(os.environ.get("APP_ROOT_PATH", "./app"))
FILES_IN_PROGRESS = APP_ROOT_PATH / "files/"
log = logging.getLogger(__name__)
logging.basicConfig(
    filename="init.log",
    level=logging.INFO,
    format="[%(asctime)s, %(filename)s:%(lineno)s -> %(funcName)10s() ]%(levelname)s: %(message)s",
)
debug = log.debug
info = log.info
error = log.error

db = mariadb.connect(
    host=os.environ.get("DB_HOST", "db"),
    user=os.environ.get("DB_USER", "user"),
    password=os.environ.get("DB_PASSWORD", "DBPASSWORD"),
    database=os.environ.get("DB_DATABASE", "unizeug"),
)
c = db.cursor()
try:
    c.execute("DROP TABLE LVAs")
except mariadb.OperationalError:
    pass
c.execute(
    "CREATE TABLE LVAs(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,lvid VARCHAR(6), lvname VARCHAR(256), lvpath VARCHAR(256),PRIMARY KEY(id))"
)
try:
    c.execute("DROP TABLE Profs")
except mariadb.OperationalError:
    pass
c.execute(
    "CREATE TABLE Profs(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,name VARCHAR(256),PRIMARY KEY(id))"
)
try:
    c.execute("DROP TABLE LPLink")
except mariadb.OperationalError:
    pass
c.execute(
    "CREATE TABLE LPLink(id BIGINT(20) unsigned NOT NULL AUTO_INCREMENT,LId bigint(20),PId bigint(20),PRIMARY KEY(id))"
)
try:
    c.execute("DROP TABLE SubCats")
except mariadb.OperationalError:
    pass
c.execute(
    "CREATE TABLE SubCats(id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,LId BIGINT(20),PId BIGINT(20),cat TINYINT UNSIGNED,name VARCHAR(256), PRIMARY KEY(id))"
)
try:
    c.execute(
        "CREATE TABLE FIP(id UUID DEFAULT(UUID()), filename VARCHAR(256), filetype VARCHAR(8),initTimeStamp DATETIME, PRIMARY KEY(id))"
    )
except mariadb.OperationalError:
    pass
db.commit()


def remove_old_FIP_entrys():
    cur = db.cursor(dictionary=True)
    cur.execute(
        "SELECT id,filename FROM FIP WHERE HOUR(TIMEDIFF(NOW(),initTimeStamp)) > 24 "
    )
    files = cur.fetchall()
    info(f"Remove Files: {files}")
    for file in files:
        c.execute("DELETE FROM FIP WHERE id=?", (file["id"],))
        os.remove(FILES_IN_PROGRESS / file["filename"])
    db.commit()


def get_dirstruct():
    # with open("app/pwfile.json", "r") as f:
    # cred = json.load(f)
    # ssh = paramiko.SSHClient()
    # print(cred["sftpurl"])
    # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    # key=paramiko.RSAKey(data=decodebytes(bytes(cred["key"],"utf-8")))
    # ssh.get_host_keys().add(cred["sftpurl"], 'ssh-rsa', key)
    # ssh.connect(cred["sftpurl"], username=cred["sftpuser"], password=cred["sftpPW"])
    # sftp = ssh.open_sftp()
    # folders = sftp.listdir_attr(unizeug_path)
    folders = pathlib.Path(unizeug_path)
    for entry in folders.iterdir():
        if entry is None:
            continue
        if not entry.is_dir():
            continue
        fname = str(entry.name)
        regex = re.compile(r"Multimedia_only")
        if regex.search(fname):
            continue
        # print(fname)
        lvid = re.search(r"[a-zA-Z0-9]{3}\.[a-zA-Z0-9]{3}", fname)
        # print(lvid)
        if lvid is None:
            error(f"Didnt Find LVA ID in Directory {fname}")
            continue
        lvid = lvid.group()[:3] + lvid.group()[4:]
        # name = fname[:-8]
        name = re.sub(r"[a-zA-Z0-9]{3}\.[a-zA-Z0-9]{3}", "", fname)
        # print(name)
        # print(lvid)
        cur = db.cursor()
        cur.execute(
            "INSERT INTO LVAs (lvid, lvname, lvpath) VALUES(?,?,?)", (lvid, name, fname)
        )
        cur.execute("SELECT id FROM LVAs WHERE lvid=?", (lvid,))
        lid = cur.fetchone()[0]
        db.commit()
        for profsdir in entry.iterdir():
            if profsdir is None:
                continue
            if not profsdir.is_dir():
                continue
            # print(profsdir.filename)
            try:
                lastname, firstname = re.split(r"[_\-\s]", str(profsdir.name))
                pid = link_prof(firstname, lastname, lid)
            except ValueError:
                error(f"Couldnt get Profs from {fname}")
                continue
            for cat in profsdir.iterdir():
                if cat is None:
                    continue
                if not cat.is_dir():
                    continue
                if cat.name not in SUBCAT_CATEGORIES:
                    continue
                idx = CATEGORIES.index(cat.name)
                for subcat in cat.iterdir():
                    if subcat is None:
                        continue
                    if not subcat.is_dir():
                        continue
                    cur = db.cursor()
                    cur.execute(
                        "INSERT INTO SubCats (LId,PId,cat,name) VALUES(?,?,?,?)",
                        (lid, pid, idx, subcat.name),
                    )
                db.commit()
    remove_old_FIP_entrys()


def link_prof(firstname, lastname, lid):
    cur = db.cursor()
    cur.execute("SELECT id from Profs WHERE name=?", (lastname + " " + firstname,))
    res = cur.fetchone()
    if res is not None:
        cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0]))
        db.commit()
        return res[0]
    cur.execute("SELECT id from Profs WHERE name=?", (firstname + " " + lastname,))
    res = cur.fetchone()
    if res is not None:
        cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0]))
        db.commit()
        return res[0]
    cur.execute("INSERT INTO Profs (name) VALUES(?)", (lastname + " " + firstname,))
    cur.execute("SELECT id FROM Profs WHERE name=?", (lastname + " " + firstname,))
    res = cur.fetchone()
    cur.execute("INSERT INTO LPLink (LId,PId) VALUES(?,?)", (lid, res[0]))
    db.commit()
    return res[0]


if __name__ == "__main__":
    get_dirstruct()
    info("Database updated")
    schedule.every().day.at("04:00", "Europe/Vienna").do(get_dirstruct)
    while True:
        schedule.run_pending()
        time.sleep(1)