#!/usr/bin/python3
# MAmas TEsting LIBrary
# Copyright (c) Jan Otte, Dwarf Technologies s.r.o. All rights reserved.

import sys
import getopt
import os
import subprocess
import time
import datetime
import re
import random
import string
import uuid
import json
from enum import IntEnum
import MySQLdb
import dwarfg_lib as dgl
from threading import Lock

dgvars = None
brandfile_name_default = ".brand_selection"
mtl_output_enabled = True
dbpasswd_default = "gogogegegigigugu"
mtl_allowed_chars = set("ABCDFGOPRSTUVXYZ")
mtl_devt_dir = [ "nonexistent", "default_linux", "default_advrouter", "default_owrt", "default_teltrouter", "default_cstech", "default_snmp" ]
mtl_example_file_base = "dwarfg-cache"
mtl_example_file_varied = "dwarfg-cache-varied"
mtl_example_ext_small = ".small"
mtl_example_ext_big = ".big"
mtl_data_defaults = "defaults.tgz"
mtl_data_bcfg = "cfgbackup.tgz"
mtl_example_datadir = "example_data"
mtl_register_inv_resp = ["EEEE", "NNNN", "JJJJ", "LLLL", "KKKK", "IIII"]
mtl_server = "127.0.0.1"
mtl_mampath = ""
mtl_mamctl = ""
mtl_mamclean = ""
mtl_maminst = "devdeploy.sh"
mtl_emulfile_separator = '^'

mdb = None

class Linetype(IntEnum):
    PLAINTEXT = 0
    RANDINT = 1        # I:
    RANDREAL = 2       # R:
    RANDSET = 3        # S:
    RANDINT_ONCE = 4   # i:
    RANDREAL_ONCE = 5  # r:
    RANDSET_ONCE = 6   # s:
    RANDADD = 7        # A:

linetypes = set(Linetype)

class Devtype(IntEnum):
    ANYDEVICE = 0
    LINUXBOX = 1
    ADVROUTER = 2
    OPENWRT = 3
    TELTROUTER = 4
    TENECT_GW = 5
    SNMPDEVICE = 6
    MAXDEVICES = 7

devtypes = set(Devtype)

# cline is a tuple with meaning: (linetype, templace line to substitute or orig line from file, [ substitutable or range values ], None or cached value)
# the list [substitutable or range values ] depends on linetype:
#   in RANDINT it is [ min, max ]
#   in RANDREAL it is [ min, max, decimal precision ]
#   in RANDSER it is [ str1, ... strn ] set of possible string values
#   in RANDADD it is [ last, min, max ] where last is the actual value and min max is range for randint

def cline_expand(cline, dict_keep, debug):
    if cline[0] == Linetype.PLAINTEXT:
        return cline[1]
    if cline[0] in (Linetype.RANDSET, Linetype.RANDINT, Linetype.RANDREAL) or (cline[3] is None and cline[0] in (Linetype.RANDREAL_ONCE, Linetype.RANDINT_ONCE, Linetype.RANDSET_ONCE)):
        if cline[0] in (Linetype.RANDINT, Linetype.RANDINT_ONCE):
            if Linetype.RANDINT_ONCE == cline[0] and cline[1] in dict_keep:
                cline[3] = dict_keep[cline[1]]
                if debug: tprint("Caching once-randint line", cline[1], "from dict_keep to", cline[3])
            else:
                cline[3] = re.sub('<num>', lambda _: str(random.randint(cline[2][0],cline[2][1])), cline[1])
                if Linetype.RANDINT_ONCE == cline[0]:
                    dict_keep[cline[1]] = cline[3]
                    if debug: tprint("New dict_keep randint for line", cline[1], "to", cline[3])
        elif cline[0] in (Linetype.RANDREAL, Linetype.RANDREAL_ONCE):
            if Linetype.RANDREAL_ONCE == cline[0] and cline[1] in dict_keep:
                cline[3] = dict_keep[cline[1]]
                if debug: tprint("Caching once-randreal line", cline[1], "from dict_keep to", cline[3])
            else:
                cline[3] = re.sub('<num>', lambda _: str(round(random.uniform(cline[2][0], cline[2][1]), int(cline[2][2]))), cline[1])
                if Linetype.RANDREAL_ONCE == cline[0]:
                    dict_keep[cline[1]] = cline[3]
                    if debug: tprint("New dict_keep randreal for line", cline[1], "to", cline[3])
        elif cline[0] in (Linetype.RANDSET, Linetype.RANDSET_ONCE):
            if Linetype.RANDSET_ONCE == cline[0] and cline[1] in dict_keep:
                cline[3] = dict_keep[cline[1]]
                if debug: tprint("Caching once-randset line", cline[1], "from dict_keep to", cline[3])
            else:
                cline[3] = re.sub('<occ>', lambda _: cline[2][random.randint(0, len(cline[2])-1)], cline[1])
                if Linetype.RANDSET_ONCE == cline[0]:
                    dict_keep[cline[1]] = cline[3]
                    if debug: tprint("New dict_keep randset for line", cline[1], "to", cline[3])
        if debug:
            if cline[0] in (Linetype.RANDREAL_ONCE, Linetype.RANDINT_ONCE, Linetype.RANDSET_ONCE):
                tprint("Replaced ONCE line ", cline[1], "to", cline[3])
            else:
                tprint("Replaced line", cline[1], "to", cline[3])
    elif Linetype.RANDADD == cline[0]:
        if cline[1] not in dict_keep: dict_keep[cline[1]] = cline[2][0]
        dict_keep[cline[1]] += random.randint(cline[2][1], cline[2][2])
        cline[3] = re.sub('<num>', str(dict_keep[cline[1]]), cline[1])
        if debug:
            tprint("Updated ADDITION on line", cline[1], "to", cline[3])
    elif cline[3] is not None and cline[0] in (Linetype.RANDREAL_ONCE, Linetype.RANDINT_ONCE, Linetype.RANDSET_ONCE):
        pass
    else:
        raise Exception("Unsupported type in line def: ", str(cline))
    return cline[3]

def cfile_load(filename, debug):
    rdata = []
    lineno = 0
    if filename is None:
        return None
    try:
        if debug:
            tprint("Reading data from file:", filename)
        f = open(filename, 'rb')
        for line in f:
            lineno += 1
            mline = line.decode('utf-8', errors='replace').rstrip()
            if len(mline) > 1 and mline[1] == ':':
                paramstr = mline.partition(':')[2].partition(';')[0]
                if 0 == len(paramstr):
                    tprint("Lineno", lineno, ': syntax error - parameter string is empty\n')
                    return None
                if mline[0] == 'I' or mline[0] == 'i':
                    # randint - 3 / 2nd is array of 2: min, max; 3rd is text with one <num>
                    try:
                        pars = [ int(x) for x in paramstr.split(',') ]
                    except ValueError:
                        tprint("Lineno", lineno, ": value error - expecting list of integers after :\n")
                        return None
                    if len(pars) != 2:
                        tprint("Lineno", lineno, ": syntax error - must provide min, max\n")
                        return None
                    if pars[0] > pars[1]:
                        tprint("Lineno", lineno, ": value error - first integer must be >= second integer\n")
                        return None
                    if mline[0] == 'I':
                        mytype = Linetype.RANDINT
                    else:
                        mytype = Linetype.RANDINT_ONCE
                    rdata.append([mytype, mline.partition(';')[2], pars, None])
                elif mline[0] == 'R' or mline[0] == 'r':
                    # randreal - 3 / 2nd is array of 3: min, max, dec; 3rd is text with one <num>
                    try:
                        pars = [ float(x) for x in paramstr.split(',') ]
                    except ValueError:
                        tprint("Lineno", lineno, ": value error - expecting list of numbers after :\n")
                        return None
                    if len(pars) != 3:
                        tprint("Lineno", lineno, ": syntax error - must provide min, max, decimals\n")
                        return None
                    if pars[0] > pars[1]:
                        tprint("Lineno", lineno, ": value error - first number must be >= second number\n")
                        return None
                    if mline[0] == 'R':
                        mytype = Linetype.RANDREAL
                    else:
                        mytype = Linetype.RANDREAL_ONCE
                    rdata.append([mytype, mline.partition(';')[2], pars, None])
                elif mline[0] == 'S' or mline[0] == 's':
                    # randset - 3 / 2nd is array of n: val1, val2, ..., valn; 3rd is text with one <occ>
                    pars = paramstr.split(',')
                    if mline[0] == 'S':
                        mytype = Linetype.RANDSET
                    else:
                        mytype = Linetype.RANDSET_ONCE
                    rdata.append([mytype, mline.partition(';')[2], pars, None])
                elif mline[0] == 'A':
                    # addset - 3 / 2nd is array of 3: lastnum, val1, val2; 3rd is thext with <num>
                    try:
                        pars = [ int(x) for x in paramstr.split(',') ]
                    except ValueError:
                        tprint("Lineno", lineno, ": value error - expecting list of integers after :\n")
                        return None
                    if len(pars) != 3:
                        tprint("Lineno", lineno, ": syntax error - must provide origval, min, max\n")
                        return None
                    if pars[1] > pars[2]:
                        tprint("Lineno", lineno, ": value error - second integer must be >= third integer\n")
                        return None
                    rdata.append([Linetype.RANDADD, mline.partition(';')[2], pars, None])
                else:
                    tprint("Lineno", lineno, ' syntax error - non understood line type: ', mline[0], "\n")
                    return None
            else:
                # simple fixed text line
                rdata.append([Linetype.PLAINTEXT, mline, None, None])
        f.close()
    except:
        tprint("Error opening data file", filename)
        return None
    return rdata

def tprint(*args):
    if mtl_output_enabled: print(" ... ", *args)

def devid_int(devid):
    if isinstance(devid, int):
        return devid
    return dgl.pathstringtonum(devid)

def devid_str(devid):
    if isinstance(devid, str):
        return devid
    return dgl.pathstring(devid)

def mdb_connect(host = "localhost", db_passwd = None, dbname = None):
    global mdb
    mypasswd = db_passwd
    mydbname = dbname
    if None == mypasswd: mypasswd = dgvars.md_dbpwd
    if None == mydbname: mydbname = dgvars.md['dwarfg_dbn']
    mdb = MySQLdb.connect(host=host, user=dgvars.mp_brand_short, passwd=mypasswd, db=mydbname)
    if not mdb:
        tprint("Unable to connect to DB!")
        return False
    mdb.autocommit(True)
    return True

def mdb_disconnect():
    mdb.close()
    return True

def mdb_get_devicecount():
    c = mdb.cursor()
    c.execute("SELECT COUNT(*) FROM devices")
    r = c.fetchone()
    c.close()
    if r is None:
        return -1
    try:
        i = int(r[0])
        return i
    except Exception as e:
        tprint(e)
        return -2

def mdb_get_devices():
    c = mdb.cursor()
    c.execute("SELECT * FROM devices")
    r = c.fetchall()
    c.close()
    return r

def mdb_get_device(devid):
    c = mdb.cursor()
    c.execute("SELECT * FROM devices WHERE id_device = %s", (devid_int(devid),))
    r = c.fetchone()
    c.close()
    return r

def mdb_get_devicestat(devid):
    c = mdb.cursor()
    c.execute("SELECT * FROM device_stats WHERE id_device = %s", (devid_int(devid),))
    r = c.fetchone()
    c.close()
    return r

def mdb_get_fwvers():
    c = mdb.cursor()
    c.execute("SELECT * FROM os_versions")
    r = c.fetchall()
    c.close()
    return r

def mdb_get_fwver(osverid):
    c = mdb.cursor()
    c.execute("SELECT * FROM os_versions WHERE id = %s", (osverid,))
    r = c.fetchone()
    c.close()
    return r

def mdb_set_fwver(osverid, approved):
    c = mdb.cursor()
    current = int(c.execute("SELECT approved FROM os_versions WHERE id=%s", (osverid,)))
    if current == approved:
        return True # set to the right approval state already
    res = c.execute("UPDATE os_versions set approved=%s WHERE id=%s", (approved, osverid,))
    c.close()
    if 0 < res:
        return True
    return False

def mdb_set_device_fwver(devid, osverid):
    c = mdb.cursor()
    res = c.execute("UPDATE devices set os_ver_desired=(%s) WHERE id_device=%s", (osverid, devid_int(devid),))
    c.close()
    if 0 < res:
        return True
    return False

def mdb_set_device_hostname(devid, devname):
    c = mdb.cursor()
    res = c.execute("UPDATE devices set hostname_desired=(%s) WHERE id_device=%s", (devname, devid_int(devid),))
    c.close()
    if 0 < res:
        return True
    return False

def mdb_set_device(devid, osver_desired, hostname_desired):
    c = mdb.cursor()
    c.execute("SELECT version from max_gui_change WHERE id=%s", ("uniq",))
    row = c.fetchone()
    ver = int(row[0])
    if osver_desired is not None:
        mdb_set_device_fwver(devid, osver_desired)
    if hostname_desired is not None:
        mdb_set_device_hostname(devid, hostname_desired)
    c.execute("UPDATE devices set gui_version=%s WHERE id_device=%s", (ver+1, devid_int(devid)))
    c.execute("UPDATE max_gui_change set version=%s WHERE id=%s", (ver+1,"uniq",))
    return True

def mdb_device_toupdate(devid):
    res = mdb_get_device(devid)
    if res is None:
        return False
    if res[5] is None:
        return False
    if res[4] is None:
        return False
    if res[5] == res[4]:
        return False
    return True

def mdb_devicedata_waiting(devid):
    if os.path.isfile(dgvars.md_srvdir_srv + '/' + dgl.expand_path(devid_str(devid)) + '/' + dgvars.md_sfn_data):
        return True
    return False

def get_agent_type(intype, maxtype=None):
    maxt = maxtype
    if maxtype is None:
        maxt = Devtype.TENECT_GW+1
    if intype not in devtypes or Devtype.ANYDEVICE == intype or intype == Devtype.MAXDEVICES:
        return random.randrange(Devtype.ANYDEVICE+1, maxt+1)
    return intype

def get_iccid():
    return ''.join(random.choice(string.digits) for _ in range(19))

def get_machid(devt = Devtype.ANYDEVICE):
    if devt == Devtype.ADVROUTER:
        #"6602882;000A1485E3BC;;SPECTRE-v3L-LTE"
        m = ''.join(random.choice(string.digits) for _ in range(7)) # serial
        m = m + ';' + ''.join(random.choice(string.digits+'ABCDEF') for _ in range(12)) # MAC1
        m = m + ',' + ''.join(random.choice(string.digits+'ABCDEF') for _ in range(12)) # MAC2
        m = m + ',' + ''.join(random.choice(string.digits+'ABCDEF') for _ in range(12)) # MAC3
        m = m + ';;SPECTRE-v3L-LTE' # prodname
        return m
    elif devt == Devtype.TENECT_GW:
        #g_machid="NA;867267496916;NA;NA" -> Serial, MAC1, HW UUID, Product type
        m = ''.join(random.choice(string.digits) for _ in range(7)) # serial
        m = m + ';' + ''.join(random.choice(string.digits+'ABCDEF') for _ in range(12)) # MAC1
        m = m + ';' + str(uuid.uuid4()) # UUID
        m = m + ';CSG-3xx' # prodname
        return m
    elif devt == Devtype.LINUXBOX:
        #"7d461dac87e84043a3360614c96496d8;e4b31843d1c7,507b9dea1a57;52f04894-f403-4ea1-b5dd-4ad04deb9cbd;pecka"
        m = ''.join(random.choice(string.digits+string.ascii_lowercase) for _ in range(32))
        m = m + ';' + ''.join(random.choice(string.digits+'abcdef') for _ in range(12)) # MAC1
        m = m + ';' + str(uuid.uuid4())
        m = m + ';' + ''.join(random.choice(string.ascii_lowercase) for _ in range(random.randint(6,12))) # hostname
        return m
    elif devt == Devtype.OPENWRT:
        #"000000001711bd41,a01041,BCM2835;b827eb11bd41;;OpenWrt"
        m = ''.join(random.choice(string.digits+'abcdef') for _  in range(16))
        m = m + ',' + ''.join(random.choice(string.digits+'abcdef') for _ in range(6))
        m = m + ',BCM' + ''.join(random.choice(string.digits) for _ in range(4)) # chipset :-)
        m = m + ';' + ''.join(random.choice(string.digits+'abcdef') for _ in range(12)) # MAC1
        m = m + ';;' + ''.join(random.choice(string.ascii_lowercase) for _ in range(random.randint(6,12))) # hostname
        return m
    elif devt == Devtype.TELTROUTER:
        #"09544095,RUT955H7V3B0;001e4216b08a;;Teltonika-RUT955.com"
        m = ''.join(random.choice(string.digits) for _ in range(8))
        m = m + ',RUT955H7V3B0;' + ''.join(random.choice(string.digits+'abcdef')) # MAC1
        m = m + ';;' + ''.join(random.choice(string.ascii_lowercase) for _ in range(random.randint(6,12))) # hostname
        return m
    elif devt == Devtype.SNMPDEVICE:
        m = ".".join(str(random.randint(0, 255)) for _ in range(4))+';snmp;snmp;snmp'
    else:
        raise Exception("Machine-ID generation not implemented for this device type:", str(devt))

# devicelist structure
#   devicelist is an array of devices
#   each array containst these elements:
#       DevID
#       Device type
#       Machine ID
#       Device token
#       ICCID
#       dict_keep - one-time (during first push) randomized data that stays the same after the randomization
#       array of small data push clines
#       array of big data push clines
#       boolean - is first push done?

def register(devicelist, idevt = Devtype.ANYDEVICE, count = 1, server = mtl_server, debug = False, use_ssl = True, servid = "", silent = False, prefdevid = None, initmachids = None, superdebug = False, iccid = None, emulate_iccid = True, maxdevt = None):
    myiccid = iccid
    if myiccid is None and emulate_iccid:
        myiccid = get_iccid()
    devt = get_agent_type(idevt, maxdevt)
    if server is None:
        server = mtl_server
    if servid is None or len(servid) == 0:
        if 'servid' not in dgvars.md or dgvars.md['servid'] is None or len(dgvars.md['servid']) == 0:
            try:
                bfile = open("/opt/"+dgvars.mp_brand_short+"_"+"/base_defs", "r")
                for line in bfile:
                    if re.search("^SERVID=", line):
                        servid = line.split("=")[1]
            except:
                pass
        else:
            servid = dgvars.md['servid']
    if servid is None or len(servid) == 0:
        servid = "0badc0de-beef-f00d-cafe-4badf001babe"
    devtype = "device_type=" + dgvars.devtypes[devt]
    protov = "protocol_version=2.0"
    servidstr = "server_id=" + servid
    htproto = "https://"
    machid=""
    if not use_ssl:
        htproto = "http://"
    url = htproto + server + "/"+dgvars.mp_brand_short+"/d/r"
    cmds = ['curl', '-s', '-k', '-F', devtype, '-F', protov, '-F', servidstr, '-F', '', url]
    if prefdevid is not None:
        cmds[-3:-3] = ['-F', "preferred_devid="]
    if debug:
        tprint("Commands are ", cmds)
    for i in range(count):
        if initmachids is None:
            machid = str(get_machid(devt))
            cmds[-2] = 'machine_id="' + str(machid) + '"'
        else:
            cmds[-2] = 'machine_id="' + str(initmachids[i]) + '"'
        if prefdevid is not None:
            cmds[-4] = "preferred_devid=" + prefdevid[i]
        if debug:
            tprint("Commands are ", cmds)
        res = subprocess.run(cmds, stdout=subprocess.PIPE, text=True)
        answ = res.stdout.strip()
        for j in mtl_register_inv_resp:
            if answ.startswith(j):
                print("Server returned app error " + str(j))
                return False, j
        if not silent:
            print(answ)
        if len(answ)<5 or set(answ[:4]) > mtl_allowed_chars or ':' != answ[4]:
            print("Server provided invalid (non-MAMAS) answer:", answ)
            return False, "non-MAMAS"
        else:
            fields = answ.split(':')
            if prefdevid is not None:
                if prefdevid[i] != fields[0]:
                    print("Server declined re-registration of ", prefdevid[i], " returning ", fields[0], " - maybe you forgot to run deploy in the meantime?")
                    return False, "blank-failed"
            devicelist.append([fields[0], devt, machid, fields[1], myiccid, {}, False, [], []])
            if not silent:
                print(mtl_emulfile_separator.join([fields[0], str(devt), machid, fields[1], "" if myiccid is None else myiccid, json.dumps({}), str(False)]) + "\n")
    return True, "OK"

def readdevice(devicelist, txt, txt_dict):
    items = txt.split(mtl_emulfile_separator)
    dict_items = txt_dict.split(mtl_emulfile_separator)
    if 6 != len(items) or 2 != len(dict_items):
        tprint("Incorrect number of items (", str(len(items)), ",", str(len(dict_items)), ")from one of the input files. Lines are:\n", txt, "\n", txt_dict)
        return False
    if 4 != len(items[0]) or 4 != len(dict_items[0]):
        tprint("Invalid DevID length from one of the input files. Lines are:\n", txt, "\n", txt_dict)
        return False
    if 1 != len(items[1]):
        tprint("Invalid length of type indicator. Line is:\n", txt)
        return False
    if not set(items[0]).issubset(mtl_allowed_chars) or not set(dict_items[0]).issubset(mtl_allowed_chars):
        tprint("Invalid DevID from one of the input files. Lines are:\n", txt, "\n", txt_dict)
        return False
    if items[0] != dict_items[0]:
        tprint("Mismatched DevID from input files. Lines are:\n", txt, "\n", txt_dict)
        return False
    try:
        mv = int(items[1])
    except:
        return False
    if mv not in devtypes or 0 == mv:
        tprint("Invalid device type. Line is:\n", txt)
        return False
    if not isinstance(bool(items[5]), bool):
        tprint("Invalid bool flag. Line is:\n", txt)
        return False
    devicelist.append([items[0], int(items[1]), items[2], items[3], items[4], json.loads(dict_items[1]), bool(items[5]), [], []])
    return True

def readdevsfromfile(devicelist, filename):
    count = 0
    try:
        with open(filename) as f1, open(filename + ".dict") as f2:
            for linef1, linef2 in zip(f1, f2):
                if readdevice(devicelist, linef1.strip(), linef2.strip()):
                    count += 1
                else:
                    tprint("Invalid device entry, file1/file2/lineno/lines:", str(filename), "/", str(filename + ".dict"), "/", str(count+1), "/\n", linef1.strip(), "\n", linef2.strip())
    except Exception as e:
        tprint(e)
        return False
    tprint("Read", str(count), "IDs.")
    return True

def writedevstofile(devicelist, filename, start = 0, appendfile = False):
    tprint("Adding agents (" + str(len(devicelist[start:])) + " new, " + str(len(devicelist)) + " total) to file " + str(filename))
    if not appendfile: start = 0
    try:
        if appendfile:
            tprint("Appending to IDS...")
            f = open(filename, "a")
        else:
            tprint("Overwriting IDS...")
            f = open(filename, "w")
        for agent in devicelist[start:]:
            f.write(mtl_emulfile_separator.join([str(agent[0]), str(agent[1]), str(agent[2]), str(agent[3]), str(agent[4]), str(agent[6])]) + "\n")
        if not appendfile:
            f.truncate()
        f.close()
        if appendfile:
            tprint("Appending to IDS dictionary...")
            f = open(filename + ".dict", "a")
        else:
            tprint("Overwriting IDS dictionary...")
            f = open(filename + ".dict", "w")
        for agent in devicelist[start:]:
            f.write(mtl_emulfile_separator.join([str(agent[0]), json.dumps(agent[5])]) + "\n")
        if not appendfile:
            f.truncate()
        f.close()
        return True
    except Exception as e:
        print("Exception when writing to file:", str(e))
        return False

def data_replace_machid(machid, data):
    machid_string = "g_machid="
    for i in range(len(data)):
        if data[i].startswith(machid_string):
            data[i] = re.sub(machid_string+'.*',machid_string+'"'+str(machid)+'"', data[i])

def data_replace_iccid(iccid, data):
    iccid_re="^ICCID *:"
    for i in range(len(data)):
        if re.match(iccid_re, data[i]):
            data[i] = re.sub("(" + iccid_re +').*','\\1 '+str(iccid), data[i])

def data_replace_time(data):
    now = datetime.datetime.now()
    for i in range(len(data)):
        if re.match("^Time[ ]+:", data[i]):
            data[i] = re.sub(": .*", ": "+str(now.strftime('%Y-%m-%d %H:%M:%S')), data[i])
    for i in range(len(data)):
        if re.match("^sysclock=", data[i]):
            data[i] = re.sub("=[0-9]*", "="+str(int(time.time())), data[i])

def dwarfg_cleanup():
    res = subprocess.run(mtl_mamclean, shell=True, text=True, input="Y")
    if res.returncode:
        tprint("MAMAS cleanup ended with error return code: " + res.returncode)
        return False
    return True

def dwarfg_setup():
    res = subprocess.run("cd .. && ./" + mtl_maminst, shell=True, text=True)
    if res.returncode:
        tprint("MAMAS deploy ended with error return code: " + res.returncode)
        return False
    return True

def dwarfg_start():
    res = subprocess.run(["sudo", mtl_mamctl, "--start"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, text=True)
    if res.returncode:
        tprint("Failed to start MAMAS.")
        return False
    return True

def dwarfg_stop():
    res = subprocess.run(["sudo", mtl_mamctl, "--stop", ">/dev/null"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, text=True)
    if res.returncode:
        tprint("Failed to stop MAMAS.")
        return False
    return True

def dwarfg_check():
    res = subprocess.run(["sudo", mtl_mamctl, "--pid"], text=True, stdout=subprocess.PIPE)
    try:
        x = int(res.stdout.strip())
    except:
        tprint("Unable to parse "+mtl_mamctl+" output.")
        return False
    if (0 == x):
        tprint("MAMAS is not running.")
        return False
    return True

def getfilo(fname):
    return "files/" + fname + ".txt"

def comparefilo(data, filepath):
    fdata = []
    try:
        f = open(filepath, 'rb')
        for line in f:
            fdata.append(line.decode('utf-8', errors='replace').rstrip('\n'))
        f.close()
    except:
        tprint("Error opening/reading file " + filepath)
        return False
    if data == fdata:
        return True
    return False


def push_data(devid, idevt, machine_id, device_token, iccid, dict_keep, first_push_done = False, clines = [None, None], data = None, datafname = None, datafpath = None, server = mtl_server, touch = True, send_defaults = False, send_bcfg = False, debug = False, file_defaults = None, file_bcfg = None, use_ssl = True, force_bigcache = False, silent = False, varied = False, update_time = True, superdebug = False, maxdevt=None):
    cindex = 0
    if force_bigcache:
        cindex = 1
    my_send_defaults = send_defaults
    my_send_bcfg = send_bcfg
    mtl_example = mtl_example_file_base
    if varied:
        mtl_example = mtl_example_file_varied
    # performs data push
    # returns (HTTP-push-ok (Bool), dwarfg-response-code (string/int), response-content)
    devt = get_agent_type(idevt, maxdevt)
    if server is None:
        server = mtl_server
    protov = "protocol_version=2.0"
    mydata = []
    htproto = "https://"
    if not use_ssl:
        htproto = "http://"
    real_file_defaults = "" if not my_send_defaults else mtl_example_datadir + "/" + mtl_devt_dir[devt] + "/" + mtl_data_defaults
    real_file_bcfg = "" if not my_send_bcfg else mtl_example_datadir + "/" + mtl_devt_dir[devt] + "/" + mtl_data_bcfg
    if file_defaults is not None:
        if os.path.isfile(file_defaults):
            real_file_defaults = file_defaults
        else:
            if not silent:
                tprint("File " + file_defaults + " not found, using default file...")
    if file_bcfg is not None:
        if os.path.isfile(file_bcfg):
            real_file_bcfg = file_bcfg
        else:
            if not silent:
                tprint("File " + file_bcfg + " not found, using default file...")
    if my_send_defaults and not os.path.isfile(real_file_defaults):
        if not silent:
            tprint("Defaults file (", real_file_defaults, ") is not present, cancelling sending the file.")
        my_send_defaults = False
    if my_send_bcfg and not os.path.isfile(real_file_bcfg):
        if not silent:
            tprint("Backup CFG file (", real_file_bcfg, ") is not present, cancelling sending the file.")
        my_send_bcfg = False
    ftopn = None
    devid_string = "DEVICE_ID="
    url = htproto + server + "/"+dgvars.mp_brand_short+"/d/d"
    cmds = ['curl', '-s', '-k', '-X', 'POST', '-H', "Content-Type: multipart/form-data", '-w', "\n%{http_code}", '-F', protov, '-F', "device_id="+devid_str(devid), "-F", "device_token=" + device_token, '-F', "filedata=@-;filename=data.txt"]
    param_file_defaults = [] if not my_send_defaults else ['-F', "defdata=@" + real_file_defaults + ";filename=" + mtl_data_defaults]
    param_file_bcfg = [] if not my_send_bcfg else ['-F', "cfgdata=@" + real_file_bcfg + ";filename=" + mtl_data_bcfg]
    # NOTE: if file is in case, get data from the cache entry (should expand randoms though...)
    if data is not None:
        mydata = data
    else:
        if datafname is not None:
            ftopn = getfilo(datafname)
        elif datafpath is not None:
            ftopn = datafpath
        else:
            if force_bigcache:
                ftopn = mtl_example_datadir + "/" + mtl_devt_dir[devt] + "/" + mtl_example + mtl_example_ext_big
            else:
                ftopn = mtl_example_datadir + "/" + mtl_devt_dir[devt] + "/" + mtl_example + mtl_example_ext_small
        if len(clines[cindex]) == 0:
            if ftopn is None:
                tprint("No data to push.")
                return False, "matelib-file-error", None
            clines[cindex] += cfile_load(ftopn, debug)
        for idx in range(len(clines[cindex])):
            if debug: old = str(clines[cindex][idx])
            token = cline_expand(clines[cindex][idx], dict_keep, debug)
            if debug and clines[cindex][idx][0] != Linetype.PLAINTEXT:
                tprint("pre-expand: ", old)
                tprint("post-expand:", str(clines[cindex][idx]))
            if token is not None: mydata.append(token)
        if mydata is None:
            tprint("Error getting data for '" + ftopn + "'")
            return False, "matelib-file-error", None
    # update time
    if update_time: data_replace_time(mydata)
    # update machid
    if machine_id is not None:
        data_replace_machid(machine_id, mydata)
    # update iccid
    if iccid is not None:
        data_replace_iccid(iccid, mydata)
    # execute the real data push
    if debug:
        tprint("Pushing data for device "+devid_str(devid)+"...")
    if my_send_defaults:
        cmds += param_file_defaults
    if my_send_bcfg:
        cmds += param_file_bcfg
    cmds += [url]
    if debug or superdebug:
        tprint("Commands are ", cmds)
    if superdebug:
        print("----- dumping data ------")
        for sline in mydata:
            print(sline)
        print("----- datadump end ------")
    res = subprocess.run(cmds, stdout=subprocess.PIPE, text=True, input="\n".join(mydata))
    outlines = res.stdout.splitlines()
    if 2>len(outlines):
        tprint("Invalid response received from server - answer too short. Is MAMAS running?")
        if (debug) :
            print("--- Complete output -- 8< --- 8< -- 8< ----")
            for ln in outlines:
                print(ln)
            print("----- >8 ----- >8 ----- >8 ----- >8 -----")
        return False, "response-error", res.stdout.splitlines()
    else:
        fline = outlines[0]
        lline = outlines[-1]
        if (debug) :
            print("HTTP/MAMAS return code for data push: " + lline + "/" + fline)
            print("--- Complete output -- 8< --- 8< -- 8< ----")
            for ln in outlines:
                print(ln)
            print("----- >8 ----- >8 ----- >8 ----- >8 -----")
        if 200 != int(lline) :
            tprint("HTTP Error received ("+lline+"), data push failed! [" + str(time.time()) + "]")
            return False, "HTTP: " + lline, None
        else:
            if 0 != int(fline.split(':')[0]) :
                tprint("MAMAS application error ("+fline+"), data push failed!")
        return True, int(fline.split(':')[0]), outlines[2:-3]

def init(alt_base_defs_dict = None, output_enabled = True, brandfile = None, brand_short = None, brand_long = None, brand_opt = None, brand_email = None, brand_web = None, brand_manuf = None):
    global dgvars, mtl_mampath, mtl_mamctl, mtl_mamclean, mtl_output_enabled
    bf = brandfile
    mtl_output_enabled = output_enabled
    if brand_short is not None:
        dgvars = dgl.LibVars(defs_dict = alt_base_defs_dict)
        dgvars.load_brand(brand_short, brand_long, brand_opt, brand_email, brand_web, brand_manuf)
    else:
        if bf is None: bf = brandfile_name_default
        dgvars = dgl.LibVars(brand_fn = bf, defs_dict = alt_base_defs_dict, init_brand = True)
    mtl_mampath = "/opt/" + dgvars.mp_brand_short + "_"
    mtl_mamctl = mtl_mampath + "/"+dgvars.mp_brand_short+"_ctl.sh"
    mtl_mamclean = mtl_mampath + "/cleanup.sh"