#!/usr/bin/python3
# MAMAS device emulator tool.
# Copyright (c) Jan Otte, Dwarf Technologies s.r.o. All rights reserved.

import sys
import getopt
import os
import subprocess
import time
import datetime
import matelib
from multiprocessing import Pool
import threading
import random
import math
import signal
from collections import Counter

devices = []
devices_untouched = []
devices_read = []
file_data = None

# To implement:
#   1. update data during data-push (small stuff like actual time; complex stuff needs to be prepared in file)
#   2. When number of data pushes is big (over 8000) it sometimes takes ages to pick up all the processes (has not finished, interrupted) - examine the culprit

def print_help():
    print("""

Agent emulator tool. There is a number of arguments supported.

******0 Verbosity and similar arguments:

 -q
    ... quiet mode, registration does not print tokens to standard output

 -v
    ... verbose = show some of the commands being executed

 --vv
    ... superverbose, show also data being sent


******1 Connection arguments:

 -s <server-IP address> 
    ... Use provided IP address instead of the default one (127.0.0.1).

 --nossl
    ... use non-SSL URLs

 --servid <server_uuid>
    ... use server_uuid as Server ID. Note that server ID is mandatory for registration since MAMAS 0.6. If not provided, emulator attempts to read serverID from ./base_defs and /opt/dwarfg/base_defs in that order. As last resort, default testing UUID is attempted (0badc0de-beef-f00d-cafe-4badf001babe)


******2 Device type, number of devices and other basic arguments:

 -t <agents_type>
    ... specifies agent type FOR NEW AGENTS. Four agent types and random choice are supported:
        0 ... choose random device type among 1 ... 4 (DEFAULT)
        1 ... emulate Linux agent(s)
        2 ... emulate Advantech router agent(s)
        3 ... emulate OpenWRT agent(s)
        4 ... emulate Teltonika router agent(s)
        5 ... emulate Tectonic gateway agent(s) - not fully supported ATM
        6 ... emulate SNMP gateway handled device(s) - do not use unless you know what exactly are you doing

 -n <number_of_agents>
    ... specifies number of agents to start (DEFAULT: 1)
    ... note that specifying both -n and -P results in -n being ignored during data pust (still effective for register phase though)


******3 Arguments regarding data options and retention

 -r
    ... Read (and write) agents from (to) a file (ids_file.txt by default). Preserves agents IDs. Agents read from the file are neither registered again in the server not their existence in the server is checked.

 -f <filename>
    ... Use <filename> for a file storing device IDs. Implies -r

 -B
    ... Send big cycle cached data instead of any small data on data push (always big).
    ... NOTE that for intelligent multi-run, first data push is big cache anyway.
    ... NOTE that when sending varied data, the big data push has no way how to account for counters increments in data (e.g. traffic on a network interface) so when using varied data, big pushes are discouraged except the initial big data push

 -F
    ... freeze time - do not update Time and sysclock to actual system time

 -V
    ... send data varied by randomization where possible
    ... you MUST use this consistently - e.g. using -V with -r where the original registration and data push were done without -V can lead to non-sensical data on server (e.g. changing of serial numbers and so on) and to emulator crashes and vice versa. Either use -V always (for all runs when working with devices stored in a file) or never.

 --df <file>
    ... use specified file to send data, not the default example file

 -d <datatype>
    ... send also data of following type:
       0 ... skip any extra data sending (default behavior)
       1 ... send defaults data (once per agent ID per session)
       2 ... send backup data (first time and then after every 4 times - NOTE: only single-sending mode implemented ATM)
       3 ... send both defaults and backup data (NOTE: only single sending mode implemented)

 --dd <file>
    ... specific defaults file to be sent (if sending defaults, use this file)

 --dc <file>
    ... specific cfg backup file to be sent (if sending cfg backup, use this file)

 -b
    ... blank / re-register. Read agent IDs from file and use original devids as preferred devids for registration. NOTE this works only for singlethreaded use (do not use with --p<x> parameters). Re-registers up to the original number of agents (up to the number of agents requested) and continues with new registrations until required number of agents (-n <number>) is reached. Possible unused old devids in IDS file are discarded.

 -P <devid[,devid...]>
    ... specify a preferred DevID(s) when registering new devices.
    ... note that this effectively disables effect of -n for pushing data (but not for registering)


******4 Arguments for parallell run and continuous execution:

 --p<mode> <number of processes>
    ... parallel/peak testing; mode is one of 
       r ... register peak test. A number of agents is registered as fast as possible
       R ... registration including initial data upload peak test. After registration it continually sends data, defaults and config.
       d ... data peak test. A nuber of agents do send data.
       D ... data long performance test - a number of agents is sending data periodically

 --e<mode> <number of threads>
    ... emulate real devices. Implies -d 3 -I -R
    ... mode:
       u ... uniform data (push exactly same data for each and every device)
       v ... varied data (randomize some of the data based on definitions - see -V for reference)

 -c
    ... continuous mode. Keep sending data (behaving similar to the real agent) for indefinite time.

 -l <number>
    ... Do a number of loops of data pushing. Exit after all loops are performed.

 -T <number>
    ... run for <number> of seconds at minimum. Real time is usually a bit longer.


******5 Arguments affecting behavior, data and time-dependent behavior:

 -w <seconds>
    ... set wait (cycle) time to a number of <seconds>. DEFAULT: 260
    ... minimal time between data sending cycles (if cycle faster, sleep for a while)

 -W <time>
    ... wait with startup until time. Time must be in this format: "hour:minute:second"
       the script sleep until time is due

 -I
    ... Intelligent - if sending defaults and config is requested, do it but for FIRST PUSH ONLY
    ... First data push is BIG data
    ... If varied data are requested, the first data push does no addition on addition lines, but uses the predefined value to not mess up the server counters for subsequent (small) data pushes.

 -R
    ... Randomize requests over the given period. Usually in conjunction with -n -w and -c


******6 Planned but not yet supported aruments:

 --cert <certificate_file>
    ... specify path to server.pem. If no path is provided, ./server.pem and /opt/dwarfg/server.pem is attempted in case certificate check is performed. (not yet supported)


******7 Usage examples:

 <no options> ... Runs one-off random-type agent: registers, makes one small data push, ends (device forgotten).

 -n 10 ... register 10 new IDs (devices) to the server and make small data push, forget devices.

 -r ... read agent ID(s) from file (ids_file.txt). If file missing or empty or too few devices there, register new ID(s). Any newly registered device will be appended to the file.

 -n 5 -r ... Reads agents from file (if there). Register new agents if less than 5 IDs present. Push data file for these 5 agents (TBD). Store new IDs into the file.

 -R -n 50 -w 100 -c ... Emulate 50 devices within interval of 100 seconds each connecting once, run this cycle continuously. Randomize device contacts over the 100 seconds. NOTE that a device may come at start or end of the interval so the delay between two contacts may be anywhere between 1 second and double the interval (200 seconds in this case). Only one process is spawned this way so this is not very useful when you need to emulate many devices.

 -R -n 50 -w 100 -l 10 ... Like above but end after 10 loops of pushing the data.

 --pd 8 -n 2000 -w 260 -c -R ... Like above but split this between 8 threads, each needing to emulate 250 contacts in 260 seconds. Repeats indefinitely (-c).

 --pD 4 -n 400 -d 3 -I -w 120 -R -T 600 ... Push data in 4 threads, for 400 devices, send full profile intelligently (defaults and backup on first push only), resend for each device once every 120 seconds, spread device's data pushes at random during the interval, end at first timecheck after 600 seconds elapsed.

""")




agents_type = 0
agents_number = 1
agents_mode = 0
agents_lifetime = 0
agents_readfile = 0
read_from_file = 0
delayint = 260
deadline = 0
terminate = False
intelligent = False
freeze = False
ids_filename = "ids_file.txt"
agents_added = 0
first_new_agent = 0
send_defaults = False
send_bcfg = False
use_ssl = True
verbose = False
superverbose = False
varied_data = False
file_defaults = None
file_bcfg = None
multirun = False
mul_proc = 1
mul_register = False
mul_data = False
mul_enhanced = False
continuous = False
myserver = None
maxtype = None
myiccid = None
silent = False
blank = False
prefids = None
randomize = False
time_start = time.time()
time_actual = time_start
time_last = time_start
big_force = False
servid = ""
loops = -1
aloop = 0
startup_time = ""
targettime = datetime.datetime.now()
emulate = False
total_ok = total_fail = total_sum = 0
total_resstrs = Counter()
total_resstrs2 = Counter()
cyclewait = threading.Event()

def ctrl_c_handle(signum, frame):
    global terminate, cyclewait
    terminate = True
    cyclewait.set()

# register count devices in a sequence, put result in queue
def proc_register(count):
    global terminate
    mrc = Counter()
    retnums = [0, 0]
    retlist = []
    for i in range(count):
        result, resstr = matelib.register(retlist, agents_type, debug = verbose, use_ssl = use_ssl, server = myserver, servid = servid, silent = silent, superdebug = superverbose, iccid = myiccid, maxdevt = maxtype)
        if result:
            retnums[0] += 1
        else:
            retnums[1] += 1
        mrc[resstr] += 1
        if terminate:
            break
    return [retnums, retlist, mrc]

def gimmemyrand(seconds, todo):
    if 0 == seconds:
        return todo
    myrand = random.randint(-50,100)
    if 0 == myrand:
        return min(todo,int(todo/seconds))
    elif 0 < myrand:
        return min(todo, int(todo/seconds*(1+myrand**2/25/100)))
    else:
        return min(todo, int(todo/seconds*((100-abs(myrand)**2/25)/100)))

def proc_emulate(devlist, count, single = False):
    global terminate
    print("emulation thread, devlist len/cont", str(len(devlist)), "/", str(count))
    done = 0
    seconds = delayint
    retlist = []
    newdevs = []
    retnums = [0, 0]
    retnums_push = [0, 0]
    mrc = Counter()
    mdc = Counter()
    while done < count: # until we have all requested...
        thiscycle = 0
        if terminate:
            break
        thiscycle = gimmemyrand(seconds, count-done)
        if seconds > 0: # there is time left...
            time.sleep(1) # otherwise sleep a bit
            seconds -= 1
        if single and not silent:
            if 0 == thiscycle:
                sys.stdout.write('.')
            else:
                if 10 > thiscycle:
                    sys.stdout.write(chr(48+thiscycle))
                else:
                    sys.stdout.write('+')
            sys.stdout.flush()
        dts = 0
        while dts < thiscycle:
            mybig = False
            if terminate:
                break
            if (retnums[1] > count):
                terminate = True
                continue
            if done >= len(devlist): # this device is not yet registered
                result, resstr = matelib.register(devlist, agents_type, debug = verbose, use_ssl = use_ssl, server = myserver, servid = servid, silent = silent, superdebug = superverbose, iccid = myiccid, maxdevt = maxtype)
                mrc[resstr] += 1
                mybig = True
                if result:
                    # registration successfull, remember, data push is below...
                    retnums[0] += 1
                    newdevs.append(devlist[-1])
                else:
                    # not successfull registration, iterate the same one again
                    retnums[1] += 1
                    continue
            else:
                mybig = False
            # now push the data
            push_code, dwarfg_code, _ = matelib.push_data(devlist[done][0], devlist[done][1], devlist[done][2], devlist[done][3], devlist[done][4], devlist[done][5], devlist[done][6], [devlist[done][7], devlist[done][8]], datafpath = file_data, send_defaults = send_defaults, send_bcfg = send_bcfg, debug = verbose, file_defaults = file_defaults, file_bcfg = file_bcfg, server = myserver, use_ssl = use_ssl, force_bigcache = mybig or big_force, silent = silent, varied = varied_data, update_time = not freeze, superdebug = superverbose, maxdevt = maxtype)
            if push_code:
                retnums_push[0] += 1
                if not devlist[done][6]: # first push done
                    devlist[done][6] = True
            else:
                retnums_push[1] += 1
            mdc[dwarfg_code] += 1
            done += 1 # global done
            dts += 1 # this cycle done
    cyclewait.wait(seconds)
    # wait rest of time
    if not silent and single: # chatty about how long are we waiting...
        if 0 < seconds:
            print(" ... wait ", str(seconds), " ...")
        else:
            print()
    cyclewait.wait(seconds) # interruptible wait
    return [retnums_push, devlist, mdc, retnums, mrc, newdevs]

# push data for devices in devlist in a sequence (or randomized over a time), return results
def proc_push(devlist, single = False):
    global terminate
    mrc = Counter()
    seconds = delayint
    todo = len(devlist)
    done = 0
    retnums = [0, 0]
    while True:
        myrand = 0
        if not randomize:
            actlist = devlist
        else:
            thiscycle = gimmemyrand(seconds, todo)
            if seconds > 0:
                time.sleep(1)
                seconds -= 1
            if 0 == thiscycle:
                if single:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                continue
            actlist = devlist[done:][:thiscycle]
            done += thiscycle
            todo = len(devlist)-done
        for i in actlist:
            push_code, dwarfg_code, _ = matelib.push_data(i[0], i[1], i[2], i[3], i[4], i[5], i[6], [i[7], i[8]], datafpath = file_data, send_defaults = send_defaults, send_bcfg = send_bcfg, debug = verbose, file_defaults = file_defaults, file_bcfg = file_bcfg, server = myserver, use_ssl = use_ssl, force_bigcache = big_force, silent = silent, varied = varied_data, update_time = not freeze, superdebug = superverbose, maxdevt = maxtype)
            if push_code:
                retnums[0] += 1
                i[6] = True
            else:
                retnums[1] += 1
            mrc[dwarfg_code] += 1
        if randomize and single and not silent:
            if len(actlist) < 10:
                sys.stdout.write(chr(48+len(actlist)))
            else:
                sys.stdout.write('+')
            sys.stdout.flush()
        if not randomize or 0 == todo:
            if randomize and not silent:
                if single:
                    print(" ... wait ", str(seconds), " ...")
                if 0 < seconds:
                    cyclewait.wait(seconds)
                    #time.sleep(seconds)
            break
        if terminate:
            break
    return [retnums, devlist, mrc]

# run either register XOR data process for devcount devices in mul_proc threads, possibly applying randomization...
def multirun_exec(register, data, emulate, mul_proc, devlist, devcount):
    if verbose:
        print("multirun_exe, register/data/emulate/mul_proc: ", register, "/", data, "/", emulate, "/", mul_proc)
    if (register, data, emulate).count(True) != 1:
        print("multirun_exec: no (or more han one) multirun mode(s) requested")
        return False
    proclist = []
    global total_ok, total_fail, total_sum, total_resstrs, aloop, devices
    totresnums = [0, 0]
    completelist = []
    resstrs = Counter()
    resstrs2 = Counter()
    if register and (0 == mul_proc or 0 >= devcount):
        print("Invalid parameters for register multirun - both number of processes and number of devices must be greater than zero")
        return
    if data and (0 == mul_proc or 0 >= len(devlist)):
        print("Invalid parameters for data multirun - both data and number of devices must be greater than zero")
        return
    num = int(devcount / mul_proc)
    elist = mul_proc * [num]
    data_lists = []
    for i in range(1, 1+mul_proc):
        data_lists.append(devlist[(i-1)*num:i*num])
    if 0 < (devcount % mul_proc):
        elist.append(devcount % mul_proc)
        data_lists.append(devlist[i*num:])
    time_start = time.time()
    if 1 != mul_proc:
        print("Starting process(es), time is ", datetime.datetime.now());
    if register or data or emulate:
        with Pool(len(elist)) as p:
            if emulate:
                resultarray = p.starmap(proc_emulate, zip(data_lists, elist, len(elist)*[1 == mul_proc]))
            elif register:
                resultarray = p.starmap(proc_register, zip(elist))
            else:
                resultarray = p.starmap(proc_push, zip(data_lists, len(elist) * [1 == mul_proc]))
        completeok = 0
        completefail = 0
        if emulate:
            opstring = "emulations"
        elif register:
            opstring = "registrations"
        else:
            opstring = "data pushes"
        for i in range(len(resultarray)):
            result = resultarray[i]
            if 1 != mul_proc:
                if emulate:
                    print("Thread " + str(i) + " returned pushes succ/fail: " + str(result[0][0]) + '/' + str(result[0][1]) + " registers succ/fail: " + str(result[3][0]) + "/" + str(result[3][1]) + " handled " + str(len(result[1])) + " devices")
                else:
                    print("Thread " + str(i) + " returned " + str(result[0][0]) + " successful and " + str(result[0][1]) + " failed " + opstring + ". Total results per thread: " + str(len(result[1])))
            completeok += result[0][0]
            completefail += result[0][1]
            completelist += result[1]
            resstrs.update(result[2])
            if emulate:
                resstrs2.update(result[4])
                if len(result[5]) > 0:
                    print("Got ", str(len(result[5])), " devices")
                    for dev in result[5]:
                        devices.append(dev)
        time_regdone = time.time()
        if 1 != mul_proc:
            print("Collected results, time is ", datetime.datetime.now(), " tot.time diff ", str(time_regdone-time_start))
        if data or emulate: aloop += 1
        print("LOOP " + str(aloop) + " results (success/fail/sum): " + str(completeok) + "/" + str(completefail) + "/" + str(len(completelist)))
        total_ok += completeok
        total_fail += completefail
        total_sum += len(completelist)
        total_resstrs.update(resstrs)
        if emulate:
            total_resstrs2.update(resstrs2)
        print("Returned strings: <string : count>")
        for key in resstrs:
            print(key, " : ", resstrs[key])
        if emulate:
            for key in resstrs2:
                print("+reg: ", key, " : ", resstrs2[key])
        if register:
            for i in completelist:
                devices.append(i)
        else:
            devices = completelist
    return 0

#  -----  main program -----
matelib.init()
usefile = False
try:
    opts, args = getopt.getopt(sys.argv[1:], 'w:W:t:n:f:d:s:i:l:T:P:bBIrRqvVcF', [ 'df=', 'dd=', 'dc=', 'pr=', 'pR=', 'pd=', 'pD=', 'eu=', 'ev=', 'nossl', 'servid=', 'vv', 'maxtype=' ])
    #print("Number of options: " + str(len(opts)) + "\nNumber of non-recognized options: " + str(len(args)))
    for opt, arg in opts:
        if opt == "-r":
            usefile = True
        if opt == "-I":
            intelligent = True
        if opt == "-T":
            deadline = time_start + int(arg)
        if opt == "-R":
            randomize = True
        if opt == "-t":
            agents_type = int(arg)
        if opt == "-b":
            blank = True
            usefile = True
        if opt == "-P":
            prefids = str(arg).split(',')
        if opt == "-B":
            big_force = True
        if opt == "-l":
            loops = int(arg)
        if opt == "-n":
            agents_number = int(arg)
        if opt == "-f":
            ids_filename = str(arg)
            usefile = True
        if opt == "-F":
            freeze = True
        if opt == "-w":
            delayint = int(arg)
        if opt == "-W":
            values = arg.split(":")
            if len(values) != 3 or int(values[0]) < 0 or int(values[0]) > 24 or int(values[1]) < 0 or int(values[1]) > 59 or int(values[2]) < 0 or int(values[2])>59:
                print("Invalid startup time given. Should be \"hour:minute:second\" is:", arg)
                sys.exit(1)
            startup_time = arg
            now = datetime.datetime.now()
            targettime = now.replace(hour=int(values[0]), minute=int(values[1]), second=int(values[2]))
        if opt == "-s":
            myserver = str(arg)
        if opt == "-i":
            myiccid = str(arg)
        if opt == "-d":
            if 1 == int(arg) or 3 == int(arg):
                send_defaults = True
            if 1 < int(arg):
                send_bcfg = True
        if opt == "-c":
            continuous = True
        if opt == "-v":
            verbose = True
        if opt == "--vv":
            verbose = True
            superverbose = True
        if opt == "-V":
            varied_data = True
        if opt == "-q":
            silent = True
        if opt == "--dd":
            file_defaults = str(arg)
        if opt == "--nossl":
            use_ssl = False
        if opt == "--servid":
            servid = str(arg)
            print("Servid is ", str(servid))
        if opt == "--maxtype":
            maxtype = int(arg)
        if opt == "--dc":
            file_bcfg = str(arg)
        if opt == "--df":
            file_data = str(arg)
        if opt == "--pr" or opt == "--pR" or opt == "--pd" or opt == '--pD':
            multirun = True
            mul_proc = int(arg)
            if 'r' == opt[3].lower():
                mul_register = True
            if 'd' == opt[3].lower():
                mul_data = True
            if opt[3].isupper():
                mul_enhanced = True
                if mul_register:
                    send_defaults = True
                    send_bcfg = True
        if opt == "--eu" or opt == "--ev":
            intelligent = True
            randomize = True
            send_defaults = True
            send_bcfg = True
            emulate = True
            multirun = True
            mul_proc = int(arg)
            if opt == "--ev":
                varied_data = True
except getopt.GetoptError:
    print("Error when parsing cmdline arguments from: ", ' '.join(sys.argv))
    print_help()
    sys.exit(1)
ids_prep = 0
if randomize:
    random.seed(None)
if mul_enhanced:
    if mul_data:
        if -1 == loops:
            continuous = True
    if mul_register:
        mul_data = True
# wait for startup time if given
if 0<len(startup_time):
    print("Delaying startup to ", startup_time, " - ", str((targettime-datetime.datetime.now()).total_seconds()), " seconds...")
    time.sleep((targettime-datetime.datetime.now()).total_seconds())
for sig in ('TERM', 'HUP', 'INT'):
        signal.signal(getattr(signal, 'SIG'+sig), ctrl_c_handle);
# Get number of ready agent IDs (read from file if -r or set to 0)
script_start_time = datetime.datetime.now()
print("Datetime start: ", script_start_time)
if mul_register:
    multirun_exec(mul_register, False, False, mul_proc, devices, agents_number)
    if not mul_data:
        if usefile:
            matelib.writedevstofile(devices, ids_filename, 0, True)
        script_finish_time = datetime.datetime.now()
        print("Datetime finish: ", script_finish_time, "\nElapsed:", (script_finish_time - script_start_time).total_seconds(), "seconds.")
        sys.exit(0) # this is a one-time operation, never to be repeated or followed by another operation unless 'R' option is implemented
else:
    if usefile and not mul_register:
        print("Reading agents from file...")
        if (not matelib.readdevsfromfile(devices_read, ids_filename)):
            print("Failed to read devices from file " + ids_filename)
        else:
            print("Read " + str(len(devices_read)) + " agents from file.")
    # if prefids were given, filter out what was read by prefids
    if prefids is not None and len(prefids) > 0:
        for dev in devices_read:
            if dev[0] in prefids:
                devices.append(dev)
                prefids.remove(dev[0])
            else:
                devices_untouched.append(dev)
        print("Filtered", len(devices), "device(s) out of", len(devices_read), "read from file.")
    else:
        devices = devices_read
    if len(devices) > agents_number:
        devices_untouched += devices[agents_number:]
        devices = devices[:agents_number]
    # If agents are missing, register new ones (adding them to the file later on when -r was used)
    first_new_agent = len(devices)
if first_new_agent > 0 and blank:
    # re-register: prepare old agent IDS
    agent_oldids=[]
    agent_machids=[]
    for i in devices:
        agent_oldids.append(i[0])
        agent_machids.append(i[2])
    # empty devices list
    print("Trying to blank/re-register " + str(len(agent_oldids)) + " devices...")
    devices=[]
    # re-register up to requested number of agents or max number of old agent IDS (dropping additional ones)
    result, resstr = matelib.register(devices, agents_type, min(agents_number, len(agent_oldids)), server = myserver, debug = verbose, use_ssl = use_ssl, servid = servid, silent = silent, prefdevid = agent_oldids, initmachids = agent_machids, iccid = myiccid, maxdevt = maxtype)
    if not result:
        sys.exit("Error when blanking/re-registering devices: " + resstr)
if not emulate and len(devices)<agents_number: # registers missing agents
    print("There are", str(len(devices)), "devices and", str(agents_number), "are requested.")
    attempts = 0
    while attempts<3 and agents_number - len(devices) > 0:
        print("Registering (seq) " + str(agents_number-len(devices)) + " agents...")
        iteration_start = len(devices)
        result, resstr = matelib.register(devices, agents_type, agents_number - len(devices), server = myserver, debug = verbose, use_ssl = use_ssl, servid = servid, silent = silent, iccid = myiccid, prefdevid = prefids, maxdevt = maxtype)
        if not result and len(devices) == iteration_start: # if none registered this round, increase # of failed attempts
            attempts += 1
        agents_added += len(devices) - iteration_start # increase number of added agents
        if len(devices) < agents_number: # if some agents are missing (there were some errors actually), wait a bit
            time.sleep(1)
    if len(devices) < agents_number:
        print("Failed to register enough agents before number of failed register cycles too high, bailing out")
        sys.exit(1)
if (agents_added):
    print("Registered " + str(agents_added) + " new agents.")
    if 1 == agents_added:
        print("DEVID_SINGLE/STR:", str(devices[0][0]))
        print("DEVID_SINGLE/INT:", str(matelib.devid_int(devices[0][0])))
loop = True
print("Pushing data...")
cycle = 0
first_big = False
if intelligent and not big_force:
    first_big = True # remember to switch big data off after first cycle
    big_force = True # enable big for first cycle
while True:
    cycle += 1
    if 2 == cycle and first_big:
        big_force = False # disable big after first cycle if should do it
    if (send_bcfg or send_defaults) and 0 < aloop and intelligent:
        send_bcfg = False
        send_defaults = False
    if emulate:
        multirun_exec(False, False, True, mul_proc, devices, agents_number)
    elif mul_data:
        multirun_exec(False, mul_data, False, mul_proc, devices, agents_number)
    else:
        multirun_exec(False, True, False, 1, devices, agents_number)
#        while i<agents_number:
#            print("Pushing data for device no. " + str(i) + " id: " + str(devices[i][0]) + " which is type " + str(devices[i][1]))
#            matelib.push_data(devices[i][0], devices[i][1], devices[i][2], datafpath = file_data, send_defaults = send_defaults, send_bcfg = send_bcfg, debug = verbose, file_defaults = file_defaults, file_bcfg = file_bcfg, server = myserver, use_ssl = use_ssl)
#            i = i+1
    if deadline:
        if deadline < time.time():
            print("Time elapsed, ending emlator run...")
            break
        else:
            print("Remaining time:", str(deadline-time.time()), "(roughly max of", str(int(1+(deadline-time.time())/delayint)), "loops)")
    if 0 < loops:
        loops -= 1
    if terminate or (not continuous and 0 >= loops):
        break
    if not randomize:
        time_actual = time.time()
        if time_last + delayint > time_actual:
            print("Sleeping for " + str(time_last + delayint - time_actual) + " seconds...")
            cyclewait.wait(int(time_last + delayint - time_actual))
            #time.sleep(time_last + delayint - time_actual)
        time_last = time.time();
    if terminate:
        break
    if deadline and deadline < time.time():
        print("Time elapsed, ending emlator run...")
        break
# Write extra agents and all used dicts to the file
if usefile and (varied_data or agents_added or blank or len(devices)>first_new_agent):
    if blank:
        print("Blanking file with " + str(len(devices)) + " agents.")
        first_new_agent = 0
    else:
        print("Adding agents to file (" + str(len(devices) - first_new_agent) + " newly registered " + str(len(devices_untouched)) + " old untouched and " + str(len(devices)) + " agents total)")
    agents_added = 0
    devstowrite = devices
    if blank:
        appendfile = False
    else:
        appendfile = 0 == len(devices_untouched) and first_new_agent == 0
        devstowrite += devices_untouched
        print("Untouched:", str(len(devices_untouched)), "first new:", str(first_new_agent), "total:", str(len(devstowrite)), "append:", str(appendfile))
    if not matelib.writedevstofile(devstowrite, ids_filename, first_new_agent, appendfile):
        print("Error writing agents to file: " + ids_filename)
if total_ok >0 or total_fail >0 or total_sum >0:
    print("\nTOTAL results (success/fail/sum): " + str(total_ok) + "/" + str(total_fail) + "/" + str(total_sum))
    print("Returned strings: <string : count>")
    for key in total_resstrs:
        print(key, " : ", total_resstrs[key])
    for key in total_resstrs2:
        print("+reg: ", key, " : ", total_resstrs2[key])
script_finish_time = datetime.datetime.now()
print("Datetime finish: ", script_finish_time, "\nElapsed:", (script_finish_time - script_start_time).total_seconds(), "seconds.")