def execute(self, execstr, args=None, retry=2):
     """ execute string on database. """
     time.sleep(0.001)
     result = 0
     execstr = execstr.strip()
     if 'sqlite' in self.dbtype: execstr = execstr.replace('%s', '?')
     if self.dbtype == 'mysql':
         try: self.ping()
         except AttributeError: self.reconnect()                
         except Exception as ex:
             try: self.reconnect()
             except Exception as ex: logging.error('failed reconnect: %s' % str(ex)) ; return
     logging.debug('exec %s %s' % (execstr, args))
     got = False
     counter = 0
     for i in range(retry):
         if not self.connection: self.reconnect()
         if not self.connection: raise NoDbConnection()
         if "sqlite" in self.dbtype:
             try: result = self.doit(execstr, args) ; got = True ; break
             except sqlite3.OperationalError as ex: logging.info(str(ex))
             except Exception as ex: handle_exception() ; logging.error(str(ex))
         else:
             try: result = self.doit(execstr, args) ; got = True ; break
             except Exception as ex: logging.error(str(ex))
         counter += 1
     if got:
         if counter == 1: logging.warn("db query is ok - 1 retry" % counter) 
         else: logging.warn("db query is ok - %s retries" % counter)
     return result

 def doit(self, execstr, args=None):
     logging.debug("%s/%s" % (execstr, str(args)))
     cursor = self.cursor()
     nr = 0
     try:
         if args != None:
             if type(args) == tuple or type(args) == list: nr = cursor.execute(execstr, args)
             else: nr = cursor.execute(execstr, (args, ))
         else: nr = cursor.execute(execstr)
     except Exception as ex:
         logging.warn(str(ex))
         if self.dbtype == 'postgres': cursor.execute(""" ROLLBACK """)
         if 'sqlite' in self.dbtype: cursor.close() ; del cursor
         raise 
     got = False
     if execstr.startswith('INSERT'): nr = cursor.lastrowid or nr ; got = True
     elif execstr.startswith('UPDATE'): nr = cursor.rowcount ; got = True
     elif execstr.startswith('DELETE'): nr = cursor.rowcount ; got = True
     if got: self.commit()
     if 'sqlite' in self.dbtype and not got and type(nr) != int():
         nr = cursor.rowcount or cursor.lastrowid
         if nr == -1: nr = 0
     result = None
     try:
         result = cursor.fetchall()
         if not result: result = nr
     except Exception as ex:
         if 'no results to fetch' in str(ex): logging.warn("no results to fetch")
         else: handle_exception()
         result = nr
     cursor.close()
     return result

def plusscan(skip=False):
    global teller
    teller += 1
    if teller % 5 != 0: return 
    logging.warn("running plus scan")
    fleet = getfleet()
    for id, channels in state.data.ids.items():
        if not id in state.data.seen: state.data.seen[id] = []
        for botname, chan in channels:
            try:
                res = getplus(id)
                if not res: logging.warn("no result from %s" % id) ; continue
                bot = fleet.byname(botname)
                if bot:
                    todo = []
                    for r in res:
                        stamp = uuid.uuid3(uuid.NAMESPACE_URL, str(r)).hex
                        if stamp not in state.data.seen[id]:
                            state.data.seen[id].append(stamp)
                            todo.append(r)
                    if todo: bot.say(chan, "new plus update: " , todo)
                else: logging.warn("no %s bot in fleet" % botname)
            except AttributeError as ex: logging.error(str(ex))
            except Exception as ex: handle_exception()
    state.save()

 def incoming(self, data):
     """ process incoming hubbub data. """
     result = feedparser.parse(data)
     url = find_self_url(result.feed.links)
     logging.debug("%s - %s" % (url, data))
     try:
         item = self.byurl(url)
         if not item: logging.warn("can't find feed for %s" % url) ; return
         if not item.data.running: logging.warn("%s is not in running mode" % item.data.url) ; return
         if not item.data.url or item.data.url == 'urlnotset':
             item.data.url = url
             item.save()
         if item:
             loopover = item.data.watchchannels
             name = item.data.name
         else: logging.warn("can't find %s item" % url) ; return
         logging.warn("loopover in %s peek is: %s" % (name, loopover))
         counter = 1
         for i in loopover:
             if len(i) == 3:
                 try: (botname, type, channel) = i
                 except:
                     try: (botname, type, channel) = loads(i)
                     except:
                         logging.warn('%s is not in the format (botname, bottype, channel)' % str(item))
                         continue
             else:
                 logging.warn('%s is not in the format (botname, bottype, channel)' % item.data.url)
                 continue
             counter += 1
             start_new_thread(self.work, (botname, type, channel, result.entries, url), {"_countdown": counter})
     except Exception as ex: handle_exception(txt=url)
     return True

 def reloadcheck(self, bot, event, target=None):
     """ check if plugin need to be reloaded for callback, """
     from .boot import isdisabled
     plugloaded = []
     done = []
     target = target or event.cbtype or event.cmnd
     if not event.nolog: logging.debug("%s - checking for %s events" % (bot.cfg.name, target))
     try:
         from .boot import getcallbacktable   
         p = getcallbacktable()[target]
     except KeyError:
         if not event.nolog: logging.debug("can't find plugin to reload for %s" % event.cmnd)
         return
     if not event.nolog: logging.debug("found %s" % str(p))
     for name in p:
         if name in bot.plugs: done.append(name) ; continue
         if isdisabled(name):
             logging.info("%s - %s is disabled" % (bot.cfg.name, name))
             continue
         elif bot.cfg.loadlist and name not in bot.cfg.loadlist:
             logging.info("%s - %s is not in loadlist" % (bot.cfg.name, name))
             continue
         if not event.nolog: logging.debug("%s - on demand reloading of %s" % (bot.cfg.name, name))
         try:
             mod = bot.plugs.reload(name, force=True, showerror=True)
             if mod: plugloaded.append(mod) ; continue
         except Exception as ex: handle_exception(event)
     if done and not event.nolog: logging.debug("%s - %s is already loaded" % (bot.cfg.name, str(done)))
     return plugloaded

 def _listen(self):
     """ listen for udp messages .. /msg via bot"""
     if not cfg['udp']: return
     fleet = getfleet()
     for botname in cfg['udpbots']:
         if not fleet.byname(botname): logging.info("udp - can't find %s bot" % botname)
     try:
         fleet.startok.wait(5)
         logging.warn('udp listening on %s %s' % (cfg['udphost'], cfg['udpport']))
         self.sock.bind((cfg['udphost'], cfg['udpport']))
         self.stop = 0
     except IOError:
         handle_exception()
         self.sock = None
         self.stop = 1
         return
     # loop on listening udp socket
     while not self.stop:
         try: input, addr = self.sock.recvfrom(64000)
         except socket.timeout: continue
         except Exception as ex:
             try: (errno, errstr) = ex
             except ValueError: errno = 0 ; errstr = str(ex)
             if errno == 4: logging.warn("udp - %s - %s" % (self.name, str(ex))) ; break
             if errno == 35: continue
             else: handle_exception() ; break
         if self.stop: break
         self.queue.put((input, addr))
     logging.info('udp - shutting down main loop')

 def docbs(self, bot, event):
     """ do the actual callback .. put callback on the waitrunner for execution. """
     if not self.cbs: return
     logging.warn("%s - found wait match: %s" % (bot.cfg.name, event.dump()))
     for cb in self.cbs:
         try: cb(bot, event) #waitrunner.put(event.speed, (self.modname, cb, bot, event))
         except Exception as ex: handle_exception()

def handle_rc(bot, event):
    """ arguments: <file>|<url> - execute a .tl resource file with bot commands. """
    if not event.rest: event.missing("<file>|<url>") ; return
    if not getmainconfig().allowrc: event.reply("rc mode is not enabled") ; return
    teller = 0
    t = event.rest
    waiting = []
    try:
        try:
            if getmainconfig().allowremoterc and t.startswith("http"): data = geturl2(t)
            else: data = open(t, 'r').read()
        except IOError as ex: event.reply("I/O error: %s" % str(ex)) ; return
        if not data: event.reply("can't get data from %s" % event.rest) ; return
        for d in data.split("\n"):
            i = d.strip()
            if not i: continue
            if i.startswith("#"): continue
            e = cpy(event)
            e.txt = "%s" % i.strip()
            e.direct = True
            bot.put(e)
            waiting.append(e)
            teller += 1
        event.reply("%s commands executed" % teller)
    except Exception as ex: event.reply("an error occured: %s" % str(ex)) ; handle_exception()

 def work(self, botname, type, channel, entries, url, *args, **kwargs):
     logging.debug("in work .. %s" % str(entries))
     try:
         item = self.byurl(url)
         name = item.data.name
         try:
             fleet = getfleet()
             bot = fleet.byname(botname)
             if not bot: logging.warn("can't find %s bot in fleet" % botname) ; return
         except NoSuchBotType as ex: logging.warn("%s" % str(ex)) ; return
         if not bot: logging.error("can't find %s bot in fleet" % type) ; return
         res2 = entries
         if not res2: logging.info("no updates for %s (%s) feed available" % (item.data.name, channel)) ; return
         if item.markup.get(jsonstring([name, channel]), 'reverse-order'): res2 = res2[::-1]
         if item.markup.get(jsonstring([name, channel]), 'all-lines'):
             for i in res2:
                 response = self.makeresponse(name, [i, ], channel)
                 try: bot.say(channel, response)
                 except Exception as ex: handle_exception()
         else:
             sep =  item.markup.get(jsonstring([name, channel]), 'separator')
             if sep: response = self.makeresponse(name, res2, channel, sep=sep)
             else: response = self.makeresponse(name, res2, channel)
             bot.say(channel, response)
     except Exception as ex: handle_exception()

def handle_geoJOIN(bot, event):
    event.reply("geo - doing query on %s" % event.hostname)
    try:
        result = querygeoipserver(host2ip(event.hostname))
        if result: event.reply("%s lives in %s, %s (%s)" % (event.nick, result['city'], result['country_name'], result['country_code']))
        else: event.reply("no result")
    except: handle_exception()

 def startshell(self, connect=True):
     """ start the console bot. """
     self.start(False)
     print(getfullversion("CONSOLE"))
     while not self.stopped: 
         try: 
             mainhandler.handle_one()
             input = console.raw_input("\n> ")
             if self.stopped: return
             event = ConsoleEvent()
             event.parse(self, input, console)
             event.nooutput = True
             event.nodispatch = False
             e = self.put(event)
             res = e.wait()
             if res:
                 sys.stdout.write("\n")
                 txt = self.makeresponse(res, dot="<br>")
                 self.out(e.userhost, txt)
             sys.stdout.flush()
         except TLStop: break
         except IOError: break
         except NoInput: continue
         except (KeyboardInterrupt, EOFError): break
         except Exception as ex: handle_exception() ; break
     console.save_history()

def handle_adminreloadconfig(bot, event):
    """ no arguments - reload bot config and mainconfig files. """
    try:
        bot.cfg.reload()
        getmainconfig().reload()
    except Exception as ex: handle_exception()
    event.done()

    def handle(self, job):
        speed, event, url, depth, spiderspeed = job.args
        if not url: logging.error("no url provided") ; return
        if depth < 0: return
        if not self.url.base in url: logging.warn("skipping %s (%s)" % (url, self.url.base)) ; return
        if url in self.errors: logging.warn("skipping %s" % url) ; return
        urls = []
        linknr = 0
        follownr = 0
        n = 0
        try:
            if url not in self.urls:

                self.urls.append(url)
                page = Url(url)
                time.sleep(10-spiderspeed)
                content = page.fetch()
                event.reply("fetched %s - %s - %s" % (url, len(content), content.status))
                try:
                    urldata = UrlData(url, striphtml(content))
                    if urldata.data.txt: urldata.save()
                except Exception as ex: handle_exception()
                for p in page.geturls():
                    if not p in self.errors:
                        self.put(6, event, p, depth-1, spiderspeed-1)
            if not self.queue.qsize(): self.stop()
        except urllib.error.URLError as ex: logging.warn("error fetching %s url: %s" % (url, str(ex)))
        except Exception as e:
            logging.warn("ERROR: Can't process url '%s' (%s)" % (url, e))
            self.errors.append(url)
            handle_exception()
            if len(self.errors) > 10: self.stop()

def get_tinyurl(url):
    """ grab a tinyurl. """
    res = get(url, namespace='tinyurl') ; logging.debug('tinyurl - cache - %s' % str(res))
    if res and res[0] == '[': return json.loads(res)
    postarray = [
        ('submit', 'submit'),
        ('url', url),
        ]
    postdata = urllib.parse.urlencode(postarray)
    req = urllib.request.Request(url=plugcfg.url, data=bytes(postdata, "utf-8"))
    req.add_header('User-agent', useragent())
    try: res = urllib.request.urlopen(req).readlines()
    except urllib.error.URLError as e: logging.warn('tinyurl - %s - URLError: %s' % (url, str(e))) ; return
    except urllib.error.HTTPError as e: logging.warn('tinyurl - %s - HTTP error: %s' % (url, str(e))) ; return
    except Exception as ex:
        if "DownloadError" in str(ex): logging.warn('tinyurl - %s - DownloadError: %s' % (url, str(e)))
        else: handle_exception()
        return
    urls = []
    for line in res:
        l = str(line, "utf-8")
        if l.startswith('<blockquote><b>'): urls.append(striphtml(l.strip()).split('[Open')[0])
    if len(urls) == 3: urls.pop(0)
    set(url, json.dumps(urls), namespace='tinyurl')
    return urls

def handle_alarmadd(bot, ievent):
    """ alarm <txt-with-time> | <+delta> <txt> .. add an alarm """
    if not ievent.rest: ievent.reply('alarm <txt-with-time> or alarm <+delta> <txt>') ; return
    else: alarmtxt = ievent.rest
    if alarmtxt[0] == '+':
        try: sec = int(ievent.args[0][1:]) 
        except ValueError: ievent.reply('use +nrofsecondstosleep') ; return
        if len(ievent.args) < 2: ievent.reply('i need txt to remind you') ; return
        try:
            ttime = time.time() + sec
            if ttime > 2**31: ievent.reply("time overflow") ; return
            if bot.type == "xmpp":
                if ievent.groupchat: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
                else: nrid = alarms.add(bot.cfg.name, ievent.stripped, ttime, ' '.join(ievent.args[1:]))
            elif bot.type == "irc":
                if ievent.msg:  nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.nick)
                else:  nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
            else: nrid = alarms.add(bot.cfg.name, ievent.nick, ttime, ' '.join(ievent.args[1:]), ievent.channel)
            ievent.reply("alarm %s set at %s" % (nrid, time.ctime(ttime)))
            return
        except Exception as ex: handle_exception(ievent) ; return
    alarmtime = strtotime(alarmtxt)
    if not alarmtime: ievent.reply("can't detect time") ; return
    txt = striptime(alarmtxt).strip()
    if not txt: ievent.reply('i need txt to remind you') ; return
    if time.time() > alarmtime: ievent.reply("we are already past %s" % time.ctime(alarmtime)) ; return
    if bot.jabber: nrid = alarms.add(bot.cfg.name, ievent.nick, alarmtime, txt, ievent.channel)
    else: nrid = alarms.add(bot.cfg.name, ievent.nick, alarmtime, txt, ievent.channel)
    ievent.reply("alarm %s set at %s" % (nrid, time.ctime(alarmtime)))

    def loadall(self, paths=[], force=True):
        """
            load all plugins from given paths, if force is true .. 
            otherwise load all plugins for default_plugins list.

        """
        if not paths: paths = plugin_packages
        imp = None
        old = list(self.keys())
        new = []
        logging.warn("loading all from paths %s" % paths)
        for module in paths:
            try: imp = _import(module)
            except ImportError as ex: logging.warn("no %s plugin package found - %s" % (module, str(ex))) ; continue
            except Exception as ex: handle_exception() ; continue
            logging.warn("got plugin package %s" % module)
            try: todo = imp.__plugs__
            except AttributeError as ex:
                if "__plugs__" in str(ex):
                    logging.error("no plugins in %s .. define __plugs__ in __init__.py" % module)
                    continue
                raise
            todo.sort()
            for plug in todo:
                mod = "%s.%s" % (module, plug)
                if not force and isdisabled(mod): logging.warn("%s is not enabled. not loading" % mod) ; continue
                try: self.reload(mod, force=force, showerror=True)
                except RequireError as ex: logging.info(str(ex)) ; continue
                except KeyError: logging.debug("failed to load plugin package %s" % module) ; continue
                except Exception as ex: handle_exception() ; continue
                new.append(mod)
        remove = [x for x in old if x not in new and x not in default_plugins]
        logging.info("unload list is:  %s" % ", ".join(remove))
        for mod in remove: self.unload(mod)
        return new

def handle_chatlogoff(bot, ievent):
    """ no arguments - disable chatlog. """
    try: cfg['channels'].remove([bot.cfg.name, ievent.channel]) ; cfg.save()
    except ValueError: ievent.reply('chatlog is not enabled in (%s,%s)' % (bot.cfg.name, ievent.channel)) ; return
    try: del loggers["%s-%s" % (bot.cfg.name, stripname(ievent.channel))]
    except KeyError: pass
    except Exception as ex: handle_exception()
    ievent.reply('chatlog disabled on (%s,%s)' % (bot.cfg.name, ievent.channel))

 def search(self, field, target):
     res = []
     for obj in list(self.objects().values()):
         try: item = getattr(obj.data, field)
         except AttributeError: handle_exception() ; continue
         if not item: continue
         if target in item: res.append(obj)
     return res

 def makeoptions(self):
     """ check the given txt for options. """
     try: self.options = makeeventopts(self.txt)
     except: handle_exception() ; return 
     if not self.options: return
     if self.options.channel: self.target = self.options.channel
     logging.info("options - %s" % str(self.options))
     self.txt = ' '.join(self.options.args)
     self.makeargs()

def handle_size(bot, event):
    res = []
    mods = dict(sys.modules)
    for name, mod in mods.items():
       if not 'tl' in name: continue
       try: res.append("<i><%s></i> %s" % (name.split(".")[-1], str(getattr(mod, 'size')())))
       except (TypeError, AttributeError): continue
       except Exception as ex: handle_exception()
    event.reply("sizes in %s modules scanned: " % len(res), res, dot=", ")