def _ensure_transport(self):
        """
        Ensures this Channel has been activated with the Node.
        """
#        log.debug("BaseChannel._ensure_transport (current: %s)", self._transport is not None)
        if not self._transport:
            raise ChannelError("No transport attached")

        if not self._lock:
            raise ChannelError("No lock available")

        # is lock already acquired? spit out a notice
        if self._lock._is_owned():
            log.warn("INTERLEAVE DETECTED:\n\nCURRENT STACK:\n%s\n\nSTACK THAT LOCKED: %s\n",
                    "".join(traceback.format_stack()), "".join(self._lock_trace))

        with self._lock:
            # we could wait and wait, and it gets closed, and unless we check again, we'd never know!
            if not self._transport:
                raise ChannelError("No transport attached")

            self._lock_trace = traceback.format_stack()
            try:
                yield
            finally:
                self._lock_trace = None

def _submit(runConfig):
    if registry.has_run(runConfig.runid):
        msg = "Runid %s does already exist" % runConfig.runid
        return SubmissionResponse(runConfig.runid, SUBM_ALREADY_SUBMITTED, msg)
        
    try:
        pipeline_exec = PipelineExecution(runConfig, server_model.config)
        logger.debug("PipelineExecution object for runid %s created."%runConfig.runid)
        pipeline_exec.initialize()
        logger.debug("PipelineExecution object for runid %s initialized."%runConfig.runid)
        registry.add_run(runConfig.runid, pipeline_exec)
        logger.debug("PipelineExecution object for runid %s registered."%runConfig.runid)
        _ = pipeline_exec.start()
        msg = "Pipeline execution for runid %s started."%runConfig.runid
        logger.info(msg)
    except ConfigurationError as ce:
        msg = 'Submission failed with exception %s.' % str(ce)
        stacktrace= 'Stacktrace: \n%s'%'\n'.join(traceback.format_stack())
        logger.warn(msg)
        return SubmissionResponse(runConfig.runid, SUBM_FAILED, msg, stacktrace)
    except Exception as e:
        msg = 'Submission failed with exception %s.' % str(e)
        stacktrace= 'Stacktrace: \n%s'%'\n'.join(traceback.format_stack())
        logger.error(msg)
        return SubmissionResponse(runConfig.runid, SUBM_ERROR, msg, stacktrace)
    try:
        history.add_entry("Runid %s submitted to the system" % runConfig.runid, datetime.datetime.now())
    except Exception as e:
        logger.warn("Runid %s could not be appended to the history."%runConfig.runid)
        msg=msg+"\n But not entry could be added the server history."

    return SubmissionResponse(runConfig.runid, SUBM_EXECUTING, msg)

def queuedb_query_execute( cur, query, values ):
    """
    Execute a query.  If it fails, exit.

    DO NOT CALL THIS DIRECTLY.
    """
    timeout = 1.0
    while True:
        try:
            ret = cur.execute( query, values )
            return ret
        except sqlite3.OperationalError as oe:
            if oe.message == "database is locked":
                timeout = timeout * 2 + timeout * random.random()
                log.error("Query timed out due to lock; retrying in %s: %s" % (timeout, namedb_format_query( query, values )))
                time.sleep(timeout)
            
            else:
                log.exception(oe)
                log.error("FATAL: failed to execute query (%s, %s)" % (query, values))
                log.error("\n".join(traceback.format_stack()))
                os.abort()

        except Exception, e:
            log.exception(e)
            log.error("FATAL: failed to execute query (%s, %s)" % (query, values))
            log.error("\n".join(traceback.format_stack()))
            os.abort()

	def deprecated(self, method):
		"""Output a deprecation warning.

		The implementation of WebKit sometimes invokes this method which prints
		a warning that the method you are using has been deprecated.
		This method expects that deprecated methods say so at the beginning of
		their doc string and terminate that msg with @. For example:

			DEPRECATED: Class.foo() on 01/24/01 in ver 0.5. Use Class.bar() instead. @

		Putting this information in the doc string is important for accuracy
		in the generated docs.

		Example call:
			self.deprecated(self.foo)

		"""
		docString = method.__doc__
		if docString:
			msg = docString.split('@')[0]
			msg = '\n'.join(map(lambda s: s.strip(), msg.splitlines()))
		else:
			msg = 'DEPRECATED: %s (no doc string)' % method
		print msg
		try:
			from traceback import format_stack
			print format_stack(limit =3)[0]
		except Exception:
			print 'Could not determine calling function.'

    def __init__(self, *args, **kwargs):
        if len(args) == 1 and not kwargs and isinstance(args[0], Exception):
            # we shall just wrap a non-caused exception
            self.stack = traceback.format_stack()[:-2] + traceback.format_tb(sys.exc_info()[2])
            # ^^^ let's hope the information is still there; caller must take
            #     care of this.
            self.wrapped = args[0]
            self.cause = ()
            super(CausedException, self).__init__(repr(args[0]))
            # ^^^ to display what it is wrapping, in case it gets printed or similar
            return

        self.wrapped = None
        # cut off frames we don't care about
        class_depth = max(ancestor_class_depth(CausedException, self.__class__), 0)
        self.stack = traceback.format_stack()[: -(class_depth + 1)]

        try:
            cause = kwargs["cause"]
            del kwargs["cause"]
        except:
            cause = ()

        if isinstance(cause, Exception) and not isinstance(cause, CausedException):
            cause = CausedException(cause)
        self.cause = cause if isinstance(cause, tuple) else (cause,)
        super(CausedException, self).__init__(*args, **kwargs)

 def trace(self, output=''):
     if output == 'stderr':
         sys.stderr.write(''.join(traceback.format_stack()))
         sys.stderr.flush()
     else:
         sys.stdout.write(''.join(traceback.format_stack()))
         sys.stdout.flush()

    def entryExitLoggingWrap(*args, **kwargs):

      if entryExitLogLevel is None:
        enabled = False
      else:
        logger = getLoggerCallback()
        enabled = logger.isEnabledFor(entryExitLogLevel)

      if not enabled:
        return func(*args, **kwargs)
      
      funcName = str(func)
      
      if logArgs:
        argsRepr = ', '.join(
          [repr(a) for a in args] +
          ['%s=%r' % (k,v,) for k,v in kwargs.iteritems()])
      else:
        argsRepr = ''
      
      logger.log(
        entryExitLogLevel, "ENTERING: %s(%s)%s", funcName, argsRepr,
        '' if not logTraceback else '; ' + repr(traceback.format_stack()))

      try:
        return func(*args, **kwargs)
      finally:
        logger.log(
          entryExitLogLevel, "LEAVING: %s(%s)%s", funcName, argsRepr,
          '' if not logTraceback else '; ' + repr(traceback.format_stack()))

def error(event, raises=False, e_type=Exception, log_trace=True, limit=None):
    """A level of information that may have caused an error.

    :param event: error event instance
    :param raises: boolean of whether or not an error should be raised
    :param e_type: exception type to be raised
    :param log_trace: boolean of whether or not log traceback
    :param limit: integer of traceback limit
    """
    if raises:
        if hasattr(event, "info"):
            logged_exception = e_type(event.info)
        else:
            logged_exception = e_type(event)

        if log_trace:
            try:
                raise logged_exception
            except Exception as err:
                given_traceback = traceback.format_stack(limit=limit)[:-1]
                given_traceback.append(traceback.format_exc(limit=limit))
                event.info += '\n' + ''.join(given_traceback)
            finally:
                raise logged_exception
        else:
            raise logged_exception

    elif log_trace:
        event.info += ''.join(traceback.format_stack())

 def endrun(self, status):
     """By convention for now, status is one of:
         'DEAD'
         'UNSUBMITTED' (a pseudo-status defined in the ready-loop of alive())
         'EXIT rc'
     """
     name = status.split()[0]
     if name == 'DEAD':
         log.warning(''.join(traceback.format_stack()))
         log.error('Task {}\n is DEAD, meaning no HEARTBEAT, but this can be a race-condition. If it was not killed, then restarting might suffice. Otherwise, you might have excessive clock-skew.'.format(self.brief()))
         self.setTargetStatus(pypeflow.task.TaskFail) # for lack of anything better
     elif name == 'UNSUBMITTED':
         log.warning(''.join(traceback.format_stack()))
         log.error('Task {}\n is UNSUBMITTED, meaning job-submission somehow failed. Possibly a re-start would work. Otherwise, you need to investigate.'.format(self.brief()))
         self.setTargetStatus(pypeflow.task.TaskFail) # for lack of anything better
     elif name != 'EXIT':
         raise Exception('Unexpected status {!r}'.format(name))
     else:
         code = int(status.split()[1])
         if 0 == code:
             self.__target.check_missing()
             # TODO: If missing, just leave the status as TaskInitialized?
         else:
             log.error('Task {} failed with exit-code={}'.format(self.brief(), code))
             self.setTargetStatus(pypeflow.task.TaskFail) # for lack of anything better
     self.__target.finish()

  def __formatFailure(self, failure, logMessage, extras):
    """Generates a dict from the given Failure object."""

    parts = ['Traceback (most recent call last):']
    if not failure.frames:
      parts += traceback.format_stack()
    else:
      for functionName, filename, lineNumber, _, _ in failure.frames:
        parts.append('File "%s", line %s, in %s' % (filename, lineNumber, functionName))
    backtrace = '\n'.join(parts)

    result = {
      'project': self.__project,
      'type': failure.type.__module__ + '.' + failure.type.__name__,
      'message': str(failure.value),
      'environment': self.__environment,
      'serverName': self.__serverName,
      'logMessage': logMessage,
      'backtrace': backtrace,
      'loggedFrom': '\n'.join(traceback.format_stack())
    }

    if extras and 'level' in extras:
      result['errorLevel'] = extras['level']
      del extras['level']

    if context and context.all():
      result['context'] = context.all()
      if extras:
        result['context'] = result['context'].copy()
        result['context'].update(extras)
    elif extras:
      result['context'] = extras

    return result

    def upsert(self, pgUrl, commit=True, **kwargs):
        if "url" in kwargs and "drive_web" in kwargs["url"]:
            self.log.error("")
            self.log.error("")
            self.log.error("WAT")
            self.log.error("")
            self.log.error(traceback.format_stack())
            self.log.error("")

        if "url" in kwargs and not kwargs["url"].startswith("http"):
            self.log.error("")
            self.log.error("")
            self.log.error("WAT")
            self.log.error("")
            self.log.error(traceback.format_stack())
            self.log.error("")

            # print("Upserting!")

        with self.transaction(commit=commit) as cur:
            # Do everything in one transaction

            try:
                self.insertIntoDb(url=pgUrl, cursor=cur, **kwargs)

            except psycopg2.IntegrityError:
                if kwargs:
                    cur.execute("ROLLBACK")
                    cur.execute("BEGIN")
                    self.updateDbEntry(url=pgUrl, cursor=cur, **kwargs)

    def drop_transaction(self, transaction):
        """Drops the specified transaction, validating that it is
        actually saved away under the current executing thread.

        """

        thread_id = transaction.thread_id

        if not thread_id in self._cache:
            _logger.error('Runtime instrumentation error. Attempt to '
                    'to drop the transaction but where none is active. '
                    'Report this issue to New Relic support.\n%s',
                    ''.join(traceback.format_stack()[:-1]))

            raise RuntimeError('no active transaction')

        current = self._cache.get(thread_id)

        if transaction != current:
            _logger.error('Runtime instrumentation error. Attempt to '
                    'to drop the transaction when it is not the current '
                    'transaction. Report this issue to New Relic support.\n%s',
                    ''.join(traceback.format_stack()[:-1]))

            raise RuntimeError('not the current transaction')

        transaction._greenlet = None

        del self._cache[thread_id]

def checkExceptionPerGreenlet(outfileName=None, ignoreHealthyOnes=True):
    mylog("Trying to detect greenlets...", verboseLevel=-2)
    if not outfileName:
        for ob in gc.get_objects():
            if not hasattr(ob, 'parent_args'):
                continue
            if not ob:
                continue
            if ignoreHealthyOnes and (not ob.exception):
                 continue
            mylog('%s[%s] called with parent arg\n(%s)\n%s' % (ob.name, repr(ob.args), repr(ob.parent_args),
                ''.join(traceback.format_stack(ob.gr_frame))), verboseLevel=-2)
            mylog(ob.exception, verboseLevel=-2)
    else:
        handler = open(outfileName, 'w')
        for ob in gc.get_objects():
            if not hasattr(ob, 'parent_args'):
                continue
            if not ob:
                continue
            if ignoreHealthyOnes and (not ob.exception):
                 continue
            handler.write('%s[%s] called with parent arg\n(%s)\n%s' % (ob.name, repr(ob.args), repr(ob.parent_args),
                ''.join(traceback.format_stack(ob.gr_frame))))
            handler.write(ob.exception)

 def __init__(self, *args, **kwargs):
     if len(args) == 1 and not kwargs and isinstance(args[0], Exception):
         # we shall just wrap a non-caused exception
         self.stack = (
             traceback.format_stack()[:-2] +
             traceback.format_tb(sys.exc_info()[2]))
         # ^^^ let's hope the information is still there; caller must take
         # care of this.
         self.wrapped = args[0]
         self._cause = []
         super(CausedException, self).__init__(repr(args[0]))
         # ^^^ to display what it is wrapping, in case it gets printed or similar
         return
     self.wrapped = None
     self.stack = traceback.format_stack()[:-1]  # cut off current frame
     try:
         cause = kwargs['cause']
         del kwargs['cause']
     except KeyError:
         cause = []
     if not isinstance(cause, list):
         cause = [cause]
     self._cause = [CausedException(e) if not isinstance(e, CausedException) else e for e in cause]
     self.kwargs = kwargs
     super(CausedException, self).__init__(*args)

def dump_stacks():
    dump = []

    # threads
    threads = dict([(th.ident, th.name)
                    for th in threading.enumerate()])

    for thread, frame in sys._current_frames().items():
        if thread not in threads:
            continue
        dump.append('Thread 0x%x (%s)\n' % (thread, threads[thread]))
        dump.append(''.join(traceback.format_stack(frame)))
        dump.append('\n')

    # greenlets
    try:
        from greenlet import greenlet
    except ImportError:
        return dump

    # if greenlet is present, let's dump each greenlet stack
    for ob in gc.get_objects():
        if not isinstance(ob, greenlet):
            continue
        if not ob:
            continue   # not running anymore or not started
        dump.append('Greenlet\n')
        dump.append(''.join(traceback.format_stack(ob.gr_frame)))
        dump.append('\n')

    return dump

	def updateObject(self, table_name, flt, values, turn_n = None):
		#table_name = '%s_%s'%(table, turn_n) if turn_n else table
		try:
			self.cur.execute('update %s SET %s WHERE %s'%(table_name, ','.join(['%s=%s'%(k,v) for k,v in values.iteritems()]), ' AND '.join(flt)))
			self.conn.commit()
		except sqlite3.Error, e:
			log.error('Error "%s", when executing: delete from %s WHERE %s'%(e, table_name, ' AND '.join(data)))
			print traceback.format_stack()		

	def eraseObject(self, table_name, data, turn_n = None):
		#table_name = '%s_%s'%(table, turn_n) if turn_n else table
		try:
			self.cur.execute('delete from %s WHERE %s'%(table_name, ' AND '.join(data)))
			self.conn.commit()
		except sqlite3.Error, e:
			log.error('Error "%s", when executing: delete from %s WHERE %s'%(e, table_name, ' AND '.join(data)))
			print traceback.format_stack()

 def printTrace(self, output=''):
     if output == 'stderr':
         sys.stderr.write(''.join(traceback.format_stack()))
         if self.__log_level >= self.__levels['debug']:
             sys.stderr.flush()
     else:
         sys.stdout.write(''.join(traceback.format_stack()))
         if self.__log_level >= self.__levels['debug']:
             sys.stdout.flush()

				def timeout(self):
					try:
						import time

						now = time.strftime('%c')
						self.journal.info("## %s: %s ##" % (now, self.message))
					except:
						import traceback
						traceback.format_stack()

def prlk_stack(siPrevio=False):
    import traceback

    if siPrevio:
        prlk("-" * 80 + "\n")
        prlk(traceback.format_stack())
        prlk("\n" + "-" * 80 + "\n")
    for line in traceback.format_stack()[:-1]:
        prlk(line.strip() + "\n")