def start_stream(self, conf):
     if not self.fact.__init_timeout__():
         returnD(False)
     queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query'])
     track = []
     skip = []
     k = 0
     for query in queries:
         q = str(query['query'].encode('utf-8')).lower()
         # queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip
         if self.re_twitter_account.match(q):
             continue
         elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6:
             skip.append(q)
             continue
         track.append(q)
         k += 1
         if k > 395:
             break
     if self.fact.twuser not in track:
         track.append(self.fact.twuser)
     if len(skip):
         self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True)
     self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True)
     conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
     # tries to find users corresponding with queries to follow with stream
     users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users'])
     deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
     self.depiler = LoopingCall(self.flush_tweets)
     self.depiler.start(1)
     returnD(True)

    def test_contemporaneous_requests(self):

        '''
        We're going to create two request-response cycles here:

        Cycle 1 will begin.
        Cycle 2 will begin.
        Cycle 2 will return.
        Cycle 1 will return.

        This way, we can prove that the crosstown_traffic created
        by cycle 1 is not resolved by the return of cycle 2.
        '''
        tp = ThreadPool(maxthreads=20)
        tp.start()
        self.addCleanup(tp.stop)


        log.debug("\n\nStarting the two stream stuff.")

        request1 = DummyRequest('r1')
        request1.isSecure = lambda: False
        request1.content = "Nothing really here."
        request1.headers['llamas'] = 'dingo'

        nameSpace.test_case = self

        hr = HendrixWSGIResource(reactor, tp, wsgi_application)
        d1 = deferToThreadPool(reactor, tp, hr.render, request1)

        request2 = DummyRequest('r2')
        request2.isSecure = lambda: False
        request2.content = "Nothing really here."
        request2.headers['llamas'] = 'dingo'

        d2 = deferToThreadPool(reactor, tp, hr.render, request2)

        def woah_stop(failure):
            nameSpace.async_task_was_done.put_nowait(False)
            nameSpace.second_cycle_complete.put_nowait(False)
            nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)

        d1.addErrback(woah_stop)
        d2.addErrback(woah_stop)

        combo_deferred = gatherResults([d1, d2])

        def wait_for_queue_resolution():
            nameSpace.async_task_was_done.get(True, 3)

        combo_deferred.addCallback(
            lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution)
        )

        combo_deferred.addCallback(
            lambda _: self.assertTrue(nameSpace.async_task_was_run)
        )

        return combo_deferred

    def process_item(self, domain, item):
        args = dict(item)
        random_numbers = args['random_numbers'].split(',')
        for num in random_numbers:
            #log.msg('dispatching : %s' % num,level=log.ERROR)
            trackable = TrackableObject()
            threads.deferToThreadPool(reactor, self.threadpool, trackable.takes_long_time, num,self)

        return item

    def run(self, threadpool):
        # See if status code is a go
        self.check_status_code_against_no_go_list()
        if self.no_go:
            return

        if self.same_thread:
            self.crosstown_task()
        else:
            deferToThreadPool(reactor, threadpool, self.crosstown_task)

    def run(self, threadpool=None):
        if self.no_go:
            return

        if not threadpool:
            threadpool = reactor.threadpool or ThreadPool()

        if self.same_thread:
            self.crosstown_task()
        else:
            deferToThreadPool(reactor, threadpool, self.crosstown_task)

 def _handshake(self, msg, line):
     if self.state.current != 'WAIT':
         self.error("Invalid state for a handshake command: %s",
                    self.state.current)
         self._sendError("Invalid state")
         return
     mysha = self.host.workflow.checksum
     your_sha = msg.get("checksum")
     if not your_sha:
         self.error("Did not receive the workflow checksum")
         self._sendError("Workflow checksum is missing")
         return
     if mysha != your_sha:
         self._sendError("Workflow checksum mismatch: "
                         "expected %s, got %s" % (mysha, your_sha))
         return
     must_reply = False
     msgid = msg.get("id")
     if msgid is None:
         self.id = str(uuid.uuid4())
         must_reply = True
     else:
         self.id = msgid
         if not self.nodes.get(self.id):
             self.warning("Did not recognize the received ID %s")
             must_reply = True
         else:
             self.sendLine({'reconnect': "ok"})
     if must_reply:
         try:
             _, mid, pid = self._extractClientInformation(msg)
         except Exception as e:
             self.error(str(e))
             return
         data = self.host.workflow.generate_initial_data_for_slave(
             SlaveDescription.make(self.nodes[self.id]))
         endpoint = self.host.choose_endpoint(self.id, mid, pid, self.hip)
         self.nodes[self.id]['endpoint'] = self._endpoint = endpoint
         retmsg = {'endpoint': endpoint, 'data': data}
         if not msgid:
             retmsg['id'] = self.id
         retmsg['log_id'] = self.host.launcher.log_id
         self.sendLine(retmsg)
     data = msg.get('data')
     if data is not None:
         threads.deferToThreadPool(
             reactor, self.host.workflow.thread_pool,
             self.host.workflow.apply_initial_data_from_slave,
             data, SlaveDescription.make(self.nodes[self.id])) \
             .addErrback(errback)
         self.nodes[self.id]['data'] = [d for d in data if d is not None]
     self.state.identify()

    def test_threadLocality(self):
        """
        An 'Record' repr()'d in two separate threads at the same time should
        look the same (i.e. the repr state tracking for '...' should be
        thread-local).
        """
        pool = ThreadPool(2, 2)
        pool.start()
        self.addCleanup(pool.stop)

        class StickyRepr(object):
            """
            This has a __repr__ which will block until a separate thread
            notifies it that it should return.  We use this to create a race
            condition.
            """
            waited = False
            def __init__(self):
                self.set = threading.Event()
                self.wait = threading.Event()
            def __repr__(self):
                if not self.waited:
                    self.set.set()
                    self.wait.wait()
                return 'sticky'
        r = StickyRepr()
        mr = MyRecord(something=1, somethingElse=r)
        d = deferToThreadPool(reactor, pool, repr, mr)
        def otherRepr():
            # First we wait for the first thread doing a repr() to enter its
            # __repr__()...
            r.set.wait()
            # OK, now it's blocked.  Let's make sure that subsequent calls to
            # this repr() won't block.
            r.waited = True
            # Do it!  This is a concurrent repr().
            result = repr(mr)
            # Now we're done, wake up the other repr and let it complete.
            r.wait.set()
            return result
        d2 = deferToThreadPool(reactor, pool, otherRepr)

        def done(xxx_todo_changeme):
            (thread1repr, thread2repr) = xxx_todo_changeme
            knownGood = 'MyRecord(something=1, somethingElse=sticky)'
            # self.assertEquals(thread1repr, thread2repr)
            self.assertEqual(thread1repr, knownGood)
            self.assertEqual(thread2repr, knownGood)
        return gatherResults([d, d2]).addCallback(done)

def _clean_redir_urls(text, urls={}, first=True, pool=None):
    for res in URL_REGEX.findall(text):
        url00 = res[2].encode('utf-8')
        url0 = url00
        if not url00.startswith('http'):
            if "@" in url00 or url00.startswith('#'):
                continue
            url0 = "http://%s" % url00
        if url0 in urls:
            url1 = urls[url0]
            if url1 == url0:
                continue
        else:
            try:
                url1 = yield deferToThreadPool(reactor, pool, get_url, url0, timeout=8)
                url1 = clean_url(url1)
                urls[url0] = url1
                urls[url1] = url1
            except Exception as e:
                if config.DEBUG and not first:
                    loggerr("trying to resolve %s : %s" % (url0, e))
                if "403" in str(e) or "Error 30" in str(e):
                    urls[url0] = url00
                url1 = url00
        if first and not url1 == url00:
            url1 = url1.replace('http', '##HTTP##')
        try:
            url1 = url1.decode('utf-8')
            text = text.replace(res[0], '%s%s%s' % (res[1], url1, res[4]))
        except:
            if config.DEBUG:
                logerr("encoding %s" % url1)
    if not first:
        text = text.replace('##HTTP##', 'http')
    defer.returnValue((text, urls))

    def run(self, function, *args, **kwargs):
        """Run C{function} in a thread.

        The function is run in a thread by a function wrapper, which
        commits the transaction if the function runs successfully. If it
        raises an exception the transaction is aborted.

        If the named variable 'async' is set to False we don't run the
        function in the ThreadPool but in the main thread.

        @param function: The function to run.
        @param args: Positional arguments to pass to C{function}.
        @param kwargs: Keyword arguments to pass to C{function}.
        @return: A C{Deferred} that will fire after the function has been run.
        """
        run_async = kwargs.pop('async', True)
        if run_async:
            # Inline the reactor import here for sake of safeness, in case a
            # custom reactor needs to be installed
            from twisted.internet import reactor
            return deferToThreadPool(
                reactor, self._threadpool, self._wrap,
                function, *args, **kwargs)

        return self._wrap(function, *args, **kwargs)

    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            # Run the master.cfg in thread, so that it cas use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

 def start_stream(self, conf):
     self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])
     queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query']))
     track = []
     follow = []
     skip = []
     k = 0
     f = 0
     for query in queries:
         q = str(query['query'].encode('utf-8'))
         if self.re_twitter_account.match(q):
             q = q.lstrip('@')
             follow.append(q)
             f += 1
         elif " OR " in q or " -" in q or '"' in q or len(q) > 60:
             skip.append(q)
             continue
         track.append(q)
         k += 1
         if k > 395 or f > 4995:
             break
     user = conf["TWITTER"]["USER"]
     if user not in follow:
         follow.append(user)
     if user not in track:
         track.append(user)
     if len(skip):
         self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True)
     self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True)
     conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
     users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users)
     return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)

def watch_children(kzclient,
                   path, func, allow_session_lost=True, send_event=False,
                   ChildrenWatch=ChildrenWatch):
    """
    Install a Kazoo :obj:`ChildrenWatch` on the given path.

    The given `func` will be called in the reactor thread when any children are
    created or deleted, or if the node itself is deleted.

    Returns a Deferred which usually has no result, but may fail with an
    exception if e.g. the path does not exist.
    """

    def wrapped_func(*args, **kwargs):
        return blockingCallFromThread(kzclient.reactor, func, *args, **kwargs)

    return deferToThreadPool(
        kzclient.reactor,
        kzclient.pool,
        lambda: ChildrenWatch(
            kzclient.kazoo_client,
            path,
            func=wrapped_func,
            allow_session_lost=allow_session_lost,
            send_event=send_event))

    def render_POST(self, req):
        uid = self.transcriber.next_id()

        tran = req.args['transcript'][0]
        audio = req.args['audio'][0]

        async = True
        if 'async' in req.args and req.args['async'][0] == 'false':
            async = False

        result_promise = threads.deferToThreadPool(
            self.reactor, self.reactor.getThreadPool(),
            self.transcriber.transcribe,
            uid, tran, audio)

        if not async:
            def write_result(result):
                '''Write JSON to client on completion'''
                req.headers["Content-Type"] = "application/json"
                req.write(json.dumps(result, indent=2))
                req.finish()
            result_promise.addCallback(write_result)
            result_promise.addErrback(lambda _: None) # ignore errors

            req.notifyFinish().addErrback(lambda _: result_promise.cancel())

            return NOT_DONE_YET

        req.setResponseCode(FOUND)
        req.setHeader(b"Location", "/transcriptions/%s" % (uid))
        return ''

 def __safe_call(*args, **kwargs):
     client = None
     keepalive = True
     if "_nokeepalive" in kwargs:
         keepalive = not kwargs.pop("_nokeepalive")
     try:
         corpus = kwargs.pop("corpus")
     except:
         corpus = ""
         fail = format_error("corpus argument missing")
     else:
         fail = format_error({"corpus_id": corpus,
           "ready": self.factory.test_corpus(corpus),
           "status": self.factory.status_corpus(corpus),
           "message": "Corpus is not started"})
         if corpus in self.factory.corpora:
             if keepalive:
                 self.factory.corpora[corpus].lastcall = time.time()
             client = getattr(self.factory.corpora[corpus],
               "client_%s" % type_client)
             if fail["message"]["status"] == "error":
                 fail["message"]["message"] = self.factory.corpora[corpus].error
     if hasattr(client, 'threadpool'):
         if self.factory.test_corpus(corpus):
             return deferToThreadPool(reactor, client.threadpool,
               client.__thrift_call__, call, *args, **kwargs)
         return defer.succeed(fail)
     if self.factory.test_corpus(corpus):
         return client.__thrift_call__(call, *args, **kwargs)
     return fail

    def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
        self._buildSocket(filter, iface, nofilter)
        self._buildPacketQueues(pkts)
        if not self.last_answer:
            self.last_answer = time.time()

        def sent(cb):
            if self.cthreads < self.mthreads and not self.done:
                pkt = None
                try:
                    pkt = self.outqueue.pop()
                except:
                    self.done = True
                    if not self.recv:
                        self.deferred.callback(None)
                    return
                d = threads.deferToThreadPool(reactor, self.threadpool,
                                    self.sendPkt, pkt)
                d.addCallback(sent)
                return d

        for x in range(self.mthreads):
            try:
                pkt = self.outqueue.pop()
            except:
                self.done = True
                return
            if self.cthreads >= self.mthreads and self.done:
                return
            d = threads.deferToThreadPool(reactor, self.threadpool,
                                self.sendPkt, pkt)
            d.addCallback(sent)
            return d

 def runWithConnection(self, func, *args, **kwargs):
     """
     Executes a function with database connection and return the result.
     """
     return threads.deferToThreadPool(reactor, self.threadpool,
                                      self._runWithConnection,
                                      func, *args, **kwargs)

    def connect(self, masterUrl, deferred):
        """ Connect to RCE.

            @param masterUrl:   URL of Master process.
            @type  masterUrl:   str

            @param deferred:    Deferred which is called as soon as the
                                connection was successfully established.
            @type  deferred:    twisted.internet.defer.Deferred

            @raise:             ConnectionError, if no connection could be
                                established.
        """
        self._connectedDeferred = deferred

        def eb(e):
            print(e.getErrorMessage())

            if self._connectedDeferred:
                self._connectedDeferred.errback(e)
                self._connectedDeferred = None

        d = deferToThreadPool(self._reactor, self._reactor.getThreadPool(),
                              self._getRobotURL, masterUrl)
        d.addCallback(self._robotConnect)
        d.addErrback(eb)

    def get_id(self, model, unique, fields):
        ''' Get an ID from the cache or from the database.
        If doesn't exist - create an item.
        All database operations are done from
        the separate thread
        '''
        assert isInIOThread()

        fval = fields[unique]

        try:
            result = self.cache[model][fval]
            self.counters['hit'][model] += 1
            returnValue(result)
        except KeyError:
            self.counters['miss'][model] += 1

        selectors = {unique: fval}

        result, created = yield deferToThreadPool(
            self.reactor, self.read_pool,
            get_or_create,
            model, fields, **selectors)

        result = result.id

        if created:
            self.counters['db_create'][model] += 1
        else:
            self.counters['db_hit'][model] += 1

        self.cache[model][fval] = result
        returnValue(result)

  def _retry(self, method_name, call):
    """Retries |call| on transient errors and access token expiration.

    Args:
      method_name (str): name of the remote method, for logging.
      call (func() -> any): a function that makes an RPC call and returns
        result.
    """
    attempts = self.retry_attempt_count
    wait = self.retry_wait_seconds
    while attempts > 0:
      attempts -= 1
      try:
        if not self.started:
          raise NotStartedError('DeferredResource is not started')
        res = yield threads.deferToThreadPool(reactor, self._pool, call)
        defer.returnValue(res)
      except Exception as ex:
        if not self.started:
          raise ex
        if attempts > 0 and is_transient(ex):
          self.log('Transient error while calling %s. '
                   'Will retry in %d seconds.' % (method_name, wait))
          # TODO(nodir), optimize: stop waiting if the resource is stopped.
          yield sleep(wait)
          if not self.started:
            raise ex
          wait = min(wait * 2, 30)
          continue
        self.log('RPC "%s" failed: %s'% (method_name, traceback.format_exc()))
        raise ex

    def runWithConnection(self, func, *args, **kw):
        """
        Execute a function with a database connection and return the result.

        @param func: A callable object of one argument which will be executed
            in a thread with a connection from the pool. It will be passed as
            its first argument a L{Connection} instance (whose interface is
            mostly identical to that of a connection object for your DB-API
            module of choice), and its results will be returned as a
            L{Deferred}. If the method raises an exception the transaction will
            be rolled back. Otherwise, the transaction will be committed.
            B{Note} that this function is B{not} run in the main thread: it
            must be threadsafe.

        @param *args: positional arguments to be passed to func

        @param **kw: keyword arguments to be passed to func

        @return: a L{Deferred} which will fire the return value of
            C{func(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runWithConnection,
                                         func, *args, **kw)