• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

Python utils.GreenAsyncPile类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中swift.common.utils.GreenAsyncPile的典型用法代码示例。如果您正苦于以下问题:Python GreenAsyncPile类的具体用法?Python GreenAsyncPile怎么用?Python GreenAsyncPile使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



在下文中一共展示了GreenAsyncPile类的4个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: _get_put_responses

    def _get_put_responses(self, req, conns, nodes):
        statuses = []
        reasons = []
        bodies = []
        etags = set()

        def get_conn_response(conn):
            try:
                with Timeout(self.app.node_timeout):
                    if conn.resp:
                        return (conn, conn.resp)
                    else:
                        return (conn, conn.getresponse())
            except (Exception, Timeout):
                self.app.exception_occurred(
                    conn.node, _('Object'),
                    _('Trying to get final status of PUT to %s') % req.path)
            return (None, None)

        pile = GreenAsyncPile(len(conns))
        for conn in conns:
            pile.spawn(get_conn_response, conn)

        def _handle_response(conn, response):
            statuses.append(response.status)
            reasons.append(response.reason)
            bodies.append(response.read())
            if response.status == HTTP_INSUFFICIENT_STORAGE:
                self.app.error_limit(conn.node,
                                     _('ERROR Insufficient Storage'))
            elif response.status >= HTTP_INTERNAL_SERVER_ERROR:
                self.app.error_occurred(
                    conn.node,
                    _('ERROR %(status)d %(body)s From Object Server '
                      're: %(path)s') %
                    {'status': response.status,
                     'body': bodies[-1][:1024], 'path': req.path})
            elif is_success(response.status):
                etags.add(response.getheader('etag').strip('"'))

        for (conn, response) in pile:
            if response:
                _handle_response(conn, response)
                if self.have_quorum(statuses, len(nodes)):
                    break

        # give any pending requests *some* chance to finish
        finished_quickly = pile.waitall(self.app.post_quorum_timeout)
        for (conn, response) in finished_quickly:
            if response:
                _handle_response(conn, response)

        while len(statuses) < len(nodes):
            statuses.append(HTTP_SERVICE_UNAVAILABLE)
            reasons.append('')
            bodies.append('')
        return statuses, reasons, bodies, etags
开发者ID:dpgoetz,项目名称:swift,代码行数:57,代码来源:obj.py


示例2: _get_put_responses

    def _get_put_responses(self, req, conns, nodes):
        statuses = []
        reasons = []
        bodies = []
        etags = set()

        def get_conn_response(conn):
            try:
                with Timeout(self.app.node_timeout):
                    if conn.resp:
                        return conn.resp
                    else:
                        return conn.getresponse()
            except (Exception, Timeout):
                self.app.exception_occurred(
                    conn.node, _("Object"), _("Trying to get final status of PUT to %s") % req.path
                )

        pile = GreenAsyncPile(len(conns))
        for conn in conns:
            pile.spawn(get_conn_response, conn)
        for response in pile:
            if response:
                statuses.append(response.status)
                reasons.append(response.reason)
                bodies.append(response.read())
                if response.status >= HTTP_INTERNAL_SERVER_ERROR:
                    self.app.error_occurred(
                        conn.node,
                        _("ERROR %(status)d %(body)s From Object Server " "re: %(path)s")
                        % {"status": response.status, "body": bodies[-1][:1024], "path": req.path},
                    )
                elif is_success(response.status):
                    etags.add(response.getheader("etag").strip('"'))
                if self.have_quorum(statuses, len(nodes)):
                    break
        # give any pending requests *some* chance to finish
        pile.waitall(self.app.post_quorum_timeout)
        while len(statuses) < len(nodes):
            statuses.append(HTTP_SERVICE_UNAVAILABLE)
            reasons.append("")
            bodies.append("")
        return statuses, reasons, bodies, etags
开发者ID:krishna-kashyap,项目名称:swift,代码行数:43,代码来源:obj.py


示例3: make_requests

    def make_requests(self, req, ring, part, method, path, headers,
                      query_string=''):
        """
        Sends an HTTP request to multiple nodes and aggregates the results.
        It attempts the primary nodes concurrently, then iterates over the
        handoff nodes as needed.

        :param req: a request sent by the client
        :param ring: the ring used for finding backend servers
        :param part: the partition number
        :param method: the method to send to the backend
        :param path: the path to send to the backend
                     (full path ends up being  /<$device>/<$part>/<$path>)
        :param headers: a list of dicts, where each dict represents one
                        backend request that should be made.
        :param query_string: optional query string to send to the backend
        :returns: a swob.Response object
        """
        start_nodes = ring.get_part_nodes(part)
        nodes = GreenthreadSafeIterator(self.app.iter_nodes(ring, part))
        pile = GreenAsyncPile(len(start_nodes))
        for head in headers:
            pile.spawn(self._make_request, nodes, part, method, path,
                       head, query_string, self.app.logger.thread_locals)
        response = []
        statuses = []
        for resp in pile:
            if not resp:
                continue
            response.append(resp)
            statuses.append(resp[0])
            if self.have_quorum(statuses, len(start_nodes)):
                break
        # give any pending requests *some* chance to finish
        pile.waitall(self.app.post_quorum_timeout)
        while len(response) < len(start_nodes):
            response.append((HTTP_SERVICE_UNAVAILABLE, '', '', ''))
        statuses, reasons, resp_headers, bodies = zip(*response)
        return self.best_response(req, statuses, reasons, bodies,
                                  '%s %s' % (self.server_type, req.method),
                                  headers=resp_headers)
开发者ID:10389030,项目名称:swift,代码行数:41,代码来源:base.py


示例4: reconstruct_fa

    def reconstruct_fa(self, job, node, datafile_metadata):
        """
        Reconstructs a fragment archive - this method is called from ssync
        after a remote node responds that is missing this object - the local
        diskfile is opened to provide metadata - but to reconstruct the
        missing fragment archive we must connect to multiple object servers.

        :param job: job from ssync_sender
        :param node: node that we're rebuilding to
        :param datafile_metadata:  the datafile metadata to attach to
                                   the rebuilt fragment archive
        :returns: a DiskFile like class for use by ssync
        :raises DiskFileError: if the fragment archive cannot be reconstructed
        """

        part_nodes = job['policy'].object_ring.get_part_nodes(
            job['partition'])
        part_nodes.remove(node)

        # the fragment index we need to reconstruct is the position index
        # of the node we're rebuilding to within the primary part list
        fi_to_rebuild = node['index']

        # KISS send out connection requests to all nodes, see what sticks
        headers = self.headers.copy()
        headers['X-Backend-Storage-Policy-Index'] = int(job['policy'])
        pile = GreenAsyncPile(len(part_nodes))
        path = datafile_metadata['name']
        for node in part_nodes:
            pile.spawn(self._get_response, node, job['partition'],
                       path, headers, job['policy'])
        responses = []
        etag = None
        for resp in pile:
            if not resp:
                continue
            resp.headers = HeaderKeyDict(resp.getheaders())
            if str(fi_to_rebuild) == \
                    resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index'):
                continue
            if resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') in set(
                    r.headers.get('X-Object-Sysmeta-Ec-Frag-Index')
                    for r in responses):
                continue
            responses.append(resp)
            etag = sorted(responses, reverse=True,
                          key=lambda r: Timestamp(
                              r.headers.get('X-Backend-Timestamp')
                          ))[0].headers.get('X-Object-Sysmeta-Ec-Etag')
            responses = [r for r in responses if
                         r.headers.get('X-Object-Sysmeta-Ec-Etag') == etag]

            if len(responses) >= job['policy'].ec_ndata:
                break
        else:
            self.logger.error(
                'Unable to get enough responses (%s/%s) '
                'to reconstruct %s with ETag %s' % (
                    len(responses), job['policy'].ec_ndata,
                    self._full_path(node, job['partition'],
                                    datafile_metadata['name'], job['policy']),
                    etag))
            raise DiskFileError('Unable to reconstruct EC archive')

        rebuilt_fragment_iter = self.make_rebuilt_fragment_iter(
            responses[:job['policy'].ec_ndata], path, job['policy'],
            fi_to_rebuild)
        return RebuildingECDiskFileStream(datafile_metadata, fi_to_rebuild,
                                          rebuilt_fragment_iter)
开发者ID:harrisonfeng,项目名称:swift,代码行数:69,代码来源:reconstructor.py



注:本文中的swift.common.utils.GreenAsyncPile类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Python utils.Timestamp类代码示例发布时间:2022-05-27
下一篇:
Python utils.write_pickle函数代码示例发布时间:2022-05-27
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap