• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

Python conf.combine_values函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中mrjob.conf.combine_values函数的典型用法代码示例。如果您正苦于以下问题:Python combine_values函数的具体用法?Python combine_values怎么用?Python combine_values使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。



在下文中一共展示了combine_values函数的16个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: __init__

    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                 is_secure=True, port=None, proxy=None, proxy_port=None,
                 proxy_user=None, proxy_pass=None, host='iam.amazonaws.com',
                 debug=0, https_connection_factory=None, path='/',
                 security_token=None, validate_certs=True, profile_name=None,
                 mock_iam_instance_profiles=None, mock_iam_roles=None,
                 mock_iam_role_policies=None):
        """Mock out connection to IAM.

        mock_iam_instance_profiles maps profile name to a dictionary containing:
            create_date -- ISO creation datetime
            path -- IAM path
            role_name -- name of single role for this instance profile, or None

        mock_iam_roles maps role name to a dictionary containing:
            assume_role_policy_document -- a JSON-then-URI-encoded policy doc
            create_date -- ISO creation datetime
            path -- IAM path

        mock_iam_role_policies maps policy name to a dictionary containing:
            policy_document -- JSON-then-URI-encoded policy doc
            role_name -- name of single role for this policy (always defined)

        We don't currently support role IDs or ARNs because our code doesn't
        use them.
        """
        self.mock_iam_instance_profiles = combine_values(
            {}, mock_iam_instance_profiles)
        self.mock_iam_roles = combine_values({}, mock_iam_roles)
        self.mock_iam_role_policies = combine_values(
            {}, mock_iam_role_policies)
开发者ID:seatgeek,项目名称:mrjob,代码行数:31,代码来源:mockboto.py


示例2: __init__

    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                 is_secure=True, port=None, proxy=None, proxy_port=None,
                 proxy_user=None, proxy_pass=None, debug=0,
                 https_connection_factory=None, region=None,
                 mock_s3_fs=None, mock_emr_job_flows=None,
                 mock_emr_failures=None, mock_emr_output=None,
                 max_days_ago=DEFAULT_MAX_DAYS_AGO,
                 max_job_flows_returned=DEFAULT_MAX_JOB_FLOWS_RETURNED,
                 max_simulation_steps=100):
        """Create a mock version of EmrConnection. Most of these args are
        the same as for the real EmrConnection, and are ignored.

        By default, jobs will run to conclusion, and if their output dir
        is on S3, create a single empty output file. You can manually
        decide that some jobs will fail, or give them different output
        by setting mock_emr_failures/mock_emr_output.

        Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc.
        Step numbers are 0-indexed.

        Extra args:
        :param mock_s3_fs: a mock S3 filesystem to point to (just a dictionary
                           mapping bucket name to key name to bytes)
        :param mock_emr_job_flows: a mock set of EMR job flows to point to
                                   (just a map from job flow ID to a
                                   :py:class:`MockEmrObject` representing a job
                                   flow)
        :param mock_emr_failures: a map from ``(job flow ID, step_num)`` to a
                                  failure message (or ``None`` for the default
                                  message)
        :param mock_emr_output: a map from ``(job flow ID, step_num)`` to a
                                list of ``str``s representing file contents to
                                output when the job completes
        :type max_job_flows_returned: int
        :param max_job_flows_returned: the maximum number of job flows that
                                       :py:meth:`describe_jobflows` can return,
                                       to simulate a real limitation of EMR
        :type max_days_ago: int
        :param max_days_ago: the maximum amount of days that EMR will go back
                             in time
        :type max_simulation_steps: int
        :param max_simulation_steps: the maximum number of times we can
                                     simulate the progress of EMR job flows (to
                                     protect against simulating forever)
        """
        self.mock_s3_fs = combine_values({}, mock_s3_fs)
        self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows)
        self.mock_emr_failures = combine_values({}, mock_emr_failures)
        self.mock_emr_output = combine_values({}, mock_emr_output)
        self.max_days_ago = max_days_ago
        self.max_job_flows_returned = max_job_flows_returned
        self.simulation_steps_left = max_simulation_steps
        if region is not None:
            self.endpoint = region.endpoint
        else:
            self.endpoint = 'elasticmapreduce.amazonaws.com'
开发者ID:inncapsule,项目名称:mrjob,代码行数:56,代码来源:mockboto.py


示例3: __init__

    def __init__(
        self,
        aws_access_key_id=None,
        aws_secret_access_key=None,
        is_secure=True,
        port=None,
        proxy=None,
        proxy_port=None,
        proxy_user=None,
        proxy_pass=None,
        host=None,
        debug=0,
        https_connection_factory=None,
        calling_format=None,
        path="/",
        provider="aws",
        bucket_class=None,
        mock_s3_fs=None,
    ):
        """Mock out a connection to S3. Most of these args are the same
        as for the real S3Connection, and are ignored.

        You can set up a mock filesystem to share with other objects
        by specifying mock_s3_fs. The mock filesystem is just a map
        from bucket name to key name to bytes.
        """
        # use mock_s3_fs even if it's {}
        self.mock_s3_fs = combine_values({}, mock_s3_fs)
        self.endpoint = host or "s3.amazonaws.com"
开发者ID:bchess,项目名称:mrjob,代码行数:29,代码来源:mockboto.py


示例4: test_falseish_values

 def test_falseish_values(self):
     # everything but None is a legit value
     assert_equal(combine_values(True, False), False)
     assert_equal(combine_values(1, 0), 0)
     assert_equal(combine_values('full', ''), '')
     assert_equal(combine_values([1, 2, 3], []), [])
     assert_equal(combine_values((1, 2, 3), ()), ())
     assert_equal(combine_values({'a': 'b'}, {}), {})
     assert_equal(combine_values(set([1]), set()), set())
开发者ID:gimlids,项目名称:LTPM,代码行数:9,代码来源:conf_test.py


示例5: test_falseish_values

 def test_falseish_values(self):
     # everything but None is a legit value
     self.assertEqual(combine_values(True, False), False)
     self.assertEqual(combine_values(1, 0), 0)
     self.assertEqual(combine_values("full", ""), "")
     self.assertEqual(combine_values([1, 2, 3], []), [])
     self.assertEqual(combine_values((1, 2, 3), ()), ())
     self.assertEqual(combine_values({"a": "b"}, {}), {})
     self.assertEqual(combine_values(set([1]), set()), set())
开发者ID:nyccto,项目名称:mrjob,代码行数:9,代码来源:test_conf.py


示例6: __init__

    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                 is_secure=True, port=None, proxy=None, proxy_port=None,
                 proxy_user=None, proxy_pass=None, debug=0,
                 https_connection_factory=None, region=None,
                 mock_s3_fs=None, mock_emr_job_flows=None,
                 mock_emr_failures=None, mock_emr_output=None,
                 max_simulation_steps=100):
        """Create a mock version of EmrConnection. Most of these args are
        the same as for the real EmrConnection, and are ignored.

        By default, jobs will run to conclusion, and if their output dir
        is on S3, create a single empty output file. You can manually
        decide that some jobs will fail, or give them different output
        by setting mock_emr_failures/mock_emr_output.

        Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc.
        Step numbers are 0-indexed.

        Extra args:
        mock_s3_fs -- a mock S3 filesystem to point to (just a dictionary
            mapping bucket name to key name to bytes)
        mock_emr_job_flows -- a mock set of EMR job flows to point to
            (just a map from job flow ID to a MockEmrObject representing
            a job flow)
        mock_emr_failures -- a map from (job flow ID, step_num) to a failure
            message (or None for the default message)
        mock_emr_output -- a map from (job flow ID, step_num) to a list of
            strs representing file contents to output when the job completes
        max_simulation_steps -- the maximum number of times we can simulate the
            the progress of EMR job flows (to protect against simulating
            forever)
        """
        self.mock_s3_fs = combine_values({}, mock_s3_fs)
        self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows)
        self.mock_emr_failures = combine_values({}, mock_emr_failures)
        self.mock_emr_output = combine_values({}, mock_emr_output)
        self.simulation_steps_left = max_simulation_steps
开发者ID:atiw003,项目名称:mrjob,代码行数:37,代码来源:mockboto.py


示例7: __init__

    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                 is_secure=True, port=None, proxy=None, proxy_port=None,
                 proxy_user=None, proxy_pass=None, host='iam.amazonaws.com',
                 debug=0, https_connection_factory=None, path='/',
                 security_token=None, validate_certs=True, profile_name=None,
                 mock_iam_instance_profiles=None, mock_iam_roles=None,
                 mock_iam_role_policies=None,
                 mock_iam_role_attached_policies=None):
        """Mock out connection to IAM.

        mock_iam_instance_profiles maps profile name to a dictionary containing:
            create_date -- ISO creation datetime
            path -- IAM path
            role_name -- name of single role for this instance profile, or None

        mock_iam_roles maps role name to a dictionary containing:
            assume_role_policy_document -- a JSON-then-URI-encoded policy doc
            create_date -- ISO creation datetime
            path -- IAM path

        mock_iam_role_policies maps policy name to a dictionary containing:
            policy_document -- JSON-then-URI-encoded policy doc
            role_name -- name of single role for this policy (always defined)

        mock_iam_role_attached_policies maps role to a list of ARNs for
        attached (managed) policies.

        We don't track which managed policies exist or what their contents are.
        We also don't support role IDs.
        """
        self.mock_iam_instance_profiles = combine_values(
            {}, mock_iam_instance_profiles)
        self.mock_iam_roles = combine_values({}, mock_iam_roles)
        self.mock_iam_role_policies = combine_values(
            {}, mock_iam_role_policies)
        self.mock_iam_role_attached_policies = combine_values(
            {}, mock_iam_role_attached_policies)
开发者ID:DanisHack,项目名称:mrjob,代码行数:37,代码来源:mockboto.py


示例8: test_skips_None

 def test_skips_None(self):
     self.assertEqual(combine_values(None, 'one'), 'one')
     self.assertEqual(combine_values('one', None), 'one')
     self.assertEqual(combine_values(None, None, 'one', None), 'one')
开发者ID:icio,项目名称:mrjob,代码行数:4,代码来源:test_conf.py


示例9: __init__

    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                 is_secure=True, port=None, proxy=None, proxy_port=None,
                 proxy_user=None, proxy_pass=None, debug=0,
                 https_connection_factory=None, region=None,
                 mock_s3_fs=None, mock_emr_job_flows=None,
                 mock_emr_failures=None, mock_emr_output=None,
                 max_days_ago=DEFAULT_MAX_DAYS_AGO,
                 max_job_flows_returned=DEFAULT_MAX_JOB_FLOWS_RETURNED,
                 simulation_iterator=None, security_token=None):
        """Create a mock version of EmrConnection. Most of these args are
        the same as for the real EmrConnection, and are ignored.

        By default, jobs will run to conclusion, and if their output dir
        is on S3, create a single empty output file. You can manually
        decide that some jobs will fail, or give them different output
        by setting mock_emr_failures/mock_emr_output.

        Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc.
        Step numbers are 0-indexed.

        Extra args:
        :param mock_s3_fs: a mock S3 filesystem to point to (just a dictionary
                           mapping bucket name to key name to bytes)
        :param mock_emr_job_flows: a mock set of EMR job flows to point to
                                   (just a map from job flow ID to a
                                   :py:class:`MockEmrObject` representing a job
                                   flow)
        :param mock_emr_failures: a map from ``(job flow ID, step_num)`` to a
                                  failure message (or ``None`` for the default
                                  message)
        :param mock_emr_output: a map from ``(job flow ID, step_num)`` to a
                                list of ``bytes``s representing file contents to
                                output when the job completes
        :type max_job_flows_returned: int
        :param max_job_flows_returned: the maximum number of job flows that
                                       :py:meth:`describe_jobflows` can return,
                                       to simulate a real limitation of EMR
        :type max_days_ago: int
        :param max_days_ago: the maximum amount of days that EMR will go back
                             in time
        :param simulation_iterator: we call ``next()`` on this each time
                                    we simulate progress. If there is
                                    no next element, we bail out.
        """
        # check this now; strs will cause problems later in Python 3
        if mock_emr_output and any(
                any(not isinstance(part, bytes) for part in parts)
                for parts in mock_emr_output.values()):
            raise TypeError('mock EMR output must be bytes')

        self.mock_s3_fs = combine_values({}, mock_s3_fs)
        self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows)
        self.mock_emr_failures = combine_values({}, mock_emr_failures)
        self.mock_emr_output = combine_values({}, mock_emr_output)
        self.max_days_ago = max_days_ago
        self.max_job_flows_returned = max_job_flows_returned
        self.simulation_iterator = simulation_iterator
        if region is not None:
            self.endpoint = region.endpoint
        else:
            self.endpoint = 'elasticmapreduce.amazonaws.com'
开发者ID:datascientist1976,项目名称:mrjob,代码行数:61,代码来源:mockboto.py


示例10: test_skips_None

 def test_skips_None(self):
     assert_equal(combine_values(None, ['cat']), ['cat'])
     assert_equal(combine_values(['cat'], None), ['cat'])
     assert_equal(combine_values(None, None, ['cat'], None), ['cat'])
开发者ID:gimlids,项目名称:LTPM,代码行数:4,代码来源:conf_test.py


示例11: test_all_None

 def test_all_None(self):
     assert_equal(combine_values(None, None, None), None)
开发者ID:gimlids,项目名称:LTPM,代码行数:2,代码来源:conf_test.py


示例12: test_picks_last_value

 def test_picks_last_value(self):
     assert_equal(combine_values(1, 2, 3), 3)
开发者ID:gimlids,项目名称:LTPM,代码行数:2,代码来源:conf_test.py


示例13: test_empty

 def test_empty(self):
     assert_equal(combine_values(), None)
开发者ID:gimlids,项目名称:LTPM,代码行数:2,代码来源:conf_test.py


示例14: test_all_None

 def test_all_None(self):
     self.assertEqual(combine_values(None, None, None), None)
开发者ID:icio,项目名称:mrjob,代码行数:2,代码来源:test_conf.py


示例15: test_picks_last_value

 def test_picks_last_value(self):
     self.assertEqual(combine_values(1, 2, 3), 3)
开发者ID:icio,项目名称:mrjob,代码行数:2,代码来源:test_conf.py


示例16: test_empty

 def test_empty(self):
     self.assertEqual(combine_values(), None)
开发者ID:icio,项目名称:mrjob,代码行数:2,代码来源:test_conf.py



注:本文中的mrjob.conf.combine_values函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Python conf.dump_mrjob_conf函数代码示例发布时间:2022-05-27
下一篇:
Python conf.combine_path_lists函数代码示例发布时间:2022-05-27
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap