• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号
登陆 注册

Python runner.MRJobRunner类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中mrjob.runner.MRJobRunner类的典型用法代码示例。如果您正苦于以下问题：Python MRJobRunner类的具体用法？Python MRJobRunner怎么用？Python MRJobRunner使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



在下文中一共展示了MRJobRunner类的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: test_jobconf

 def test_jobconf(self):
     jobconf = {'FOO': 'bar', 'BAZ': 'qux', 'BAX': 'Arnold'}
     runner = MRJobRunner(conf_path=False, jobconf=jobconf)
     assert_equal(runner._hadoop_conf_args(0, 1),
                  ['-jobconf', 'BAX=Arnold',
                   '-jobconf', 'BAZ=qux',
                   '-jobconf', 'FOO=bar',])
开发者ID:Jyrsa，项目名称:mrjob，代码行数:7，代码来源:runner_test.py


示例2: test_cmdenv

 def test_cmdenv(self):
     cmdenv = {'FOO': 'bar', 'BAZ': 'qux', 'BAX': 'Arnold'}
     runner = MRJobRunner(conf_path=False, cmdenv=cmdenv)
     assert_equal(runner._hadoop_conf_args(0, 1),
                  ['-cmdenv', 'BAX=Arnold',
                   '-cmdenv', 'BAZ=qux',
                   '-cmdenv', 'FOO=bar',])
开发者ID:Jyrsa，项目名称:mrjob，代码行数:7，代码来源:runner_test.py


示例3: test_one_file

    def test_one_file(self):
        runner = MRJobRunner(conf_paths=[])
        runner._invoke_sort([self.a], self.out)

        self.assertEqual(list(open(self.out)),
                         ['A\n',
                          'alligator\n',
                          'apple\n'])
开发者ID:duedil-ltd，项目名称:mrjob，代码行数:8，代码来源:test_runner.py


示例4: test_two_files

    def test_two_files(self):
        runner = MRJobRunner(conf_paths=[])
        self.addCleanup(runner.cleanup)

        runner._invoke_sort([self.a, self.b], self.out)

        with open(self.out) as out_f:
            self.assertEqual(list(out_f), ["A\n", "B\n", "alligator\n", "apple\n", "ball\n", "banana\n"])
开发者ID:irskep，项目名称:mrjob，代码行数:8，代码来源:test_runner.py


示例5: test_hadoop_output_format

 def test_hadoop_output_format(self):
     format = 'org.apache.hadoop.mapred.SequenceFileOutputFormat'
     runner = MRJobRunner(conf_path=False, hadoop_output_format=format)
     assert_equal(runner._hadoop_conf_args(0, 1),
                  ['-outputformat', format])
     # test multi-step job
     assert_equal(runner._hadoop_conf_args(0, 2), [])
     assert_equal(runner._hadoop_conf_args(1, 2),
                  ['-outputformat', format])
开发者ID:Jyrsa，项目名称:mrjob，代码行数:9，代码来源:runner_test.py


示例6: test_two_files

    def test_two_files(self):
        runner = MRJobRunner(conf_paths=[])
        runner._invoke_sort([self.a, self.b], self.out)

        self.assertEqual(list(open(self.out)),
                         ['A\n',
                          'B\n',
                          'alligator\n',
                          'apple\n',
                          'ball\n',
                          'banana\n'])
开发者ID:duedil-ltd，项目名称:mrjob，代码行数:11，代码来源:test_runner.py


示例7: test_one_file

    def test_one_file(self):
        runner = MRJobRunner(conf_paths=[])
        self.addCleanup(runner.cleanup)

        runner._invoke_sort([self.a], self.out)

        with open(self.out) as out_f:
            self.assertEqual(list(out_f),
                             ['A\n',
                              'alligator\n',
                              'apple\n'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:11，代码来源:test_runner.py


示例8: test_hadoop_extra_args_comes_first

 def test_hadoop_extra_args_comes_first(self):
     runner = MRJobRunner(
         conf_path=False,
         cmdenv={'FOO': 'bar'},
         hadoop_input_format='FooInputFormat',
         hadoop_output_format='BarOutputFormat',
         jobconf={'baz': 'quz'},
         hadoop_extra_args=['-libjar', 'qux.jar'])
     # hadoop_extra_args should come first
     conf_args = runner._hadoop_conf_args(0, 1)
     assert_equal(conf_args[:2], ['-libjar', 'qux.jar'])
     assert_equal(len(conf_args), 10)
开发者ID:Jyrsa，项目名称:mrjob，代码行数:12，代码来源:runner_test.py


示例9: test_interpreter_overrides_steps_python_bin

 def test_interpreter_overrides_steps_python_bin(self):
     runner = MRJobRunner(interpreter=['ruby'],
                          steps_python_bin=['python', '-v'])
     self.assertEqual(runner._interpreter(), ['ruby'])
     self.assertEqual(runner._interpreter(steps=True), ['ruby'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:5，代码来源:test_runner.py


示例10: test_steps_interpreter

 def test_steps_interpreter(self):
     # including whether steps_interpreter overrides interpreter
     runner = MRJobRunner(interpreter=['ruby', '-v'],
                          steps_interpreter=['ruby'])
     self.assertEqual(runner._interpreter(), ['ruby', '-v'])
     self.assertEqual(runner._interpreter(steps=True), ['ruby'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:6，代码来源:test_runner.py


示例11: test_interpreter

 def test_interpreter(self):
     runner = MRJobRunner(interpreter=['ruby'])
     self.assertEqual(runner._interpreter(), ['ruby'])
     self.assertEqual(runner._interpreter(steps=True), ['ruby'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:4，代码来源:test_runner.py


示例12: test_steps_python_bin

 def test_steps_python_bin(self):
     runner = MRJobRunner(steps_python_bin=['python', '-v'])
     self.assertEqual(runner._interpreter(),
                      self.default_python_bin())
     self.assertEqual(runner._interpreter(steps=True), ['python', '-v'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:5，代码来源:test_runner.py


示例13: test_python_bin

 def test_python_bin(self):
     runner = MRJobRunner(python_bin=['python', '-v'])
     self.assertEqual(runner._interpreter(), ['python', '-v'])
     self.assertEqual(runner._interpreter(steps=True), [sys.executable])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:4，代码来源:test_runner.py


示例14: test_default

 def test_default(self):
     runner = MRJobRunner()
     self.assertEqual(runner._interpreter(),
                      self.default_python_bin())
     self.assertEqual(runner._interpreter(steps=True),
                      [sys.executable])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:6，代码来源:test_runner.py


示例15: test_environment_variables_windows

    def test_environment_variables_windows(self):
        runner = MRJobRunner(conf_paths=[])
        self.addCleanup(runner.cleanup)

        runner._sort_is_windows_sort = True
        self.environment_variable_checks(runner, ['TMP'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:6，代码来源:test_runner.py


示例16: test_hadoop_extra_args

 def test_hadoop_extra_args(self):
     extra_args = ['-foo', 'bar']
     runner = MRJobRunner(conf_path=False, hadoop_extra_args=extra_args)
     assert_equal(runner._hadoop_conf_args(0, 1), extra_args)
开发者ID:Jyrsa，项目名称:mrjob，代码行数:4，代码来源:runner_test.py


示例17: test_interpreter_overrides_steps_python_bin

 def test_interpreter_overrides_steps_python_bin(self):
     runner = MRJobRunner(interpreter=["ruby"], steps_python_bin=["python", "-v"])
     self.assertEqual(runner._interpreter(), ["ruby"])
     self.assertEqual(runner._interpreter(steps=True), ["ruby"])
开发者ID:irskep，项目名称:mrjob，代码行数:4，代码来源:test_runner.py


示例18: test_steps_interpreter

 def test_steps_interpreter(self):
     # including whether steps_interpreter overrides interpreter
     runner = MRJobRunner(interpreter=["ruby", "-v"], steps_interpreter=["ruby"])
     self.assertEqual(runner._interpreter(), ["ruby", "-v"])
     self.assertEqual(runner._interpreter(steps=True), ["ruby"])
开发者ID:irskep，项目名称:mrjob，代码行数:5，代码来源:test_runner.py


示例19: test_no_bootstrap_mrjob

 def test_no_bootstrap_mrjob(self):
     runner = MRJobRunner(conf_paths=[], bootstrap_mrjob=False)
     self.assertEqual(runner._bootstrap_mrjob(), False)
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:3，代码来源:test_runner.py


示例20: test_environment_variables_windows

 def test_environment_variables_windows(self):
     runner = MRJobRunner(conf_path=False)
     runner._sort_is_windows_sort = True
     self.environment_variable_checks(runner, ['TMP'])
开发者ID:duedil-ltd，项目名称:mrjob，代码行数:4，代码来源:test_runner.py



注：本文中的mrjob.runner.MRJobRunner类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。

鲜花
握手
雷人
路过
鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
More+
上一篇：
Python setup.name_uniquely函数代码示例发布时间：2022-05-27
下一篇：
Python py2.StringIO类代码示例发布时间：2022-05-27
热门推荐
More+
热门话题
More+
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服（服务时间 9:00～18:00）

在线QQ客服
地址：深圳市南山区西丽大学城创智工业园
电邮：jeky_zhao#qq.com
移动电话：139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap