• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号
登陆 注册

Python fs.HDFSHelper类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中twitter.common.fs.HDFSHelper类的典型用法代码示例。如果您正苦于以下问题：Python HDFSHelper类的具体用法？Python HDFSHelper怎么用？Python HDFSHelper使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



在下文中一共展示了HDFSHelper类的14个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: test_copy_to_local

 def test_copy_to_local(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.copy_to_local('text_file','dest')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \
     "-copyToLocal text_file dest"
   self.assertEqual(cmd, expected_cmd)
开发者ID:jayeye，项目名称:commons，代码行数:7，代码来源:hdfs_test.py


示例2: test_cat

 def test_cat(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.cat('text_file', 'local')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cat " + \
     "text_file"
   self.assertEqual(cmd, expected_cmd)
开发者ID:jayeye，项目名称:commons，代码行数:7，代码来源:hdfs_test.py


示例3: test_read

 def test_read(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   read_text = hdfs_helper.read('somefile')
   self.assertEqual("read_test", read_text)
   read_text = hdfs_helper.read('non_exist')
   self.assertEqual(None, read_text)
开发者ID:jayeye，项目名称:commons，代码行数:7，代码来源:hdfs_test.py


示例4: test_hdfs_lsr

 def test_hdfs_lsr(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
     expected_output_dir = [["hadoop_dir/test_dir", 68]]
     cmd = hdfs_helper.lsr("path")
     self.assertEqual(cmd, expected_output)
     cmd = hdfs_helper.lsr("path", True)
     self.assertEqual(cmd, expected_output_dir)
开发者ID:UrbanCompass，项目名称:commons，代码行数:8，代码来源:hdfs_test.py


示例5: test_hadoop_v1

 def test_hadoop_v1(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil,
                            use_hadoop_v1=True)
   cmd = hdfs_helper.get(['src'],"dst")
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
   self.assertEqual(cmd, expected_cmd)
   cmd = hdfs_helper.mkdir('dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mkdir dest"
   self.assertEqual(cmd, expected_cmd)
   cmd = hdfs_helper.cp('src','dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
   self.assertEqual(cmd, expected_cmd)
开发者ID:jayeye，项目名称:commons，代码行数:13，代码来源:hdfs_test.py


示例6: test_hdfs_ls

 def test_hdfs_ls(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.ls("path", True)
     expected_output_dir = [["hadoop_dir/test_dir", 68]]
     expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
     self.assertEqual(cmd, expected_output_dir)
     cmd = hdfs_helper.ls("path")
     self.assertEqual(cmd, expected_output)
     # Empty path
     cmd = hdfs_helper.ls("empty", True)
     self.assertTrue(not cmd)
     # Return code 255
     self.assertRaises(HDFSHelper.InternalError, hdfs_helper.ls, "non_existing", True)
开发者ID:UrbanCompass，项目名称:commons，代码行数:13，代码来源:hdfs_test.py


示例7: fetch

  def fetch(self, uri, directory):
    log.info("Fetching %s from HDFS" % uri)

    if "JAVA_HOME" in os.environ:
      log.info("Using JAVA_HOME '%s' for HDFS commands" % os.environ["JAVA_HOME"])

    config = os.environ.get("HADOOP_CONF_DIR", HADOOP_CONF_DIR)
    h = HDFSHelper(config, heap_limit=Amount(256, Data.MB))
    try:
      f = lambda: h.copy_to_local(uri, directory)
      deadline(f, timeout=self._timeout, propagate=True, daemon=True)
    except HDFSHelper.InternalError as e:
      raise self.Error('Unable to fetch HDFS package: %s' % e)
    except Timeout as e:
      raise self.Error("Failed to fetch package from HDFS within : %s" % (self._timeout, e))
开发者ID:GavinHwa，项目名称:mysos，代码行数:15，代码来源:hdfs.py


示例8: test_mv

 def test_mv(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.mv('src', 'dest')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mv src dest"
   self.assertEqual(cmd, expected_cmd)
开发者ID:jayeye，项目名称:commons，代码行数:6，代码来源:hdfs_test.py


示例9: test_write

 def test_write(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   self.assertEqual(True, hdfs_helper.write('somefile',"write_text"))
开发者ID:jayeye，项目名称:commons，代码行数:4，代码来源:hdfs_test.py


示例10: test_exists

 def test_exists(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   self.assertEquals(0, hdfs_helper.exists('hadoop_dir'))
开发者ID:jayeye，项目名称:commons，代码行数:4，代码来源:hdfs_test.py


示例11: test_get

 def test_get(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.get(['src'],"dst")
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
   self.assertEqual(cmd, expected_cmd)
开发者ID:jayeye，项目名称:commons，代码行数:6，代码来源:hdfs_test.py


示例12: test_rm

 def test_rm(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.rm('dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -rm dest"
   self.assertEqual(cmd, expected_cmd)
开发者ID:EricCen，项目名称:commons，代码行数:6，代码来源:hdfs_test.py


示例13: test_copy_from_local

 def test_copy_from_local(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.copy_from_local("text_file", "dest")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + "-copyFromLocal text_file dest"
     self.assertEqual(cmd, expected_cmd)
开发者ID:UrbanCompass，项目名称:commons，代码行数:5，代码来源:hdfs_test.py


示例14: test_cp

 def test_cp(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.cp("src", "dest")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
     self.assertEqual(cmd, expected_cmd)
开发者ID:UrbanCompass，项目名称:commons，代码行数:5，代码来源:hdfs_test.py



注：本文中的twitter.common.fs.HDFSHelper类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。

鲜花
握手
雷人
路过
鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
More+
上一篇：
Python log.debug函数代码示例发布时间：2022-05-27
下一篇：
Python exceptions.ExceptionalThread类代码示例发布时间：2022-05-27
热门推荐
More+
热门话题
More+
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服（服务时间 9:00～18:00）

在线QQ客服
地址：深圳市南山区西丽大学城创智工业园
电邮：jeky_zhao#qq.com
移动电话：139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap