本文整理汇总了Python中pyeclib.ec_iface.ECDriver类的典型用法代码示例。如果您正苦于以下问题：Python ECDriver类的具体用法？Python ECDriver怎么用？Python ECDriver使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了ECDriver类的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: Eraser
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
raw_strips = self.driver.encode(data)
strips = []
for raw_strip in raw_strips:
strip = Strip()
strip.data = raw_strip
strips.append(strip)
return strips
def decode(self, playcloud_strips):
"""Decode byte strips in a string of bytes"""
strips = convert_strips_to_bytes_list(playcloud_strips)
return self.driver.decode(strips)
开发者ID:safecloud-project，项目名称:playcloud，代码行数:27，代码来源:coding_servicer.py
示例2: ECStorageMethod
class ECStorageMethod(StorageMethod):
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
@property
def quorum(self):
return self._ec_quorum_size
@classmethod
def build(cls, params):
ec_nb_data = params.pop('k')
ec_nb_parity = params.pop('m')
ec_type = params.pop('algo')
return cls('ec', ec_segment_size=EC_SEGMENT_SIZE,
ec_type=ec_type, ec_nb_data=ec_nb_data,
ec_nb_parity=ec_nb_parity)
@property
def ec_type(self):
return self._ec_type
@property
def ec_nb_data(self):
return self._ec_nb_data
@property
def ec_nb_parity(self):
return self._ec_nb_parity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def ec_fragment_size(self):
return self.driver.get_segment_info(
self.ec_segment_size, self.ec_segment_size)['fragment_size']
开发者ID:lanweichang，项目名称:oio-sds，代码行数:57，代码来源:storage_method.py
示例3: __init__
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
开发者ID:safecloud-project，项目名称:playcloud，代码行数:10，代码来源:coding_servicer.py
示例4: TestNullDriver
class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
def test_null_driver(self):
self.null_driver.encode("")
self.null_driver.decode([])
开发者ID:tsg-，项目名称:pyeclib，代码行数:10，代码来源:test_pyeclib_api.py
示例5: check_metadata_formatted
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
if ec_type not in VALID_EC_TYPES:
return
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters)
for i in range(filesize))
file_bytes = file_str.encode('utf-8')
pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if 'index' in metadata:
self.assertEqual(metadata['index'], f)
else:
self.assertTrue(False)
if 'chksum_mismatch' in metadata:
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(False)
if 'backend_id' in metadata:
self.assertEqual(metadata['backend_id'], ec_type)
else:
self.assertTrue(False)
if 'orig_data_size' in metadata:
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(False)
if 'chksum_type' in metadata:
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(False)
if 'backend_version' not in metadata:
self.assertTrue(False)
if 'chksum' not in metadata:
self.assertTrue(False)
if 'size' not in metadata:
self.assertTrue(False)
f += 1
开发者ID:openstack，项目名称:pyeclib，代码行数:53，代码来源:test_pyeclib_api.py
示例6: check_metadata_formatted
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
if ec_type not in VALID_EC_TYPES:
return
filesize = 1024 * 1024 * 3
file_str = "".join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode("utf-8")
pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if "index" in metadata:
self.assertEqual(metadata["index"], f)
else:
self.assertTrue(False)
if "chksum_mismatch" in metadata:
self.assertEqual(metadata["chksum_mismatch"], 0)
else:
self.assertTrue(False)
if "backend_id" in metadata:
self.assertEqual(metadata["backend_id"], ec_type)
else:
self.assertTrue(False)
if "orig_data_size" in metadata:
self.assertEqual(metadata["orig_data_size"], 3145728)
else:
self.assertTrue(False)
if "chksum_type" in metadata:
self.assertEqual(metadata["chksum_type"], "crc32")
else:
self.assertTrue(False)
if "backend_version" not in metadata:
self.assertTrue(False)
if "chksum" not in metadata:
self.assertTrue(False)
if "size" not in metadata:
self.assertTrue(False)
f += 1
开发者ID:tsg-，项目名称:pyeclib，代码行数:51，代码来源:test_pyeclib_api.py
示例7: test_get_metadata_formatted
def test_get_metadata_formatted(self):
pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
fragments = pyeclib_driver.encode(file_bytes)
i = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if metadata.has_key('index'):
self.assertEqual(metadata['index'], i)
else:
self.assertTrue(false)
if metadata.has_key('chksum_mismatch'):
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(false)
if metadata.has_key('backend_id'):
self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
else:
self.assertTrue(false)
if metadata.has_key('orig_data_size'):
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(false)
if metadata.has_key('chksum_type'):
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(false)
if not metadata.has_key('backend_version'):
self.assertTrue(false)
if not metadata.has_key('chksum'):
self.assertTrue(false)
if not metadata.has_key('size'):
self.assertTrue(false)
i += 1
开发者ID:wawani82，项目名称:SwiftER-pyeclib，代码行数:47，代码来源:test_pyeclib_api.py
示例8: encode
def encode(k,m,data_to_encode,base_filename_to_store,ec_type) :
home = expanduser("~")
key_list = base_filename_to_store.split('.')
Disk_base_dir = home + '/Disk'
jencoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
# Create a replicated copy first.
#replica_dir = Disk_base_dir + "/Disk_0"
#if not os.path.exists(replica_dir):
# os.makedirs(replica_dir)
#with open(replica_dir + "/" + base_filename_to_store,'wb') as fp :
# fp.write(data_to_encode)
#fp.close()
if ec_type in jencoder_types :
jencoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jencoder.so')
w = 8
key = key_list[0]
jencoder.encode(key,data_to_encode,Disk_base_dir,k,m,ec_type,w,0)
else :
ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)
# encode
fragments = ec_driver.encode(data_to_encode)
# store
i = 1
for fragment in fragments:
fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i)
if not os.path.exists(fragment_dir):
os.makedirs(fragment_dir)
with open("%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp:
fp.write(fragment)
fp.close()
i += 1
开发者ID:Vignesh2208，项目名称:Secure-AMI-Data-Store，代码行数:46，代码来源:encoder.py
示例9: decode
def decode(k,m,base_filename_to_decode,ec_type) :
#print("k = %d, m = %d" % (k, m))
#print("ec_type = %s" % ec_type)
home = expanduser("~")
Disk_base_dir = home + '/Disk'
key_list = base_filename_to_decode.split('.')
jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
if ec_type in jdecoder_types :
print "Decoding started "
jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
key = key_list[0]
decoded_data = str(jdecoder.decode(key,Disk_base_dir))
print "decoded data = ", decoded_data
else :
ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)
fragment_list = []
i = 0
while i <=k :
if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) :
curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
if i == 0 :
if os.path.exists(curr_dir + "/" + base_filename_to_decode) :
with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp :
decoded_data = fp.read()
fp.close()
return decoded_data
else :
if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) :
with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp :
fragment = fp.read()
fp.close()
fragment_list.append(fragment)
i = i + 1
if len(fragment_list) < k :
return -1 # Not enough fragments to decode
else :
decoded_data = ec_driver.decode(fragment_list)
return decoded_data
开发者ID:Vignesh2208，项目名称:Secure-AMI-Data-Store，代码行数:45，代码来源:decoder.py
示例10: test_ec_fragment_size_cached
def test_ec_fragment_size_cached(self):
policy = ECStoragePolicy(
0, 'ec2-1', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1, object_ring=FakeRing(replicas=3),
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, is_default=True)
ec_driver = ECDriver(ec_type=DEFAULT_TEST_EC_TYPE,
k=2, m=1)
expected_fragment_size = ec_driver.get_segment_info(
DEFAULT_EC_OBJECT_SEGMENT_SIZE,
DEFAULT_EC_OBJECT_SEGMENT_SIZE)['fragment_size']
with mock.patch.object(
policy.pyeclib_driver, 'get_segment_info') as fake:
fake.return_value = {
'fragment_size': expected_fragment_size}
for x in range(10):
self.assertEqual(expected_fragment_size,
policy.fragment_size)
# pyeclib_driver.get_segment_info is called only once
self.assertEqual(1, fake.call_count)
开发者ID:aureliengoulon，项目名称:swift，代码行数:22，代码来源:test_storage_policy.py
示例11: __init__
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
开发者ID:lanweichang，项目名称:oio-sds，代码行数:22，代码来源:storage_method.py
示例12: ECDriver
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse
parser = argparse.ArgumentParser(
description='PyECLib tool to determine fragment indexes needed to '
'recover missing fragments.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
nargs='+', help='missing_fragments')
args = parser.parse_args()
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments)
print(fragments_needed)
开发者ID:openstack，项目名称:pyeclib，代码行数:30，代码来源:pyeclib_fragments_needed.py
示例13: print
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('file_dir', help='directory with the file')
parser.add_argument('filename', help='file to encode')
parser.add_argument('fragment_dir', help='directory to drop encoded fragments')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)
start = time.time()
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
whole_file_str = fp.read()
# encode
fragments = ec_driver.encode(whole_file_str)
# store
i = 0
for fragment in fragments:
with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
fp.write(fragment)
i += 1
开发者ID:wawani82，项目名称:SwiftER-pyeclib，代码行数:30，代码来源:pyeclib_encode.py
示例14: setUp
def setUp(self):
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
开发者ID:openstack，项目名称:pyeclib，代码行数:3，代码来源:test_pyeclib_api.py
示例15: ECStoragePolicy
class ECStoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'erasure_coding'.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
@property
def ec_type(self):
return self._ec_type
@property
def ec_ndata(self):
return self._ec_ndata
@property
def ec_nparity(self):
return self._ec_nparity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def fragment_size(self):
"""
Maximum length of a fragment, including header.
NB: a fragment archive is a sequence of 0 or more max-length
fragments followed by one possibly-shorter fragment.
"""
# Technically pyeclib's get_segment_info signature calls for
# (data_len, segment_size) but on a ranged GET we don't know the
#.........这里部分代码省略.........
开发者ID:HoratiusTang，项目名称:swift，代码行数:101，代码来源:storage_policy.py
示例16: __init__
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
开发者ID:HoratiusTang，项目名称:swift，代码行数:64，代码来源:storage_policy.py
示例17: print
# Results will be List[(ec_type, throughput)]
results = []
# Num iterations
num_iterations = 10
for scheme in schemes:
print(scheme)
# Generate a new string for each test
file_str = ''.join(
random.choice(
string.ascii_uppercase + string.digits) for x in range(args.s))
try:
ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue
timer.start()
for i in range(num_iterations):
ec_driver.encode(file_str)
duration = timer.stop_and_return()
results.append((scheme, duration))
timer.reset()
开发者ID:openstack，项目名称:pyeclib，代码行数:30，代码来源:pyeclib_conf_tool.py
示例18: print
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments', metavar='fragment', nargs='+',
help='fragments to decode')
parser.add_argument('filename', help='output file')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []
# read fragments
for fragment in args.fragments:
with open(("%s" % fragment), "rb") as fp:
fragment_list.append(fp.read())
# decode
decoded_file = ec_driver.decode(fragment_list)
# write
with open("%s.decoded" % args.filename, "wb") as fp:
fp.write(decoded_file)
开发者ID:openstack，项目名称:pyeclib，代码行数:30，代码来源:pyeclib_decode.py
示例19: ECStoragePolicy
class ECStoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'erasure_coding'.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
diskfile_module='egg:swift#erasure_coding.fs',
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None,
ec_duplication_factor=1):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring,
diskfile_module=diskfile_module)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5:
logger = logging.getLogger("swift.common.storage_policy")
if not logger.handlers:
# If nothing else, log to stderr
logger.addHandler(logging.StreamHandler(sys.__stderr__))
logger.warning(
'Storage policy %s uses an EC configuration known to harm '
'data durability. Any data in this policy should be migrated. '
'See https://bugs.launchpad.net/swift/+bug/1639691 for '
'more information.' % self.name)
if not is_deprecated:
raise PolicyError(
'Storage policy %s uses an EC configuration known to harm '
'data durability. This policy MUST be deprecated.'
% self.name)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
self._fragment_size = None
self._ec_duplication_factor = \
config_positive_int_value(ec_duplication_factor)
@property
def ec_type(self):
return self._ec_type
#.........这里部分代码省略.........
开发者ID:mahak，项目名称:swift，代码行数:101，代码来源:storage_policy.py
示例20: len
print ""
print "A script that benchmarks pyeclib libraries reconstruction capabilities"
print ""
print "Arguments:"
print "\tsize Size of the payload to encode in bytes"
if __name__ == "__main__":
if len(sys.argv) != 2:
print_usage()
sys.exit(0)
SIZE = int(sys.argv[1])
EC_K = int(os.environ.get("EC_K", 10))
EC_M = int(os.environ.get("EC_M", 4))
EC_TYPE = os.environ.get("EC_TYPE", "liberasurecode_rs_vand")
DRIVER = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
DATA = os.urandom(SIZE)
STRIPS = DRIVER.encode(DATA)
LENGTH = EC_K + EC_M
SUPPORTED_DISTANCE = LENGTH - EC_K + 1
print "About to reconstruct ", REQUESTS, " times a payload of size ", SIZE, " bytes (", \
(DRIVER.ec_type if hasattr(DRIVER, "ec_type") else EC_TYPE), ", k =", DRIVER.k, \
", m =", DRIVER.m, ") from 0 to", SUPPORTED_DISTANCE, "missing blocks"
random.seed(0)
for missing_blocks in range(SUPPORTED_DISTANCE):
for i in range(REQUESTS):
missing_indices = range(missing_blocks)
start = time.clock()
开发者ID:safecloud-project，项目名称:playcloud，代码行数:31，代码来源:microbench_local_reconstruct.py
注：本文中的pyeclib.ec_iface.ECDriver类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论