本文整理汇总了Python中swift.common.utils.fallocate函数的典型用法代码示例。如果您正苦于以下问题:Python fallocate函数的具体用法?Python fallocate怎么用?Python fallocate使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了fallocate函数的9个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: writer
def writer(self, size=None):
"""
Context manager to write a file. We create a temporary file first, and
then return a DiskWriter object to encapsulate the state.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not os.path.exists(self.tmpdir):
mkdirs(self.tmpdir)
fd, tmppath = mkstemp(dir=self.tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskWriter(self, fd, tmppath, self.threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
开发者ID:iBeacons,项目名称:swift,代码行数:28,代码来源:diskfile.py
示例2: _preallocate
def _preallocate(self):
"""
The idea is to allocate space in front of an expanding db. If it gets
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
if not DB_PREALLOCATION or self.db_file == ':memory:':
return
MB = (1024 * 1024)
def prealloc_points():
for pm in (1, 2, 5, 10, 25, 50):
yield pm * MB
while True:
pm += 50
yield pm * MB
stat = os.stat(self.db_file)
file_size = stat.st_size
allocated_size = stat.st_blocks * 512
for point in prealloc_points():
if file_size <= point - MB / 2:
prealloc_size = point
break
if allocated_size < prealloc_size:
with open(self.db_file, 'rb+') as fp:
fallocate(fp.fileno(), int(prealloc_size))
开发者ID:bebule,项目名称:swift,代码行数:27,代码来源:db.py
示例3: mkstemp
def mkstemp(self, size=None):
"""
Contextmanager to make a temporary file.
:param size: optional initial size of file to allocate on disk
:raises DiskFileNoSpace: if a size is specified and fallocate fails
"""
if not os.path.exists(self.tmpdir):
mkdirs(self.tmpdir)
fd, self.tmppath = mkstemp(dir=self.tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield fd
finally:
try:
os.close(fd)
except OSError:
pass
tmppath, self.tmppath = self.tmppath, None
try:
os.unlink(tmppath)
except OSError:
pass
开发者ID:hswayne77,项目名称:swift,代码行数:27,代码来源:server.py
示例4: create
def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not exists(self._tmpdir):
mkdirs(self._tmpdir)
fd, tmppath = mkstemp(dir=self._tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskFileWriter(self._name, self._datadir, fd, tmppath,
self._bytes_per_sync, self._threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
开发者ID:yohsuke,项目名称:swift,代码行数:35,代码来源:diskfile.py
示例5: int
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
self.logger.increment('PUT.errors')
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
file = DiskFile(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):
if 'content-length' in request.headers:
try:
fallocate(fd, int(request.headers['content-length']))
except OSError:
return HTTPInsufficientStorage(drive=device,
request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
开发者ID:greenCloud123,项目名称:StorageAllocation,代码行数:31,代码来源:server.py
示例6: mkstemp
def mkstemp(self, size=None):
"""
Contextmanager to make a temporary file, optionally of a specified
initial size.
For Gluster, we first optimistically create the temporary file using
the "rsync-friendly" .NAME.random naming. If we find that some path to
the file does not exist, we then create that path and then create the
temporary file again. If we get file name conflict, we'll retry using
different random suffixes 1,000 times before giving up.
"""
data_file = os.path.join(self.put_datadir, self._obj)
# Assume the full directory path exists to the file already, and
# construct the proper name for the temporary file.
for i in range(0, 1000):
tmpfile = '.' + self._obj + '.' + md5(self._obj +
str(random.random())).hexdigest()
tmppath = os.path.join(self.put_datadir, tmpfile)
try:
fd = do_open(tmppath,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
except GlusterFileSystemOSError as gerr:
if gerr.errno == errno.EEXIST:
# Retry with a different random number.
continue
if gerr.errno == errno.EIO:
# FIXME: Possible FUSE issue or race condition, let's
# sleep on it and retry the operation.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
continue
if gerr.errno != errno.ENOENT:
# FIXME: Other cases we should handle?
raise
if not self._obj_path:
# No directory hierarchy and the create failed telling us
# the container or volume directory does not exist. This
# could be a FUSE issue or some race condition, so let's
# sleep a bit and retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
continue
if i != 0:
# Got ENOENT after previously making the path. This could
# also be a FUSE issue or some race condition, nap and
# retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs" % gerr)
continue
# It looks like the path to the object does not already exist
self._create_dir_object(self._obj_path)
continue
else:
break
else:
# We failed after 1,000 attempts to create the temporary file.
raise DiskFileError('DiskFile.mkstemp(): failed to successfully'
' create a temporary file without running'
' into a name conflict after 1,000 attempts'
' for: %s' % (data_file,))
self.tmppath = tmppath
try:
# Ensure it is properly owned before we make it available.
do_fchown(fd, self.uid, self.gid)
if _preallocate and size:
# For XFS, fallocate() turns off speculative pre-allocation
# until a write is issued either to the last block of the file
# before the EOF or beyond the EOF. This means that we are
# less likely to fragment free space with pre-allocated
# extents that get truncated back to the known file size.
# However, this call also turns holes into allocated but
# unwritten extents, so that allocation occurs before the
# write, not during XFS writeback. This effectively defeats
# any allocation optimizations the filesystem can make at
# writeback time.
fallocate(fd, size)
yield fd
finally:
try:
do_close(fd)
except OSError:
pass
if self.tmppath:
tmppath, self.tmppath = self.tmppath, None
do_unlink(tmppath)
开发者ID:navidshaikh,项目名称:gluster-swift,代码行数:91,代码来源:DiskFile.py
示例7: int
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
file = DiskFile(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
elasped_time = 0
with file.mkstemp() as fd:
try:
fallocate(fd, int(request.headers.get('content-length', 0)))
except OSError:
return HTTPInsufficientStorage(drive=device, request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
start_time = time.time()
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
开发者ID:OSLL,项目名称:swift,代码行数:31,代码来源:server.py
示例8: int
return error_response
new_delete_at = int(request.headers.get("X-Delete-At") or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain")
file = DiskFile(
self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size
)
orig_timestamp = file.metadata.get("X-Timestamp")
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):
if "content-length" in request.headers:
try:
fallocate(fd, int(request.headers["content-length"]))
except OSError:
return HTTPInsufficientStorage(drive=device, request=request)
reader = request.environ["wsgi.input"].read
for chunk in iter(lambda: reader(self.network_chunk_size), ""):
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment("PUT.timeouts")
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
tpool.execute(fsync, fd)
开发者ID:a3linux,项目名称:swift,代码行数:31,代码来源:server.py
示例9: create
#.........这里部分代码省略.........
:raises DiskFileNoSpace: if a size is specified and allocation fails
:raises AlreadyExistsAsFile: if path or part of a path is not a \
directory
"""
# Create /account/container directory structure on mount point root
try:
os.makedirs(self._container_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
data_file = os.path.join(self._put_datadir, self._obj)
# Assume the full directory path exists to the file already, and
# construct the proper name for the temporary file.
attempts = 1
while True:
# To know more about why following temp file naming convention is
# used, please read this GlusterFS doc:
# https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa
tmpfile = '.' + self._obj + '.' + uuid4().hex
tmppath = os.path.join(self._put_datadir, tmpfile)
try:
fd = do_open(tmppath,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
# Raise DiskFileNoSpace to be handled by upper layers when
# there is no space on disk OR when quota is exceeded
raise DiskFileNoSpace()
if gerr.errno == errno.ENOTDIR:
raise AlreadyExistsAsFile('do_open(): failed on %s,'
' path or part of a'
' path is not a directory'
% (tmppath))
if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
# FIXME: Other cases we should handle?
raise
if attempts >= MAX_OPEN_ATTEMPTS:
# We failed after N attempts to create the temporary
# file.
raise DiskFileError('DiskFile.mkstemp(): failed to'
' successfully create a temporary file'
' without running into a name conflict'
' after %d of %d attempts for: %s' % (
attempts, MAX_OPEN_ATTEMPTS,
data_file))
if gerr.errno == errno.EEXIST:
# Retry with a different random number.
attempts += 1
elif gerr.errno == errno.EIO:
# FIXME: Possible FUSE issue or race condition, let's
# sleep on it and retry the operation.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif not self._obj_path:
# No directory hierarchy and the create failed telling us
# the container or volume directory does not exist. This
# could be a FUSE issue or some race condition, so let's
# sleep a bit and retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif attempts > 1:
# Got ENOENT after previously making the path. This could
# also be a FUSE issue or some race condition, nap and
# retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs" % gerr)
attempts += 1
else:
# It looks like the path to the object does not already
# exist; don't count this as an attempt, though, since
# we perform the open() system call optimistically.
self._create_dir_object(self._obj_path)
else:
break
dw = None
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError as err:
if err.errno in (errno.ENOSPC, errno.EDQUOT):
raise DiskFileNoSpace()
raise
# Ensure it is properly owned before we make it available.
do_fchown(fd, self._uid, self._gid)
dw = DiskFileWriter(fd, tmppath, self)
yield dw
finally:
if dw:
dw.close()
if dw._tmppath:
do_unlink(dw._tmppath)
开发者ID:jeffrolland,项目名称:swiftonfile,代码行数:101,代码来源:diskfile.py
注:本文中的swift.common.utils.fallocate函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论