本文整理汇总了Python中mrq.job.Job类的典型用法代码示例。如果您正苦于以下问题:Python Job类的具体用法?Python Job怎么用?Python Job使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Job类的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: run
def run(self, params):
additional_timeout = params.get("timeout", 300)
stats = {
"requeued": 0,
"started": 0
}
# There shouldn't be that much "started" jobs so we can quite safely
# iterate over them.
fields = {"_id": 1, "datestarted": 1, "queue": 1, "path": 1, "retry_count": 1}
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"status": "started"}, projection=fields):
job = Job(job_data["_id"])
job.set_data(job_data)
stats["started"] += 1
expire_date = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=job.timeout + additional_timeout)
if job_data["datestarted"] < expire_date:
log.debug("Requeueing job %s" % job.id)
job.requeue()
stats["requeued"] += 1
return stats
开发者ID:AshBT,项目名称:mrq,代码行数:29,代码来源:cleaning.py
示例2: test_cancel_by_path
def test_cancel_by_path(worker):
# Start the worker with only one greenlet so that tasks execute sequentially
worker.start(flags="--gevent 1")
job_id1 = worker.send_task("tests.tasks.general.MongoInsert", {"a": 41, "sleep": 2}, block=False)
worker.send_task("mrq.basetasks.utils.JobAction", {
"path": "tests.tasks.general.MongoInsert",
"status": "queued",
"action": "cancel"
}, block=False)
job_id2 = worker.send_task("tests.tasks.general.MongoInsert", {"a": 43}, block=False)
Job(job_id2).wait(poll_interval=0.01)
# Leave some time to unqueue job_id2 without executing.
time.sleep(1)
worker.stop(deps=False)
job1 = Job(job_id1).fetch().data
job2 = Job(job_id2).fetch().data
assert job1["status"] == "success"
assert job1["result"] == {"a": 41, "sleep": 2}
assert job2["status"] == "cancel"
assert job2["dateexpires"] > job2["dateupdated"]
assert job2.get("result") is None
assert worker.mongodb_logs.tests_inserts.count() == 1
assert Queue("default").size() == 0
开发者ID:bossjones,项目名称:mrq,代码行数:35,代码来源:test_cancel.py
示例3: test_interrupt_worker_double_sigint
def test_interrupt_worker_double_sigint(worker, p_flags):
""" Test what happens when we interrupt a running worker with 2 SIGINTs. """
start_time = time.time()
worker.start(flags=p_flags)
job_id = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 20}, block=False)
while Job(job_id).fetch().data["status"] == "queued":
time.sleep(0.1)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task("tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 20}, block=False)
time.sleep(1)
job2 = Job(job_id2).fetch().data
assert job2.get("status") == "queued"
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Sending a second kill -2 should make it stop
worker.stop(block=True, deps=False, force=True)
while Job(job_id).fetch().data["status"] == "started":
time.sleep(0.1)
job = Job(job_id).fetch().data
assert job["status"] == "interrupt"
assert time.time() - start_time < 15
# Then try the cleaning task that requeues interrupted jobs
assert Queue("default").size() == 1
worker.start(queues="cleaning", deps=False, flush=False)
res = worker.send_task("mrq.basetasks.cleaning.RequeueInterruptedJobs", {}, block=True, queue="cleaning")
assert res["requeued"] == 1
assert Queue("default").size() == 2
Queue("default").list_job_ids() == [str(job_id2), str(job_id)]
job = Job(job_id).fetch().data
assert job["status"] == "queued"
assert job["queue"] == "default"
开发者ID:pricingassistant,项目名称:mrq,代码行数:59,代码来源:test_interrupts.py
示例4: run
def run(self, params):
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {"fetched": 0, "requeued": 0}
# This was only checking in Redis and wasn't resistant to a redis-wide flush.
# Doing Queue.all() is slower but covers more edge cases.
# all_queues = Queue.all_known()
all_queues = Queue.all()
log.info("Checking %s queues" % len(all_queues))
for queue_name in all_queues:
queue = Queue(queue_name)
queue_size = queue.size()
# If the queue is raw, the jobs were only stored in redis so they are lost for good.
if queue.is_raw:
continue
log.info("Checking queue %s" % queue_name)
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue_name, queue_size))
continue
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info(
"Stopping because queue %s actually had more than %s items" % (queue_name, len(queue_jobs_ids))
)
continue
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"queue": queue_name, "status": "queued"}, projection={"_id": 1}
).sort([["_id", 1]]):
stats["fetched"] += 1
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Found job %s on queue %s. Stopping" % (job_data["_id"], queue.id))
break
# At this point, this job is not on the queue and we're sure
# the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=queue_name)
return stats
开发者ID:pricingassistant,项目名称:mrq,代码行数:58,代码来源:cleaning.py
示例5: test_retry
def test_retry(worker):
job_id = worker.send_task("mrq.basetasks.tests.general.Retry", {"queue": "noexec", "countdown": 60}, block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["queue"] == "noexec"
assert job_data["status"] == "retry"
assert job_data["dateretry"] > datetime.datetime.utcnow()
assert job_data.get("result") is None
开发者ID:nfredrik,项目名称:mrq,代码行数:10,代码来源:test_retry.py
示例6: get_job
def get_job():
job_id = request.args['job_id']
job = Job(job_id)
job.fetch()
if job.data["params"].get("user"):
if not g.user.is_authenticated() or (job.data["params"].get("user") != str(g.user.id)):
return "Unauthorized."
return json.dumps({k: v for k, v in job.data.iteritems() if k in ("status", "result")})
开发者ID:PUNTOZERO,项目名称:imgfab,代码行数:12,代码来源:app.py
示例7: test_retry_cancel_on_retry
def test_retry_cancel_on_retry(worker):
job_id = worker.send_task("mrq.basetasks.tests.general.Retry", {
"queue": "noexec",
"countdown": 60,
"cancel_on_retry": True
}, block=False)
job_data = Job(job_id).wait(poll_interval=0.01, full_data=True)
assert job_data["status"] == "cancel"
assert job_data["queue"] == "default"
assert job_data.get("result") is None
开发者ID:nfredrik,项目名称:mrq,代码行数:13,代码来源:test_retry.py
示例8: wait_for_tasks_results
def wait_for_tasks_results(self, job_ids, block=True, accept_statuses=["success"]):
if not block:
return job_ids
results = []
for job_id in job_ids:
job = Job(job_id).wait(poll_interval=0.01)
assert job.get("status") in accept_statuses, "Job had status %s, not in %s. Dump: %s" % (job.get("status"), accept_statuses, job)
results.append(job.get("result"))
return results
开发者ID:bossjones,项目名称:mrq,代码行数:14,代码来源:conftest.py
示例9: test_cli_run_nonblocking
def test_cli_run_nonblocking(worker):
worker.start_deps()
job_id1 = worker.send_task_cli("tests.tasks.general.Add", {"a": 41, "b": 1}, block=False)
job1 = Job(job_id1).fetch()
job1.wait(poll_interval=0.01)
job1.fetch()
assert job1.data["status"] == "success"
assert job1.data["result"] == 42
开发者ID:bossjones,项目名称:mrq,代码行数:14,代码来源:test_cli.py
示例10: test_interrupt_maxconcurrency
def test_interrupt_maxconcurrency(worker):
# The worker will raise a maxconcurrency on the second job
worker.start(flags="--greenlets=2")
job_ids = worker.send_tasks(
"tests.tasks.concurrency.LockedAdd", [{"a": i, "b": 1, "sleep": 2} for i in range(2)], block=False
)
worker.wait_for_tasks_results(job_ids, accept_statuses=["success", "failed", "maxconcurrency"])
job_statuses = [Job(job_id).fetch().data["status"] for job_id in job_ids]
assert set(job_statuses) == set(["success", "maxconcurrency"])
# the job concurrency key must be equal to 0
last_job_id = worker.send_task("tests.tasks.concurrency.LockedAdd", {"a": 1, "b": 1, "sleep": 2}, block=False)
last_job = Job(last_job_id).wait(poll_interval=0.01)
assert last_job.get("status") == "success"
开发者ID:pricingassistant,项目名称:mrq,代码行数:18,代码来源:test_interrupts.py
示例11: run
def run(self, params):
self.collection = connections.mongodb_jobs.mrq_jobs
# If there are more than this much items on the queue, we don't try to check if our mongodb
# jobs are still queued.
max_queue_items = params.get("max_queue_items", 1000)
stats = {
"fetched": 0,
"requeued": 0
}
for job_data in self.collection.find({
"status": "queued"
}, fields={"_id": 1, "queue": 1}).sort([("_id", 1)]):
stats["fetched"] += 1
queue = Queue(job_data["queue"])
queue_size = queue.size()
if queue_size > max_queue_items:
log.info("Stopping because queue %s has %s items" % (queue, queue_size))
break
queue_jobs_ids = set(queue.list_job_ids(limit=max_queue_items + 1))
if len(queue_jobs_ids) >= max_queue_items:
log.info("Stopping because queue %s actually had more than %s items" % (queue, len(queue_jobs_ids)))
break
if str(job_data["_id"]) in queue_jobs_ids:
log.info("Stopping because we found job %s in redis" % job_data["_id"])
break
# At this point, this job is not on the queue and we're sure the queue is less than max_queue_items
# We can safely requeue the job.
log.info("Requeueing %s on %s" % (job_data["_id"], queue.id))
stats["requeued"] += 1
job = Job(job_data["_id"])
job.requeue(queue=job_data["queue"])
return stats
开发者ID:nfredrik,项目名称:mrq,代码行数:43,代码来源:cleaning.py
示例12: test_interrupt_worker_gracefully
def test_interrupt_worker_gracefully(worker, p_flags):
""" Test what happens when we interrupt a running worker gracefully. """
worker.start(flags=p_flags)
job_id = worker.send_task(
"tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 5}, block=False)
time.sleep(2)
job = Job(job_id).fetch().data
assert job["status"] == "started"
# Stop the worker gracefully. first job should still finish!
worker.stop(block=False, deps=False)
time.sleep(1)
# Should not be accepting new jobs!
job_id2 = worker.send_task(
"tests.tasks.general.Add", {"a": 42, "b": 1, "sleep": 4}, block=False)
time.sleep(1)
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
time.sleep(4)
job = Job(job_id).fetch().data
assert job["status"] == "success"
assert job["result"] == 42
job = Job(job_id2).fetch().data
assert job.get("status") == "queued"
开发者ID:iorlas,项目名称:mrq,代码行数:35,代码来源:test_interrupts.py
示例13: test_raw_no_storage
def test_raw_no_storage(worker):
""" Test tasks that don't store unless they go to error status like 'failed' """
worker.start(
flags="--config tests/fixtures/config-raw1.py",
queues="default testnostorage_raw"
)
jobs_collection = worker.mongodb_jobs.mrq_jobs
test_collection = worker.mongodb_logs.tests_inserts
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.MongoInsert 3"
], block=False)
time.sleep(2)
# No started inserted.
assert jobs_collection.count() == 0
time.sleep(2)
# No success either, but we did insert
assert test_collection.count() == 1
assert jobs_collection.count() == 0
test_collection.remove({})
# However failed tasks get stored.
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.RaiseException 0"
], block=False)
time.sleep(2)
# Failed was inserted.
assert jobs_collection.count({"status": "failed", "path": "tests.tasks.general.RaiseException"}) == 1
# If we requeue and don't raise, should be OK and inserted this time, even in success
# no_storage depends on a raw queue, not a task path.
_id = jobs_collection.find_one()["_id"]
jobs_collection.update({"_id": _id}, {"$set": {"path": "tests.tasks.general.MongoInsert"}})
job = Job(_id).fetch(full_data=True)
job.requeue(queue="default")
time.sleep(1)
assert test_collection.count() == 1
assert jobs_collection.count() == 1
assert jobs_collection.count({"status": "success"}) == 1
jobs_collection.remove({})
# Test with retry: should be inserted
worker.send_raw_tasks("testnostorage_raw", [
"tests.tasks.general.Retry 0"
], block=False)
assert jobs_collection.count({"status": "started"}) == 0
time.sleep(2)
assert jobs_collection.count({"status": "retry"}) == 1
开发者ID:AshBT,项目名称:mrq,代码行数:62,代码来源:test_raw.py
注:本文中的mrq.job.Job类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论