Commit 88517f23 authored by Michael Salim's avatar Michael Salim
Browse files

Minor changes to kill() in launcher.dag: defaults to recursive=True

cli_commands: fixed small user error message bug
model saving: more robust retry: wait and check if save succeeded in
spite of the exception
parent 48deead9
......@@ -72,7 +72,7 @@ __all__ = ['JOB_ID', 'TIMEOUT', 'ERROR',
os.environ['DJANGO_SETTINGS_MODULE'] = 'balsam.django_config.settings'
django.setup()
from balsam.service.models import BalsamJob as _BalsamJob
from balsam.service.models import BalsamJob
from django.conf import settings
current_job = None
......@@ -88,7 +88,7 @@ ERROR = _envs.get('BALSAM_JOB_ERROR', False) == "TRUE"
if JOB_ID:
JOB_ID = uuid.UUID(JOB_ID)
try:
current_job = _BalsamJob.objects.get(pk=JOB_ID)
current_job = BalsamJob.objects.get(pk=JOB_ID)
except:
raise RuntimeError(f"The environment specified current job: "
f"BALSAM_JOB_ID {JOB_ID}\n but this does not "
......@@ -114,7 +114,7 @@ def add_job(**kwargs):
Raises:
- ``ValueError``: if an invalid field name is provided to *kwargs*
'''
job = _BalsamJob()
job = BalsamJob()
for k,v in kwargs.items():
try:
getattr(job, k)
......@@ -158,10 +158,10 @@ def add_dependency(parent,child):
if isinstance(parent, str): parent = uuid.UUID(parent)
if isinstance(child, str): child = uuid.UUID(child)
if not isinstance(parent, _BalsamJob):
parent = _BalsamJob.objects.get(pk=parent)
if not isinstance(child, _BalsamJob):
child = _BalsamJob.objects.get(pk=child)
if not isinstance(parent, BalsamJob):
parent = BalsamJob.objects.get(pk=parent)
if not isinstance(child, BalsamJob):
child = BalsamJob.objects.get(pk=child)
existing_parents = child.get_parents_by_id()
new_parents = existing_parents.copy()
......@@ -197,14 +197,14 @@ def spawn_child(clone=False, **kwargs):
- ``RuntimeError``: If no BalsamJob detected on module-load
- ``ValueError``: if an invalid field name is passed into *kwargs*
'''
if not isinstance(current_job, _BalsamJob):
if not isinstance(current_job, BalsamJob):
raise RuntimeError("No current BalsamJob detected in environment")
if 'workflow' not in kwargs:
kwargs['workflow'] = current_job.workflow
if clone:
child = _BalsamJob.objects.get(pk=current_job.pk)
child = BalsamJob.objects.get(pk=current_job.pk)
child.pk = None
for k,v in kwargs.items():
try:
......@@ -223,7 +223,7 @@ def spawn_child(clone=False, **kwargs):
child.update_state("CREATED", f"spawned by {current_job.cute_id}")
return child
def kill(job, recursive=False):
def kill(job, recursive=True):
'''Kill a job or its entire subtree in the DAG
Mark a job (and optionally all jobs that depend on it) by the state
......@@ -232,7 +232,7 @@ def kill(job, recursive=False):
Args:
- ``job`` (*BalsamJob*): the job (or subtree root) to kill
- ``recursive`` (*bool*): if *True*, then traverse the DAG recursively
to kill all jobs in the subtree. Defaults to *False*
to kill all jobs in the subtree. Defaults to *True*
'''
job.update_state('USER_KILLED')
if recursive:
......
......@@ -38,7 +38,7 @@ def newapp(args):
for arg in (args.executable,args.preprocess,args.postprocess):
paths = arg.split()
if arg and not all(os.path.exists(p) for p in paths):
raise RuntimeError(f"{path} not found")
raise RuntimeError(f"{paths} not found")
app = AppDef()
app.name = args.name
......
......@@ -5,6 +5,7 @@ import sys
from datetime import datetime
from socket import gethostname
import uuid
import time
from django.core.exceptions import ValidationError,ObjectDoesNotExist
from django.conf import settings
......@@ -254,7 +255,17 @@ class BalsamJob(models.Model):
# Work around sqlite3 DB locked error
while True:
try: models.Model.save(self, force_insert, force_update, using, update_fields)
except OperationalError: pass
except OperationalError:
try:
time.sleep(1)
newjob = BalsamJob.objects.get(pk=self.pk)
if newjob.version == self.version: break
except ObjectDoesNotExist: pass
except RecordModifiedError:
newjob = BalsamJob.objects.get(pk=self.pk)
logger.error(f'RecordModifiedError when saving {self.cute_id}')
logger.error(f'Trying to save:\n{str(self)}\nIn DB:\n{str(newjob)}\n')
raise
else: break
def __str__(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment