Commit 036cf45b authored by Michael Salim's avatar Michael Salim
Browse files

Merged theta-changes, feature/user-kill, and feature/sqlite-writer

all back into development branch.

Merge remote-tracking branch 'origin/theta-changes' into develop
parents aa5cc7b5 5e1d1b25
......@@ -196,7 +196,7 @@ def rm(args):
def qsub(args):
job = Job()
job.name = args.name
job.name = args.name if args.name else "default"
job.description = 'Added by balsam qsub'
job.workflow = 'qsub'
job.allowed_work_sites = settings.BALSAM_SITE
......
Jobid: 169162
qsub --mode script -A Performance -n 2 -t 30 -q debug-cache-quad ./job.sh
Fri Jan 12 23:21:16 2018 +0000 (UTC) submitted with cwd set to: /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta
jobid 169162 submitted from terminal /dev/pts/0
Fri Jan 12 23:22:16 2018 +0000 (UTC)
Fri Jan 12 23:22:16 2018 +0000 (UTC) Command: '/gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh'
Fri Jan 12 23:22:16 2018 +0000 (UTC)
Fri Jan 12 23:22:16 2018 +0000 (UTC) Environment:
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTCORES=64
Fri Jan 12 23:22:16 2018 +0000 (UTC) SHELL=/bin/bash
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PROJECT=Performance
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTNAME=3830,3832
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_JOBID=169162
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_STARTTIME=1515799303
Fri Jan 12 23:22:16 2018 +0000 (UTC) LOGNAME=harms
Fri Jan 12 23:22:16 2018 +0000 (UTC) USER=harms
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_ENDTIME=1515801103
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTSIZE=2
Fri Jan 12 23:22:16 2018 +0000 (UTC) HOME=/home/harms
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_JOBSIZE=2
Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_QUEUE=debug-cache-quad
Fri Jan 12 23:22:16 2018 +0000 (UTC)
Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stdin received from /dev/null
Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stdout sent to /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/169162.output
Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stderr sent to /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/169162.error
Fri Jan 12 23:22:16 2018 +0000 (UTC)
Fri Jan 12 23:23:17 2018 +0000 (UTC) Info: user delete requested with signal SIGTERM by user harms
+ module load intelpython35
++ /opt/cray/pe/modules/3.2.10.6/bin/modulecmd bash load intelpython35
Intel Python 3.5 version 2017.0.035 loaded
+ eval C_INCLUDE_PATH=/opt/intel/python/2017.0.035/intelpython35/include ';export' 'C_INCLUDE_PATH;LD_LIBRARY_PATH=/opt/intel/python/2017.0.035/intelpython35/lib:/soft/perftools/darshan/darshan-3.1.5/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/mic/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/ipp/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mkl/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/tbb/lib/intel64/gcc4.4:/opt/intel/debugger_2017/libipt/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/../compiler/lib/intel64_lin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/lib64:/dbhome/db2cat/sqllib/lib64:/dbhome/db2cat/sqllib/lib64/gskit:/dbhome/db2cat/sqllib/lib32:' ';export' 'LD_LIBRARY_PATH;LOADEDMODULES=modules/3.2.10.6:alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:intel/18.0.0.128:craype-network-aries:craype/2.5.13:cray-libsci/17.09.1:pmi/5.0.12:atp/2.1.1:rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:perftools-base/6.5.2:PrgEnv-intel/6.0.4:craype-mic-knl:cray-mpich/7.6.3:nompirun/nompirun:darshan/3.1.5:trackdeps:xalt:intelpython35/2017.0.035' ';export' 'LOADEDMODULES;MANPATH=/opt/intel/python/2017.0.035/intelpython35/share/man:/opt/cray/pe/mpt/7.6.3/gni/man/mpich:/opt/cray/pe/perftools/6.5.2/man:/opt/cray/pe/papi/5.5.1.3/share/pdoc/man:/opt/cray/pe/atp/2.1.1/man:/opt/cray/pe/pmi/5.0.12/man:/opt/cray/pe/libsci/17.09.1/man:/opt/cray/pe/man/csmlversion:/opt/cray/pe/craype/2.5.13/man:/opt/intel/man/common:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/man:/opt/intel/documentation_2017/en/debugger//gdb-ia/man/:/opt/intel/documentation_2017/en/debugger//gdb-mic/man/:/opt/intel/documentation_2017/en/debugger//gdb-igfx/man/:/opt/cray/cfgutils/man:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/man:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/man:/opt/cray/pe/modules/3.2.10.6/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/pe/man' ';export' 'MANPATH;PATH=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin' ';export' 'PATH;PYTHON_HOME=/opt/intel/python/2017.0.035/intelpython35' ';export' 'PYTHON_HOME;PYTHON_VERSION=3.5' ';export' 'PYTHON_VERSION;TCL_LIBRARY=/opt/intel/python/2017.0.035/intelpython35/lib/tcl8.6' ';export' 'TCL_LIBRARY;_LMFILES_=/opt/cray/pe/modulefiles/modules/3.2.10.6:/opt/cray/ari/modulefiles/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:/opt/cray/ari/modulefiles/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:/opt/cray/ari/modulefiles/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:/opt/cray/ari/modulefiles/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:/opt/cray/ari/modulefiles/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:/opt/cray/ari/modulefiles/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:/opt/cray/ari/modulefiles/dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:/opt/cray/ari/modulefiles/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:/opt/cray/ari/modulefiles/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:/opt/cray/ari/modulefiles/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:/opt/cray/ari/modulefiles/system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:/opt/cray/ari/modulefiles/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:/opt/modulefiles/Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:/opt/modulefiles/intel/18.0.0.128:/opt/cray/pe/craype/2.5.13/modulefiles/craype-network-aries:/opt/cray/pe/modulefiles/craype/2.5.13:/opt/cray/pe/modulefiles/cray-libsci/17.09.1:/opt/cray/pe/modulefiles/pmi/5.0.12:/opt/cray/pe/modulefiles/atp/2.1.1:/opt/cray/ari/modulefiles/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:/opt/cray/pe/modulefiles/perftools-base/6.5.2:/opt/cray/pe/modulefiles/PrgEnv-intel/6.0.4:/opt/cray/pe/craype/2.5.13/modulefiles/craype-mic-knl:/opt/cray/pe/modulefiles/cray-mpich/7.6.3:/opt/modulefiles/nompirun/nompirun:/soft/environment/modules/modulefiles/darshan/3.1.5:/soft/environment/modules/modulefiles/trackdeps:/soft/environment/modules/modulefiles/xalt:/opt/modulefiles/intelpython35/2017.0.035' ';export' '_LMFILES_;unset' 'PYTHONSTARTUP;'
++ C_INCLUDE_PATH=/opt/intel/python/2017.0.035/intelpython35/include
++ export C_INCLUDE_PATH
++ LD_LIBRARY_PATH=/opt/intel/python/2017.0.035/intelpython35/lib:/soft/perftools/darshan/darshan-3.1.5/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/mic/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/ipp/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mkl/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/tbb/lib/intel64/gcc4.4:/opt/intel/debugger_2017/libipt/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/../compiler/lib/intel64_lin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/lib64:/dbhome/db2cat/sqllib/lib64:/dbhome/db2cat/sqllib/lib64/gskit:/dbhome/db2cat/sqllib/lib32:
++ export LD_LIBRARY_PATH
++ LOADEDMODULES=modules/3.2.10.6:alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:intel/18.0.0.128:craype-network-aries:craype/2.5.13:cray-libsci/17.09.1:pmi/5.0.12:atp/2.1.1:rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:perftools-base/6.5.2:PrgEnv-intel/6.0.4:craype-mic-knl:cray-mpich/7.6.3:nompirun/nompirun:darshan/3.1.5:trackdeps:xalt:intelpython35/2017.0.035
++ export LOADEDMODULES
++ MANPATH=/opt/intel/python/2017.0.035/intelpython35/share/man:/opt/cray/pe/mpt/7.6.3/gni/man/mpich:/opt/cray/pe/perftools/6.5.2/man:/opt/cray/pe/papi/5.5.1.3/share/pdoc/man:/opt/cray/pe/atp/2.1.1/man:/opt/cray/pe/pmi/5.0.12/man:/opt/cray/pe/libsci/17.09.1/man:/opt/cray/pe/man/csmlversion:/opt/cray/pe/craype/2.5.13/man:/opt/intel/man/common:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/man:/opt/intel/documentation_2017/en/debugger//gdb-ia/man/:/opt/intel/documentation_2017/en/debugger//gdb-mic/man/:/opt/intel/documentation_2017/en/debugger//gdb-igfx/man/:/opt/cray/cfgutils/man:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/man:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/man:/opt/cray/pe/modules/3.2.10.6/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/pe/man
++ export MANPATH
++ PATH=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
++ export PATH
++ PYTHON_HOME=/opt/intel/python/2017.0.035/intelpython35
++ export PYTHON_HOME
++ PYTHON_VERSION=3.5
++ export PYTHON_VERSION
++ TCL_LIBRARY=/opt/intel/python/2017.0.035/intelpython35/lib/tcl8.6
++ export TCL_LIBRARY
++ _LMFILES_=/opt/cray/pe/modulefiles/modules/3.2.10.6:/opt/cray/ari/modulefiles/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:/opt/cray/ari/modulefiles/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:/opt/cray/ari/modulefiles/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:/opt/cray/ari/modulefiles/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:/opt/cray/ari/modulefiles/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:/opt/cray/ari/modulefiles/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:/opt/cray/ari/modulefiles/dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:/opt/cray/ari/modulefiles/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:/opt/cray/ari/modulefiles/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:/opt/cray/ari/modulefiles/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:/opt/cray/ari/modulefiles/system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:/opt/cray/ari/modulefiles/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:/opt/modulefiles/Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:/opt/modulefiles/intel/18.0.0.128:/opt/cray/pe/craype/2.5.13/modulefiles/craype-network-aries:/opt/cray/pe/modulefiles/craype/2.5.13:/opt/cray/pe/modulefiles/cray-libsci/17.09.1:/opt/cray/pe/modulefiles/pmi/5.0.12:/opt/cray/pe/modulefiles/atp/2.1.1:/opt/cray/ari/modulefiles/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:/opt/cray/pe/modulefiles/perftools-base/6.5.2:/opt/cray/pe/modulefiles/PrgEnv-intel/6.0.4:/opt/cray/pe/craype/2.5.13/modulefiles/craype-mic-knl:/opt/cray/pe/modulefiles/cray-mpich/7.6.3:/opt/modulefiles/nompirun/nompirun:/soft/environment/modules/modulefiles/darshan/3.1.5:/soft/environment/modules/modulefiles/trackdeps:/soft/environment/modules/modulefiles/xalt:/opt/modulefiles/intelpython35/2017.0.035
++ export _LMFILES_
++ unset PYTHONSTARTUP
+ source activate balsam
++ [[ -n 4.3.42(1)-release ]]
++ _SCRIPT_LOCATION=/opt/intel/python/2017.0.035/intelpython35/bin/activate
++ _SHELL=bash
+++ dirname /opt/intel/python/2017.0.035/intelpython35/bin/activate
++ _CONDA_DIR=/opt/intel/python/2017.0.035/intelpython35/bin
++ '[' 1 -gt 1 ']'
++ case "$(uname -s)" in
+++ uname -s
++ EXT=
++ [[ -n 4.3.42(1)-release ]]
+++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
++ [[ job.sh == \a\c\t\i\v\a\t\e ]]
++ '[' 1 -eq 0 ']'
++ args=balsam
++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..checkenv bash balsam
++ (( 0 != 0 ))
++ source /opt/intel/python/2017.0.035/intelpython35/bin/deactivate
+++ [[ -n 4.3.42(1)-release ]]
+++ _SCRIPT_LOCATION=/opt/intel/python/2017.0.035/intelpython35/bin/deactivate
+++ _SHELL=bash
++++ dirname /opt/intel/python/2017.0.035/intelpython35/bin/deactivate
+++ _CONDA_DIR=/opt/intel/python/2017.0.035/intelpython35/bin
+++ case "$(uname -s)" in
++++ uname -s
+++ EXT=
+++ [[ 1 > 0 ]]
+++ key=balsam
+++ case $key in
+++ shift
+++ [[ 0 > 0 ]]
+++ [[ -n 4.3.42(1)-release ]]
++++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
+++ [[ job.sh == \d\e\a\c\t\i\v\a\t\e ]]
+++ [[ -z '' ]]
+++ [[ -n 4.3.42(1)-release ]]
++++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
+++ [[ job.sh == \d\e\a\c\t\i\v\a\t\e ]]
+++ return 0
+++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..activate bash balsam
++ _NEW_PART=/home/harms/.conda/envs/balsam/bin
++ (( 0 == 0 ))
++ export CONDA_PATH_BACKUP=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
++ CONDA_PATH_BACKUP=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
++ export CONDA_PS1_BACKUP=
++ CONDA_PS1_BACKUP=
++ export PATH=/home/harms/.conda/envs/balsam/bin:/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
++ PATH=/home/harms/.conda/envs/balsam/bin:/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
++ [[ '' == */* ]]
++ export CONDA_DEFAULT_ENV=balsam
++ CONDA_DEFAULT_ENV=balsam
++ firstpath=/home/harms/.conda/envs/balsam/bin
+++ echo /home/harms/.conda/envs/balsam/bin
+++ sed 's|/bin$||'
++ export CONDA_PREFIX=/home/harms/.conda/envs/balsam
+++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..changeps1
++ '[' 1 = 1 ']'
+++ grep -q CONDA_DEFAULT_ENV
++ export 'PS1=(balsam) '
++ PS1='(balsam) '
++ _CONDA_D=/home/harms/.conda/envs/balsam/etc/conda/activate.d
++ [[ -d /home/harms/.conda/envs/balsam/etc/conda/activate.d ]]
++ unset CONDA_PATH
++ [[ -n 4.3.42(1)-release ]]
++ hash -r
+ grep '^No'
+ balsam ls
+ '[' 1 -eq 0 ']'
+ balsam ls
+ balsam launcher --consume-all --num-workers 2 --nodes-per-worker 1 --max-ranks-per-node 64
job_id | name | workflow | application | latest update
--------------------------------------------------------------------------------------------------------------------------------------------
c0315dc3-e0a2-42e5-80b4-9a743acc47b1 | default | qsub | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:20:52 CREATED]
e8afc054-41c0-48e2-b64c-2b40e0358689 | default | qsub | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:20:58 CREATED]
eed322bd-1a32-4777-b956-fd88435e89d0 | default | qsub | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:21:03 CREATED]
Theta Example
-------------
This is a simple example of submitting a job on Theta which executes
the Balsam launcher that runs all of the ensemble jobs.
First, create a set of jobs to run using balsam qsub. Next, verify the
jobs are listed within the database using balsam ls. Finally, submit the
test job.sh script to execute the balsam lanucher.
.. code:: bash
# build simple example
cc -qopenmp -o hello.x ./hello.c
# verify no jobs are found
balsam ls jobs
# queue work
balsam qsub -n 1 -N 2 -t 0 -d 64 -j 2 $PWD/hello.x
balsam qsub -n 2 -N 64 -t 0 -d 1 -j 1 $PWD/hello.x
balsam qsub -n 2 -N 32 -t 0 -d 2 -j 1 $PWD/hello.x
# verify jobs have been loaded
balsam ls jobs
# submit the job to theta
qsub ./job.sh
Balsam Status after Cobalt job completes
----------------------------------------
job_id | name | workflow | application | latest update
--------------------------------------------------------------------------------------------------------------------------------------------
c0315dc3-e0a2-42e5-80b4-9a743acc47b1 | default | qsub | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:23:00 JOB_FINISHED]
e8afc054-41c0-48e2-b64c-2b40e0358689 | default | qsub | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:22:52 JOB_FINISHED]
eed322bd-1a32-4777-b956-fd88435e89d0 | default | qsub | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:22:44 JOB_FINISHED]
Job Output
----------
The output from the job.sh script is in 169162.output and 169162.error but
the output from the individual subjobs are located in the balsam data directory.
.. code:: bash
ls $PWD/../../../../data/balsamjobs/qsub/
There is one directory for each subjob with the named `default_<job_id>` where
the job_id is shown in the balsam ls output.
.. code:: bash
tail $PWD/../../../../data/balsamjobs/qsub/default_eed322bd/default.out
hello from thread: 0
hello from thread: 13
hello from thread: 14
hello from thread: 23
hello from thread: 56
hello from thread: 52
hello from thread: 3
hello from thread: 61
hello from thread: 0
hello from thread: 55
hello from thread: 32
hello from thread: 63
hello from thread: 16
hello from thread: 17
hello from thread: 4
hello from thread: 12
hello from thread: 48
hello from thread: 5
hello from thread: 63
hello from thread: 45
hello from thread: 35
hello from thread: 21
hello from thread: 4
hello from thread: 40
hello from thread: 41
hello from thread: 28
hello from thread: 15
hello from thread: 44
hello from thread: 28
hello from thread: 30
hello from thread: 37
hello from thread: 5
hello from thread: 1
hello from thread: 36
hello from thread: 60
hello from thread: 25
hello from thread: 19
hello from thread: 30
hello from thread: 18
hello from thread: 41
hello from thread: 29
hello from thread: 24
hello from thread: 60
hello from thread: 14
hello from thread: 62
hello from thread: 51
hello from thread: 21
hello from thread: 7
hello from thread: 37
hello from thread: 39
hello from thread: 10
hello from thread: 26
hello from thread: 57
hello from thread: 34
hello from thread: 1
hello from thread: 2
hello from thread: 16
hello from thread: 47
hello from thread: 20
hello from thread: 46
hello from thread: 36
hello from thread: 53
hello from thread: 17
hello from thread: 9
hello from thread: 38
hello from thread: 26
hello from thread: 38
hello from thread: 39
hello from thread: 43
hello from thread: 48
hello from thread: 13
hello from thread: 31
hello from thread: 9
hello from thread: 29
hello from thread: 8
hello from thread: 18
hello from thread: 20
hello from thread: 19
hello from thread: 15
hello from thread: 3
hello from thread: 24
hello from thread: 2
hello from thread: 40
hello from thread: 6
hello from thread: 7
hello from thread: 49
hello from thread: 11
hello from thread: 23
hello from thread: 22
hello from thread: 27
hello from thread: 33
hello from thread: 62
hello from thread: 6
hello from thread: 52
hello from thread: 31
hello from thread: 45
hello from thread: 25
hello from thread: 55
hello from thread: 54
hello from thread: 56
hello from thread: 11
hello from thread: 42
hello from thread: 42
hello from thread: 51
hello from thread: 50
hello from thread: 27
hello from thread: 54
hello from thread: 59
hello from thread: 58
hello from thread: 58
hello from thread: 22
hello from thread: 61
hello from thread: 10
hello from thread: 34
hello from thread: 43
Job: 3944152 Ranks: 2 Threads: 1
hello from thread: 59
hello from thread: 33
hello from thread: 44
hello from thread: 49
hello from thread: 47
hello from thread: 46
hello from thread: 57
hello from thread: 35
hello from thread: 53
hello from thread: 12
hello from thread: 50
hello from thread: 8
hello from thread: 32
Application 3944152 resources: utime ~4s, stime ~1s, Rss ~8276, inblocks ~0, outblocks ~8
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
Job: 3944150 Ranks: 128 Threads: 1
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
hello from thread: 0
Application 3944150 resources: utime ~30s, stime ~16s, Rss ~14064, inblocks ~0, outblocks ~8
hello from thread: 0
hello from thread: 1
Job: 3944145 Ranks: 64 Threads: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 0
hello from thread: 1
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 0
hello from thread: 1
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 0
hello from thread: 1
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
hello from thread: 0
hello from thread: 1
Application 3944145 resources: utime ~26s, stime ~8s, Rss ~12180, inblocks ~0, outblocks ~8
#include <mpi.h>
#include <omp.h>
#include <string.h>
int main (int argc, char **argv)
{
int rank;
int size;
int thread;
int threads;
int jid;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
jid = atoi(getenv("ALPS_APP_ID"));
threads = omp_get_num_threads();
#pragma omp parallel private(thread)
{
thread = omp_get_thread_num();
printf("hello from thread: %d\n", thread);
}
if (rank == 0)
printf("Job: %d Ranks: %d Threads: %d\n", jid, size, threads);
MPI_Finalize();
return 0;
}
#!/bin/bash -x
#COBALT -A Performance