Add Theta user example

docs/quick/examples/theta/169162.cobaltlog

+Jobid: 169162
+qsub --mode script -A Performance -n 2 -t 30 -q debug-cache-quad ./job.sh
+Fri Jan 12 23:21:16 2018 +0000 (UTC) submitted with cwd set to: /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta
+jobid 169162 submitted from terminal /dev/pts/0
+Fri Jan 12 23:22:16 2018 +0000 (UTC) 
+Fri Jan 12 23:22:16 2018 +0000 (UTC) Command: '/gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh'
+Fri Jan 12 23:22:16 2018 +0000 (UTC) 
+Fri Jan 12 23:22:16 2018 +0000 (UTC) Environment:
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTCORES=64
+Fri Jan 12 23:22:16 2018 +0000 (UTC) SHELL=/bin/bash
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PROJECT=Performance
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTNAME=3830,3832
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_JOBID=169162
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_STARTTIME=1515799303
+Fri Jan 12 23:22:16 2018 +0000 (UTC) LOGNAME=harms
+Fri Jan 12 23:22:16 2018 +0000 (UTC) USER=harms
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_ENDTIME=1515801103
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_PARTSIZE=2
+Fri Jan 12 23:22:16 2018 +0000 (UTC) HOME=/home/harms
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_JOBSIZE=2
+Fri Jan 12 23:22:16 2018 +0000 (UTC) COBALT_QUEUE=debug-cache-quad
+Fri Jan 12 23:22:16 2018 +0000 (UTC) 
+Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stdin received from /dev/null
+Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stdout sent to /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/169162.output
+Fri Jan 12 23:22:16 2018 +0000 (UTC) Info: stderr sent to /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/169162.error
+Fri Jan 12 23:22:16 2018 +0000 (UTC) 
+Fri Jan 12 23:23:17 2018 +0000 (UTC) Info: user delete requested with signal SIGTERM by user harms

docs/quick/examples/theta/169162.error

++ module load intelpython35
+++ /opt/cray/pe/modules/3.2.10.6/bin/modulecmd bash load intelpython35
+Intel Python 3.5 version 2017.0.035 loaded
++ eval C_INCLUDE_PATH=/opt/intel/python/2017.0.035/intelpython35/include ';export' 'C_INCLUDE_PATH;LD_LIBRARY_PATH=/opt/intel/python/2017.0.035/intelpython35/lib:/soft/perftools/darshan/darshan-3.1.5/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/mic/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/ipp/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mkl/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/tbb/lib/intel64/gcc4.4:/opt/intel/debugger_2017/libipt/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/../compiler/lib/intel64_lin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/lib64:/dbhome/db2cat/sqllib/lib64:/dbhome/db2cat/sqllib/lib64/gskit:/dbhome/db2cat/sqllib/lib32:' ';export' 'LD_LIBRARY_PATH;LOADEDMODULES=modules/3.2.10.6:alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:intel/18.0.0.128:craype-network-aries:craype/2.5.13:cray-libsci/17.09.1:pmi/5.0.12:atp/2.1.1:rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:perftools-base/6.5.2:PrgEnv-intel/6.0.4:craype-mic-knl:cray-mpich/7.6.3:nompirun/nompirun:darshan/3.1.5:trackdeps:xalt:intelpython35/2017.0.035' ';export' 'LOADEDMODULES;MANPATH=/opt/intel/python/2017.0.035/intelpython35/share/man:/opt/cray/pe/mpt/7.6.3/gni/man/mpich:/opt/cray/pe/perftools/6.5.2/man:/opt/cray/pe/papi/5.5.1.3/share/pdoc/man:/opt/cray/pe/atp/2.1.1/man:/opt/cray/pe/pmi/5.0.12/man:/opt/cray/pe/libsci/17.09.1/man:/opt/cray/pe/man/csmlversion:/opt/cray/pe/craype/2.5.13/man:/opt/intel/man/common:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/man:/opt/intel/documentation_2017/en/debugger//gdb-ia/man/:/opt/intel/documentation_2017/en/debugger//gdb-mic/man/:/opt/intel/documentation_2017/en/debugger//gdb-igfx/man/:/opt/cray/cfgutils/man:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/man:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/man:/opt/cray/pe/modules/3.2.10.6/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/pe/man' ';export' 'MANPATH;PATH=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin' ';export' 'PATH;PYTHON_HOME=/opt/intel/python/2017.0.035/intelpython35' ';export' 'PYTHON_HOME;PYTHON_VERSION=3.5' ';export' 'PYTHON_VERSION;TCL_LIBRARY=/opt/intel/python/2017.0.035/intelpython35/lib/tcl8.6' ';export' 'TCL_LIBRARY;_LMFILES_=/opt/cray/pe/modulefiles/modules/3.2.10.6:/opt/cray/ari/modulefiles/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:/opt/cray/ari/modulefiles/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:/opt/cray/ari/modulefiles/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:/opt/cray/ari/modulefiles/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:/opt/cray/ari/modulefiles/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:/opt/cray/ari/modulefiles/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:/opt/cray/ari/modulefiles/dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:/opt/cray/ari/modulefiles/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:/opt/cray/ari/modulefiles/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:/opt/cray/ari/modulefiles/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:/opt/cray/ari/modulefiles/system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:/opt/cray/ari/modulefiles/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:/opt/modulefiles/Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:/opt/modulefiles/intel/18.0.0.128:/opt/cray/pe/craype/2.5.13/modulefiles/craype-network-aries:/opt/cray/pe/modulefiles/craype/2.5.13:/opt/cray/pe/modulefiles/cray-libsci/17.09.1:/opt/cray/pe/modulefiles/pmi/5.0.12:/opt/cray/pe/modulefiles/atp/2.1.1:/opt/cray/ari/modulefiles/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:/opt/cray/pe/modulefiles/perftools-base/6.5.2:/opt/cray/pe/modulefiles/PrgEnv-intel/6.0.4:/opt/cray/pe/craype/2.5.13/modulefiles/craype-mic-knl:/opt/cray/pe/modulefiles/cray-mpich/7.6.3:/opt/modulefiles/nompirun/nompirun:/soft/environment/modules/modulefiles/darshan/3.1.5:/soft/environment/modules/modulefiles/trackdeps:/soft/environment/modules/modulefiles/xalt:/opt/modulefiles/intelpython35/2017.0.035' ';export' '_LMFILES_;unset' 'PYTHONSTARTUP;'
+++ C_INCLUDE_PATH=/opt/intel/python/2017.0.035/intelpython35/include
+++ export C_INCLUDE_PATH
+++ LD_LIBRARY_PATH=/opt/intel/python/2017.0.035/intelpython35/lib:/soft/perftools/darshan/darshan-3.1.5/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/mic/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/ipp/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mkl/lib/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/tbb/lib/intel64/gcc4.4:/opt/intel/debugger_2017/libipt/intel64/lib:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.0.128/linux/daal/../compiler/lib/intel64_lin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/lib64:/dbhome/db2cat/sqllib/lib64:/dbhome/db2cat/sqllib/lib64/gskit:/dbhome/db2cat/sqllib/lib32:
+++ export LD_LIBRARY_PATH
+++ LOADEDMODULES=modules/3.2.10.6:alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:intel/18.0.0.128:craype-network-aries:craype/2.5.13:cray-libsci/17.09.1:pmi/5.0.12:atp/2.1.1:rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:perftools-base/6.5.2:PrgEnv-intel/6.0.4:craype-mic-knl:cray-mpich/7.6.3:nompirun/nompirun:darshan/3.1.5:trackdeps:xalt:intelpython35/2017.0.035
+++ export LOADEDMODULES
+++ MANPATH=/opt/intel/python/2017.0.035/intelpython35/share/man:/opt/cray/pe/mpt/7.6.3/gni/man/mpich:/opt/cray/pe/perftools/6.5.2/man:/opt/cray/pe/papi/5.5.1.3/share/pdoc/man:/opt/cray/pe/atp/2.1.1/man:/opt/cray/pe/pmi/5.0.12/man:/opt/cray/pe/libsci/17.09.1/man:/opt/cray/pe/man/csmlversion:/opt/cray/pe/craype/2.5.13/man:/opt/intel/man/common:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/man:/opt/intel/documentation_2017/en/debugger//gdb-ia/man/:/opt/intel/documentation_2017/en/debugger//gdb-mic/man/:/opt/intel/documentation_2017/en/debugger//gdb-igfx/man/:/opt/cray/cfgutils/man:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/man:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/man:/opt/cray/pe/modules/3.2.10.6/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/pe/man
+++ export MANPATH
+++ PATH=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
+++ export PATH
+++ PYTHON_HOME=/opt/intel/python/2017.0.035/intelpython35
+++ export PYTHON_HOME
+++ PYTHON_VERSION=3.5
+++ export PYTHON_VERSION
+++ TCL_LIBRARY=/opt/intel/python/2017.0.035/intelpython35/lib/tcl8.6
+++ export TCL_LIBRARY
+++ _LMFILES_=/opt/cray/pe/modulefiles/modules/3.2.10.6:/opt/cray/ari/modulefiles/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari:/opt/cray/ari/modulefiles/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari:/opt/cray/ari/modulefiles/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari:/opt/cray/ari/modulefiles/udreg/2.3.2-6.0.4.0_12.2__g2f9c3ee.ari:/opt/cray/ari/modulefiles/ugni/6.0.14-6.0.4.0_14.1__ge7db4a2.ari:/opt/cray/ari/modulefiles/gni-headers/5.0.11-6.0.4.0_7.2__g7136988.ari:/opt/cray/ari/modulefiles/dmapp/7.1.1-6.0.4.0_46.2__gb8abda2.ari:/opt/cray/ari/modulefiles/xpmem/2.2.2-6.0.4.0_3.1__g43b0535.ari:/opt/cray/ari/modulefiles/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari:/opt/cray/ari/modulefiles/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari:/opt/cray/ari/modulefiles/system-config/3.4.2448-6.0.4.1_9.3__gc628d7f.ari:/opt/cray/ari/modulefiles/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari:/opt/modulefiles/Base-opts/2.4.123-6.0.4.0_10.1__g6460790.ari:/opt/modulefiles/intel/18.0.0.128:/opt/cray/pe/craype/2.5.13/modulefiles/craype-network-aries:/opt/cray/pe/modulefiles/craype/2.5.13:/opt/cray/pe/modulefiles/cray-libsci/17.09.1:/opt/cray/pe/modulefiles/pmi/5.0.12:/opt/cray/pe/modulefiles/atp/2.1.1:/opt/cray/ari/modulefiles/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari:/opt/cray/pe/modulefiles/perftools-base/6.5.2:/opt/cray/pe/modulefiles/PrgEnv-intel/6.0.4:/opt/cray/pe/craype/2.5.13/modulefiles/craype-mic-knl:/opt/cray/pe/modulefiles/cray-mpich/7.6.3:/opt/modulefiles/nompirun/nompirun:/soft/environment/modules/modulefiles/darshan/3.1.5:/soft/environment/modules/modulefiles/trackdeps:/soft/environment/modules/modulefiles/xalt:/opt/modulefiles/intelpython35/2017.0.035
+++ export _LMFILES_
+++ unset PYTHONSTARTUP
++ source activate balsam
+++ [[ -n 4.3.42(1)-release ]]
+++ _SCRIPT_LOCATION=/opt/intel/python/2017.0.035/intelpython35/bin/activate
+++ _SHELL=bash
++++ dirname /opt/intel/python/2017.0.035/intelpython35/bin/activate
+++ _CONDA_DIR=/opt/intel/python/2017.0.035/intelpython35/bin
+++ '[' 1 -gt 1 ']'
+++ case "$(uname -s)" in
++++ uname -s
+++ EXT=
+++ [[ -n 4.3.42(1)-release ]]
++++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
+++ [[ job.sh == \a\c\t\i\v\a\t\e ]]
+++ '[' 1 -eq 0 ']'
+++ args=balsam
+++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..checkenv bash balsam
+++ ((  0 != 0  ))
+++ source /opt/intel/python/2017.0.035/intelpython35/bin/deactivate
++++ [[ -n 4.3.42(1)-release ]]
++++ _SCRIPT_LOCATION=/opt/intel/python/2017.0.035/intelpython35/bin/deactivate
++++ _SHELL=bash
+++++ dirname /opt/intel/python/2017.0.035/intelpython35/bin/deactivate
++++ _CONDA_DIR=/opt/intel/python/2017.0.035/intelpython35/bin
++++ case "$(uname -s)" in
+++++ uname -s
++++ EXT=
++++ [[ 1 > 0 ]]
++++ key=balsam
++++ case $key in
++++ shift
++++ [[ 0 > 0 ]]
++++ [[ -n 4.3.42(1)-release ]]
+++++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
++++ [[ job.sh == \d\e\a\c\t\i\v\a\t\e ]]
++++ [[ -z '' ]]
++++ [[ -n 4.3.42(1)-release ]]
+++++ basename /gpfs/mira-home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/./job.sh
++++ [[ job.sh == \d\e\a\c\t\i\v\a\t\e ]]
++++ return 0
++++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..activate bash balsam
+++ _NEW_PART=/home/harms/.conda/envs/balsam/bin
+++ ((  0 == 0  ))
+++ export CONDA_PATH_BACKUP=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
+++ CONDA_PATH_BACKUP=/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
+++ export CONDA_PS1_BACKUP=
+++ CONDA_PS1_BACKUP=
+++ export PATH=/home/harms/.conda/envs/balsam/bin:/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
+++ PATH=/home/harms/.conda/envs/balsam/bin:/opt/intel/python/2017.0.035/intelpython35/bin:/home/harms/perl5/bin:/home/harms/perl5/bin:/opt/xalt/bin:/soft/buildtools/trackdeps/bin:/soft/perftools/darshan/darshan-3.1.5/bin:/opt/modulefiles/nompirun/bin:/opt/cray/pe/mpt/7.6.3/gni/bin:/opt/cray/pe/perftools/6.5.2/bin:/opt/cray/pe/papi/5.5.1.3/bin:/opt/cray/rca/2.2.15-6.0.4.1_13.1__g46acb0f.ari/bin:/opt/cray/pe/craype/2.5.13/bin:/opt/intel/compilers_and_libraries_2018.0.128/linux/bin/intel64:/opt/intel/compilers_and_libraries_2018.0.128/linux/mpi/intel64/bin:/opt/intel/debugger_2017/gdb/intel64_mic/bin:/opt/cray/sysadm/2.4.119-6.0.4.0_14.2__gcab7125.ari/bin:/opt/cray/cfgutils/bin:/opt/cray/nodehealth/5.4.0-6.0.4.0_12.4__g3427370.ari/bin:/opt/cray/llm/21.3.446-6.0.4.0_20.1__gbe30105.ari/bin:/opt/cray/sdb/3.3.729-6.0.4.0_16.2__gb405b22.ari/bin:/opt/cray/nodestat/2.3.78-6.0.4.0_7.2__gbe57af8.ari/bin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/sbin:/opt/cray/alps/6.4.1-6.0.4.0_7.2__g86d0f3d.ari/bin:/opt/cray/pe/modules/3.2.10.6/bin:/home/harms/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc::/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/home/harms/bin
+++ [[ '' == */* ]]
+++ export CONDA_DEFAULT_ENV=balsam
+++ CONDA_DEFAULT_ENV=balsam
+++ firstpath=/home/harms/.conda/envs/balsam/bin
++++ echo /home/harms/.conda/envs/balsam/bin
++++ sed 's|/bin$||'
+++ export CONDA_PREFIX=/home/harms/.conda/envs/balsam
++++ /opt/intel/python/2017.0.035/intelpython35/bin/conda ..changeps1
+++ '[' 1 = 1 ']'
++++ grep -q CONDA_DEFAULT_ENV
+++ export 'PS1=(balsam) '
+++ PS1='(balsam) '
+++ _CONDA_D=/home/harms/.conda/envs/balsam/etc/conda/activate.d
+++ [[ -d /home/harms/.conda/envs/balsam/etc/conda/activate.d ]]
+++ unset CONDA_PATH
+++ [[ -n 4.3.42(1)-release ]]
+++ hash -r
++ grep '^No'
++ balsam ls
++ '[' 1 -eq 0 ']'
++ balsam ls
++ balsam launcher --consume-all --num-workers 2 --nodes-per-worker 1 --max-ranks-per-node 64

docs/quick/examples/theta/169162.output

+ job_id                               | name                       | workflow                   | application                | latest update
+--------------------------------------------------------------------------------------------------------------------------------------------
+ c0315dc3-e0a2-42e5-80b4-9a743acc47b1 | default                    | qsub                       | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:20:52 CREATED]
+ e8afc054-41c0-48e2-b64c-2b40e0358689 | default                    | qsub                       | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:20:58 CREATED]
+ eed322bd-1a32-4777-b956-fd88435e89d0 | default                    | qsub                       | /home/harms/working/theta/workflow/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:21:03 CREATED]

docs/quick/examples/theta/README.md

+Theta Example
+-------------
+This is a simple example of submitting a job on Theta which executes
+the Balsam launcher that runs all of the ensemble jobs.
+First, create a set of jobs to run using balsam qsub. Next, verify the
+jobs are listed within the database using balsam ls. Finally, submit the
+test job.sh script to execute the balsam lanucher.
+
+.. code:: bash
+    # build simple example
+    cc -qopenmp -o hello.x ./hello.c
+
+    # verify no jobs are found
+    balsam ls jobs
+
+    # queue work
+    balsam qsub -n 1 -N 2 -t 0 -d 64 -j 2 $PWD/hello.x
+    balsam qsub -n 2 -N 64 -t 0 -d 1 -j 1 $PWD/hello.x
+    balsam qsub -n 2 -N 32 -t 0 -d 2 -j 1 $PWD/hello.x
+
+    # verify jobs have been loaded
+    balsam ls jobs
+
+    # submit the job to theta
+    qsub ./job.sh    
+
+
+Balsam Status after Cobalt job completes
+----------------------------------------
+
+  job_id                               | name                       | workflow                   | application                | latest update
+  --------------------------------------------------------------------------------------------------------------------------------------------
+  c0315dc3-e0a2-42e5-80b4-9a743acc47b1 | default                    | qsub                       | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:23:00 JOB_FINISHED]
+  e8afc054-41c0-48e2-b64c-2b40e0358689 | default                    | qsub                       | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:22:52 JOB_FINISHED]
+  eed322bd-1a32-4777-b956-fd88435e89d0 | default                    | qsub                       | /<...>/hpc-edge-service/docs/quick/examples/theta/hello.x | [01-12-2018 23:22:44 JOB_FINISHED]
+
+Job Output
+----------
+The output from the job.sh script is in 169162.output and 169162.error but
+the output from the individual subjobs are located in the balsam data directory.
+
+.. code:: bash
+  ls $PWD/../../../../data/balsamjobs/qsub/
+
+There is one directory for each subjob with the named `default_<job_id>` where
+the job_id is shown in the balsam ls output.
+
+.. code:: bash
+  tail $PWD/../../../../data/balsamjobs/qsub/default_eed322bd/default.out
+

docs/quick/examples/theta/default_c0315dc3/default.out

+hello from thread: 0
+hello from thread: 13
+hello from thread: 14
+hello from thread: 23
+hello from thread: 56
+hello from thread: 52
+hello from thread: 3
+hello from thread: 61
+hello from thread: 0
+hello from thread: 55
+hello from thread: 32
+hello from thread: 63
+hello from thread: 16
+hello from thread: 17
+hello from thread: 4
+hello from thread: 12
+hello from thread: 48
+hello from thread: 5
+hello from thread: 63
+hello from thread: 45
+hello from thread: 35
+hello from thread: 21
+hello from thread: 4
+hello from thread: 40
+hello from thread: 41
+hello from thread: 28
+hello from thread: 15
+hello from thread: 44
+hello from thread: 28
+hello from thread: 30
+hello from thread: 37
+hello from thread: 5
+hello from thread: 1
+hello from thread: 36
+hello from thread: 60
+hello from thread: 25
+hello from thread: 19
+hello from thread: 30
+hello from thread: 18
+hello from thread: 41
+hello from thread: 29
+hello from thread: 24
+hello from thread: 60
+hello from thread: 14
+hello from thread: 62
+hello from thread: 51
+hello from thread: 21
+hello from thread: 7
+hello from thread: 37
+hello from thread: 39
+hello from thread: 10
+hello from thread: 26
+hello from thread: 57
+hello from thread: 34
+hello from thread: 1
+hello from thread: 2
+hello from thread: 16
+hello from thread: 47
+hello from thread: 20
+hello from thread: 46
+hello from thread: 36
+hello from thread: 53
+hello from thread: 17
+hello from thread: 9
+hello from thread: 38
+hello from thread: 26
+hello from thread: 38
+hello from thread: 39
+hello from thread: 43
+hello from thread: 48
+hello from thread: 13
+hello from thread: 31
+hello from thread: 9
+hello from thread: 29
+hello from thread: 8
+hello from thread: 18
+hello from thread: 20
+hello from thread: 19
+hello from thread: 15
+hello from thread: 3
+hello from thread: 24
+hello from thread: 2
+hello from thread: 40
+hello from thread: 6
+hello from thread: 7
+hello from thread: 49
+hello from thread: 11
+hello from thread: 23
+hello from thread: 22
+hello from thread: 27
+hello from thread: 33
+hello from thread: 62
+hello from thread: 6
+hello from thread: 52
+hello from thread: 31
+hello from thread: 45
+hello from thread: 25
+hello from thread: 55
+hello from thread: 54
+hello from thread: 56
+hello from thread: 11
+hello from thread: 42
+hello from thread: 42
+hello from thread: 51
+hello from thread: 50
+hello from thread: 27
+hello from thread: 54
+hello from thread: 59
+hello from thread: 58
+hello from thread: 58
+hello from thread: 22
+hello from thread: 61
+hello from thread: 10
+hello from thread: 34
+hello from thread: 43
+Job: 3944152 Ranks: 2 Threads: 1
+hello from thread: 59
+hello from thread: 33
+hello from thread: 44
+hello from thread: 49
+hello from thread: 47
+hello from thread: 46
+hello from thread: 57
+hello from thread: 35
+hello from thread: 53
+hello from thread: 12
+hello from thread: 50
+hello from thread: 8
+hello from thread: 32
+Application 3944152 resources: utime ~4s, stime ~1s, Rss ~8276, inblocks ~0, outblocks ~8

docs/quick/examples/theta/default_e8afc054/default.out

+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+Job: 3944150 Ranks: 128 Threads: 1
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+hello from thread: 0
+Application 3944150 resources: utime ~30s, stime ~16s, Rss ~14064, inblocks ~0, outblocks ~8

docs/quick/examples/theta/default_eed322bd/default.out

+hello from thread: 0
+hello from thread: 1
+Job: 3944145 Ranks: 64 Threads: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 0
+hello from thread: 1
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 0
+hello from thread: 1
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 0
+hello from thread: 1
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+hello from thread: 0
+hello from thread: 1
+Application 3944145 resources: utime ~26s, stime ~8s, Rss ~12180, inblocks ~0, outblocks ~8

docs/quick/examples/theta/hello.c

+#include <mpi.h>
+#include <omp.h>
+#include <string.h>
+
+int main (int argc, char **argv)
+{
+    int rank;
+    int size;
+    int thread;
+    int threads;
+    int jid;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    jid = atoi(getenv("ALPS_APP_ID"));
+    threads = omp_get_num_threads();
+
+#pragma omp parallel private(thread)
+    {
+        thread = omp_get_thread_num();
+        printf("hello from thread: %d\n", thread);
+    }
+
+    if (rank == 0)
+      printf("Job: %d Ranks: %d Threads: %d\n", jid, size, threads);
+
+    MPI_Finalize();
+    return 0;
+}

docs/quick/examples/theta/job.sh

+#!/bin/bash -x
+#COBALT -A Performance
+#COBALT -n 2
+#COBALT -t 30
+#COBALT -q debug-cache-quad
+
+#
+# See docs/quickstart.rst for how to setup and run on Theta
+#
+
+module load intelpython35
+source activate balsam
+
+balsam ls | grep "^No"