Update ssg 'separate-group' test

- test now shuts down cleanly: no need to 'kill' anything
- found a few problems with the job script

perf-regression/ssg-test-separate-group-attach.c

@@ -5,6 +5,7 @@
 int main(int argc, char **argv)
 {
     int ret;
+    hg_addr_t remote_addr = HG_ADDR_NULL;
     ssg_group_id_t gid;
     margo_instance_id mid;
 
@@ -25,7 +26,15 @@ int main(int argc, char **argv)
     ssg_group_dump(gid);
     fprintf(stderr, "        dumped...\n");
 
+    remote_addr = ssg_get_group_member_addr(gid, ssg_get_group_member_id_from_rank(gid, 0));
+    assert(remote_addr != HG_ADDR_NULL);
+
+    ret = margo_shutdown_remote_instance(mid, remote_addr);
+    assert (ret == HG_SUCCESS);
+
     ssg_group_unobserve(gid);
+    ssg_finalize();
+
     margo_finalize(mid);
     MPI_Finalize();
 }

perf-regression/ssg-test-separate-group-create.c

@@ -4,12 +4,19 @@
 #define GROUP_NAME "test-group"
 #define ASSERT(__cond, __msg, ...) { if(!(__cond)) { fprintf(stderr, "[%s:%d] " __msg, __FILE__, __LINE__, __VA_ARGS__); exit(-1); } }
 
-static void finalized_ssg_group_cb(void* data)
-{
-    ssg_group_id_t gid = *((ssg_group_id_t*)data);
-    ssg_group_destroy(gid);
-}
+typedef struct {
+    ssg_group_id_t g_id;
+    margo_instance_id mid;
+} finalize_args_t;
 
+void ssg_finalize_callback(void* arg) {
+    fprintf(stderr, "Entering ssg_finalize_callback\n");
+    finalize_args_t* a = (finalize_args_t*)arg;
+    ssg_group_destroy(a->g_id);
+    fprintf(stderr, "Successfully destroyed ssg group\n");
+    ssg_finalize();
+    fprintf(stderr, "Successfully finalized ssg\n");
+}
 
 int main(int argc, char **argv)
 {
@@ -35,17 +42,24 @@ int main(int argc, char **argv)
         fprintf(stderr, "ssg_group_create_mpi failed\n");
         exit(-1);
     }
-    margo_push_finalize_callback(mid, &finalized_ssg_group_cb, (void*)&gid);
 
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
     if (rank == 0)
 	ssg_group_id_store(argv[2], gid, 1);
 
-    fprintf(stderr, "   just hanging out...\n");
+    finalize_args_t args = {
+        .g_id = gid,
+        .mid = mid
+    };
+    margo_push_prefinalize_callback(mid, ssg_finalize_callback, (void*)&args);
+
+    fprintf(stderr, "   just hanging out waiting for shutdown\n");
+
     margo_wait_for_finalize(mid);
+    fprintf(stderr, "   .. done waiting: now cleaning up\n");
     ssg_group_destroy(gid);
-    margo_finalize(mid);
+    fprintf(stderr, "bye\n");
 
     MPI_Finalize();
 }

perf-regression/theta/separate-ssg.qsub

@@ -13,6 +13,8 @@ module swap PrgEnv-intel PrgEnv-gnu
 #spack load -r ssg/lfdi2fo
 
 spack load -r ssg
+# ssg doesn't need bake, but these ssg tests were built as part of sds-tests, which does
+spack load -r bake
 
 module list
 
@@ -27,7 +29,7 @@ apmgr pdomain -c -u $SSG_PDOMAIN
 SSG_STATEFILE=ssg-state.cfg
 rm -f $SSG_STATEFILE
 
-aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-create ${SSG_STATEFILE} &
+aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-create ofi+gni:// ${SSG_STATEFILE} &
 
 sleep_time=1;
 while [ ! -e $SSG_STATEFILE ] ; do
@@ -37,7 +39,6 @@ done
 
 xxd $SSG_STATEFILE
 
-aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-attach ofi+gni ${SSG_STATEFILE}
-kill %1
-apmgr pdomain -r  -u ${SSG_PDOMAIN}
+aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-attach ofi+gni:// ${SSG_STATEFILE}
+apmgr pdomain -r ${SSG_PDOMAIN}