move server daemon to src, and integrate ssg

Makefile.am

 AUTOMAKE_OPTIONS = foreign
 ACLOCAL_AMFLAGS="-Im4"
 
-check_PROGRAMS = 
+bin_PROGRAMS =
+check_PROGRAMS =
 TESTS =
 EXTRA_DIST = prepare.sh
 
@@ -9,18 +10,18 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 AM_CFLAGS =
 AM_CXXFLAGS = $(AM_CFLAGS)
 
-lib_LTLIBRARIES = src/libmobject-store.la
-lib_LTLIBRARIES += src/libmobject-store-server.la
+lib_LTLIBRARIES = \
+  src/libmobject-store.la \
+  src/libmobject-server.la
 
-include_HEADERS = include/libmobject-store.h \
-		  include/librados-mobject-store.h \
-		  include/mobject-server.h
+include_HEADERS = \
+  include/libmobject-store.h \
+  include/librados-mobject-store.h \
+  include/mobject-server.h
 
 # setting this to empty so subdir makefiles can use +=
 noinst_HEADERS =
 
-src_libmobject_la_CPPFLAGS = -I${srcdir}/include -I${srcdir}/src
-
 include Make.rules
 
 include $(top_srcdir)/src/Makefile.subdir

include/mobject-server.h

@@ -11,6 +11,10 @@
 /* server-side utilities and routines.  Clients are looking for either
  * libmobject-store.h or librados-mobject-store.h */
 
+#define MOBJECT_SERVER_GROUP_NAME "mobject-store-servers"
+
+int mobject_server_init(margo_instance_id mid);
+
 /**
  * Start a mobject server instance
  *
@@ -19,5 +23,6 @@
  * @returns 0 on success, negative error code on failure */
 int mobject_server_register(margo_instance_id mid, const char *poolname);
 
+void mobject_server_shutdown(margo_instance_id mid);
 
 #endif

src/Makefile.subdir

-src_libmobject_store_la_SOURCES = \
- src/libmobject-store.c \
- src/completion.c \
- src/write-op-impl.c \
- src/read-op-impl.c \
- src/proc-write-actions.c \
- src/proc-read-actions.c \
- src/prepare-write-op.c \
- src/prepare-read-op.c \
- src/write-op-visitor.c \
- src/read-op-visitor.c
-
 noinst_HEADERS += \
   src/log.h \
   src/completion.h \
@@ -28,6 +16,25 @@ noinst_HEADERS += \
   src/args-read-actions.h \
   src/read-op-visitor.h
 
+src_libmobject_store_la_SOURCES = \
+  src/libmobject-store.c \
+  src/completion.c \
+  src/write-op-impl.c \
+  src/read-op-impl.c \
+  src/proc-write-actions.c \
+  src/proc-read-actions.c \
+  src/prepare-write-op.c \
+  src/prepare-read-op.c \
+  src/write-op-visitor.c \
+  src/read-op-visitor.c
+
+src_libmobject_server_la_SOURCES = \
+  src/mobject-server.c
+
+src_mobject_server_daemon_SOURCES = \
+  src/mobject-server-daemon.c
+src_mobject_server_daemon_LDADD = \
+  src/libmobject-server.la
 
-src_libmobject_store_server_la_SOURCES = \
-					 src/mobject-server.c
+bin_PROGRAMS += \
+  src/mobject-server-daemon

src/mobject-server-daemon.c

+/*
+ * (C) 2017 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include <mpi.h>
+#include <margo.h>
+#include <ssg.h>
+
+#include "mobject-server.h"
+
+void usage(void)
+{
+    fprintf(stderr, "Usage: mobject-server-daemon <listen_addr> <gid_file>\n");
+    fprintf(stderr, "  <listen_addr>    the Mercury address to listen on\n");
+    fprintf(stderr, "  <gid_file>       the file to write the server SSG group ID to\n");
+    exit(-1);
+}
+
+int main(int argc, char *argv[])
+{
+    char *listen_addr;
+    char *gid_file;
+    margo_instance_id mid;
+    int ret;
+
+    /* check args */
+    if (argc != 3)
+        usage();
+    listen_addr = argv[1];
+    gid_file = argv[2];
+
+    /* XXX: MPI required for SSG bootstrapping */
+    MPI_Init(&argc, &argv);
+
+    mid = margo_init(listen_addr, MARGO_SERVER_MODE, 0, -1);
+    if (mid == MARGO_INSTANCE_NULL)
+    {
+        fprintf(stderr, "Error: Unable to initialize margo\n");
+        return -1;
+    }
+
+    ret = mobject_server_init(mid);
+    if (ret != 0)
+    {
+        fprintf(stderr, "Error: Unable to initialize mobject server\n");
+        margo_finalize(mid);
+        return -1;
+    }
+
+    /* XXX write GID to file... where does gid come from? */
+
+    /* shutdown */
+    mobject_server_shutdown(mid);
+    margo_finalize(mid);
+    //margo_wait_for_finalize(mid);
+    MPI_Finalize();
+
+    return 0;
+}

src/mobject-server.c

-#include <sds-keyval.h>
-#include <bake-bulk.h>
-#include <bake-bulk-server.h>
+/*
+ * (C) 2017 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include <assert.h>
+#include <mpi.h>
 #include <margo.h>
-#include <libpmemobj.h>
+//#include <sds-keyval.h>
+//#include <bake-bulk-server.h>
+//#include <libpmemobj.h>
+#include <ssg-mpi.h>
+
+#include "mobject-server.h"
+
+typedef struct mobject_server_context
+{
+    /* XXX bake, sds-keyval stuff */
+    ssg_group_id_t gid;
+} mobject_server_context_t;
+
+/* XXX one global mobject server state struct */
+mobject_server_context_t *g_srv_ctx = NULL;
+
+int mobject_server_init(margo_instance_id mid)
+{
+    int ret;
+
+    if (g_srv_ctx)
+    {
+        fprintf(stderr, "Error: mobject server has already been initialized\n");
+        return -1;
+    }
+
+    g_srv_ctx = malloc(sizeof(*g_srv_ctx));
+    if (!g_srv_ctx)
+        return -1;
+    memset(g_srv_ctx, 0, sizeof(*g_srv_ctx));
+
+    /* TODO bake-bulk */
+    /* TODO sds-keyval */
 
-#include <mobject-server.h>
+    ret = ssg_init(mid);
+    if (ret != SSG_SUCCESS)
+    {
+        fprintf(stderr, "Error: Unable to initialize SSG\n");
+        return -1;
+    }
 
+    /* server group create */
+    g_srv_ctx->gid = ssg_group_create_mpi(MOBJECT_SERVER_GROUP_NAME, MPI_COMM_WORLD,
+        NULL, NULL); /* XXX membership change callbacks unused currently */
+    if (g_srv_ctx->gid == SSG_GROUP_ID_NULL)
+    {
+        fprintf(stderr, "Error: Unable to create the mobject server group\n");
+        ssg_finalize();
+        return -1;
+    }
+
+    /* XXX cleanup? */
+
+    return 0;
+}
+
+#if 0
 int mobject_server_register(margo_instance_id mid, const char *poolname)
 {
     int ret=0;
@@ -18,12 +76,17 @@ int mobject_server_register(margo_instance_id mid, const char *poolname)
     metadata = kv_server_register(mid);
     return ret;
 }
+#endif
 
-
-int mobject_shutdown(margo_instance_id mid)
+void mobject_server_shutdown(margo_instance_id mid)
 {
-    margo_wait_for_finalize(mid);
-    pmemobj_close(NULL);
-    return 0;
+    assert(g_srv_ctx);
+
+    ssg_group_destroy(g_srv_ctx->gid);
+    ssg_finalize();
+
+    //margo_wait_for_finalize(mid);
+    //pmemobj_close(NULL);
 
+    return;
 }