mobject-server-daemon.c 5.29 KB
Newer Older
1 2 3 4 5 6
/*
 * (C) 2017 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */

7
#include <unistd.h>
8 9 10
#include <mpi.h>
#include <margo.h>
#include <ssg.h>
11
#include <ssg-mpi.h>
12 13
#include <bake-client.h>
#include <bake-server.h>
Matthieu Dorier's avatar
Matthieu Dorier committed
14 15
#include <sdskv-client.h>
#include <sdskv-server.h>
16 17 18

#include "mobject-server.h"

19 20
#define ASSERT(__cond, __msg, ...) { if(!(__cond)) { fprintf(stderr, "[%s:%d] " __msg, __FILE__, __LINE__, __VA_ARGS__); exit(-1); } }

21 22
void usage(void)
{
23
    fprintf(stderr, "Usage: mobject-server-daemon <listen_addr> <cluster_file>\n");
24
    fprintf(stderr, "  <listen_addr>    the Mercury address to listen on\n");
25
    fprintf(stderr, "  <cluster_file>   the file to write mobject cluster connect info to\n");
26 27 28
    exit(-1);
}

29 30 31 32 33
typedef struct {
    bake_client_t          client;
    bake_provider_handle_t provider_handle;
} bake_client_data;

Matthieu Dorier's avatar
Matthieu Dorier committed
34 35 36 37 38
typedef struct {
    sdskv_client_t         client;
    sdskv_provider_handle_t provider_handle;
} sdskv_client_data;

39
static void finalize_ssg_cb(void* data);
40
static void finalize_bake_client_cb(void* data);
Matthieu Dorier's avatar
Matthieu Dorier committed
41 42
static void finalize_sdskv_client_cb(void* data);

43 44 45
int main(int argc, char *argv[])
{
    char *listen_addr;
46
    char *cluster_file;
47 48 49 50 51 52 53
    margo_instance_id mid;
    int ret;

    /* check args */
    if (argc != 3)
        usage();
    listen_addr = argv[1];
54
    cluster_file = argv[2];
55

56
    /* MPI required for SSG bootstrapping */
57
    MPI_Init(&argc, &argv);
58 59
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
60

61
    /* Margo initialization */
62 63 64 65 66 67
    mid = margo_init(listen_addr, MARGO_SERVER_MODE, 0, -1);
    if (mid == MARGO_INSTANCE_NULL)
    {
        fprintf(stderr, "Error: Unable to initialize margo\n");
        return -1;
    }
68
    margo_enable_remote_shutdown(mid);
69

70 71
    /* SSG initialization */
    ret = ssg_init(mid);
72
    ASSERT(ret == 0, "ssg_init() failed (ret = %d)\n", ret);
73 74 75 76 77 78 79

    hg_addr_t self_addr;
    margo_addr_self(mid, &self_addr);

    /* Bake provider initialization */
    /* XXX mplex id and target name should be taken from config file */
    uint8_t bake_mplex_id = 1;
80 81 82 83 84 85 86 87
    char bake_target_name[128];
    sprintf(bake_target_name, "/dev/shm/mobject.%d.dat", rank);
    /* create the bake target if it does not exist */
    if(-1 == access(bake_target_name, F_OK)) {
        // XXX creating a pool of 10MB - this should come from a config file
        ret = bake_makepool(bake_target_name, 10*1024*1024, 0664);
        ASSERT(ret == 0, "bake_makepool() failed (ret = %d)\n", ret);
    }
88 89
    bake_provider_t bake_prov;
    bake_target_id_t bake_tid;
90 91 92 93 94
    ret = bake_provider_register(mid, bake_mplex_id, BAKE_ABT_POOL_DEFAULT, &bake_prov);
    ASSERT(ret == 0, "bake_provider_register() failed (ret = %d)\n", ret);
    ret = bake_provider_add_storage_target(bake_prov, bake_target_name, &bake_tid);
    ASSERT(ret == 0, "bake_provider_add_storage_target() failed to add target %s (ret = %d)\n",
            bake_target_name, ret);
95 96 97

    /* Bake provider handle initialization from self addr */
    bake_client_data bake_clt_data;
98 99 100 101
    ret = bake_client_init(mid, &(bake_clt_data.client));
    ASSERT(ret == 0, "bake_client_init() failed (ret = %d)\n", ret);
    ret = bake_provider_handle_create(bake_clt_data.client, self_addr, bake_mplex_id, &(bake_clt_data.provider_handle));
    ASSERT(ret == 0, "bake_provider_handle_create() failed (ret = %d)\n", ret);
102 103
    margo_push_finalize_callback(mid, &finalize_bake_client_cb, (void*)&bake_clt_data);

Matthieu Dorier's avatar
Matthieu Dorier committed
104 105 106
    /* SDSKV provider initialization */
    uint8_t sdskv_mplex_id = 1;
    sdskv_provider_t sdskv_prov;
107 108 109
    ret = sdskv_provider_register(mid, sdskv_mplex_id, SDSKV_ABT_POOL_DEFAULT, &sdskv_prov);
    ASSERT(ret == 0, "sdskv_provider_register() failed (ret = %d)\n", ret);

Matthieu Dorier's avatar
Matthieu Dorier committed
110
    ret = mobject_sdskv_provider_setup(sdskv_prov);
Matthieu Dorier's avatar
Matthieu Dorier committed
111 112 113

    /* SDSKV provider handle initialization from self addr */
    sdskv_client_data sdskv_clt_data;
114 115 116 117
    ret = sdskv_client_init(mid, &(sdskv_clt_data.client));
    ASSERT(ret == 0, "sdskv_client_init() failed (ret = %d)\n", ret);
    ret = sdskv_provider_handle_create(sdskv_clt_data.client, self_addr, sdskv_mplex_id, &(sdskv_clt_data.provider_handle));
    ASSERT(ret == 0, "sdskv_provider_handle_create() failed (ret = %d)\n", ret);
Matthieu Dorier's avatar
Matthieu Dorier committed
118 119
    margo_push_finalize_callback(mid, &finalize_sdskv_client_cb, (void*)&sdskv_clt_data);

120 121
    /* Mobject provider initialization */
    mobject_provider_t mobject_prov;
Matthieu Dorier's avatar
Matthieu Dorier committed
122 123 124 125 126
    ret = mobject_provider_register(mid, 1, 
            MOBJECT_ABT_POOL_DEFAULT, 
            bake_clt_data.provider_handle, 
            sdskv_clt_data.provider_handle, 
            cluster_file, &mobject_prov);
127
    if (ret != 0)
128
    {
129
        fprintf(stderr, "Error: Unable to initialize mobject provider\n");
130 131 132 133
        margo_finalize(mid);
        return -1;
    }

134
    margo_addr_free(mid, self_addr);
135

136 137
    margo_wait_for_finalize(mid);

138 139 140 141
    MPI_Finalize();

    return 0;
}
142

143 144 145 146 147 148
static void finalize_ssg_cb(void* data)
{
    ssg_group_id_t* gid = (ssg_group_id_t*)data;
    ssg_group_destroy(*gid);
}

149 150 151 152 153 154
static void finalize_bake_client_cb(void* data)
{
    bake_client_data* clt_data = (bake_client_data*)data;
    bake_provider_handle_release(clt_data->provider_handle);
    bake_client_finalize(clt_data->client);
}
Matthieu Dorier's avatar
Matthieu Dorier committed
155 156 157 158 159 160 161

static void finalize_sdskv_client_cb(void* data)
{
    sdskv_client_data* clt_data = (sdskv_client_data*)data;
    sdskv_provider_handle_release(clt_data->provider_handle);
    sdskv_client_finalize(clt_data->client);
}