bake-bulk-server.c 17.1 KB
Newer Older
Philip Carns's avatar
Philip Carns committed
1 2 3 4 5 6 7
/*
 * (C) 2015 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */

#include <assert.h>
8
#include <libpmemobj.h>
9
#include <bake-bulk-server.h>
Philip Carns's avatar
Philip Carns committed
10 11
#include "bake-bulk-rpc.h"

12
/* definition of bake root data structure (just a target_id for now) */
13
typedef struct
14 15 16 17
{   
    bake_target_id_t target_id;
} bake_bulk_root_t;
 
18
/* definition of internal region_id_t identifier for libpmemobj back end */
19 20
typedef struct
{
21 22 23
    PMEMoid oid;
    uint64_t size;
} pmemobj_region_id_t;
24

25 26 27 28 29 30 31 32 33 34 35 36
typedef struct
{
    PMEMobjpool *pmem_pool;
    bake_bulk_root_t *pmem_root;

    /* server shutdown conditional logic */
    ABT_mutex shutdown_mutex;
    ABT_cond shutdown_cond;
    int shutdown_flag;
    int ref_count;
} bake_bulk_server_context_t;

37 38
static void bake_server_cleanup(bake_bulk_server_context_t *svr_ctx);

Philip Carns's avatar
Philip Carns committed
39 40 41
/* TODO: this should not be global in the long run; server may provide access
 * to multiple targets
 */
42
static bake_bulk_server_context_t *g_svr_ctx = NULL;
43

44 45 46 47
int bake_server_makepool(
	const char *pool_name,
    size_t pool_size,
    mode_t pool_mode)
48
{
49
    PMEMobjpool *pool;
50
    PMEMoid root_oid;
51
    bake_bulk_root_t *root;
52

53 54
    pool = pmemobj_create(pool_name, NULL, pool_size, pool_mode);
    if(!pool)
55
    {
56 57
        fprintf(stderr, "pmemobj_create: %s\n", pmemobj_errormsg());
        return(-1);
58 59 60
    }

    /* find root */
61 62 63 64 65 66
    root_oid = pmemobj_root(pool, sizeof(bake_bulk_root_t));
    root = pmemobj_direct(root_oid);

    /* store the target id for this bake pool at the root */
    uuid_generate(root->target_id.id);
    pmemobj_persist(pool, root, sizeof(bake_bulk_root_t));
Shane Snyder's avatar
Shane Snyder committed
67
#if 0
68
    char target_string[64];
69 70
    uuid_unparse(root->target_id.id, target_string);
    fprintf(stderr, "created BAKE target ID: %s\n", target_string);
71
#endif
72 73 74 75 76 77 78 79 80 81 82 83 84

    pmemobj_close(pool);

    return(0);
}

int bake_server_init(
    margo_instance_id mid,
    const char *pool_name)
{
    PMEMobjpool *pool;
    PMEMoid root_oid;
    bake_bulk_root_t *root;
85 86
    bake_bulk_server_context_t *tmp_svr_ctx;
    
87
    /* make sure to initialize the server only once */
88
    if(g_svr_ctx)
89
    {
90 91
        fprintf(stderr, "Error: bake-bulk server already initialized\n");
        return(-1);
92 93
    }

94 95 96 97 98
    tmp_svr_ctx = malloc(sizeof(*tmp_svr_ctx));
    if(!tmp_svr_ctx)
        return(-1);
    memset(tmp_svr_ctx, 0, sizeof(*tmp_svr_ctx));

99 100 101 102 103 104 105
    /* open the given pmem pool */
    pool = pmemobj_open(pool_name, NULL);
    if(!pool)
    {
        fprintf(stderr, "pmemobj_open: %s\n", pmemobj_errormsg());
        return(-1);
    }
106

107 108 109 110 111 112 113 114 115
    /* check to make sure the root is properly set */
    root_oid = pmemobj_root(pool, sizeof(bake_bulk_root_t));
    root = pmemobj_direct(root_oid);
    if(uuid_is_null(root->target_id.id))
    {
        fprintf(stderr, "Error: bake-bulk pool is not properly initialized\n");
        pmemobj_close(pool);
        return(-1);
    }
Shane Snyder's avatar
Shane Snyder committed
116
#if 0
117
    char target_string[64];
118 119
    uuid_unparse(root->target_id.id, target_string);
    fprintf(stderr, "opened BAKE target ID: %s\n", target_string);
120
#endif
Philip Carns's avatar
Philip Carns committed
121

122
    /* register RPCs */
123 124 125
    MARGO_REGISTER(mid, "bake_bulk_shutdown_rpc", void, void,
        bake_bulk_shutdown_ult);
    MARGO_REGISTER(mid, "bake_bulk_create_rpc", bake_bulk_create_in_t,
126
        bake_bulk_create_out_t,
127 128
        bake_bulk_create_ult);
    MARGO_REGISTER(mid, "bake_bulk_write_rpc", bake_bulk_write_in_t,
129
        bake_bulk_write_out_t,
130 131
        bake_bulk_write_ult);
    MARGO_REGISTER(mid, "bake_bulk_eager_write_rpc", bake_bulk_eager_write_in_t,
132
        bake_bulk_eager_write_out_t,
133 134
        bake_bulk_eager_write_ult);
    MARGO_REGISTER(mid, "bake_bulk_eager_read_rpc", bake_bulk_eager_read_in_t,
135
        bake_bulk_eager_read_out_t,
136 137
        bake_bulk_eager_read_ult);
    MARGO_REGISTER(mid, "bake_bulk_persist_rpc", bake_bulk_persist_in_t,
138
        bake_bulk_persist_out_t,
139 140
        bake_bulk_persist_ult);
    MARGO_REGISTER(mid, "bake_bulk_get_size_rpc", bake_bulk_get_size_in_t,
141
        bake_bulk_get_size_out_t,
142 143
        bake_bulk_get_size_ult);
    MARGO_REGISTER(mid, "bake_bulk_read_rpc", bake_bulk_read_in_t,
144
        bake_bulk_read_out_t,
145 146
        bake_bulk_read_ult);
    MARGO_REGISTER(mid, "bake_bulk_probe_rpc", void,
147
        bake_bulk_probe_out_t,
148 149
        bake_bulk_probe_ult);
    MARGO_REGISTER(mid, "bake_bulk_noop_rpc", void,
150
        void,
151
        bake_bulk_noop_ult);
152

153 154 155 156 157 158 159
    /* set global server context */
    tmp_svr_ctx->pmem_pool = pool;
    tmp_svr_ctx->pmem_root = root;
    tmp_svr_ctx->ref_count = 1;
    ABT_mutex_create(&tmp_svr_ctx->shutdown_mutex);
    ABT_cond_create(&tmp_svr_ctx->shutdown_cond);
    g_svr_ctx = tmp_svr_ctx;
160

161
    return(0);
162 163
}

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
void bake_server_shutdown()
{
    bake_bulk_server_context_t *svr_ctx = g_svr_ctx;
    int do_cleanup;

    assert(svr_ctx);

    ABT_mutex_lock(svr_ctx->shutdown_mutex);
    svr_ctx->shutdown_flag = 1;
    ABT_cond_broadcast(svr_ctx->shutdown_cond);

    svr_ctx->ref_count--;
    do_cleanup = svr_ctx->ref_count == 0;

    ABT_mutex_unlock(svr_ctx->shutdown_mutex);

    if (do_cleanup)
    {
        bake_server_cleanup(svr_ctx);
        g_svr_ctx = NULL;
    }

    return;
}

void bake_server_wait_for_shutdown()
{
    bake_bulk_server_context_t *svr_ctx = g_svr_ctx;
    int do_cleanup;

    assert(svr_ctx);

    ABT_mutex_lock(svr_ctx->shutdown_mutex);

    svr_ctx->ref_count++;
    while(!svr_ctx->shutdown_flag)
        ABT_cond_wait(svr_ctx->shutdown_cond, svr_ctx->shutdown_mutex);
    svr_ctx->ref_count--;
    do_cleanup = svr_ctx->ref_count == 0;

    ABT_mutex_unlock(svr_ctx->shutdown_mutex);

    if (do_cleanup)
    {
        bake_server_cleanup(svr_ctx);
        g_svr_ctx = NULL;
    }

    return;
}

215 216 217 218
/* service a remote RPC that instructs the server daemon to shut down */
static void bake_bulk_shutdown_ult(hg_handle_t handle)
{
    hg_return_t hret;
Philip Carns's avatar
Philip Carns committed
219 220
    margo_instance_id mid;

221 222
    assert(g_svr_ctx);

223
    mid = margo_hg_handle_get_instance(handle);
224

225
    hret = margo_respond(handle, NULL);
226 227
    assert(hret == HG_SUCCESS);

228
    margo_destroy(handle);
229 230

    /* NOTE: we assume that the server daemon is using
231 232
     * bake_server_wait_for_shutdown() to suspend until this RPC executes, so
     * there is no need to send any extra signal to notify it.
233
     */
234
    bake_server_shutdown();
235 236 237 238 239 240 241 242 243 244 245 246 247

    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_shutdown_ult)

/* service a remote RPC that creates a bulk region */
static void bake_bulk_create_ult(hg_handle_t handle)
{
    bake_bulk_create_out_t out;
    bake_bulk_create_in_t in;
    hg_return_t hret;
    pmemobj_region_id_t* prid;

248 249
    assert(g_svr_ctx);

250 251 252 253 254
    /* TODO: this check needs to be somewhere else */
    assert(sizeof(pmemobj_region_id_t) <= BAKE_BULK_REGION_ID_DATA_SIZE);
    
    memset(&out, 0, sizeof(out));

255
    hret = margo_get_input(handle, &in);
256 257 258
    if(hret != HG_SUCCESS)
    {
        out.ret = -1;
259 260
        margo_respond(handle, &out);
        margo_destroy(handle);
261 262 263 264 265
        return;
    }

    prid = (pmemobj_region_id_t*)out.rid.data;
    prid->size = in.region_size;
266 267
    out.ret = pmemobj_alloc(g_svr_ctx->pmem_pool, &prid->oid,
        in.region_size, 0, NULL, NULL);
268

269 270 271
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
272 273 274 275 276 277 278 279 280 281
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_create_ult)

/* service a remote RPC that writes to a bulk region */
static void bake_bulk_write_ult(hg_handle_t handle)
{
    bake_bulk_write_out_t out;
    bake_bulk_write_in_t in;
    hg_return_t hret;
282
    hg_addr_t src_addr;
283 284
    char* buffer;
    hg_bulk_t bulk_handle;
285
    const struct hg_info *hgi;
286 287 288
    margo_instance_id mid;
    pmemobj_region_id_t* prid;

289 290
    assert(g_svr_ctx);

291 292
    memset(&out, 0, sizeof(out));

293
    hgi = margo_get_info(handle);
294
    assert(hgi);
295
    mid = margo_hg_info_get_instance(hgi);
296

297
    hret = margo_get_input(handle, &in);
298 299 300
    if(hret != HG_SUCCESS)
    {
        out.ret = -1;
301 302
        margo_respond(handle, &out);
        margo_destroy(handle);
303 304 305 306 307 308 309 310 311 312
        return;
    }

    prid = (pmemobj_region_id_t*)in.rid.data;

    /* find memory address for target object */
    buffer = pmemobj_direct(prid->oid);
    if(!buffer)
    {
        out.ret = -1;
313 314 315
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
316 317 318 319
        return;
    }

    /* create bulk handle for local side of transfer */
320
    hret = margo_bulk_create(mid, 1, (void**)(&buffer), &in.bulk_size, 
321 322 323 324
        HG_BULK_WRITE_ONLY, &bulk_handle);
    if(hret != HG_SUCCESS)
    {
        out.ret = -1;
325 326 327
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
328 329 330
        return;
    }

331 332
    if(in.remote_addr_str)
    {
333
        /* a proxy address was provided to pull write data from */
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
        hret = margo_addr_lookup(mid, in.remote_addr_str, &src_addr);
        if(hret != HG_SUCCESS)
        {
            out.ret = -1;
            margo_bulk_free(bulk_handle);
            margo_free_input(handle, &in);
            margo_respond(handle, &out);
            margo_destroy(handle);
            return;
        }
    }
    else
    {
        /* no proxy write, use the source of this request */
        src_addr = hgi->addr;
    }

    hret = margo_bulk_transfer(mid, HG_BULK_PULL, src_addr, in.bulk_handle,
352
        in.bulk_offset, bulk_handle, 0, in.bulk_size);
353
    if(hret != HG_SUCCESS)
Philip Carns's avatar
Philip Carns committed
354
    {
355
        out.ret = -1;
356 357
        if(in.remote_addr_str)
            margo_addr_free(mid, src_addr);
358 359 360 361
        margo_bulk_free(bulk_handle);
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
362
        return;
Philip Carns's avatar
Philip Carns committed
363 364
    }

365 366
    out.ret = 0;

367 368
    if(in.remote_addr_str)
        margo_addr_free(mid, src_addr);
369 370 371 372
    margo_bulk_free(bulk_handle);
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_write_ult)


/* service a remote RPC that writes to a bulk region in eager mode */
static void bake_bulk_eager_write_ult(hg_handle_t handle)
{
    bake_bulk_eager_write_out_t out;
    bake_bulk_eager_write_in_t in;
    hg_return_t hret;
    char* buffer;
    hg_bulk_t bulk_handle;
    pmemobj_region_id_t* prid;

388 389
    assert(g_svr_ctx);

390 391
    memset(&out, 0, sizeof(out));

392
    hret = margo_get_input(handle, &in);
393 394 395
    if(hret != HG_SUCCESS)
    {
        out.ret = -1;
396 397
        margo_respond(handle, &out);
        margo_destroy(handle);
398 399 400 401 402 403 404 405
        return;
    }

    prid = (pmemobj_region_id_t*)in.rid.data;

    /* find memory address for target object */
    buffer = pmemobj_direct(prid->oid);
    if(!buffer)
406
    {
407
        out.ret = -1;
408 409 410
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
411
        return;
412
    }
413 414 415 416 417

    memcpy(buffer, in.buffer, in.size);

    out.ret = 0;

418 419 420
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
421 422 423 424 425 426 427 428 429 430 431 432 433
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_eager_write_ult)

/* service a remote RPC that persists to a bulk region */
static void bake_bulk_persist_ult(hg_handle_t handle)
{
    bake_bulk_persist_out_t out;
    bake_bulk_persist_in_t in;
    hg_return_t hret;
    char* buffer;
    pmemobj_region_id_t* prid;

434
    assert(g_svr_ctx);
435
    
436 437
    memset(&out, 0, sizeof(out));

438
    hret = margo_get_input(handle, &in);
439
    if(hret != HG_SUCCESS)
440
    {
441
        out.ret = -1;
442 443
        margo_respond(handle, &out);
        margo_destroy(handle);
444
        return;
445
    }
446

447 448 449 450 451
    prid = (pmemobj_region_id_t*)in.rid.data;

    /* find memory address for target object */
    buffer = pmemobj_direct(prid->oid);
    if(!buffer)
Philip Carns's avatar
Philip Carns committed
452
    {
453
        out.ret = -1;
454 455 456
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
457
        return;
Philip Carns's avatar
Philip Carns committed
458
    }
459 460

    /* TODO: should this have an abt shim in case it blocks? */
461
    pmemobj_persist(g_svr_ctx->pmem_pool, buffer, prid->size);
462 463 464

    out.ret = 0;

465 466 467
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
468 469 470 471 472 473 474 475 476 477 478 479
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_persist_ult)

/* service a remote RPC that retrieves the size of a bulk region */
static void bake_bulk_get_size_ult(hg_handle_t handle)
{
    bake_bulk_get_size_out_t out;
    bake_bulk_get_size_in_t in;
    hg_return_t hret;
    pmemobj_region_id_t* prid;

480
    assert(g_svr_ctx);
481 482 483
    
    memset(&out, 0, sizeof(out));

484
    hret = margo_get_input(handle, &in);
485
    if(hret != HG_SUCCESS)
Philip Carns's avatar
Philip Carns committed
486
    {
487
        out.ret = -1;
488 489
        margo_respond(handle, &out);
        margo_destroy(handle);
490
        return;
Philip Carns's avatar
Philip Carns committed
491 492
    }

493 494 495 496 497 498
    prid = (pmemobj_region_id_t*)in.rid.data;

    /* kind of cheating here; the size is encoded in the RID */
    out.size = prid->size;
    out.ret = 0;

499 500 501
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
502 503 504 505 506 507 508
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_get_size_ult)

/* service a remote RPC for a no-op */
static void bake_bulk_noop_ult(hg_handle_t handle)
{
509 510
    assert(g_svr_ctx);
    
511 512
    margo_respond(handle, NULL);
    margo_destroy(handle);
513 514 515 516 517 518 519 520 521 522 523
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_noop_ult)

/* TODO consolidate with write handler; read and write are nearly identical */
/* service a remote RPC that reads to a bulk region */
static void bake_bulk_read_ult(hg_handle_t handle)
{
    bake_bulk_read_out_t out;
    bake_bulk_read_in_t in;
    hg_return_t hret;
524
    hg_addr_t src_addr;
525 526
    char* buffer;
    hg_bulk_t bulk_handle;
527
    const struct hg_info *hgi;
528 529 530
    margo_instance_id mid;
    pmemobj_region_id_t* prid;

531
    assert(g_svr_ctx);
532 533 534
    
    memset(&out, 0, sizeof(out));

535
    hgi = margo_get_info(handle);
536
    assert(hgi);
537
    mid = margo_hg_info_get_instance(hgi);
538

539
    hret = margo_get_input(handle, &in);
540
    if(hret != HG_SUCCESS)
Philip Carns's avatar
Philip Carns committed
541
    {
542
        out.ret = -1;
543 544
        margo_respond(handle, &out);
        margo_destroy(handle);
545
        return;
Philip Carns's avatar
Philip Carns committed
546 547
    }

548 549 550 551 552
    prid = (pmemobj_region_id_t*)in.rid.data;

    /* find memory address for target object */
    buffer = pmemobj_direct(prid->oid);
    if(!buffer)
Philip Carns's avatar
Philip Carns committed
553
    {
554
        out.ret = -1;
555 556 557
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
558
        return;
Philip Carns's avatar
Philip Carns committed
559 560
    }

561
    /* create bulk handle for local side of transfer */
562
    hret = margo_bulk_create(mid, 1, (void**)(&buffer), &in.bulk_size, 
563 564
        HG_BULK_READ_ONLY, &bulk_handle);
    if(hret != HG_SUCCESS)
Philip Carns's avatar
Philip Carns committed
565
    {
566
        out.ret = -1;
567 568 569
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
570
        return;
Philip Carns's avatar
Philip Carns committed
571
    }
572

573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
    if(in.remote_addr_str)
    {
        /* a proxy address was provided to push read data to */
        hret = margo_addr_lookup(mid, in.remote_addr_str, &src_addr);
        if(hret != HG_SUCCESS)
        {
            out.ret = -1;
            margo_bulk_free(bulk_handle);
            margo_free_input(handle, &in);
            margo_respond(handle, &out);
            margo_destroy(handle);
            return;
        }
    }
    else
    {
        /* no proxy write, use the source of this request */
        src_addr = hgi->addr;
    }

    hret = margo_bulk_transfer(mid, HG_BULK_PUSH, src_addr, in.bulk_handle,
        in.bulk_offset, bulk_handle, 0, in.bulk_size);
595
    if(hret != HG_SUCCESS)
Philip Carns's avatar
Philip Carns committed
596
    {
597
        out.ret = -1;
598 599
        if(in.remote_addr_str)
            margo_addr_free(mid, src_addr);
600 601 602 603
        margo_bulk_free(bulk_handle);
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
604
        return;
Philip Carns's avatar
Philip Carns committed
605 606
    }

607
    out.ret = 0;
Philip Carns's avatar
Philip Carns committed
608

609 610
    if(in.remote_addr_str)
        margo_addr_free(mid, src_addr);
611 612 613 614
    margo_bulk_free(bulk_handle);
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
615 616 617
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_read_ult)
Philip Carns's avatar
Philip Carns committed
618

619 620 621 622 623 624 625 626 627 628 629
/* service a remote RPC that reads to a bulk region and eagerly sends
 * response */
static void bake_bulk_eager_read_ult(hg_handle_t handle)
{
    bake_bulk_eager_read_out_t out;
    bake_bulk_eager_read_in_t in;
    hg_return_t hret;
    char* buffer;
    hg_size_t size;
    pmemobj_region_id_t* prid;

630
    assert(g_svr_ctx);
631 632 633
    
    memset(&out, 0, sizeof(out));

634
    hret = margo_get_input(handle, &in);
635 636 637
    if(hret != HG_SUCCESS)
    {
        out.ret = -1;
638 639
        margo_respond(handle, &out);
        margo_destroy(handle);
640 641 642 643
        return;
    }

    prid = (pmemobj_region_id_t*)in.rid.data;
Philip Carns's avatar
Philip Carns committed
644

645 646 647 648 649
    /* find memory address for target object */
    buffer = pmemobj_direct(prid->oid);
    if(!buffer)
    {
        out.ret = -1;
650 651 652
        margo_free_input(handle, &in);
        margo_respond(handle, &out);
        margo_destroy(handle);
653 654
        return;
    }
655

656 657 658 659
    out.ret = 0;
    out.buffer = buffer;
    out.size = in.size;

660 661 662
    margo_free_input(handle, &in);
    margo_respond(handle, &out);
    margo_destroy(handle);
663
    return;
Philip Carns's avatar
Philip Carns committed
664
}
665 666 667 668 669 670 671
DEFINE_MARGO_RPC_HANDLER(bake_bulk_eager_read_ult)

/* service a remote RPC that probes for a target id */
static void bake_bulk_probe_ult(hg_handle_t handle)
{
    bake_bulk_probe_out_t out;

672
    assert(g_svr_ctx);
673 674 675 676
    
    memset(&out, 0, sizeof(out));

    out.ret = 0;
677
    out.bti = g_svr_ctx->pmem_root->target_id;
678

679 680
    margo_respond(handle, &out);
    margo_destroy(handle);
681 682 683 684
    return;
}
DEFINE_MARGO_RPC_HANDLER(bake_bulk_probe_ult)

685 686 687 688 689 690 691 692 693
static void bake_server_cleanup(bake_bulk_server_context_t *svr_ctx)
{
    pmemobj_close(svr_ctx->pmem_pool);
    ABT_mutex_free(&svr_ctx->shutdown_mutex);
    ABT_cond_free(&svr_ctx->shutdown_cond);
    free(svr_ctx);

    return;
}
Philip Carns's avatar
Philip Carns committed
694