margo-p2p-latency.c 11.4 KB
Newer Older
1 2 3 4 5 6
/*
 * Copyright (c) 2017 UChicago Argonne, LLC
 *
 * See COPYRIGHT in top-level directory.
 */

7 8
#include "ssg-config.h"

9 10 11 12 13 14 15 16
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>

#include <mpi.h>

#include <margo.h>
17
#ifdef HAVE_ABT_SNOOZER
18
#include <abt-snoozer.h>
19
#endif
20 21 22 23 24 25 26 27
#include <mercury.h>
#include <abt.h>
#include <ssg.h>
#include <ssg-mpi.h>

struct options
{
    int iterations;
28 29
    int snoozer_flag_client;
    int snoozer_flag_server;
30 31
    unsigned int mercury_timeout_client;
    unsigned int mercury_timeout_server;
Philip Carns's avatar
Philip Carns committed
32
    char* diag_file_name;
33 34 35 36 37 38
    char* na_transport;
};

static void parse_args(int argc, char **argv, struct options *opts);
static void usage(void);
static int run_benchmark(int iterations, hg_id_t id, ssg_member_id_t target, 
Shane Snyder's avatar
Shane Snyder committed
39
    ssg_group_id_t gid, margo_instance_id mid, double *measurement_array);
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
static void bench_routine_print(const char* op, int size, int iterations, 
    double* measurement_array);
static int measurement_cmp(const void* a, const void *b);
DECLARE_MARGO_RPC_HANDLER(noop_ult);

static hg_id_t noop_id;
static int rpcs_serviced = 0;
static ABT_eventual rpcs_serviced_eventual;
static struct options g_opts;

int main(int argc, char **argv) 
{
    margo_instance_id mid;
    int nranks;
    hg_context_t *hg_context;
    hg_class_t *hg_class;
Shane Snyder's avatar
Shane Snyder committed
56 57
    ABT_xstream xstream;
    ABT_pool pool;
58 59 60 61 62
    int ret;
    ssg_group_id_t gid;
    ssg_member_id_t self;
    int rank;
    double *measurement_array;
63 64
    int namelen;
    char processor_name[MPI_MAX_PROCESSOR_NAME];
65 66 67 68 69 70 71 72 73 74 75 76

    ABT_init(argc, argv);
    MPI_Init(&argc, &argv);

    /* 2 process rtt measurements only */
    MPI_Comm_size(MPI_COMM_WORLD, &nranks);
    if(nranks != 2)
    {
        usage();
        exit(EXIT_FAILURE);
    }
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
77 78 79
    MPI_Get_processor_name(processor_name,&namelen);
    printf("Process %d of %d is on %s\n",
	rank, nranks, processor_name);
80 81 82 83 84

    parse_args(argc, argv, &g_opts);

    /* boilerplate HG initialization steps */
    /***************************************/
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104

    if((rank == 0 && g_opts.mercury_timeout_client == 0) ||
       (rank == 1 && g_opts.mercury_timeout_server == 0))
    {
        struct hg_init_info hii;
        
        /* If mercury timeout of zero is requested, then set
         * init option to NO_BLOCK.  This allows some transports to go
         * faster because they do not have to set up or maintain the data
         * structures necessary for signaling completion on blocked
         * operations.
         */
        memset(&hii, 0, sizeof(hii));
        hii.na_init_info.progress_mode = NA_NO_BLOCK;
        hg_class = HG_Init_opt(g_opts.na_transport, HG_TRUE, &hii);
    }
    else
    {
        hg_class = HG_Init(g_opts.na_transport, HG_TRUE);
    }
105 106 107 108 109 110 111 112 113 114 115 116 117
    if(!hg_class)
    {
        fprintf(stderr, "Error: HG_Init()\n");
        return(-1);
    }
    hg_context = HG_Context_create(hg_class);
    if(!hg_context)
    {
        fprintf(stderr, "Error: HG_Context_create()\n");
        HG_Finalize(hg_class);
        return(-1);
    }

118 119
    if((rank == 0 && g_opts.snoozer_flag_client) || 
        (rank == 1 && g_opts.snoozer_flag_server))
120
    {
121
#ifdef HAVE_ABT_SNOOZER
122 123 124 125 126 127 128
        /* set primary ES to idle without polling in scheduler */
        ret = ABT_snoozer_xstream_self_set();
        if(ret != 0)
        {
            fprintf(stderr, "Error: ABT_snoozer_xstream_self_set()\n");
            return(-1);
        }
129 130 131 132
#else
        fprintf(stderr, "Error: abt-snoozer scheduler is not supported\n");
        return(-1);
#endif
133 134
    }

Shane Snyder's avatar
Shane Snyder committed
135 136 137 138 139 140 141 142 143 144 145 146 147 148
    /* get main pool for running mercury progress and RPC handlers */
    ret = ABT_xstream_self(&xstream);
    if(ret != 0)
    {
        fprintf(stderr, "Error: ABT_xstream_self()\n");
        return(-1);
    }   
    ret = ABT_xstream_get_main_pools(xstream, 1, &pool);
    if(ret != 0)
    {
        fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
        return(-1);
    }

149
    /* actually start margo */
Shane Snyder's avatar
Shane Snyder committed
150
    mid = margo_init_pool(pool, pool, hg_context);
151 152
    assert(mid);

Philip Carns's avatar
Philip Carns committed
153 154 155
    if(g_opts.diag_file_name)
        margo_diag_start(mid);

156 157
    /* adjust mercury timeout in Margo if requested */
    if(rank == 0 && g_opts.mercury_timeout_client != UINT_MAX)
Shane Snyder's avatar
Shane Snyder committed
158
        margo_set_param(mid, MARGO_PARAM_PROGRESS_TIMEOUT_UB, &g_opts.mercury_timeout_client);
159
    if(rank == 1 && g_opts.mercury_timeout_server != UINT_MAX)
Shane Snyder's avatar
Shane Snyder committed
160
        margo_set_param(mid, MARGO_PARAM_PROGRESS_TIMEOUT_UB, &g_opts.mercury_timeout_server);
161

Philip Carns's avatar
Philip Carns committed
162
    noop_id = MARGO_REGISTER(
163 164 165 166
        mid, 
        "noop_rpc", 
        void,
        void,
Philip Carns's avatar
Philip Carns committed
167
        noop_ult);
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188

    /* set up group */
    ret = ssg_init(mid);
    assert(ret == 0);
    gid = ssg_group_create_mpi("margo-p2p-latency", MPI_COMM_WORLD, NULL, NULL);
    assert(gid != SSG_GROUP_ID_NULL);

    assert(ssg_get_group_size(gid) == 2);

    self = ssg_get_group_self_id(gid);
#if 0
    printf("MPI rank %d has SSG ID %lu\n", rank, self);
#endif

    if(self == 0)
    {
        /* ssg id 0 runs benchmark */

        measurement_array = calloc(g_opts.iterations, sizeof(*measurement_array));
        assert(measurement_array);

Shane Snyder's avatar
Shane Snyder committed
189
        ret = run_benchmark(g_opts.iterations, noop_id, 1, gid, mid, measurement_array);
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
        assert(ret == 0);

        printf("# <op> <iterations> <size> <min> <q1> <med> <avg> <q3> <max>\n");
        bench_routine_print("noop", 0, g_opts.iterations, measurement_array);
        free(measurement_array);
    }
    else
    {
        /* ssg id 1 acts as server, waiting until iterations have been
         * completed
         */

        ret = ABT_eventual_create(0, &rpcs_serviced_eventual);
        assert(ret == 0);

        ABT_eventual_wait(rpcs_serviced_eventual, NULL);
        assert(rpcs_serviced == g_opts.iterations);
Philip Carns's avatar
Philip Carns committed
207
        sleep(3);
208 209 210 211
    }

    ssg_group_destroy(gid);
    ssg_finalize();
Philip Carns's avatar
Philip Carns committed
212 213 214 215

    if(g_opts.diag_file_name)
        margo_diag_dump(mid, g_opts.diag_file_name, 1);

216 217
    margo_finalize(mid);
    HG_Context_destroy(hg_context);
218 219 220 221 222 223 224 225 226 227 228
    HG_Finalize(hg_class);
    MPI_Finalize();
    ABT_finalize();

    return 0;
}

static void parse_args(int argc, char **argv, struct options *opts)
{
    int opt;
    int ret;
229
    char clientflag, serverflag;
230 231 232

    memset(opts, 0, sizeof(*opts));

233
#ifdef HAVE_ABT_SNOOZER
234 235 236
    /* default to enabling snoozer scheduler on both client and server */
    opts->snoozer_flag_client = 1;
    opts->snoozer_flag_server = 1;
237 238 239 240
#else
    opts->snoozer_flag_client = 0;
    opts->snoozer_flag_server = 0;
#endif
241 242 243
    /* default to using whatever the standard timeout is in margo */
    opts->mercury_timeout_client = UINT_MAX;
    opts->mercury_timeout_server = UINT_MAX; 
244

245
    while((opt = getopt(argc, argv, "n:i:d:s:t:")) != -1)
246 247 248
    {
        switch(opt)
        {
Philip Carns's avatar
Philip Carns committed
249 250 251 252 253 254 255 256
            case 'd':
                opts->diag_file_name = strdup(optarg);
                if(!opts->diag_file_name)
                {
                    perror("strdup");
                    exit(EXIT_FAILURE);
                }
                break;
257 258 259 260 261 262 263 264
            case 'i':
                ret = sscanf(optarg, "%d", &opts->iterations);
                if(ret != 1)
                {
                    usage();
                    exit(EXIT_FAILURE);
                }
                break;
265
            case 's':
266
                ret = sscanf(optarg, "%c,%c", &clientflag, &serverflag);
267 268 269 270 271 272
                if(ret != 2)
                {
                    usage();
                    exit(EXIT_FAILURE);
                }
                if(clientflag == '0') opts->snoozer_flag_client = 0;
273
                else if(clientflag == '1') opts->snoozer_flag_client = 1;
274
                if(serverflag == '0') opts->snoozer_flag_server = 0;
275
                else if(serverflag == '1') opts->snoozer_flag_server = 1;
276
                break;
277 278 279 280 281 282 283 284
            case 't':
                ret = sscanf(optarg, "%u,%u", &opts->mercury_timeout_client, &opts->mercury_timeout_server);
                if(ret != 2)
                {
                    usage();
                    exit(EXIT_FAILURE);
                }
                break;
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
            case 'n':
                opts->na_transport = strdup(optarg);
                if(!opts->na_transport)
                {
                    perror("strdup");
                    exit(EXIT_FAILURE);
                }
                break;
            default:
                usage();
                exit(EXIT_FAILURE);
        }
    }

    if(opts->iterations < 1 || !opts->na_transport)
    {
        usage();
        exit(EXIT_FAILURE);
    }

    return;
}

static void usage(void)
{
    fprintf(stderr,
        "Usage: "
        "margo-p2p-latency -i <iterations> -n <na>\n"
        "\t-i <iterations> - number of RPC iterations\n"
        "\t-n <na> - na transport\n"
315
        "\t[-d filename] - enable diagnostics output \n"
316 317
        "\t[-s <bool,bool>] - specify if snoozer scheduler is used on client and server\n"
        "\t\t(e.g., -s 0,1 means snoozer disabled on client and enabled on server)\n"
318 319 320 321 322 323 324 325 326 327
        "\t\texample: mpiexec -n 2 ./margo-p2p-latency -i 10000 -n verbs://\n"
        "\t\t(must be run with exactly 2 processes\n");
    
    return;
}


/* service a remote RPC for a no-op */
static void noop_ult(hg_handle_t handle)
{
328 329
    margo_respond(handle, NULL);
    margo_destroy(handle);
330 331 332 333 334 335 336 337 338 339 340 341

    rpcs_serviced++;
    if(rpcs_serviced == g_opts.iterations)
    {
        ABT_eventual_set(rpcs_serviced_eventual, NULL, 0);
    }

    return;
}
DEFINE_MARGO_RPC_HANDLER(noop_ult)

static int run_benchmark(int iterations, hg_id_t id, ssg_member_id_t target, 
Shane Snyder's avatar
Shane Snyder committed
342
    ssg_group_id_t gid, margo_instance_id mid, double *measurement_array)
343 344 345 346 347 348 349 350 351 352
{
    hg_handle_t handle;
    hg_addr_t target_addr;
    int i;
    int ret;
    double tm1, tm2;

    target_addr = ssg_get_addr(gid, target);
    assert(target_addr != HG_ADDR_NULL);

Shane Snyder's avatar
Shane Snyder committed
353
    ret = margo_create(mid, target_addr, id, &handle);
354 355 356 357 358
    assert(ret == 0);

    for(i=0; i<iterations; i++)
    {
        tm1 = ABT_get_wtime();
359
        ret = margo_forward(handle, NULL);
360 361 362 363 364
        tm2 = ABT_get_wtime();
        assert(ret == 0);
        measurement_array[i] = tm2-tm1;
    }

365
    margo_destroy(handle);
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435

    return(0);
}

static void bench_routine_print(const char* op, int size, int iterations, double* measurement_array)
{
    double min, max, q1, q3, med, avg, sum;
    int bracket1, bracket2;
    int i;

    qsort(measurement_array, iterations, sizeof(double), measurement_cmp);

    min = measurement_array[0];
    max = measurement_array[iterations-1];

    sum = 0;
    for(i=0; i<iterations; i++)
    {
        sum += measurement_array[i];
    }
    avg = sum/(double)iterations;

    bracket1 = iterations/2;
    if(iterations%2)
        bracket2 = bracket1 + 1;
    else
        bracket2 = bracket1;
    med = (measurement_array[bracket1] + measurement_array[bracket2])/(double)2;

    bracket1 = iterations/4;
    if(iterations%4)
        bracket2 = bracket1 + 1;
    else
        bracket2 = bracket1;
    q1 = (measurement_array[bracket1] + measurement_array[bracket2])/(double)2;

    bracket1 *= 3;
    if(iterations%4)
        bracket2 = bracket1 + 1;
    else
        bracket2 = bracket1;
    q3 = (measurement_array[bracket1] + measurement_array[bracket2])/(double)2;

    printf("%s\t%d\t%d\t%.9f\t%.9f\t%.9f\t%.9f\t%.9f\t%.9f\n", op, iterations, size, min, q1, med, avg, q3, max);
#if 0
    for(i=0; i<iterations; i++)
    {
        printf("\t%.9f", measurement_array[i]);
    }
    printf("\n");
#endif
    fflush(NULL);

    return;
}

static int measurement_cmp(const void* a, const void *b)
{
    const double *d_a = a;
    const double *d_b = b;

    if(*d_a < *d_b)
        return(-1);
    else if(*d_a > *d_b)
        return(1);
    else
        return(0);
}