darshan-bgq.c 7.81 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <pthread.h>
#include <assert.h>

#include "uthash.h"
#include "darshan.h"
Shane Snyder's avatar
Shane Snyder committed
19
#include "darshan-dynamic.h"
20

Shane Snyder's avatar
Shane Snyder committed
21
#include <mpix.h>
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
#include <spi/include/kernel/location.h>
#include <spi/include/kernel/process.h>
#include <firmware/include/personality.h>

/*
 * Simple module which captures BG/Q hardware specific information about 
 * the job.
 * 
 * This module does not intercept any system calls. It just pulls data
 * from the personality struct at initialization.
 */


/*
 * Global runtime struct for tracking data needed at runtime
 */
struct bgq_runtime
{
40
    struct darshan_bgq_record *record;
41 42 43 44 45 46 47 48
};

static struct bgq_runtime *bgq_runtime = NULL;
static pthread_mutex_t bgq_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

/* my_rank indicates the MPI rank of this process */
static int my_rank = -1;

49
/* internal helper functions for the BGQ module */
Kevin Harms's avatar
Kevin Harms committed
50
void bgq_runtime_initialize(void);
51

52 53
/* forward declaration for shutdown function needed to interface with darshan-core */
static void bgq_shutdown(MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count, void **buffer, int *size);
54

55
/* macros for obtaining/releasing the BGQ module lock */
56 57 58 59 60 61
#define BGQ_LOCK() pthread_mutex_lock(&bgq_runtime_mutex)
#define BGQ_UNLOCK() pthread_mutex_unlock(&bgq_runtime_mutex)

/*
 * Function which updates all the counter data
 */
62
static void capture(struct darshan_bgq_record *rec, darshan_record_id rec_id)
63 64
{
    Personality_t person;
Kevin Harms's avatar
Kevin Harms committed
65
    int r;
66 67 68

    rec->counters[BGQ_CSJOBID] = Kernel_GetJobID();
    rec->counters[BGQ_RANKSPERNODE] = Kernel_ProcessCount();
Kevin Harms's avatar
Kevin Harms committed
69
    rec->counters[BGQ_INODES] = MPIX_IO_node_id();
Kevin Harms's avatar
Kevin Harms committed
70

71 72 73 74 75 76 77 78 79 80
    r = Kernel_GetPersonality(&person, sizeof(person));
    if (r == 0)
    {
        rec->counters[BGQ_NNODES] = ND_TORUS_SIZE(person.Network_Config);
        rec->counters[BGQ_ANODES] = person.Network_Config.Anodes;
        rec->counters[BGQ_BNODES] = person.Network_Config.Bnodes;
        rec->counters[BGQ_CNODES] = person.Network_Config.Cnodes;
        rec->counters[BGQ_DNODES] = person.Network_Config.Dnodes;
        rec->counters[BGQ_ENODES] = person.Network_Config.Enodes;
        rec->counters[BGQ_TORUSENABLED] =
81 82 83 84 85
            (((person.Network_Config.NetFlags & ND_ENABLE_TORUS_DIM_A) == ND_ENABLE_TORUS_DIM_A) << 0) |
            (((person.Network_Config.NetFlags & ND_ENABLE_TORUS_DIM_B) == ND_ENABLE_TORUS_DIM_B) << 1) |
            (((person.Network_Config.NetFlags & ND_ENABLE_TORUS_DIM_C) == ND_ENABLE_TORUS_DIM_C) << 2) |
            (((person.Network_Config.NetFlags & ND_ENABLE_TORUS_DIM_D) == ND_ENABLE_TORUS_DIM_D) << 3) |
            (((person.Network_Config.NetFlags & ND_ENABLE_TORUS_DIM_E) == ND_ENABLE_TORUS_DIM_E) << 4);
86 87 88 89

        rec->counters[BGQ_DDRPERNODE] = person.DDR_Config.DDRSizeMB;
    }

90 91
    rec->base_rec.id = rec_id;
    rec->base_rec.rank = my_rank;
92 93 94 95 96 97 98 99 100
    rec->fcounters[BGQ_F_TIMESTAMP] = darshan_core_wtime();

    return;
}

/**********************************************************
 * Internal functions for manipulating BGQ module state *
 **********************************************************/

Kevin Harms's avatar
Kevin Harms committed
101
void bgq_runtime_initialize()
102
{
103 104
    int bgq_buf_size;
    darshan_record_id rec_id;
105 106 107

    BGQ_LOCK();

108 109 110 111
    /* don't do anything if already initialized */
    if(bgq_runtime)
    {
        BGQ_UNLOCK();
112
        return;
113 114 115 116
    }

    /* we just need to store one single record */
    bgq_buf_size = sizeof(struct darshan_bgq_record);
117

118
    /* register the BG/Q module with the darshan-core component */
119 120
    darshan_core_register_module(
        DARSHAN_BGQ_MOD,
121 122
        &bgq_shutdown,
        &bgq_buf_size,
123
        &my_rank,
124
        NULL);
125

126 127
    /* not enough memory to fit bgq module record */
    if(bgq_buf_size < sizeof(struct darshan_bgq_record))
128
    {
129
        darshan_core_unregister_module(DARSHAN_BGQ_MOD);
130
        BGQ_UNLOCK();
131 132 133 134 135 136 137
        return;
    }

    /* initialize module's global state */
    bgq_runtime = malloc(sizeof(*bgq_runtime));
    if(!bgq_runtime)
    {
138
        darshan_core_unregister_module(DARSHAN_BGQ_MOD);
139
        BGQ_UNLOCK();
140 141 142 143
        return;
    }
    memset(bgq_runtime, 0, sizeof(*bgq_runtime));

144 145 146 147 148 149
    rec_id = darshan_core_gen_record_id("darshan-bgq-record");

    /* register the bgq file record with darshan-core */
    bgq_runtime->record = darshan_core_register_record(
        rec_id,
        NULL,
150
        DARSHAN_BGQ_MOD,
151 152 153
        sizeof(struct darshan_bgq_record),
        NULL);
    if(!(bgq_runtime->record))
154
    {
155
        darshan_core_unregister_module(DARSHAN_BGQ_MOD);
156 157 158 159 160 161
        free(bgq_runtime);
        bgq_runtime = NULL;
        BGQ_UNLOCK();
        return;
    }

162
    capture(bgq_runtime->record, rec_id);
163 164 165 166 167 168

    BGQ_UNLOCK();

    return;
}

169 170
static int cmpr(const void *p1, const void *p2)
{
Kevin Harms's avatar
Kevin Harms committed
171 172
    const uint64_t *a = (uint64_t*) p1;
    const uint64_t *b = (uint64_t*) p2;
173 174 175
    return ((*a == *b) ?  0 : ((*a < *b) ? -1 : 1));
}

176 177 178 179
/********************************************************************************
 * shutdown function exported by this module for coordinating with darshan-core *
 ********************************************************************************/

180
/* Pass output data for the BGQ module back to darshan-core to log to file. */
181
static void bgq_shutdown(
182 183 184
    MPI_Comm mod_comm,
    darshan_record_id *shared_recs,
    int shared_rec_count,
185 186 187
    void **buffer,
    int *size)
{
188 189 190 191
    int nprocs;
    int result;
    uint64_t *ion_ids;

192 193 194
    BGQ_LOCK();
    assert(bgq_runtime);

195 196
    if (my_rank == 0)
    {
Shane Snyder's avatar
Shane Snyder committed
197
        DARSHAN_MPI_CALL(PMPI_Comm_size)(mod_comm, &nprocs);
198 199 200
        ion_ids = malloc(sizeof(*ion_ids)*nprocs);
        result = (ion_ids != NULL); 
    }
Shane Snyder's avatar
Shane Snyder committed
201
    DARSHAN_MPI_CALL(PMPI_Bcast)(&result, 1, MPI_INT, 0, mod_comm);
202

203 204
    /* caclulate the number of I/O nodes */
    if (result)
205 206 207 208
    {
        int i, found;
        uint64_t val;

209
        DARSHAN_MPI_CALL(PMPI_Gather)(&bgq_runtime->record->counters[BGQ_INODES],
Shane Snyder's avatar
Shane Snyder committed
210 211 212 213 214 215 216
                                      1,
                                      MPI_LONG_LONG_INT,
                                      ion_ids,
                                      1,
                                      MPI_LONG_LONG_INT,
                                      0,
                                      mod_comm);
217 218 219 220 221 222 223 224 225 226 227
        if (my_rank == 0)
        {
            qsort(ion_ids, nprocs, sizeof(*ion_ids), cmpr);
            for (i = 1, val = ion_ids[0], found = 1; i < nprocs; i++)
            {
                if (val != ion_ids[i])
                {
                    val = ion_ids[i];
                    found += 1;
                }
            }
228
            bgq_runtime->record->counters[BGQ_INODES] = found;
229 230 231
        }
    }

232 233
    /* non-zero ranks throw out their BGQ record */
    if (my_rank != 0)
234 235 236 237 238
    {
        *buffer = NULL;
        *size   = 0;
    }

239 240
    free(bgq_runtime);
    bgq_runtime = NULL;
241

242
    BGQ_UNLOCK();
243 244 245 246

    return;
}

Shane Snyder's avatar
Shane Snyder committed
247
#if 0
Kevin Harms's avatar
Kevin Harms committed
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
static void bgq_record_reduction_op(
    void* infile_v,
    void* inoutfile_v,
    int* len,
    MPI_Datatype *datatype)
{
    int i;
    int j;
    struct darshan_bgq_record *infile = infile_v;
    struct darshan_bgq_record *inoutfile = inoutfile_v;

    for (i = 0; i<*len; i++)
    {
        for (j = 0; j < BGQ_NUM_INDICES; j++)
        {
            if (infile->counters[j] != inoutfile->counters[j])
            {
                // unexpected
                fprintf(stderr,
                        "%lu counter mismatch: %d [%lu] [%lu]\n",
                        infile->f_id,
                        j,
                        infile->counters[j],
                        inoutfile->counters[j]);
            }
        }
        infile++;
        inoutfile++;
    }

    return;
}
Shane Snyder's avatar
Shane Snyder committed
280
#endif
Kevin Harms's avatar
Kevin Harms committed
281

282 283 284 285 286 287 288 289
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */