darshan-null.c 14.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _XOPEN_SOURCE 500
#define _GNU_SOURCE

#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <pthread.h>
#include <assert.h>

#include "uthash.h"
#include "darshan.h"
#include "darshan-null-log-format.h"

/* The "NULL" module is an example instrumentation module implementation provided
 * with Darshan, primarily to indicate how arbitrary modules may be integrated
 * into Darshan. In particular, this module demonstrates how to develop wrapper
 * functions for intercepting functions of interest, how to best manage necessary
 * runtime data structures, and how to coordinate with the darshan-core component,
 * among other things. This module is not linked with the darshan-runtime library; 
 * it is intended mainly to serve as a basic stubbed out module implementation
 * that may be reused and expanded on by developers adding new instrumentation modules.
 */

/* The DARSHAN_FORWARD_DECL macro (defined in darshan.h) is used to provide forward
 * declarations for wrapped funcions, regardless if Darshan is used with statically
 * or dynamically linked executables.
 */
DARSHAN_FORWARD_DECL(foo, int, (const char *name, int arg1, int arg2));

/* The null_record_runtime structure maintains necessary runtime metadata
 * for a "NULL" module data record (darshan_null_record structure, defined
 * in darshan-null-log-format.h). This metadata assists with the instrumenting
 * of specific statistics in the file record.
 *
 * RATIONALE: In general, a module may need to track some stateful, volatile 
 * information regarding specific I/O statistics to aid in the instrumentation
 * process. However, this information should not be stored in the darshan_null_record
 * struct because we don't want it to appear in the final darshan log file.
 * We therefore associate a null_record_runtime structure with each darshan_null_record
 * structure in order to track this information.
 *
 * NOTE: The null_record_runtime struct contains a pointer to a darshan_null_record
 * struct (see the *record_p member) rather than simply embedding an entire
 * darshan_null_record struct.  This is done so that all of the darshan_null_record
 * structs can be kept contiguous in memory as a single array to simplify
 * reduction, compression, and storage.
 */
struct null_record_runtime
{
    /* Darshan record for the "NULL" example module */
    struct darshan_null_record* record_p;

    /* ... other runtime data ... */

    /* hash table link for this record */
    /* NOTE: it is entirely up to the module developer how to persist module
     * records in memory as the instrumented application runs. These records
     * could just as easily be stored in an array or linked list. That said,
     * the data structure selection should be mindful of the resulting memory
     * footprint and search time complexity to attempt minimize Darshan overheads.
     * hash table and linked list implementations are available in uthash.h and
     * utlist.h, respectively.
     */
    UT_hash_handle hlink;
};

/* The null_runtime structure simply encapsulates global data structures needed
 * by the module for instrumenting functions of interest and providing the output
 * I/O data for this module to the darshan-core component at shutdown time.
 */
struct null_runtime
{
    /* runtime_record_array is the array of runtime records for the "NULL" module. */
    struct null_record_runtime* runtime_record_array;
    /* record_array is the array of high-level Darshan records for the "NULL" module,
     * each corresponding to the the runtime record structure stored at the same array
     * index in runtime_record_array.
     */
    struct darshan_null_record* record_array;
    /* file_array_size is the maximum amount of records that can be stored in 
     * record_array (and consequentially, runtime_record_array).
     */
    int rec_array_size;
    /* file_array_ndx is the current index into both runtime_record_array and
     * record_array.
     */
    int rec_array_ndx;
    /* record_hash is a pointer to a hash table of null_record_runtime structures
     * currently maintained by the "NULL" module.
     */
    struct null_record_runtime* record_hash;
};

/* null_runtime is the global data structure encapsulating "NULL" module state */
static struct null_runtime *null_runtime = NULL;
/* The null_runtime_mutex is a lock used when updating the null_runtime global
 * structure (or any other global data structures). This is necessary to avoid race
 * conditions as multiple threads execute function wrappers and update module state.
 * NOTE: Recursive mutexes are used in case functions wrapped by this module call
 * other wrapped functions that would result in deadlock, otherwise. This mechanism
 * may not be necessary for all instrumentation modules.
 */
static pthread_mutex_t null_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
/* the instrumentation_disabled flag is used to toggle wrapper functions on/off */
static int instrumentation_disabled = 0;
/* my_rank indicates the MPI rank of this process */
static int my_rank = -1;

/* internal helper functions for the "NULL" module */
static void null_runtime_initialize(void);
static struct null_record_runtime* null_record_by_name(const char *name);

/* forward declaration for module functions needed to interface with darshan-core */
static void null_begin_shutdown(void);
static void null_get_output_data(void **buffer, int *size);
static void null_shutdown(void);

/* macros for obtaining/releasing the "NULL" module lock */
#define NULL_LOCK() pthread_mutex_lock(&null_runtime_mutex)
#define NULL_UNLOCK() pthread_mutex_unlock(&null_runtime_mutex)

/* macro for instrumenting the "NULL" module's foo function */
/* NOTE: this macro makes use of the DARSHAN_COUNTER_* macros defined
 * and documented in darshan.h.
 */
#define NULL_RECORD_FOO(__ret, __name, __dat, __tm1, __tm2) do{ \
    struct null_record_runtime* rec; \
    double elapsed = __tm2 - __tm1; \
    /* if foo returns error (return code < 0), don't instrument anything */ \
    if(__ret < 0) break; \
    /* use '__name' to lookup a corresponding "NULL" record */ \
    rec = null_record_by_name(__name); \
    if(!rec) break; \
    /* increment counter indicating number of calls to 'bar' */ \
142
    rec->record_p->counters[NULL_BARS] += 1; \
143
    /* store data value for most recent call to 'bar' */ \
144
    rec->record_p->counters[NULL_BAR_DAT] = __dat; \
145
    /* store timestamp of most recent call to 'bar' */ \
146
    rec->record_p->fcounters[NULL_F_BAR_TIMESTAMP] = __tm1; \
147
    /* store duration of most recent call to 'bar' */ \
148
    rec->record_p->fcounters[NULL_F_BAR_DURATION] = elapsed; \
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
} while(0)

/**********************************************************
 *    Wrappers for "NULL" module functions of interest    * 
 **********************************************************/

/* The DARSHAN_DECL macro provides the appropriate wrapper function names,
 * depending on whether the Darshan library is statically or dynamically linked.
 */
int DARSHAN_DECL(foo)(const char* name, int arg1, int arg2)
{
    ssize_t ret;
    double tm1, tm2;

    /* The MAP_OR_FAIL macro attempts to obtain the address of the actual
     * underlying foo function call (__real_foo), in the case of LD_PRELOADing
     * the Darshan library. For statically linked executables, this macro is
     * just a NOP. 
     */
    MAP_OR_FAIL(foo);

    /* In general, Darshan wrappers begin by calling the real version of the
     * given wrapper function. Timers are used to record the duration of this
     * operation. */
    tm1 = darshan_core_wtime();
    ret = __real_foo(name, arg1, arg2);
    tm2 = darshan_core_wtime();

    NULL_LOCK();

    /* Before attempting to instrument I/O statistics for function foo, make
     * sure the "NULL" module runtime environment has been initialized. 
     * NOTE: this runtime environment is initialized only once -- if the
     * appropriate structures have already been initialized, this function simply
     * returns.
     */
    null_runtime_initialize();

    /* Call macro for instrumenting data for foo function calls. */
    NULL_RECORD_FOO(ret, name, arg1+arg2, tm1, tm2);

    NULL_UNLOCK();

    return(ret);
}

/**********************************************************
 * Internal functions for manipulating POSIX module state *
 **********************************************************/

/* Initialize internal POSIX module data structures and register with darshan-core. */
static void null_runtime_initialize()
{
    /* struct of function pointers for interfacing with darshan-core */
    struct darshan_module_funcs null_mod_fns =
    {
        .begin_shutdown = &null_begin_shutdown,
        .setup_reduction = NULL,        /* "NULL" module does not do reductions */
        .record_reduction_op = NULL,    /* "NULL" module does not do reductions */
        .get_output_data = &null_get_output_data,
        .shutdown = &null_shutdown
    };
    int mem_limit; /* max. memory this module can consume, dictated by darshan-core */

    /* don't do anything if already initialized or instrumenation is disabled */
    if(null_runtime || instrumentation_disabled)
        return;

    /* register the "NULL" module with the darshan-core component */
    darshan_core_register_module(
        DARSHAN_NULL_MOD,   /* Darshan module identifier, defined in darshan-log-format.h */
        &null_mod_fns,
221
        &my_rank,
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374
        &mem_limit,
        NULL);

    /* return if no memory assigned by darshan-core */
    if(mem_limit == 0)
        return;

    /* initialize module's global state */
    null_runtime = malloc(sizeof(*null_runtime));
    if(!null_runtime)
        return;
    memset(null_runtime, 0, sizeof(*null_runtime));

    /* Set the maximum number of data records this module may track, as indicated
     * by mem_limit (set by darshan-core).
     * NOTE: We interpret the maximum memory limit to be related to the maximum
     * amount of data which may be written to log by a single process for a given
     * module. We therefore use this maximum memory limit to determine how many
     * darshan_null_record structures we can track per process.
     */
    null_runtime->rec_array_size = mem_limit / sizeof(struct darshan_null_record);
    null_runtime->rec_array_ndx = 0;

    /* allocate both record arrays (runtime and high-level records) */
    null_runtime->runtime_record_array = malloc(null_runtime->rec_array_size *
                                                sizeof(struct null_record_runtime));
    null_runtime->record_array = malloc(null_runtime->rec_array_size *
                                        sizeof(struct darshan_null_record));
    if(!null_runtime->runtime_record_array || !null_runtime->record_array)
    {
        null_runtime->rec_array_size = 0;
        return;
    }
    memset(null_runtime->runtime_record_array, 0, null_runtime->rec_array_size *
           sizeof(struct null_record_runtime));
    memset(null_runtime->record_array, 0, null_runtime->rec_array_size *
           sizeof(struct darshan_null_record));

    return;
}

/* Search for and return a "NULL" module record corresponding to name parameter. */
static struct null_record_runtime* null_record_by_name(const char *name)
{
    struct null_record_runtime *rec = NULL;
    darshan_record_id rec_id;

    /* Don't search for a record if the "NULL" module is not initialized or
     * if instrumentation has been toggled off.
     */
    if(!null_runtime || instrumentation_disabled)
        return(NULL);

    /* get a unique record identifier for this record from darshan-core */
    darshan_core_register_record(
        (void*)name,
        strlen(name),
        1,
        DARSHAN_NULL_MOD,
        &rec_id,
        NULL);

    /* search the hash table for this file record, and return if found */
    HASH_FIND(hlink, null_runtime->record_hash, &rec_id, sizeof(darshan_record_id), rec);
    if(rec)
    {
        return(rec);
    }

    if(null_runtime->rec_array_ndx < null_runtime->rec_array_size);
    {
        /* no existing record, assign a new one from the global array */
        rec = &(null_runtime->runtime_record_array[null_runtime->rec_array_ndx]);
        rec->record_p = &(null_runtime->record_array[null_runtime->rec_array_ndx]);

        /* set the darshan record id and corresponding process rank for this record */
        rec->record_p->f_id = rec_id;
        rec->record_p->rank = my_rank;

        /* add new record to file hash table */
        HASH_ADD(hlink, null_runtime->record_hash, record_p->f_id, sizeof(darshan_record_id), rec);

        null_runtime->rec_array_ndx++;
    }

    return(rec);
}

/******************************************************************************
 * Functions exported by the "NULL" module for coordinating with darshan-core *
 ******************************************************************************/

/* Perform any necessary steps prior to shutting down for the "NULL" module. */
static void null_begin_shutdown()
{
    assert(null_runtime);

    NULL_LOCK();

    /* In general, we want to disable all wrappers while Darshan shuts down. 
     * This is to avoid race conditions and ensure data consistency, as
     * executing wrappers could potentially modify module state while Darshan
     * is in the process of shutting down. 
     */
    instrumentation_disabled = 1;

    /* ... any other code which needs to be executed before beginning shutdown process ... */

    NULL_UNLOCK();

    return;
}

/* Pass output data for the "NULL" module back to darshan-core to log to file. */
static void null_get_output_data(
    void **buffer,
    int *size)
{
    assert(null_runtime);

    /* Just set the output buffer to point at the array of the "NULL" module's
     * I/O records, and set the output size according to the number of records
     * currently being tracked.
     */
    *buffer = (void *)(null_runtime->record_array);
    *size = null_runtime->rec_array_ndx * sizeof(struct darshan_null_record);

    return;
}

/* Shutdown the "NULL" module by freeing up all data structures. */
static void null_shutdown()
{
    assert(null_runtime);

    HASH_CLEAR(hlink, null_runtime->record_hash); /* these hash entries are freed all at once below */

    free(null_runtime->runtime_record_array);
    free(null_runtime->record_array);
    free(null_runtime);
    null_runtime = NULL;

    return;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */