Commit 6eae98bd authored by Shane Snyder's avatar Shane Snyder

Merge branch 'lustre-mod'

Conflicts:
	darshan-bgq-log-format.h
	darshan-runtime/darshan.h
	darshan-runtime/lib/darshan-bgq.c
	darshan-runtime/lib/darshan-core.c
	darshan-runtime/lib/darshan-posix.c
	darshan-util/darshan-convert.c
	darshan-util/darshan-diff.c
parents 301fd492 7d9b5b37
......@@ -111,6 +111,7 @@ struct darshan_base_record
#include "darshan-hdf5-log-format.h"
#include "darshan-pnetcdf-log-format.h"
#include "darshan-bgq-log-format.h"
#include "darshan-lustre-log-format.h"
/* X-macro for keeping module ordering consistent */
/* NOTE: first val used to define module enum values,
......@@ -128,7 +129,8 @@ struct darshan_base_record
X(DARSHAN_MPIIO_MOD, "MPI-IO", DARSHAN_MPIIO_VER, &mpiio_logutils) \
X(DARSHAN_HDF5_MOD, "HDF5", DARSHAN_HDF5_VER, &hdf5_logutils) \
X(DARSHAN_PNETCDF_MOD, "PNETCDF", DARSHAN_PNETCDF_VER, &pnetcdf_logutils) \
X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils)
X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils) \
X(DARSHAN_LUSTRE_MOD, "LUSTRE", DARSHAN_LUSTRE_VER, &lustre_logutils)
/* unique identifiers to distinguish between available darshan modules */
/* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
......
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_LUSTRE_LOG_FORMAT_H
#define __DARSHAN_LUSTRE_LOG_FORMAT_H
/* NOTE -- redefining the size of OST_ID will require changing the DARSHAN_BSWAP
* macro used in darshan-util/darshan-lustre-logutils.c as well
*/
typedef int64_t OST_ID;
/* current Lustre log format version */
#define DARSHAN_LUSTRE_VER 1
#define LUSTRE_COUNTERS \
/* number of OSTs for file system */\
X(LUSTRE_OSTS) \
/* number of MDTs for file system */\
X(LUSTRE_MDTS) \
/* index of first OST for file */\
X(LUSTRE_STRIPE_OFFSET) \
/* bytes per stripe for file */\
X(LUSTRE_STRIPE_SIZE) \
/* number of stripes (OSTs) for file */\
X(LUSTRE_STRIPE_WIDTH) \
/* end of counters */\
X(LUSTRE_NUM_INDICES)
#define X(a) a,
/* integer statistics for Lustre file records */
enum darshan_lustre_indices
{
LUSTRE_COUNTERS
};
#undef X
/* record structure for the Lustre module. a record is created and stored for
* every file opened that belongs to a Lustre file system. This record includes:
* - a corresponding record identifier (created by hashing the file path)
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (stripe size, width, # of OSTs, etc.)
*/
struct darshan_lustre_record
{
darshan_record_id rec_id;
int64_t rank;
int64_t counters[LUSTRE_NUM_INDICES];
OST_ID ost_ids[1];
};
/*
* helper function to calculate the size of a record
*/
#define LUSTRE_RECORD_SIZE( osts ) ( sizeof(struct darshan_lustre_record) + sizeof(OST_ID) * (osts - 1) )
#endif /* __DARSHAN_LUSTRE_LOG_FORMAT_H */
......@@ -37,6 +37,10 @@ LIBS = -lz @LIBBZ2@
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po
# TODO: make the lustre module enabled using config options
DARSHAN_STATIC_MOD_OBJS += lib/darshan-lustre.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-lustre.po
ifdef DARSHAN_USE_BGQ
DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-bgq.po
......@@ -50,63 +54,69 @@ lib::
lib/darshan-core-init-finalize.o: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-core.o: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-common.o: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
lib/darshan-common.o: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-common.po: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
lib/darshan-common.po: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-null.o: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
lib/darshan-null.o: lib/darshan-null.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-null.po: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
lib/darshan-null.po: lib/darshan-null.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-posix.o: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
lib/darshan-posix.o: lib/darshan-posix.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-posix.po: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
lib/darshan-posix.po: lib/darshan-posix.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-hdf5-stubs.o: lib/darshan-hdf5-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-pnetcdf-stubs.o: lib/darshan-pnetcdf-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-lustre.o: lib/darshan-lustre.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-lustre.po: lib/darshan-lustre.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/lookup3.o: lib/lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
......
struct lustre_record_runtime
{
struct darshan_lustre_record *record;
size_t record_size;
UT_hash_handle hlink;
};
struct lustre_runtime
{
int record_count; /* number of records defined */
size_t record_buffer_max; /* size of the allocated buffer pointed to by record_buffer */
size_t record_buffer_used; /* size of the allocated buffer actually used */
void *next_free_record; /* pointer to end of record_buffer */
void *record_buffer; /* buffer in which records are created */
struct lustre_record_runtime *record_runtime_array;
struct lustre_record_runtime *record_runtime_hash;
};
......@@ -73,6 +73,15 @@ typedef void (*darshan_module_shutdown)(
int *mod_buf_sz /* output parameter to save module buffer size */
);
/* stores FS info from statfs calls for a given mount point */
struct darshan_fs_info
{
int fs_type;
int block_size;
int ost_count;
int mdt_count;
};
/*****************************************************
* darshan-core functions exported to darshan modules *
*****************************************************/
......@@ -122,17 +131,18 @@ darshan_record_id darshan_core_gen_record_id(
* Darshan record (e.g., the full file path), which for now is just a
* string. 'mod_id' is the identifier of the calling module. 'rec_len'
* is the size of the record being registered with Darshan. If given,
* 'file_alignment' is a pointer to an integer which on return will
* contain the corresponding file system alignment of the file system
* path 'name' resides on. Returns a pointer to the address the record
* should be written to on success, NULL on failure.
* 'fs_info' is a pointer to a structure containing information on
* the underlying FS this record is associated with (determined by
* matching the file name prefix with Darshan's list of tracked mount
* points). Returns a pointer to the address the record should be
* written to on success, NULL on failure.
*/
void *darshan_core_register_record(
darshan_record_id rec_id,
const char *name,
darshan_module_id mod_id,
int rec_len,
int *file_alignment);
struct darshan_fs_info *fs_info);
/* darshan_core_wtime()
*
......
......@@ -15,6 +15,7 @@
#include "darshan.h"
#include "darshan-core.h"
#include "darshan-dynamic.h"
#ifdef DARSHAN_PRELOAD
......
......@@ -19,6 +19,8 @@
#include <limits.h>
#include <pthread.h>
#include <fcntl.h>
#include <dirent.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
......@@ -32,6 +34,9 @@
#include "darshan-core.h"
#include "darshan-dynamic.h"
/* XXX stick this into autoconf .h */
#include <lustre/lustre_user.h>
extern char* __progname;
extern char* __progname_full;
......@@ -83,9 +88,9 @@ void (*mod_static_init_fns[])(void) =
#define DARSHAN_MAX_MNT_TYPE 32
struct mnt_data
{
int block_size;
char path[DARSHAN_MAX_MNT_PATH];
char type[DARSHAN_MAX_MNT_TYPE];
struct darshan_fs_info fs_info;
};
static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;
......@@ -99,11 +104,11 @@ static void darshan_log_record_hints_and_ver(
struct darshan_core_runtime* core);
static void darshan_get_exe_and_mounts(
struct darshan_core_runtime *core, int argc, char **argv);
static void darshan_fs_info_from_path(
const char *path, struct darshan_fs_info *fs_info);
static int darshan_add_name_record_ref(
struct darshan_core_runtime *core, darshan_record_id rec_id,
const char *name, darshan_module_id mod_id);
static int darshan_block_size_from_path(
const char *path);
static void darshan_get_user_name(
char *user);
static void darshan_get_logfile_name(
......@@ -919,12 +924,42 @@ static void add_entry(char* buf, int* space_left, struct mntent* entry)
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
ret = statfs(entry->mnt_dir, &statfsbuf);
mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
mnt_data_array[mnt_data_count].block_size = 1024*1024;
mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
else
mnt_data_array[mnt_data_count].block_size = 4096;
mnt_data_array[mnt_data_count].fs_info.block_size = 4096;
/* XXX */
/* attempt to retrieve OST and MDS counts from Lustre */
mnt_data_array[mnt_data_count].fs_info.ost_count = -1;
mnt_data_array[mnt_data_count].fs_info.mdt_count = -1;
if ( statfsbuf.f_type == LL_SUPER_MAGIC )
{
int n_ost, n_mdt;
int ret_ost, ret_mdt;
DIR *mount_dir;
mount_dir = opendir( entry->mnt_dir );
if ( mount_dir )
{
/* n_ost and n_mdt are used for both input and output to ioctl */
n_ost = 0;
n_mdt = 1;
ret_ost = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_ost );
ret_mdt = ioctl( dirfd(mount_dir), LL_IOC_GETOBDCOUNT, &n_mdt );
if ( !(ret_ost < 0 || ret_mdt < 0) )
{
mnt_data_array[mnt_data_count].fs_info.ost_count = n_ost;
mnt_data_array[mnt_data_count].fs_info.mdt_count = n_mdt;
}
closedir( mount_dir );
}
}
/* store mount information with the job-level metadata in darshan log */
ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
......@@ -1059,21 +1094,55 @@ static void darshan_get_exe_and_mounts(struct darshan_core_runtime *core,
return;
}
static int darshan_block_size_from_path(const char *path)
static void darshan_fs_info_from_path(const char *path, struct darshan_fs_info *fs_info)
{
int i;
int block_size = -1;
fs_info->fs_type = -1;
fs_info->block_size = -1;
for(i=0; i<mnt_data_count; i++)
{
if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
{
block_size = mnt_data_array[i].block_size;
break;;
*fs_info = mnt_data_array[i].fs_info;
return;
}
}
return block_size;
return;
}
static int darshan_add_name_record_ref(struct darshan_core_runtime *core,
darshan_record_id rec_id, const char *name, darshan_module_id mod_id)
{
struct darshan_core_name_record_ref *ref;
int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
if((record_size + core->name_mem_used) > DARSHAN_NAME_RECORD_BUF_SIZE)
return(0);
ref = malloc(sizeof(*ref));
if(!ref)
return(0);
memset(ref, 0, sizeof(*ref));
/* initialize the name record */
ref->name_record = (struct darshan_name_record *)
((char *)core->log_name_p + core->name_mem_used);
memset(ref->name_record, 0, record_size);
ref->name_record->id = rec_id;
strcpy(ref->name_record->name, name);
DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
/* add the record to the hash table */
HASH_ADD(hlink, core->name_hash, name_record->id,
sizeof(darshan_record_id), ref);
core->name_mem_used += record_size;
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
core->log_hdr_p->name_map.len += record_size;
#endif
return(1);
}
static void darshan_get_user_name(char *cuser)
......@@ -1239,39 +1308,6 @@ static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* s
return;
}
static int darshan_add_name_record_ref(struct darshan_core_runtime *core,
darshan_record_id rec_id, const char *name, darshan_module_id mod_id)
{
struct darshan_core_name_record_ref *ref;
int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
if((record_size + core->name_mem_used) > DARSHAN_NAME_RECORD_BUF_SIZE)
return(0);
ref = malloc(sizeof(*ref));
if(!ref)
return(0);
memset(ref, 0, sizeof(*ref));
/* initialize the name record */
ref->name_record = (struct darshan_name_record *)
((char *)core->log_name_p + core->name_mem_used);
memset(ref->name_record, 0, record_size);
ref->name_record->id = rec_id;
strcpy(ref->name_record->name, name);
DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
/* add the record to the hash table */
HASH_ADD(hlink, core->name_hash, name_record->id,
sizeof(darshan_record_id), ref);
core->name_mem_used += record_size;
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
core->log_hdr_p->name_map.len += record_size;
#endif
return(1);
}
static void darshan_get_shared_records(struct darshan_core_runtime *core,
darshan_record_id **shared_recs, int *shared_rec_cnt)
{
......@@ -1889,7 +1925,7 @@ void *darshan_core_register_record(
const char *name,
darshan_module_id mod_id,
int rec_len,
int *file_alignment)
struct darshan_fs_info *fs_info)
{
struct darshan_core_name_record_ref *ref;
void *rec_buf;
......@@ -1942,8 +1978,8 @@ void *darshan_core_register_record(
#endif
DARSHAN_CORE_UNLOCK();
if(file_alignment)
*file_alignment = darshan_block_size_from_path(name);
if(fs_info)
darshan_fs_info_from_path(name, fs_info);
return(rec_buf);;
}
......
......@@ -4,6 +4,9 @@
*
*/
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
......@@ -16,7 +19,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
#define __USE_GNU
#include <pthread.h>
#include "darshan.h"
......
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <time.h>
#include <stdlib.h>
#include <assert.h>
#include <pthread.h>
#include <sys/ioctl.h>
/* XXX stick this into autoconf .h */
#include <lustre/lustre_user.h>
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-lustre.h"
struct lustre_runtime *lustre_runtime = NULL;
static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
static int my_rank = -1;
static void lustre_runtime_initialize(void);
static void lustre_begin_shutdown(void);
static void lustre_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **lustre_buf, int *lustre_buf_sz);
static void lustre_shutdown(void);
static int lustre_record_compare(const void* a_p, const void* b_p);
static void lustre_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype);
#define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
#define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
void darshan_instrument_lustre_file(const char* filepath, int fd)
{
struct lustre_record_runtime *rec_rt;
struct darshan_lustre_record *rec;
struct darshan_fs_info fs_info;
darshan_record_id rec_id;
int limit_flag;
int i;
struct lov_user_md *lum;
size_t lumsize = sizeof(struct lov_user_md) +
LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data);
size_t rec_size;
char *newname = NULL;
LUSTRE_LOCK();
if(instrumentation_disabled)
{
LUSTRE_UNLOCK();
return;
}
/* make sure the lustre module is already initialized */
lustre_runtime_initialize();
if(!lustre_runtime)
{
LUSTRE_UNLOCK();
return;
}
/* if we can't issue ioctl, we have no counter data at all */
if ( (lum = calloc(1, lumsize)) == NULL )
{
LUSTRE_UNLOCK();
return;
}
/* find out the OST count of this file so we can allocate memory */
lum->lmm_magic = LOV_USER_MAGIC;
lum->lmm_stripe_count = LOV_MAX_STRIPE_COUNT;
/* -1 means ioctl failed, likely because file isn't on Lustre */
if ( ioctl( fd, LL_IOC_LOV_GETSTRIPE, (void *)lum ) == -1 )
{
free(lum);
LUSTRE_UNLOCK();
return;
}
rec_size = LUSTRE_RECORD_SIZE( lum->lmm_stripe_count );
/* get fully qualified name for record */
newname = darshan_clean_file_path(filepath);
if(!newname)
newname = (char*)filepath;
{
/* broken out for clarity */
void *end_of_new_record = (char*)lustre_runtime->next_free_record + rec_size;
void *end_of_rec_buffer = (char*)lustre_runtime->record_buffer + lustre_runtime->record_buffer_max;
limit_flag = ( end_of_new_record > end_of_rec_buffer );
}
/* register a Lustre file record with Darshan */
fs_info.fs_type = -1;
darshan_core_register_record(
(void *)newname,
strlen(newname),
DARSHAN_LUSTRE_MOD,
1,
limit_flag,
&rec_id,
&fs_info);
/* if record id is 0, darshan has no more memory for instrumenting */
if(rec_id == 0)
{
free(lum);
LUSTRE_UNLOCK();
return;
}
/* search the hash table for this file record, and initialize if not found */
HASH_FIND(hlink, lustre_runtime->record_runtime_hash, &rec_id, sizeof(darshan_record_id), rec_rt );
if ( !rec_rt ) {
/* allocate a new lustre record and append it to the array */
rec_rt = &(lustre_runtime->record_runtime_array[lustre_runtime->record_count]);
rec_rt->record = lustre_runtime->next_free_record;
rec_rt->record_size = rec_size;
lustre_runtime->next_free_record = (char*)(lustre_runtime->next_free_record) + rec_size;
lustre_runtime->record_buffer_used += rec_size;
rec = rec_rt->record;
rec->rec_id = rec_id;
rec->rank = my_rank;
/* implicit assumption here that none of these counters will change
* after the first time a file is opened. This may not always be
* true in the future */
if ( fs_info.fs_type != -1 )
{
rec->counters[LUSTRE_OSTS] = fs_info.ost_count;
rec->counters[LUSTRE_MDTS] = fs_info.mdt_count;
}
else
{
rec->counters[LUSTRE_OSTS] = -1;
rec->counters[LUSTRE_MDTS] = -1;
}
rec->counters[LUSTRE_STRIPE_SIZE] = lum->lmm_stripe_size;
rec->counters[LUSTRE_STRIPE_WIDTH] = lum->lmm_stripe_count;
rec->counters[LUSTRE_STRIPE_OFFSET] = lum->lmm_stripe_offset;
for ( i = 0; i < lum->lmm_stripe_count; i++ )
rec->ost_ids[i] = lum->lmm_objects[i].l_ost_idx;
free(lum);
HASH_ADD(hlink, lustre_runtime->record_runtime_hash, record->rec_id, sizeof(darshan_record_id), rec_rt);
lustre_runtime->record_count++;
}
LUSTRE_UNLOCK();
return;
}
static void lustre_runtime_initialize()
{
int mem_limit;
int max_records;
struct darshan_module_funcs lustre_mod_fns =
{
.begin_shutdown = &lustre_begin_shutdown,
.get_output_data = &lustre_get_output_data,
.shutdown = &lustre_shutdown
};
/* don't do anything if already initialized or instrumenation is disabled */
if(lustre_runtime || instrumentation_disabled)
return;
/* register the lustre module with darshan-core */
darshan_core_register_module(
DARSHAN_LUSTRE_MOD,
&lustre_mod_fns,
&my_rank,
&mem_limit,
NULL);
/* return if no memory assigned by darshan core */
if(mem_limit == 0)
return;
lustre_runtime = malloc(sizeof(*lustre_runtime));
if(!lustre_runtime)
return;
memset(lustre_runtime, 0, sizeof(*lustre_runtime));
/* allocate the full size of the memory limit we are given */
lustre_runtime->record_buffer= malloc(mem_limit);
if(!lustre_runtime->record_buffer)
{
lustre_runtime->record_buffer_max = 0;
return;
}
lustre_runtime->record_buffer_max = mem_limit;
lustre_runtime->next_free_record = lustre_runtime->record_buffer;
memset(lustre_runtime->record_buffer, 0, lustre_runtime->record_buffer_max);
/* Allocate array of Lustre runtime data. We calculate the maximum possible
* number of records that will fit into mem_limit by assuming that each
* record has the minimum possible OST count, then allocate that many
* runtime records. record_buffer will always run out of memory before
* we overflow record_runtime_array.
*/
max_records = mem_limit / sizeof(struct darshan_lustre_record);
lustre_runtime->record_runtime_array =
malloc( max_records * sizeof(struct lustre_record_runtime));
if(!lustre_runtime->record_runtime_array)
{
lustre_runtime->record_buffer_max = 0;
free( lustre_runtime->record_buffer );
return;
}
memset(lustre_runtime->record_runtime_array, 0,
max_records * sizeof(struct lustre_record_runtime));
return;
}
/**************************************************************************
* Functions exported by Lustre module for coordinating with darshan-core *
**************************************************************************/
static void lustre_begin_shutdown(void)
{
assert(lustre_runtime);
LUSTRE_LOCK();
/* disable further instrumentation while Darshan shuts down */
instrumentation_disabled = 1;
LUSTRE_UNLOCK();
return;
}
static void lustre_get_output_data(
MPI_Comm mod_comm,
darshan_record_id *shared_recs,
int shared_rec_count,
void **lustre_buf,
int *lustre_buf_sz)