Commit 657c5b56 authored by Shane Snyder's avatar Shane Snyder

initial runtime lustre module implementation

the POSIX module will call into the Lustre module for each file
record created that corresponds to a file from a Lustre FS. No
counters are currently stored but most of the module strucutre
has been stubbed.
parent 21c1b5d0
......@@ -80,6 +80,13 @@ struct darshan_module_funcs
void (*shutdown)(void);
};
/* stores FS info from statfs calls for a given mount point */
struct darshan_fs_info
{
int fs_type;
int block_size;
};
/* paths that darshan will not trace */
extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
......@@ -134,7 +141,7 @@ void darshan_core_register_record(
int printable_flag,
int mod_limit_flag,
darshan_record_id *rec_id,
int *file_alignment);
struct darshan_fs_info *fs_info);
/* darshan_core_unregister_record()
*
......
......@@ -79,9 +79,9 @@ void (*mod_static_init_fns[])(void) =
#define DARSHAN_MAX_MNT_TYPE 32
struct mnt_data
{
int block_size;
char path[DARSHAN_MAX_MNT_PATH];
char type[DARSHAN_MAX_MNT_TYPE];
struct darshan_fs_info fs_info;
};
static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;
......@@ -96,8 +96,8 @@ static void darshan_get_exe_and_mounts_root(
int space_left);
static char* darshan_get_exe_and_mounts(
struct darshan_core_runtime *core);
static void darshan_block_size_from_path(
const char *path, int *block_size);
static void darshan_fs_info_from_path(
const char *path, struct darshan_fs_info *fs_info);
static void darshan_get_shared_records(
struct darshan_core_runtime *core, darshan_record_id **shared_recs,
int *shared_rec_cnt);
......@@ -952,12 +952,13 @@ static void add_entry(char* trailing_data, int* space_left, struct mntent *entry
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
ret = statfs(entry->mnt_dir, &statfsbuf);
mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
mnt_data_array[mnt_data_count].block_size = 1024*1024;
mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
else
mnt_data_array[mnt_data_count].block_size = 4096;
mnt_data_array[mnt_data_count].fs_info.block_size = 4096;
/* store mount information for use in header of darshan log */
ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
......@@ -1097,16 +1098,17 @@ static char* darshan_get_exe_and_mounts(struct darshan_core_runtime *core)
return(trailing_data);
}
static void darshan_block_size_from_path(const char *path, int *block_size)
static void darshan_fs_info_from_path(const char *path, struct darshan_fs_info *fs_info)
{
int i;
*block_size = -1;
fs_info->fs_type = -1;
fs_info->block_size = -1;
for(i=0; i<mnt_data_count; i++)
{
if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
{
*block_size = mnt_data_array[i].block_size;
*fs_info = mnt_data_array[i].fs_info;
return;
}
}
......@@ -1695,7 +1697,7 @@ void darshan_core_register_record(
int printable_flag,
int mod_limit_flag,
darshan_record_id *rec_id,
int *file_alignment)
struct darshan_fs_info *fs_info)
{
darshan_record_id tmp_rec_id;
struct darshan_core_record_ref *ref;
......@@ -1743,14 +1745,10 @@ void darshan_core_register_record(
DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
DARSHAN_CORE_UNLOCK();
/* TODO: call into lustre if functionality enabled and if lustre data has not
* already been captured for this file
*/
if(file_alignment)
darshan_block_size_from_path(name, file_alignment);
*rec_id = tmp_rec_id;
if(fs_info)
darshan_fs_info_from_path(name, fs_info);
return;
}
......
......@@ -22,15 +22,27 @@
#include "darshan.h"
#include "darshan-dynamic.h"
/* Lustre record data is currently immutable, so once it's set it should be
* final. Since we won't likely need to search for records after they've been
* created, a simple array is fine for storing them instead of a hash table
/* TODO: once mmap merge is complete, we can just use an array
* to store record data rather than a hash table -- in that
* branch, register_record() returns whether the record
* already exists, at which point we won't need to instrument
* more data, since the Lustre record data is immutable. records
* could just be appended to the array if there is no need to
* search for a specific record.
*/
struct lustre_record_ref
{
struct darshan_lustre_record *record;
UT_hash_handle hlink;
};
struct lustre_runtime
{
struct lustre_record_ref *ref_array;
struct darshan_lustre_record *record_array;
int record_array_size;
int record_array_ndx;
struct lustre_record_ref *record_hash;
};
static struct lustre_runtime *lustre_runtime = NULL;
......@@ -48,13 +60,58 @@ static void lustre_shutdown(void);
#define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
#define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
/* TODO: is there any way we can further compact Lustre data to save space?
* e.g., are all files in the same directory guaranteed same striping parameters?
* if so, can we store stripe parameters on per-directory basis and the OST
* list on a per-file basis? maybe the storage savings are small enough this isn't
* worth it, but nice to keep in mind
*/
void darshan_instrument_lustre_file(char *filepath)
{
struct lustre_record_ref *lustre_ref;
darshan_record_id rec_id;
int limit_flag;
LUSTRE_LOCK();
/* make sure the lustre module is already initialized */
lustre_runtime_initialize();
/* TODO: implement gathering of lustre data */
limit_flag = (lustre_runtime->record_array_ndx >= lustre_runtime->record_array_size);
/* register a Lustre file record with Darshan */
darshan_core_register_record(
(void *)filepath,
strlen(filepath),
DARSHAN_LUSTRE_MOD,
1,
limit_flag,
&rec_id,
NULL);
/* if record id is 0, darshan has no more memory for instrumenting */
if(rec_id == 0)
return;
HASH_FIND(hlink, lustre_runtime->record_hash, &rec_id,
sizeof(darshan_record_id), lustre_ref);
if(!lustre_ref)
{
/* no existing record, allocate a new one and add it to the hash */
lustre_ref = &(lustre_runtime->ref_array[lustre_runtime->record_array_ndx]);
lustre_ref->record = &(lustre_runtime->record_array[lustre_runtime->record_array_ndx]);
lustre_ref->record->rec_id = rec_id;
lustre_ref->record->rank = my_rank;
/* TODO: gather lustre data, store in record hash */
/* counters in lustre_ref->record->counters */
HASH_ADD(hlink, lustre_runtime->record_hash, record->rec_id,
sizeof(darshan_record_id), lustre_ref);
lustre_runtime->record_array_ndx++;
}
LUSTRE_UNLOCK();
return;
}
......@@ -89,6 +146,24 @@ static void lustre_runtime_initialize()
return;
memset(lustre_runtime, 0, sizeof(*lustre_runtime));
/* allocate array of Lustre records according to the amount of memory
* assigned by Darshan
*/
lustre_runtime->record_array_size = mem_limit / sizeof(struct darshan_lustre_record);
lustre_runtime->ref_array = malloc(lustre_runtime->record_array_size *
sizeof(struct lustre_record_ref));
lustre_runtime->record_array = malloc(lustre_runtime->record_array_size *
sizeof(struct darshan_lustre_record));
if(!lustre_runtime->ref_array || !lustre_runtime->record_array)
{
lustre_runtime->record_array_size = 0;
return;
}
memset(lustre_runtime->ref_array, 0, lustre_runtime->record_array_size *
sizeof(struct lustre_record_ref));
memset(lustre_runtime->record_array, 0, lustre_runtime->record_array_size *
sizeof(struct darshan_lustre_record));
return;
}
......
......@@ -176,6 +176,11 @@ struct posix_runtime
struct posix_file_runtime_ref* fd_hash;
};
/* XXX modules don't expose an API for other modules, so use extern to get
* Lustre instrumentation function
*/
extern void darshan_instrument_lustre_file(char *filepath);
static struct posix_runtime *posix_runtime = NULL;
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
......@@ -1505,7 +1510,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
struct posix_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
int file_alignment;
struct darshan_fs_info fs_info;
int limit_flag;
if(!posix_runtime || instrumentation_disabled)
......@@ -1525,7 +1530,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
1,
limit_flag,
&file_id,
&file_alignment);
&fs_info);
/* the file record id is set to 0 if no memory is available for tracking
* new records -- just fall through and ignore this record
......@@ -1552,12 +1557,19 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
file->file_record->f_id = file_id;
file->file_record->rank = my_rank;
file->file_record->counters[POSIX_MEM_ALIGNMENT] = darshan_mem_alignment;
file->file_record->counters[POSIX_FILE_ALIGNMENT] = file_alignment;
file->file_record->counters[POSIX_FILE_ALIGNMENT] = fs_info.block_size;
/* add new record to file hash table */
HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
posix_runtime->file_array_ndx++;
/* XXX: check for lustre and call in */
#ifndef LL_SUPER_MAGIC
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
if(fs_info.fs_type == LL_SUPER_MAGIC)
darshan_instrument_lustre_file(newname);
if(newname != name)
free(newname);
return(file);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment