Commit cc099d3e authored by Cong Xu's avatar Cong Xu

1. Remove redundant information in dxlt_file_record.

2. Refactor DXLT to be a generic IO tracing tool for files on any file system.
3. Add file type information in DXLT log parser.

Next steps:
1. Based on the file type information in DXLT log parser, add information for specific file types. (For instance, add OST information for files on Lustre)
2. In the process of parsing DXLT logs, figure out how to obtain Lustre striping information from Lustre module.
3. Rename DXLT to be "Darshan eXtended Tracing (DXT)" or "Darshan eXtended IO Tracing (DXIOT)".
Signed-off-by: Cong Xu's avatarCong Xu <cong.xu@intel.com>
parent e5df8a50
......@@ -35,17 +35,11 @@ struct dxlt_file_record {
struct darshan_base_record base_rec;
int64_t shared_record; /* -1 means it is a shared file record */
int32_t stripe_size;
int32_t stripe_count;
OST_ID *ost_ids;
int64_t write_count;
int64_t write_available_buf;
segment_info *write_traces;
int64_t read_count;
int64_t read_available_buf;
segment_info *read_traces;
segment_info *write_traces;
segment_info *read_traces;
};
#endif /* __DARSHAN_DXLT_LOG_FORMAT_H */
This diff is collapsed.
......@@ -178,6 +178,7 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
}
/* DXLT */
#if 0
void dxlt_get_lustre_stripe_info(
darshan_record_id rec_id,
struct dxlt_file_record *file_rec)
......@@ -199,6 +200,7 @@ void dxlt_get_lustre_stripe_info(
memcpy(file_rec->ost_ids, rec_ref->record->ost_ids, ost_ids_size);
}
#endif
static void lustre_runtime_initialize()
{
......
......@@ -66,16 +66,10 @@ void dxlt_swap_file_record(struct dxlt_file_record *file_rec)
DARSHAN_BSWAP64(&file_rec->base_rec.rank);
DARSHAN_BSWAP64(&file_rec->shared_record);
DARSHAN_BSWAP64(&file_rec->stripe_size);
DARSHAN_BSWAP64(&file_rec->stripe_count);
DARSHAN_BSWAP64(&file_rec->ost_ids);
DARSHAN_BSWAP64(&file_rec->write_count);
DARSHAN_BSWAP64(&file_rec->write_available_buf);
DARSHAN_BSWAP64(&file_rec->write_traces);
DARSHAN_BSWAP64(&file_rec->read_count);
DARSHAN_BSWAP64(&file_rec->read_available_buf);
DARSHAN_BSWAP64(&file_rec->write_traces);
DARSHAN_BSWAP64(&file_rec->read_traces);
}
......@@ -101,8 +95,8 @@ static int dxlt_log_get_posix_file(darshan_fd fd, void** dxlt_posix_buf)
}
}
io_trace_size = file_rec->stripe_count * sizeof(OST_ID) +
(file_rec->write_count + file_rec->read_count) * sizeof(segment_info);
io_trace_size = (file_rec->write_count + file_rec->read_count) *
sizeof(segment_info);
if (parser_buf_sz == 0) {
parser_buf = (void *)malloc(io_trace_size);
......@@ -150,8 +144,8 @@ static int dxlt_log_get_mpiio_file(darshan_fd fd, void** dxlt_mpiio_buf)
}
}
io_trace_size = file_rec->stripe_count * sizeof(OST_ID) +
(file_rec->write_count + file_rec->read_count) * sizeof(segment_info);
io_trace_size = (file_rec->write_count + file_rec->read_count) *
sizeof(segment_info);
if (parser_buf_sz == 0) {
parser_buf = (void *)malloc(io_trace_size);
......@@ -222,23 +216,15 @@ static void dxlt_log_print_posix_file(void *posix_file_rec, char *file_name,
darshan_record_id f_id = file_rec->base_rec.id;
int64_t rank = file_rec->base_rec.rank;
int32_t stripe_size = file_rec->stripe_size;
int32_t stripe_count = file_rec->stripe_count;
int64_t write_count = file_rec->write_count;
int64_t read_count = file_rec->read_count;
OST_ID *ost_ids = (OST_ID *)parser_buf;
segment_info *io_trace = (segment_info *)((void *)parser_buf +
stripe_count * sizeof(OST_ID));
segment_info *io_trace = (segment_info *)parser_buf;
printf("\n# DXLT, rank: %d, file_id: %" PRIu64 ", file_name: %s\n", rank, f_id, file_name);
printf("# DXLT, stripe_count: %d, stripe_size: %d, write_count: %d, read_count: %d\n", stripe_count, stripe_size, write_count, read_count);
printf("\n# DXLT, file_id: %" PRIu64 ", file_name: %s, file_type: %s\n", f_id, file_name, fs_type);
printf("# DXLT, rank: %d, write_count: %d, read_count: %d\n", rank, write_count, read_count);
for (i = 0; i < stripe_count; i++) {
printf("# DXLT, OST: %d\n", ost_ids[i]);
}
printf("# Module Rank Wt/Rd Segment Offset Length Start(s) End(s) [OST]\n");
printf("# Module Rank Wt/Rd Segment Offset Length Start(s) End(s)\n");
for (i = 0; i < write_count; i++) {
offset = io_trace[i].offset;
......@@ -246,26 +232,7 @@ static void dxlt_log_print_posix_file(void *posix_file_rec, char *file_name,
start_time = io_trace[i].start_time;
end_time = io_trace[i].end_time;
printf("%8s%8d%7s%9lld%16lld%13lld%12.4f%12.4f", "POSIX", rank, "write", i, offset, length, start_time, end_time);
if (stripe_count != 0 && stripe_size != 0) {
cur_offset = offset;
ost_idx = (offset / stripe_size) % stripe_count;
print_count = 0;
while (cur_offset < offset + length) {
printf(" [%3d]", ost_ids[ost_idx]);
cur_offset = (cur_offset / stripe_size + 1) * stripe_size;
ost_idx = (ost_idx == stripe_count - 1) ? 0 : ost_idx + 1;
print_count++;
if (print_count >= stripe_count)
break;
}
}
printf("\n");
printf("%8s%8d%7s%9lld%16lld%13lld%12.4f%12.4f\n", "X_POSIX", rank, "write", i, offset, length, start_time, end_time);
}
for (i = write_count; i < write_count + read_count; i++) {
......@@ -274,26 +241,7 @@ static void dxlt_log_print_posix_file(void *posix_file_rec, char *file_name,
start_time = io_trace[i].start_time;
end_time = io_trace[i].end_time;
printf("%8s%8d%7s%9lld%16lld%13lld%12.4f%12.4f", "POSIX", rank, "read", i - write_count, offset, length, start_time, end_time);
if (stripe_count != 0 && stripe_size != 0) {
cur_offset = offset;
ost_idx = (offset / stripe_size) % stripe_count;
print_count = 0;
while (cur_offset < offset + length) {
printf(" [%3d]", ost_ids[ost_idx]);
cur_offset = (cur_offset / stripe_size + 1) * stripe_size;
ost_idx = (ost_idx == stripe_count - 1) ? 0 : ost_idx + 1;
print_count++;
if (print_count >= stripe_count)
break;
}
}
printf("\n");
printf("%8s%8d%7s%9lld%16lld%13lld%12.4f%12.4f\n", "X_POSIX", rank, "read", i - write_count, offset, length, start_time, end_time);
}
return;
}
......@@ -315,18 +263,14 @@ static void dxlt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
darshan_record_id f_id = file_rec->base_rec.id;
int64_t rank = file_rec->base_rec.rank;
int32_t stripe_count = file_rec->stripe_count;
int32_t stripe_size = file_rec->stripe_size;
int64_t write_count = file_rec->write_count;
int64_t read_count = file_rec->read_count;
OST_ID *ost_ids = (OST_ID *)parser_buf;
segment_info *io_trace = (segment_info *)((void *)parser_buf +
stripe_count * sizeof(OST_ID));
segment_info *io_trace = (segment_info *)parser_buf;
printf("\n# DXLT, rank: %d, file_id: %" PRIu64 ", file_name: %s\n", rank, f_id, file_name);
printf("# DXLT, write_count: %d, read_count: %d\n",
write_count, read_count);
printf("\n# DXLT, file_id: %" PRIu64 ", file_name: %s, file_type: %s\n", f_id, file_name, fs_type);
printf("# DXLT, rank: %d, write_count: %d, read_count: %d\n",
rank, write_count, read_count);
printf("# Module Rank Wt/Rd Segment Length Start(s) End(s)\n");
......@@ -336,7 +280,7 @@ static void dxlt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
start_time = io_trace[i].start_time;
end_time = io_trace[i].end_time;
printf("%8s%8d%7s%9lld%13lld%12.4f%12.4f\n", "MPIIO", rank, "write", i, length, start_time, end_time);
printf("%8s%8d%7s%9lld%13lld%12.4f%12.4f\n", "X_MPIIO", rank, "write", i, length, start_time, end_time);
}
for (i = write_count; i < write_count + read_count; i++) {
......@@ -345,7 +289,7 @@ static void dxlt_log_print_mpiio_file(void *mpiio_file_rec, char *file_name,
start_time = io_trace[i].start_time;
end_time = io_trace[i].end_time;
printf("%8s%8d%7s%9lld%13lld%12.4f%12.4f\n", "MPIIO", rank, "read", i - write_count, length, start_time, end_time);
printf("%8s%8d%7s%9lld%13lld%12.4f%12.4f\n", "X_MPIIO", rank, "read", i - write_count, length, start_time, end_time);
}
return;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment