Skip to content

GitLab

Commit 4b369dec authored by Philip Carns's avatar Philip Carns
Browse files

complete update of workload generator for darshan3 

notes:
- this generator will issue MPI-IO events, which cannot yet be handled
  directly by CODES simulations but are honored by the
  codes-workload-mpi-replay utility
- the codes-workload-mpi-replay utility now includes a --prep argument
  that will generate files that need to be read for subsequent replay
parent e8aa61e5
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1803 additions and 1926 deletions
Makefile.am
View file @ 4b369dec
......@@ -47,7 +47,7 @@ endif
if USE_DARSHAN
AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
src_libcodes_la_SOURCES += src/workload/methods/codes-darshan-posix-io-wrkld.c
src_libcodes_la_SOURCES += src/workload/methods/codes-darshan3-io-wrkld.c
LDADD += ${DARSHAN_LIBS}
TESTS += tests/workload/darshan-dump.sh
endif
......
codes/codes-workload.h
View file @ 4b369dec
......@@ -57,7 +57,6 @@ struct iolang_params
struct darshan_params
{
char log_file_path[MAX_NAME_LENGTH_WKLD];
int64_t aggregator_cnt;
int app_cnt;
};
......@@ -147,7 +146,24 @@ enum codes_workload_op_type
/* for workloads that have events not yet handled
* (eg the workload language) */
CODES_WK_IGNORE
CODES_WK_IGNORE,
/* extended IO workload operations: MPI */
/* open */
CODES_WK_MPI_OPEN,
/* close */
CODES_WK_MPI_CLOSE,
/* write */
CODES_WK_MPI_WRITE,
/* read */
CODES_WK_MPI_READ,
/* collective open */
CODES_WK_MPI_COLL_OPEN,
/* collective_write */
CODES_WK_MPI_COLL_WRITE,
/* collective_read */
CODES_WK_MPI_COLL_READ,
};
/* I/O operation paramaters */
......@@ -276,7 +292,7 @@ int codes_workload_load(
const char* type,
const char* params,
int app_id,
int rank, int *total_time);
int rank);
/* Retrieves the next I/O operation to execute. the wkld_id is the
* identifier returned by the init() function. The op argument is a pointer
......@@ -326,7 +342,7 @@ struct codes_workload_method
void * (*codes_workload_read_config) (
ConfigHandle *handle, char const * section_name,
char const * annotation, int num_ranks);
int (*codes_workload_load)(const char* params, int app_id, int rank, int *total_time);
int (*codes_workload_load)(const char* params, int app_id, int rank);
void (*codes_workload_get_next)(int app_id, int rank, struct codes_workload_op *op);
void (*codes_workload_get_next_rc2)(int app_id, int rank);
int (*codes_workload_get_rank_cnt)(const char* params, int app_id);
......
src/workload/codes-workload-dump.c
View file @ 4b369dec
......@@ -27,7 +27,6 @@ static struct option long_opts[] =
{"num-ranks", required_argument, NULL, 'n'},
{"start-rank", required_argument, NULL, 'r'},
{"d-log", required_argument, NULL, 'l'},
{"d-aggregator-cnt", required_argument, NULL, 'a'},
{"i-meta", required_argument, NULL, 'm'},
{"i-use-relpath", no_argument, NULL, 'p'},
{"r-trace-dir", required_argument, NULL, 'd'},
......@@ -53,7 +52,6 @@ void usage(){
"-s: print final workload stats\n"
"DARSHAN OPTIONS (darshan_io_workload)\n"
"--d-log: darshan log file\n"
"--d-aggregator-cnt: number of aggregators for collective I/O in darshan\n"
"IOLANG OPTIONS (iolang_workload)\n"
"--i-meta: i/o language kernel meta file path\n"
"--i-use-relpath: use i/o kernel path relative meta file path\n"
......@@ -133,9 +131,6 @@ int main(int argc, char *argv[])
case 'l':
strcpy(d_params.log_file_path, optarg);
break;
case 'a':
d_params.aggregator_cnt = atol(optarg);
break;
case 'm':
strcpy(i_params.io_kernel_meta_path, optarg);
break;
......@@ -206,11 +201,6 @@ int main(int argc, char *argv[])
usage();
return 1;
}
else if (d_params.aggregator_cnt == 0){
fprintf(stderr, "Expected \"--d-aggregator-cnt\" argument for darshan workload\n");
usage();
return 1;
}
else{
wparams = (char*)&d_params;
}
......@@ -308,8 +298,7 @@ int main(int argc, char *argv[])
for (i = start_rank ; i < start_rank+n; i++){
struct codes_workload_op op;
//printf("loading %s, %d\n", type, i);
int total_time;
int id = codes_workload_load(type, wparams, 0, i, &total_time);
int id = codes_workload_load(type, wparams, 0, i);
double total_read_time = 0.0, total_write_time = 0.0;
int64_t total_read_bytes = 0, total_written_bytes = 0;
codes_workload_get_time(type, wparams, 0, i, &total_read_time, &total_write_time, &total_read_bytes, &total_written_bytes);
......@@ -328,16 +317,23 @@ int main(int argc, char *argv[])
num_barriers++;
break;
case CODES_WK_OPEN:
case CODES_WK_MPI_OPEN:
case CODES_WK_MPI_COLL_OPEN:
num_opens++;
break;
case CODES_WK_CLOSE:
case CODES_WK_MPI_CLOSE:
num_closes++;
break;
case CODES_WK_WRITE:
case CODES_WK_MPI_WRITE:
case CODES_WK_MPI_COLL_WRITE:
num_writes++;
write_size += op.u.write.size;
break;
case CODES_WK_READ:
case CODES_WK_MPI_READ:
case CODES_WK_MPI_COLL_READ:
num_reads++;
read_size += op.u.write.size;
break;
......
src/workload/codes-workload.c
View file @ 4b369dec
......@@ -160,7 +160,7 @@ int codes_workload_load(
const char* type,
const char* params,
int app_id,
int rank, int *total_time)
int rank)
{
init_workload_methods();
......@@ -173,7 +173,7 @@ int codes_workload_load(
if(strcmp(method_array[i]->method_name, type) == 0)
{
/* load appropriate workload generator */
ret = method_array[i]->codes_workload_load(params, app_id, rank, total_time);
ret = method_array[i]->codes_workload_load(params, app_id, rank);
if(ret < 0)
{
return(-1);
......@@ -338,6 +338,8 @@ void codes_workload_print_op(
int app_id,
int rank)
{
char *name;
switch(op->op_type){
case CODES_WK_END:
fprintf(f, "op: app:%d rank:%d type:end\n", app_id, rank);
......@@ -351,23 +353,41 @@ void codes_workload_print_op(
app_id, rank, op->u.barrier.count, op->u.barrier.root);
break;
case CODES_WK_OPEN:
fprintf(f, "op: app:%d rank:%d type:open file_id:%llu flag:%d\n",
app_id, rank, LLU(op->u.open.file_id), op->u.open.create_flag);
case CODES_WK_MPI_OPEN:
case CODES_WK_MPI_COLL_OPEN:
if(op->op_type == CODES_WK_OPEN) name = "open";
if(op->op_type == CODES_WK_MPI_OPEN) name = "mpi_open";
if(op->op_type == CODES_WK_MPI_COLL_OPEN) name = "mpi_coll_open";
fprintf(f, "op: app:%d rank:%d type:%s file_id:%llu flag:%d\n",
app_id, rank, name, LLU(op->u.open.file_id), op->u.open.create_flag);
break;
case CODES_WK_CLOSE:
fprintf(f, "op: app:%d rank:%d type:close file_id:%llu\n",
app_id, rank, LLU(op->u.close.file_id));
case CODES_WK_MPI_CLOSE:
if(op->op_type == CODES_WK_CLOSE) name = "close";
if(op->op_type == CODES_WK_MPI_CLOSE) name = "mpi_close";
fprintf(f, "op: app:%d rank:%d type:%s file_id:%llu\n",
app_id, rank, name, LLU(op->u.close.file_id));
break;
case CODES_WK_WRITE:
fprintf(f, "op: app:%d rank:%d type:write "
case CODES_WK_MPI_WRITE:
case CODES_WK_MPI_COLL_WRITE:
if(op->op_type == CODES_WK_WRITE) name = "write";
if(op->op_type == CODES_WK_MPI_WRITE) name = "mpi_write";
if(op->op_type == CODES_WK_MPI_COLL_WRITE) name = "mpi_coll_write";
fprintf(f, "op: app:%d rank:%d type:%s "
"file_id:%llu off:%llu size:%llu\n",
app_id, rank, LLU(op->u.write.file_id), LLU(op->u.write.offset),
app_id, rank, name, LLU(op->u.write.file_id), LLU(op->u.write.offset),
LLU(op->u.write.size));
break;
case CODES_WK_READ:
fprintf(f, "op: app:%d rank:%d type:read "
case CODES_WK_MPI_READ:
case CODES_WK_MPI_COLL_READ:
if(op->op_type == CODES_WK_READ) name = "read";
if(op->op_type == CODES_WK_MPI_READ) name = "mpi_read";
if(op->op_type == CODES_WK_MPI_COLL_READ) name = "mpi_coll_read";
fprintf(f, "op: app:%d rank:%d type:%s "
"file_id:%llu off:%llu size:%llu\n",
app_id, rank, LLU(op->u.read.file_id), LLU(op->u.read.offset),
app_id, rank, name, LLU(op->u.read.file_id), LLU(op->u.read.offset),
LLU(op->u.read.size));
break;
case CODES_WK_SEND:
......
src/workload/methods/codes-darshan-io-wrkld.c deleted 100644 → 0
View file @ e8aa61e5
This diff is collapsed.
tests/workload/codes-workload-mpi-replay.c
View file @ 4b369dec
This diff is collapsed.
tests/workload/darshan-dump.sh
View file @ 4b369dec
......@@ -5,4 +5,4 @@ if [ -z $srcdir ]; then
exit 1
fi
src/workload/codes-workload-dump --num-ranks 4 --type darshan_io_workload --d-log $srcdir/tests/workload/example.darshan.gz --d-aggregator-cnt 1
src/workload/codes-workload-dump --num-ranks 4 --type darshan_io_workload --d-log $srcdir/tests/workload/example.darshan --d-aggregator-cnt 1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment