Commit b4aa6ce8 authored by Xin Wang's avatar Xin Wang

adding Conceptural as online workload

parent 733e5f1a
......@@ -61,6 +61,11 @@ if USE_ONLINE
AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} ${SWM_CFLAGS} -DUSE_ONLINE=1
LDADD += ${SWM_LIBS} ${ARGOBOTS_LIBS}
src_libcodes_la_SOURCES += src/workload/methods/codes-online-comm-wrkld.C
if USE_CONC
AM_CPPFLAGS += ${CONCEPTUAL_CFLAGS}
LDADD += ${CONCEPTUAL_LIBS}
src_libcodes_la_SOURCES += src/workload/methods/conc-latency.c
endif
endif
if USE_DUMPI
......
/*
* Copyright (C) 2017 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_CONC_ADDON_H
#define CODES_CONC_ADDON_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef USE_CONC
#include <ncptl/ncptl.h>
#endif
#define MAX_CONC_ARGC 20
typedef struct conc_bench_param conc_bench_param;
/* implementation structure */
struct codes_conceptual_bench {
char *program_name; /* name of the conceptual program */
int (*conceptual_main)(int* argc, char *argv[]);
};
struct conc_bench_param {
char *conc_program;
int conc_argc;
char *conc_argv[MAX_CONC_ARGC];
};
int codes_conc_bench_load(
const char* program,
int* argc,
const char *argv[]);
void codes_conceptual_add_bench(struct codes_conceptual_bench const * method);
#ifdef __cplusplus
}
#endif
#endif /* CODES_CONC_ADDON_H */
......@@ -112,16 +112,28 @@ if test "x${use_online}" != "x" ; then
AM_CONDITIONAL(USE_ONLINE, true)
PKG_CHECK_MODULES_STATIC([ARGOBOTS], [argobots], [],
[AC_MSG_ERROR([Could not find working argobots installation via pkg-config])])
PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
[AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
[AC_MSG_ERROR[Could not find shared directory in SWM]])
AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["${use_online}"], [if using json
data files])
else
AM_CONDITIONAL(USE_ONLINE, false)
fi
#check for Conceptual
AC_ARG_WITH([conceptual],[AS_HELP_STRING([--with-conceptual@<:@=DIR@:>@],
[location of Conceptual installation])])
if test "x${with_conceptual}" != "x" ; then
AC_CHECK_FILES([${with_conceptual}/lib/libncptl.a],
AM_CONDITIONAL(USE_CONC, true),
AC_MSG_ERROR(Could not find Conceptual libraries libncptl.a))
CONCEPTUAL_CFLAGS="-I${with_conceptual}/include"
CONCEPTUAL_LIBS="-L${with_conceptual}/lib/ -lncptl"
AC_SUBST(CONCEPTUAL_LIBS)
AC_SUBST(CONCEPTUAL_CFLAGS)
else
AM_CONDITIONAL(USE_CONC, false)
fi
# check for Recorder
AM_CONDITIONAL(USE_RECORDER, true)
RECORDER_CPPFLAGS="-DUSE_RECORDER=1"
......
......@@ -16,6 +16,8 @@ boost_cflags=@BOOST_CFLAGS@
boost_libs=@BOOST_LIBS@
argobots_libs=@ARGOBOTS_LIBS@
argobots_cflags=@ARGOBOTS_CFLAGS@
conceptual_libs=@CONCEPTUAL_LIBS@
conceptual_cflags=@CONCEPTUAL_CFLAGS@
swm_libs=@SWM_LIBS@
swm_cflags=@SWM_CFLAGS@
......@@ -24,5 +26,5 @@ Description: Base functionality for CODES storage simulation
Version: @PACKAGE_VERSION@
URL: http://trac.mcs.anl.gov/projects/CODES
Requires:
Libs: -L${libdir} -lcodes ${ross_libs} ${argobots_libs} ${swm_libs} ${darshan_libs} ${dumpi_libs} ${cortex_libs}
Cflags: -I${includedir} ${swm_datarootdir} ${ross_cflags} ${darshan_cflags} ${swm_cflags} ${argobots_cflags} ${dumpi_cflags} ${cortex_cflags}
Libs: -L${libdir} -lcodes ${ross_libs} ${argobots_libs} ${conceptual_libs} ${swm_libs} ${darshan_libs} ${dumpi_libs} ${cortex_libs}
Cflags: -I${includedir} ${swm_datarootdir} ${ross_cflags} ${darshan_cflags} ${swm_cflags} ${argobots_cflags} ${conceptual_cflags} ${dumpi_cflags} ${cortex_cflags}
This diff is collapsed.
import string
import sys
import os
MPI_OPS = [ 'MPI_Send', 'MPI_Recv', 'MPI_Barrier', 'MPI_Isend', 'MPI_Irecv', 'MPI_Waitall',
'MPI_Reduce', 'MPI_Allreduce', 'MPI_Bcast', 'MPI_Alltoall', 'MPI_Alltoallv',
'MPI_Comm_size', 'MPI_Comm_rank']
LOG = [ 'logfiletmpl_default', 'ncptl_log_compute_aggregates', 'ncptl_log_commit_data']
def eliminate_logging(inLines):
for idx, line in enumerate(inLines):
if 'Generate and broadcast a UUID' in line:
for i in range(1, 3):
inLines[idx+i] = "//"+inLines[idx+i]
elif 'ncptl_free (logfile_uuid)' in line:
for i in range(0, 12):
inLines[idx-i] = "//"+inLines[idx-i]
elif 'case 1' in line:
for i in range(5, 9):
inLines[idx+i] = "//"+inLines[idx+i]
elif 'int mpiresult' in line:
for i in range(0,30):
inLines[idx+i] = "//"+inLines[idx+i]
else:
for elem in LOG:
if elem in line:
inLines[idx] = "//"+line
def manipulate_mpi_ops(inLines, program_name):
for idx, line in enumerate(inLines):
# subcomm
if 'MPI_' not in line: # not MPI
if "int main" in line:
# inLines[idx] = "static int "+program_name+"_main(int* argc, char *argv[])"
inLines[idx] = line.replace("int main", "static int "+program_name+"_main")
else:
continue
else: # MPI
if 'MPI_Init' in line:
inLines[idx] = "//"+line
elif 'MPI_Errhandler_' in line: # error handling ignored
inLines[idx] = "//"+line
elif 'mpiresult = MPI_Finalize();' in line:
inLines[idx] = "CODES_MPI_Finalize();"
inLines[idx+2] = "exitcode = 0;"
else:
for ops in MPI_OPS:
if ops in line:
inLines[idx] = line.replace(ops,"CODES_"+ops)
def adding_struct(inLines, program_name):
new_struct = [ '/* fill in function pointers for this method */' ,
'struct codes_conceptual_bench '+program_name+'_bench = ' ,
'{' ,
'.program_name = "'+program_name+'",' ,
'.conceptual_main = '+program_name+'_main,' ,
'};' ]
codes_include = '#include "codes/codes-conc-addon.h"'
for idx, line in enumerate(inLines):
if "* Include files *" in line:
inLines.insert(idx-1, codes_include)
break
for idx, line in enumerate(inLines):
if "* Global variables *" in line:
for i in range(len(new_struct)-1,-1,-1):
inLines.insert(idx-1, new_struct[i])
break
def translate_conc_to_codes(filepath, codespath):
# get program name
program_name = filepath.split("/")[-1].replace(".c","")
with open(filepath, 'r') as infile:
content = infile.read()
# print content
inLines = content.split('\n')
eliminate_logging(inLines)
manipulate_mpi_ops(inLines, program_name)
adding_struct(inLines, program_name)
# output program file
with open(codespath+"src/workload/methods/conc-"+program_name+".c","w+") as outFile:
outFile.writelines(["%s\n" % item for item in inLines])
# modify interface file
program_struct = "extern struct codes_conceptual_bench "+program_name+"_bench;\n"
program_definition = " &"+program_name+"_bench,\n"
with open(codespath+"src/workload/codes-conc-addon.c","r+") as header:
hls = header.readlines()
for idx, line in enumerate(hls):
if '/* list of available benchmarks begin */' in line and program_struct not in hls[idx+1]:
hls.insert(idx+1, program_struct)
elif '/* default benchmarks begin */' in line and program_definition not in hls[idx+1]:
hls.insert(idx+1, program_definition)
header.seek(0)
header.writelines(hls)
# modify makefile
program_compile = "src_libcodes_la_SOURCES += src/workload/methods/conc-"+program_name+".c\n"
with open(codespath+"Makefile.am","r+") as makefile:
mfls = makefile.readlines()
for idx, line in enumerate(mfls):
if "CONCEPTUAL_LIBS" in line and program_compile not in mfls[idx+1]:
mfls.insert(idx+1, program_compile)
break
makefile.seek(0)
makefile.writelines(mfls)
if __name__ == "__main__":
if len(sys.argv) != 3:
print 'Need 2 arguments: 1. path to files to be converted \t2. path to CODES directory'
sys.exit(1)
for benchfile in next(os.walk(sys.argv[1]))[2]: # for all files
translate_conc_to_codes(sys.argv[1]+benchfile, sys.argv[2])
......@@ -97,8 +97,9 @@ nobase_include_HEADERS = \
codes/net/simplep2p.h \
codes/net/express-mesh.h \
codes/net/torus.h \
codes/codes-mpi-replay.h \
codes/configfile.h
codes/codes-mpi-replay.h \
codes/configfile.h \
codes/codes-conc-addon.h
#codes/codes-nw-workload.h
......@@ -144,6 +145,7 @@ src_libcodes_la_SOURCES = \
src/util/codes-mapping-context.c \
src/util/codes-comm.c \
src/workload/codes-workload.c \
src/workload/codes-conc-addon.c \
src/workload/methods/codes-iolang-wrkld.c \
src/workload/methods/codes-checkpoint-wrkld.c \
src/workload/methods/test-workload-method.c \
......
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <assert.h>
#include <ross.h>
#include <codes/codes.h>
#include <codes/codes-conc-addon.h>
/* list of available benchmarks begin */
extern struct codes_conceptual_bench latency_bench;
/* list of available benchmarks end */
static struct codes_conceptual_bench const * bench_array_default[] =
{
/* default benchmarks begin */
&latency_bench,
/* default benchmark end */
NULL
};
// once initialized, adding a bench generator is an error
static int is_bench_init = 0;
static int num_user_benchs = 0;
static struct codes_conceptual_bench const ** bench_array = NULL;
// only call this once
static void init_bench_methods(void)
{
if (is_bench_init)
return;
if (bench_array == NULL)
bench_array = bench_array_default;
else {
// note - includes null char
int num_default_benchs =
(sizeof(bench_array_default) / sizeof(bench_array_default[0]));
printf("\n Num default methods %d ", num_default_benchs);
bench_array = realloc(bench_array,
(num_default_benchs + num_user_benchs + 1) *
sizeof(*bench_array));
memcpy(bench_array+num_user_benchs, bench_array_default,
num_default_benchs * sizeof(*bench_array_default));
}
is_bench_init = 1;
}
int codes_conc_bench_load(
const char *program,
int* argc,
const char *argv[])
{
init_bench_methods();
int i;
int ret;
for(i=0; bench_array[i] != NULL; i++)
{
if(strcmp(bench_array[i]->program_name, program) == 0)
{
/* load appropriate workload generator */
ret = bench_array[i]->conceptual_main(argc, argv);
if(ret < 0)
{
return(-1);
}
return(i);
}
}
fprintf(stderr, "Error: failed to find workload generator %s\n", program);
return(-1);
}
void codes_conceptual_add_bench(struct codes_conceptual_bench const * bench)
{
static int bench_array_cap = 10;
if (is_bench_init)
tw_error(TW_LOC,
"adding a conceptual benchmark method after initialization is forbidden");
else if (bench_array == NULL){
bench_array = malloc(bench_array_cap * sizeof(*bench_array));
assert(bench_array);
}
if (num_user_benchs == bench_array_cap) {
bench_array_cap *= 2;
bench_array = realloc(bench_array,
bench_array_cap * sizeof(*bench_array));
assert(bench_array);
}
bench_array[num_user_benchs++] = bench;
}
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment