Commit b5332aff authored by Elsa Gonsiorowski (Uranus)'s avatar Elsa Gonsiorowski (Uranus)

merge

parents 5509f22e e6c7f2e0
......@@ -15,7 +15,7 @@
#include "ross.h"
#define MAX_NAME_LENGTH_WKLD 512
typedef struct bgp_params bgp_params;
typedef struct iolang_params iolang_params;
typedef struct darshan_params darshan_params;
typedef struct recorder_params recorder_params;
typedef struct codes_workload_info codes_workload_info;
......@@ -25,18 +25,15 @@ typedef struct scala_trace_params scala_trace_params;
typedef struct dumpi_trace_params dumpi_trace_params;
struct bgp_params
struct iolang_params
{
/* We have the number of ranks passed in from the bg/p model because
* the I/O lang workloads have no information about the number of ranks.
* Only the bg/p config file knows the number of ranks. */
/* the rank count is defined in the workload config file */
int num_cns;
/* flag - use path to find kernel files relative to the metafile */
int use_relpath;
char io_kernel_meta_path[MAX_NAME_LENGTH_WKLD];
char bgp_config_file[MAX_NAME_LENGTH_WKLD];
/* set by config in the metadata path */
char io_kernel_path[MAX_NAME_LENGTH_WKLD];
char io_kernel_def_path[MAX_NAME_LENGTH_WKLD];
};
struct darshan_params
......@@ -67,8 +64,8 @@ struct scala_trace_params {
};
struct dumpi_trace_params {
int num_net_traces;
char file_name[MAX_NAME_LENGTH_WKLD];
int num_net_traces;
};
......@@ -127,6 +124,10 @@ enum codes_workload_op_type
CODES_WK_WAITANY,
/* Testall operation */
CODES_WK_TESTALL,
/* for workloads that have events not yet handled
* (eg the workload language) */
CODES_WK_IGNORE
};
/* I/O operation paramaters */
......
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/*
* (C) 2001 Clemson University and The University of Chicago
*
* See COPYING in top-level directory.
*/
/** \defgroup codeslogging codeslogging logging interface
*
* This is a basic application logging facility. It uses printf style
* formatting and provides several mechanisms for output.
*
* @{
*/
/* This code was derived from the PVFS2 codeslogging circa vs. 2.8.1 */
/** \file
*
* Declarations for the codeslogging logging interface.
*/
#ifndef SRC_COMMON_LOGGING_CODESLOGGING_H
#define SRC_COMMON_LOGGING_CODESLOGGING_H
#include <stdint.h>
#include <stdarg.h>
/********************************************************************
* Visible interface
*/
#define CODESLOGGING_BUF_SIZE 1024
/* what type of timestamp to place in msgs */
enum codeslogging_logstamp
{
CODESLOGGING_LOGSTAMP_NONE = 0,
CODESLOGGING_LOGSTAMP_USEC = 1,
CODESLOGGING_LOGSTAMP_DATETIME = 2,
CODESLOGGING_LOGSTAMP_THREAD = 3
};
#define CODESLOGGING_LOGSTAMP_DEFAULT CODESLOGGING_LOGSTAMP_USEC
/* stdio is needed by codeslogging_debug_fp declaration for FILE* */
#include <stdio.h>
int codeslogging_enable_stderr(
void);
int codeslogging_enable_file(
const char *filename,
const char *mode);
int codeslogging_disable(
void);
int codeslogging_set_debug_mask(
int debug_on,
uint64_t mask);
int codeslogging_get_debug_mask(
int *debug_on,
uint64_t * mask);
int codeslogging_set_logstamp(
enum codeslogging_logstamp ts);
void codeslogging_backtrace(
void);
#ifdef __GNUC__
/* do printf style type checking if built with gcc */
int __codeslogging_debug(
uint64_t mask,
char prefix,
const char *format,
...) __attribute__ ((format(printf, 3, 4)));
int codeslogging_err(
const char *format,
...) __attribute__ ((format(printf, 1, 2)));
int __codeslogging_debug_va(
uint64_t mask,
char prefix,
const char *format,
va_list ap);
int codeslogging_debug_fp(
FILE * fp,
char prefix,
enum codeslogging_logstamp ts,
const char *format,
...) __attribute__ ((format(printf, 4, 5)));
#ifdef CODESLOGGING_DISABLE_DEBUG
#define codeslogging_debug(mask, format, f...) do {} while(0)
#define codeslogging_perf_log(format, f...) do {} while(0)
#define codeslogging_debug_enabled(__m) 0
#else
extern int codeslogging_debug_on;
extern int codeslogging_facility;
extern uint64_t codeslogging_debug_mask;
#define codeslogging_debug_enabled(__m) \
(codeslogging_debug_on && (codeslogging_debug_mask & __m))
/* try to avoid function call overhead by checking masks in macro */
#define codeslogging_debug(mask, format, f...) \
do { \
if ((codeslogging_debug_on) && (codeslogging_debug_mask & mask) &&\
(codeslogging_facility)) \
{ \
__codeslogging_debug(mask, '?', format, ##f); \
} \
} while(0)
#define codeslogging_perf_log(format, f...) \
do { \
if ((codeslogging_debug_on) && \
(codeslogging_debug_mask & CODESLOGGING_PERFCOUNTER_DEBUG) && \
(codeslogging_facility)) \
{ \
__codeslogging_debug(CODESLOGGING_PERFCOUNTER_DEBUG, 'P', \
format, ##f); \
} \
} while(0)
#endif /* CODESLOGGING_DISABLE_DEBUG */
/* do file and line number printouts w/ the GNU preprocessor */
#define codeslogging_ldebug(mask, format, f...) \
do { \
codeslogging_debug(mask, "%s: " format, __func__ , ##f); \
} while(0)
#define codeslogging_lerr(format, f...) \
do { \
codeslogging_err("%s line %d: " format, __FILE__ , __LINE__ , ##f); \
codeslogging_backtrace(); \
} while(0)
#else /* ! __GNUC__ */
int __codeslogging_debug(
uint64_t mask,
char prefix,
const char *format,
...);
int __codeslogging_debug_stub(
uint64_t mask,
char prefix,
const char *format,
...);
int codeslogging_err(
const char *format,
...);
#ifdef CODESLOGGING_DISABLE_DEBUG
#define codeslogging_debug(__m, __f, f...) __codeslogging_debug_stub(__m, '?', __f, ##f);
#define codeslogging_ldebug(__m, __f, f...) __codeslogging_debug_stub(__m, '?', __f, ##f);
#define codeslogging_debug_enabled(__m) 0
#else
#define codeslogging_debug(__m, __f, f...) __codeslogging_debug(__m, '?', __f, ##f);
#define codeslogging_ldebug(__m, __f, f...) __codeslogging_debug(__m, '?', __f, ##f);
#define codeslogging_debug_enabled(__m) \
((codeslogging_debug_on != 0) && (__m & codeslogging_debug_mask))
#endif /* CODESLOGGING_DISABLE_DEBUG */
#define codeslogging_lerr codeslogging_err
#endif /* __GNUC__ */
#endif /* __CODESLOGGING_H */
/* @} */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
......@@ -7,9 +7,9 @@
#ifndef __CONFIGURATION_H__
#define __CONFIGURATION_H__
#include <stddef.h>
#include <inttypes.h>
#include <mpi.h>
#include "codes/txt_configfile.h"
#define CONFIGURATION_MAX_NAME 256
#define CONFIGURATION_MAX_GROUPS 10
......@@ -53,6 +53,8 @@ typedef struct config_lpgroups_s
config_anno_map_t lpannos[CONFIGURATION_MAX_TYPES];
} config_lpgroups_t;
typedef struct ConfigVTable * ConfigHandle;
/*
* Load a configuration on the system (collectively)
*
......
......@@ -43,7 +43,10 @@ typedef enum lsm_event_e
*/
/* given LP sender, find the LSM device LP in the same group */
tw_lpid lsm_find_local_device(tw_lp *sender);
tw_lpid lsm_find_local_device(
const char * annotation,
int ignore_annotations,
tw_lpid sender_gid);
/*
* lsm_event_new
......@@ -69,6 +72,21 @@ tw_event* lsm_event_new(const char* category,
size_t message_bytes,
tw_lp *sender,
tw_stime delay);
/* equivalent to lsm_event_new, except it allows to specify an annotation to
* filter by. If ignore_annotations is nonzero, A null annotation parameter
* indicates that the lsm LP to issue to has no annotation */
tw_event* lsm_event_new_annotated(
const char* category,
tw_lpid dest_gid,
uint64_t io_object,
int64_t io_offset,
uint64_t io_size_bytes,
int io_type,
size_t message_bytes,
tw_lp *sender,
tw_stime delay,
const char * annotation,
int ignore_annotations);
void lsm_event_new_reverse(tw_lp *sender);
......
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/*
* timeline.h
*
* Created on: Jun 24, 2013
* Author: wozniak
*/
#ifndef TIMELINE_H
#define TIMELINE_H
#define TIMELINE_ENABLED 0
#if TIMELINE_ENABLED == 1
#include <ross.h>
/**
Open filename for writing for timeline data
@return 1 on success, 0 on error
*/
int timeline_init(const char *filename);
/**
Write a timeline record
@return 1 on success, 0 on error
*/
int timeline_printf(const char *format, ...);
/**
Write a timeline record
@return 1 on success, 0 on error
*/
int timeline_printf_va(const char *format, va_list ap);
/**
Write a timeline record with typical ROSS metadata
@return 1 on success, 0 on error
*/
#define timeline_event(lp, format, args...) \
timeline_event_impl(lp, __func__, format, ##args)
int timeline_event_impl(const tw_lp *lp, const char *func,
const char *format, ...);
/**
Finalize the timeline module
*/
void timeline_finalize(void);
#else
// Set all functions to noops (return success value 1)
#define timeline_init(x) 1
#define timeline_printf(f, a...) 1
#define timeline_printf_va(f, a) 1
#define timeline_event(lp, f, a...) 1
#define timeline_finalize()
#endif
#endif
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef SRC_COMMON_UTIL_TOOLS_H
#define SRC_COMMON_UTIL_TOOLS_H
#ifdef UNUSED
#elif defined(__GNUC__)
# define UNUSED(x) UNUSED_ ## x __attribute__((unused))
#elif defined(__LCLINT__)
# define UNUSED(x) /*@unused@*/ x
#else
# define UNUSED(x) x
#endif /* UNUSED */
#define codesmin(a,b) ((a)<(b) ? (a):(b))
#define codesmax(a,b) ((a)>(b) ? (a):(b))
void always_assert_error(const char * expr, const char * file, int lineno);
#define ALWAYS_ASSERT(a) if (!(a)) always_assert_error(#a,__FILE__, __LINE__);
#define ARRAY_SIZEOF(a) (sizeof(a)/sizeof(a[0]))
char * safe_strncpy(char * buf, const char * source, unsigned int bufsize);
#define CODES_FLAG_ISSET(mode_, flag_) ((mode_ & flag_) != 0)
#define CODES_FLAG_SET(mode_, flag_, branch_) \
do \
{ \
mode_ = mode_ | flag_; \
branch_ = 1; \
}while(0)
#define CODES_FLAG_SET_RC(mode_, flag_, branch_) \
do \
{ \
if(branch_) \
{ \
mode_ = mode_ & (~flag_); \
} \
}while(0)
#endif
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
......@@ -123,7 +123,7 @@ if test "x${with_dumpi}" != "x" ; then
# DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/common"
# DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/libdumpi"
# DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/libundumpi"
DUMPI_LIBS="-L${with_dumpi}/lib/ -ldumpi -lundumpi"
DUMPI_LIBS="-L${with_dumpi}/lib/ -lundumpi"
AC_SUBST(DUMPI_LIBS)
AC_SUBST(DUMPI_CFLAGS)
else
......
......@@ -67,3 +67,12 @@ Notes on using the clang static analyzer
- edit Makefile, and delete the "CC = mpicc" (or similar) line
- run "scan-build --use-cc=mpicc make"
Notes on using uncrustify
-----------------
- version 0.61 is required
- build/install uncrustify from your favorite distro (0.61), OR download from
https://github.com/bengardner/uncrustify and build from source
(configure --prefix /path/to/install && make && make install)
- either use uncrustify directly (see uncrustify --help) or use the provided
reformat.sh tool which is a shim over some of the options (see reformat.sh
-h)
......@@ -96,9 +96,7 @@ IO kernel language, and in-development ScalaTrace
The synthetic IO language is a simple, interpreted set of IO and basic
arithmetic commands meant to simplify the specification and running of
application workloads. In the code it's currently called the "bgp" workload
generator but that is just a historical artifact - in the future it will be
refactored/renamed.
application workloads.
The input for the workload generator consists of an IO kernel metadata file and
a number of IO kernel files. The former specifies a set of kernel files to run
......@@ -126,7 +124,7 @@ readat/writeat.
More detailed documentation on the language is ongoing, but for now a general
example can be seen at doc/workload, which shows a simple out-of-core data
shuffle. Braver souls may wish to visit the implementation at src/iokernellang
and src/workload/codes-bgp-io-wrkld.c.
and src/workload/codes-iolang-wrkld.c.
The following restrictions currently apply to the IO language:
* all user-defined variables must be a single, lower-case letter (the symbol
......
#!/bin/bash
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
function error () {
echo "ERROR: $@" >&2
exit 1
}
function usage() {
cat \
<<EOF
USAGE: reformat.sh [-x] [-i] [-c CFG] [-b FILE] {-a | FILE...}
-i: perform in-place (will ask to confirm if being done recursively)
-c CFG: use CFG as the uncrustify config (default: $script_dir/uc-codes.cfg)
-b FILE[,...]: add FILE to the blacklist of directories (in the form *FILE/*)
multiple files can be added by separating with commas
-a: perform recursively on all files ending in .c or .h
if -a not provided, then perform on all files passed in
-x: instead of formatting, check if output format is the same as input
-h: show this help and exit
EOF
}
function build_blacklist() {
for x in $@ ; do
echo -n "-and -not -path \*$x/\* "
done
}
function getfiles() {
find . \( -name \*.c -or -name \*.h \) $(build_blacklist $@)
}
cfg=$script_dir/uc-codes.cfg
which uncrustify > /dev/null 2>&1 || error "uncrustify not found"
do_inplace=no
do_recursive=no
do_check=no
blacklist=
while getopts ":ib:axh" opt; do
case $opt in
i)
do_inplace=yes
;;
b)
blacklist=$(cat $OPTARG | sed s/,/ /g)
;;
c)
cfg=$OPTARG
;;
a)
do_recursive=yes
;;
x)
do_check=yes
;;
h)
usage
exit 1
;;
\?)
usage
error "invalid argument: -$OPTARG"
;;
esac
done
shift $((OPTIND-1))
[[ -e $cfg ]] || ( usage && error "config file $cfg not found" )
[[ $# -gt 0 && $do_recursive == yes ]] && \
usage && error "no file args expected in recursive mode"
[[ $# -eq 0 && $do_recursive == no ]] && \
usage && error "expected file args in non-recursive mode"
if [[ $do_recursive == yes ]] ; then
file_list="$(getfiles $blacklist)"
else
file_list="$@"
fi
if [[ $do_recursive == yes && $do_inplace == yes ]] ; then
echo -n "Do recursive in-place reformat? (y/n) "
read answer
[[ $answer =~ "[^y].*" ]] && echo "...aborting reformat" && exit 1
fi
[[ $do_inplace == yes ]] \
&& output_arg="--replace" \
|| output_arg=
if [[ $do_check == yes ]] ; then
check_arg=--check
output_arg=
echo "checking format..."
else
check_arg=
fi
function echolines () {
for x in $@ ; do echo $x ; done
}
echolines $file_list | uncrustify $check_arg -c $cfg $output_arg -F -
......@@ -29,33 +29,22 @@ maintainer-clean-local::
src/modelconfig/configparser.h: src/modelconfig/configparser.y
src/modelconfig/configlex.h: src/modelconfig/configlex.l
#BUILT_SOURCES += codes/codesparser.h \
#BUILT_SOURCES += src/iokernellang/codesparser.h \
# src/iokernellang/codesparser.c \
# src/iokernellang/codeslexer.h \
# codes/codeslexer.h
# src/iokernellang/codeslexer.h
#src/iokernellang/codesparser.y: src/iokernellang/codesparser.y.in Makefile
#src/iokernellang/codesparser.c codes/codesparser.h: src/iokernellang/codesparser.y
#src/iokernellang/codesparser.c src/iokernellang/codesparser.h: src/iokernellang/codesparser.y
#codes/codeslexer.h: $(top_srcdir)/src/iokernellang/codeslexer.h
#src/iokernellang/codeslexer.h: $(top_srcdir)/src/iokernellang/codeslexer.h
#mkdir -p codes
#cp $(top_srcdir)/src/iokernellang/codeslexer.h codes/codeslexer.h
#cp $(top_srcdir)/src/iokernellang/codeslexer.h src/iokernellang/codeslexer.h
nobase_include_HEADERS = \
codes/CodesKernelHelpers.h \
codes/CodesIOKernelContext.h \
codes/CodesIOKernelParser.h \
codes/CodesIOKernelTypes.h \
codes/codeslexer.h \
codes/txt_configfile.h \
codes/tools.h \
codes/codeslogging.h \
codes/timeline.h \
codes/codesparser.h \
codes/quickhash.h \
codes/configfile.h \
codes/quicklist.h \
codes/codes_mapping.h \
codes/lp-type-lookup.h \
......@@ -73,15 +62,15 @@ nobase_include_HEADERS = \
#codes/codes-nw-workload.h
src_libcodes_base_a_SOURCES = \
codes/codesparser.h \
src/iokernellang/codesparser.h \
src/iokernellang/codesparser.c \
codes/codeslexer.h \
src/iokernellang/codeslexer.h \
src/iokernellang/codeslexer.c \
src/iokernellang/codesImpl.c \
codes/CodesIOKernelContext.h \
codes/CodesIOKernelParser.h \
codes/CodesIOKernelTypes.h \
codes/CodesKernelHelpers.h \
src/iokernellang/CodesIOKernelContext.h \
src/iokernellang/CodesIOKernelParser.h \
src/iokernellang/CodesIOKernelTypes.h \
src/iokernellang/CodesKernelHelpers.h \
src/iokernellang/CodesKernelHelpers.c \
src/modelconfig/configlex.c \
src/modelconfig/configlex.h \
......@@ -90,33 +79,25 @@ src_libcodes_base_a_SOURCES = \
src/modelconfig/configfile.c \
src/modelconfig/configglue.h \
src/modelconfig/configglue.c \
codes/configfile.h \
src/modelconfig/configfile.h \
src/modelconfig/configstore.h \
src/modelconfig/configstore.c \
src/modelconfig/configstoreadapter.h \
src/modelconfig/configstoreadapter.c \
codes/txt_configfile.h \
src/modelconfig/txt_configfile.h \
src/modelconfig/txt_configfile.c \
codes/tools.h \
src/util/tools.c \
src/util/codes-callbacks.h \
src/util/codes-callbacks.c \
src/modelconfig/configuration.c \
src/util/codes_mapping.c \
src/util/lp-type-lookup.c \
src/util/configuration.c \
src/util/lp-io.c \
src/util/lp-msg.c \
src/util/lookup3.c \
src/util/resource.c \
src/util/resource-lp.c \
src/util/local-storage-model.c \
codes/codeslogging.h \
src/logging/codeslogging.c \
codes/timeline.h \
src/logging/timeline.c \
src/workload/codes-workload.c \
src/workload/codes-workload-method.h \
src/workload/methods/codes-bgp-io-wrkld.c \
src/workload/methods/codes-iolang-wrkld.c \
src/workload/methods/test-workload-method.c \
src/workload/methods/codes-scala-trace-nw-wrkld.c \
codes/rc-stack.h \
......
......@@ -7,8 +7,8 @@
#ifndef CODESIOKERNELPARSER_H
#define CODESIOKERNELPARSER_H
#include "codes/codesparser.h"
#include "codes/CodesIOKernelContext.h"
#include "codesparser.h"
#include "CodesIOKernelContext.h"
YYLTYPE *CodesIOKernel_get_lloc (yyscan_t yyscanner);
int CodesIOKernel_lex_init (yyscan_t* scanner);
......
......@@ -7,7 +7,7 @@
#ifndef CODES_H
#define CODES_H
#include "codes/CodesIOKernelContext.h"
#include "CodesIOKernelContext.h"
typedef enum { typeCon, typeId, typeOpr } nodeEnum;
......
......@@ -4,7 +4,7 @@
*
*/
#include "codes/CodesKernelHelpers.h"
#include "CodesKernelHelpers.h"
#include <stdlib.h>
#include <stdio.h>
......@@ -110,8 +110,9 @@ static int convertKLInstToEvent(int inst)
return CL_UNKNOWN;
}
static void codes_kernel_helper_parse_cf(char * io_kernel_path, char *
io_kernel_def_path, char * io_kernel_meta_path, int task_rank, int max_ranks_default, codes_workload_info * task_info, int use_relpath)
static void codes_kernel_helper_parse_cf(char * io_kernel_path,
char * io_kernel_meta_path, int task_rank, int max_ranks_default,
codes_workload_info * task_info, int use_relpath)
{
int foundit = 0;
char line[CK_LINE_LIMIT];
......@@ -193,18 +194,12 @@ static void codes_kernel_helper_parse_cf(char * io_kernel_path, char *
fclose(ikmp);
/* if we did not find the config file, set it to the default */
if(foundit == 0)
{
/* default kernel */
strcpy(io_kernel_path, io_kernel_def_path);
/* default gid and task attrs */
/* TODO can we detect the gaps instead of -1 */
task_info->group_id = CL_DEFAULT_GID;
task_info->min_rank = -1;
task_info->max_rank = -1;
task_info->local_rank = -1;
task_info->num_lrank = -1;
if(foundit == 0) {
fprintf(stderr,
"ERROR: Unable to find iolang workload file "
"from given metadata file %s... exiting\n",
io_kernel_meta_path);
exit(1);
}
return;
......@@ -301,11 +296,10 @@ int codes_kernel_helper_parse_input(CodesIOKernel_pstate * ps, CodesIOKernelCont
return codes_inst;
}
int codes_kernel_helper_bootstrap(char * io_kernel_path, char *
io_kernel_def_path, char * io_kernel_meta_path,
int rank, int num_ranks, int use_relpath, CodesIOKernelContext * c,