merge

codes/codes-workload.h

@@ -15,7 +15,7 @@
 #include "ross.h"
 #define MAX_NAME_LENGTH_WKLD 512
 
-typedef struct bgp_params bgp_params;
+typedef struct iolang_params iolang_params;
 typedef struct darshan_params darshan_params;
 typedef struct recorder_params recorder_params;
 typedef struct codes_workload_info codes_workload_info;
@@ -25,18 +25,15 @@ typedef struct scala_trace_params scala_trace_params;
 typedef struct dumpi_trace_params dumpi_trace_params;
 
 
-struct bgp_params
+struct iolang_params
 {
-    /* We have the number of ranks passed in from the bg/p model because
-     * the I/O lang workloads have no information about the number of ranks.
-     * Only the bg/p config file knows the number of ranks. */
+    /* the rank count is defined in the workload config file */
     int num_cns;
     /* flag - use path to find kernel files relative to the metafile */
     int use_relpath;
     char io_kernel_meta_path[MAX_NAME_LENGTH_WKLD];
-    char bgp_config_file[MAX_NAME_LENGTH_WKLD];
+    /* set by config in the metadata path */
     char io_kernel_path[MAX_NAME_LENGTH_WKLD];
-    char io_kernel_def_path[MAX_NAME_LENGTH_WKLD];
 };
 
 struct darshan_params
@@ -67,8 +64,8 @@ struct scala_trace_params {
 };
 
 struct dumpi_trace_params {
-   int num_net_traces;
    char file_name[MAX_NAME_LENGTH_WKLD];
+   int num_net_traces;
 };
 
 
@@ -127,6 +124,10 @@ enum codes_workload_op_type
     CODES_WK_WAITANY,
     /* Testall operation */
     CODES_WK_TESTALL,
+
+    /* for workloads that have events not yet handled
+     * (eg the workload language) */
+    CODES_WK_IGNORE
 };
 
 /* I/O operation paramaters */

codes/codeslogging.h

-/*
- * Copyright (C) 2013 University of Chicago.
- * See COPYRIGHT notice in top-level directory.
- *
- */
-
-/*
- * (C) 2001 Clemson University and The University of Chicago
- *
- * See COPYING in top-level directory.
- */
-
-/** \defgroup codeslogging codeslogging logging interface
- *
- * This is a basic application logging facility.  It uses printf style
- * formatting and provides several mechanisms for output.
- *
- * @{
- */
-
-/* This code was derived from the PVFS2 codeslogging circa vs. 2.8.1 */
-
-/** \file
- *
- *  Declarations for the codeslogging logging interface.
- */
-
-#ifndef SRC_COMMON_LOGGING_CODESLOGGING_H
-#define SRC_COMMON_LOGGING_CODESLOGGING_H
-
-#include <stdint.h>
-#include <stdarg.h>
-
-/********************************************************************
- * Visible interface
- */
-
-#define CODESLOGGING_BUF_SIZE 1024
-
-/* what type of timestamp to place in msgs */
-enum codeslogging_logstamp
-{
-    CODESLOGGING_LOGSTAMP_NONE = 0,
-    CODESLOGGING_LOGSTAMP_USEC = 1,
-    CODESLOGGING_LOGSTAMP_DATETIME = 2,
-    CODESLOGGING_LOGSTAMP_THREAD = 3
-};
-#define CODESLOGGING_LOGSTAMP_DEFAULT CODESLOGGING_LOGSTAMP_USEC
-
-/* stdio is needed by codeslogging_debug_fp declaration for FILE* */
-#include <stdio.h>
-
-int codeslogging_enable_stderr(
-    void);
-int codeslogging_enable_file(
-    const char *filename,
-    const char *mode);
-int codeslogging_disable(
-    void);
-int codeslogging_set_debug_mask(
-    int debug_on,
-    uint64_t mask);
-int codeslogging_get_debug_mask(
-    int *debug_on,
-    uint64_t * mask);
-int codeslogging_set_logstamp(
-    enum codeslogging_logstamp ts);
-
-void codeslogging_backtrace(
-    void);
-
-#ifdef __GNUC__
-
-/* do printf style type checking if built with gcc */
-int __codeslogging_debug(
-    uint64_t mask,
-    char prefix,
-    const char *format,
-    ...) __attribute__ ((format(printf, 3, 4)));
-int codeslogging_err(
-    const char *format,
-    ...) __attribute__ ((format(printf, 1, 2)));
-int __codeslogging_debug_va(
-    uint64_t mask,
-    char prefix,
-    const char *format,
-    va_list ap);
-int codeslogging_debug_fp(
-    FILE * fp,
-    char prefix,
-    enum codeslogging_logstamp ts,
-    const char *format,
-    ...) __attribute__ ((format(printf, 4, 5)));
-
-#ifdef CODESLOGGING_DISABLE_DEBUG
-#define codeslogging_debug(mask, format, f...) do {} while(0)
-#define codeslogging_perf_log(format, f...) do {} while(0)
-#define codeslogging_debug_enabled(__m) 0
-#else
-extern int codeslogging_debug_on;
-extern int codeslogging_facility;
-extern uint64_t codeslogging_debug_mask;
-
-#define codeslogging_debug_enabled(__m) \
-    (codeslogging_debug_on && (codeslogging_debug_mask & __m))
-
-/* try to avoid function call overhead by checking masks in macro */
-#define codeslogging_debug(mask, format, f...)                  \
-do {                                                      \
-    if ((codeslogging_debug_on) && (codeslogging_debug_mask & mask) &&\
-        (codeslogging_facility))                                \
-    {                                                     \
-        __codeslogging_debug(mask, '?', format, ##f);           \
-    }                                                     \
-} while(0)
-#define codeslogging_perf_log(format, f...)                     \
-do {                                                      \
-    if ((codeslogging_debug_on) &&                              \
-        (codeslogging_debug_mask & CODESLOGGING_PERFCOUNTER_DEBUG) && \
-        (codeslogging_facility))                                \
-    {                                                     \
-        __codeslogging_debug(CODESLOGGING_PERFCOUNTER_DEBUG, 'P',     \
-            format, ##f);                                 \
-    }                                                     \
-} while(0)
-
-#endif /* CODESLOGGING_DISABLE_DEBUG */
-
-/* do file and line number printouts w/ the GNU preprocessor */
-#define codeslogging_ldebug(mask, format, f...)                  \
-do {                                                       \
-    codeslogging_debug(mask, "%s: " format, __func__ , ##f); \
-} while(0)
-
-#define codeslogging_lerr(format, f...)                  \
-do {                                               \
-    codeslogging_err("%s line %d: " format, __FILE__ , __LINE__ , ##f); \
-    codeslogging_backtrace();                            \
-} while(0)
-#else /* ! __GNUC__ */
-
-int __codeslogging_debug(
-    uint64_t mask,
-    char prefix,
-    const char *format,
-    ...);
-int __codeslogging_debug_stub(
-    uint64_t mask,
-    char prefix,
-    const char *format,
-    ...);
-int codeslogging_err(
-    const char *format,
-    ...);
-
-#ifdef CODESLOGGING_DISABLE_DEBUG
-#define codeslogging_debug(__m, __f, f...) __codeslogging_debug_stub(__m, '?', __f, ##f);
-#define codeslogging_ldebug(__m, __f, f...) __codeslogging_debug_stub(__m, '?', __f, ##f);
-#define codeslogging_debug_enabled(__m) 0
-#else
-#define codeslogging_debug(__m, __f, f...) __codeslogging_debug(__m, '?', __f, ##f);
-#define codeslogging_ldebug(__m, __f, f...) __codeslogging_debug(__m, '?', __f, ##f);
-#define codeslogging_debug_enabled(__m) \
-            ((codeslogging_debug_on != 0) && (__m & codeslogging_debug_mask))
-
-#endif /* CODESLOGGING_DISABLE_DEBUG */
-
-#define codeslogging_lerr codeslogging_err
-
-#endif /* __GNUC__ */
-
-#endif /* __CODESLOGGING_H */
-
-/* @} */
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- * End:
- *
- * vim: ts=8 sts=4 sw=4 expandtab
- */

codes/configuration.h

@@ -7,9 +7,9 @@
 #ifndef __CONFIGURATION_H__
 #define __CONFIGURATION_H__
 
+#include <stddef.h>
 #include <inttypes.h>
 #include <mpi.h>
-#include "codes/txt_configfile.h"
 
 #define CONFIGURATION_MAX_NAME 256
 #define CONFIGURATION_MAX_GROUPS 10
@@ -53,6 +53,8 @@ typedef struct config_lpgroups_s
     config_anno_map_t lpannos[CONFIGURATION_MAX_TYPES];
 } config_lpgroups_t;
 
+typedef struct ConfigVTable * ConfigHandle;
+
 /*
  * Load a configuration on the system (collectively)
  *

codes/local-storage-model.h

@@ -43,7 +43,10 @@ typedef enum lsm_event_e
  */
 
 /* given LP sender, find the LSM device LP in the same group */ 
-tw_lpid lsm_find_local_device(tw_lp *sender);
+tw_lpid lsm_find_local_device(
+        const char * annotation,
+        int ignore_annotations,
+        tw_lpid sender_gid);
 
 /*
  * lsm_event_new
@@ -69,6 +72,21 @@ tw_event* lsm_event_new(const char* category,
                         size_t   message_bytes,
                         tw_lp   *sender,
                         tw_stime delay);
+/* equivalent to lsm_event_new, except it allows to specify an annotation to
+ * filter by. If ignore_annotations is nonzero, A null annotation parameter
+ * indicates that the lsm LP to issue to has no annotation */
+tw_event* lsm_event_new_annotated(
+        const char* category,
+        tw_lpid  dest_gid,
+        uint64_t io_object,
+        int64_t  io_offset,
+        uint64_t io_size_bytes,
+        int      io_type,
+        size_t   message_bytes,
+        tw_lp   *sender,
+        tw_stime delay,
+        const char * annotation,
+        int ignore_annotations);
 
 void lsm_event_new_reverse(tw_lp *sender);
 

codes/timeline.h

-/*
- * Copyright (C) 2013 University of Chicago.
- * See COPYRIGHT notice in top-level directory.
- *
- */
-
-/*
- * timeline.h
- *
- *  Created on: Jun 24, 2013
- *      Author: wozniak
- */
-
-#ifndef TIMELINE_H
-#define TIMELINE_H
-
-#define TIMELINE_ENABLED 0
-
-#if TIMELINE_ENABLED == 1
-
-#include <ross.h>
-
-/**
-   Open filename for writing for timeline data
-   @return 1 on success, 0 on error
- */
-int timeline_init(const char *filename);
-
-/**
-   Write a timeline record
-   @return 1 on success, 0 on error
- */
-int timeline_printf(const char *format, ...);
-
-/**
-   Write a timeline record
-   @return 1 on success, 0 on error
- */
-int timeline_printf_va(const char *format, va_list ap);
-
-/**
-   Write a timeline record with typical ROSS metadata
-   @return 1 on success, 0 on error
- */
-#define timeline_event(lp, format, args...) \
-    timeline_event_impl(lp, __func__, format, ##args)
-
-int timeline_event_impl(const tw_lp *lp, const char *func,
-                        const char *format, ...);
-
-/**
-   Finalize the timeline module
- */
-void timeline_finalize(void);
-
-#else
-
-// Set all functions to noops (return success value 1)
-#define timeline_init(x)            1
-#define timeline_printf(f, a...)    1
-#define timeline_printf_va(f, a)    1
-#define timeline_event(lp, f, a...) 1
-#define timeline_finalize()
-
-#endif
-
-#endif
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- * End:
- *
- * vim: ts=8 sts=4 sw=4 expandtab
- */

codes/tools.h

-/*
- * Copyright (C) 2013 University of Chicago.
- * See COPYRIGHT notice in top-level directory.
- *
- */
-
-#ifndef SRC_COMMON_UTIL_TOOLS_H
-#define SRC_COMMON_UTIL_TOOLS_H
-
-#ifdef UNUSED
-#elif defined(__GNUC__)
-# define UNUSED(x) UNUSED_ ## x __attribute__((unused))
-#elif defined(__LCLINT__)
-# define UNUSED(x) /*@unused@*/ x
-#else
-# define UNUSED(x) x
-#endif /* UNUSED */
-
-#define codesmin(a,b) ((a)<(b) ? (a):(b))
-#define codesmax(a,b) ((a)>(b) ? (a):(b))
-
-void always_assert_error(const char * expr, const char * file, int lineno);
-
-#define ALWAYS_ASSERT(a) if (!(a)) always_assert_error(#a,__FILE__, __LINE__);
-
-#define ARRAY_SIZEOF(a) (sizeof(a)/sizeof(a[0]))
-
-char * safe_strncpy(char * buf, const char * source, unsigned int bufsize);
-
-#define CODES_FLAG_ISSET(mode_, flag_) ((mode_ & flag_) != 0)
-#define CODES_FLAG_SET(mode_, flag_, branch_)   \
-do                                              \
-{                                               \
-    mode_ = mode_ | flag_;                      \
-    branch_ = 1;                                \
-}while(0)
-
-#define CODES_FLAG_SET_RC(mode_, flag_, branch_)   \
-do                                                  \
-{                                                   \
-    if(branch_)                                     \
-    {                                               \
-        mode_ = mode_ & (~flag_);                   \
-    }                                               \
-}while(0)
-
-#endif
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- * End:
- *
- * vim: ts=8 sts=4 sw=4 expandtab
- */

configure.ac

@@ -123,7 +123,7 @@ if test "x${with_dumpi}" != "x" ; then
 #	DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/common" 
 #	DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/libdumpi"
 #	DUMPI_CFLAGS+=" -I${with_dumpi}/include/dumpi/libundumpi"
-	DUMPI_LIBS="-L${with_dumpi}/lib/ -ldumpi -lundumpi"
+	DUMPI_LIBS="-L${with_dumpi}/lib/ -lundumpi"
 	AC_SUBST(DUMPI_LIBS)
 	AC_SUBST(DUMPI_CFLAGS)
 else

doc/BUILD_STEPS

@@ -67,3 +67,12 @@ Notes on using the clang static analyzer
 - edit Makefile, and delete the "CC = mpicc" (or similar) line
 - run "scan-build --use-cc=mpicc make"
 
+Notes on using uncrustify
+-----------------
+- version 0.61 is required
+- build/install uncrustify from your favorite distro (0.61), OR download from
+  https://github.com/bengardner/uncrustify and build from source
+  (configure --prefix /path/to/install && make && make install)
+- either use uncrustify directly (see uncrustify --help) or use the provided
+  reformat.sh tool which is a shim over some of the options (see reformat.sh
+  -h)

doc/GETTING_STARTED

@@ -96,9 +96,7 @@ IO kernel language, and in-development ScalaTrace
 
 The synthetic IO language is a simple, interpreted set of IO and basic
 arithmetic commands meant to simplify the specification and running of
-application workloads. In the code it's currently called the "bgp" workload
-generator but that is just a historical artifact - in the future it will be
-refactored/renamed.
+application workloads.
 
 The input for the workload generator consists of an IO kernel metadata file and
 a number of IO kernel files. The former specifies a set of kernel files to run
@@ -126,7 +124,7 @@ readat/writeat.
 More detailed documentation on the language is ongoing, but for now a general
 example can be seen at doc/workload, which shows a simple out-of-core data
 shuffle. Braver souls may wish to visit the implementation at src/iokernellang
-and src/workload/codes-bgp-io-wrkld.c.
+and src/workload/codes-iolang-wrkld.c.
 
 The following restrictions currently apply to the IO language:
 * all user-defined variables must be a single, lower-case letter (the symbol

reformat.sh

+#!/bin/bash
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function error () {
+    echo "ERROR: $@" >&2
+    exit 1
+}
+
+function usage() {
+    cat \
+<<EOF
+USAGE: reformat.sh [-x] [-i] [-c CFG] [-b FILE] {-a | FILE...}
+    -i: perform in-place (will ask to confirm if being done recursively)
+    -c CFG: use CFG as the uncrustify config (default: $script_dir/uc-codes.cfg)
+    -b FILE[,...]: add FILE to the blacklist of directories (in the form *FILE/*)
+        multiple files can be added by separating with commas
+    -a: perform recursively on all files ending in .c or .h
+        if -a not provided, then perform on all files passed in
+    -x: instead of formatting, check if output format is the same as input
+    -h: show this help and exit
+EOF
+}
+
+function build_blacklist() {
+    for x in $@ ; do
+        echo -n "-and -not -path \*$x/\* "
+    done
+}
+
+function getfiles() {
+    find . \( -name \*.c -or -name \*.h \) $(build_blacklist $@)
+}
+
+cfg=$script_dir/uc-codes.cfg
+
+which uncrustify > /dev/null 2>&1 || error "uncrustify not found"
+
+do_inplace=no
+do_recursive=no
+do_check=no
+blacklist=
+
+while getopts ":ib:axh" opt; do
+    case $opt in
+        i)
+            do_inplace=yes
+            ;;
+        b)
+            blacklist=$(cat $OPTARG | sed s/,/ /g)
+            ;;
+        c)
+            cfg=$OPTARG
+            ;;
+        a)
+            do_recursive=yes
+            ;;
+        x)
+            do_check=yes
+            ;;
+        h)
+            usage
+            exit 1
+            ;;
+        \?)
+            usage
+            error "invalid argument: -$OPTARG"
+            ;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+[[ -e $cfg ]] || ( usage && error "config file $cfg not found" )
+
+[[ $# -gt 0 && $do_recursive == yes ]] && \
+    usage && error "no file args expected in recursive mode"
+[[ $# -eq 0 && $do_recursive == no ]] && \
+    usage && error "expected file args in non-recursive mode"
+
+if [[ $do_recursive == yes ]] ; then
+    file_list="$(getfiles $blacklist)"
+else
+    file_list="$@"
+fi
+
+if [[ $do_recursive == yes && $do_inplace == yes ]] ; then
+    echo -n "Do recursive in-place reformat? (y/n) "
+    read answer
+    [[ $answer =~ "[^y].*" ]] && echo "...aborting reformat" && exit 1
+fi
+
+[[ $do_inplace == yes ]] \
+    && output_arg="--replace" \
+    || output_arg=
+
+if [[ $do_check == yes ]] ; then
+    check_arg=--check
+    output_arg=
+    echo "checking format..."
+else
+    check_arg=
+fi
+
+function echolines () {
+    for x in $@ ; do echo $x ; done
+}
+
+echolines $file_list | uncrustify $check_arg -c $cfg $output_arg -F -

src/Makefile.subdir

@@ -29,33 +29,22 @@ maintainer-clean-local::
 src/modelconfig/configparser.h: src/modelconfig/configparser.y
 src/modelconfig/configlex.h: src/modelconfig/configlex.l
 
-#BUILT_SOURCES += codes/codesparser.h \
+#BUILT_SOURCES += src/iokernellang/codesparser.h \
 #		 src/iokernellang/codesparser.c \
 #                 src/iokernellang/codeslexer.h \
-#                 codes/codeslexer.h
+#                 src/iokernellang/codeslexer.h
 
 
 #src/iokernellang/codesparser.y: src/iokernellang/codesparser.y.in Makefile
 
-#src/iokernellang/codesparser.c codes/codesparser.h: src/iokernellang/codesparser.y
+#src/iokernellang/codesparser.c src/iokernellang/codesparser.h: src/iokernellang/codesparser.y
 
-#codes/codeslexer.h: $(top_srcdir)/src/iokernellang/codeslexer.h
+#src/iokernellang/codeslexer.h: $(top_srcdir)/src/iokernellang/codeslexer.h
 #mkdir -p codes
-#cp $(top_srcdir)/src/iokernellang/codeslexer.h codes/codeslexer.h
+#cp $(top_srcdir)/src/iokernellang/codeslexer.h src/iokernellang/codeslexer.h
 
 nobase_include_HEADERS = \
-    codes/CodesKernelHelpers.h \
-    codes/CodesIOKernelContext.h \
-    codes/CodesIOKernelParser.h \
-    codes/CodesIOKernelTypes.h \
-    codes/codeslexer.h \
-    codes/txt_configfile.h \
-    codes/tools.h \
-    codes/codeslogging.h \
-    codes/timeline.h \
-    codes/codesparser.h \
     codes/quickhash.h \
-    codes/configfile.h \
     codes/quicklist.h \
     codes/codes_mapping.h \
     codes/lp-type-lookup.h \
@@ -73,15 +62,15 @@ nobase_include_HEADERS = \
 #codes/codes-nw-workload.h
 
 src_libcodes_base_a_SOURCES =  \
-    codes/codesparser.h \
+    src/iokernellang/codesparser.h \
     src/iokernellang/codesparser.c \
-    codes/codeslexer.h \
+    src/iokernellang/codeslexer.h \
     src/iokernellang/codeslexer.c \
     src/iokernellang/codesImpl.c \
-    codes/CodesIOKernelContext.h \
-    codes/CodesIOKernelParser.h \
-    codes/CodesIOKernelTypes.h \
-    codes/CodesKernelHelpers.h \
+    src/iokernellang/CodesIOKernelContext.h \
+    src/iokernellang/CodesIOKernelParser.h \
+    src/iokernellang/CodesIOKernelTypes.h \
+    src/iokernellang/CodesKernelHelpers.h \
     src/iokernellang/CodesKernelHelpers.c \
     src/modelconfig/configlex.c \
     src/modelconfig/configlex.h \
@@ -90,33 +79,25 @@ src_libcodes_base_a_SOURCES =  \
     src/modelconfig/configfile.c \
     src/modelconfig/configglue.h \
     src/modelconfig/configglue.c \
-    codes/configfile.h \
+    src/modelconfig/configfile.h \
     src/modelconfig/configstore.h \
     src/modelconfig/configstore.c \
     src/modelconfig/configstoreadapter.h \
     src/modelconfig/configstoreadapter.c \
-    codes/txt_configfile.h \
+    src/modelconfig/txt_configfile.h \
     src/modelconfig/txt_configfile.c \
-    codes/tools.h \
-    src/util/tools.c \
-    src/util/codes-callbacks.h \
-    src/util/codes-callbacks.c \
+    src/modelconfig/configuration.c \
     src/util/codes_mapping.c \