Merge codes-base

.gitignore

+# automake artifacts
+/Makefile.in
+/aclocal.m4
+/autom4te.cache/
+/codes_base_config.h.in
+/compile
+/config.guess
+/config.sub
+/configure
+/depcomp
+/install-sh
+/missing
+
+# configure artifacts
+/config.log
+/config.status
+/Makefile
+/stamp-h1
+/codes_base_config.h
+/maint/codes-base.pc
+.deps
+
+# make generated artifacts
+.dirstamp
+*.o
+*.a
+/src/modelconfig/configparser.h
+/src/modelconfig/configparser.c
+/src/modelconfig/configlex.h
+/src/modelconfig/configlex.c
+/tests/lp-io-test
+
+# hide backups
+*~
+
+# generated files from test runs
+ross.csv

COPYRIGHT

+
+				  COPYRIGHT
+
+The following is a notice of limited availability of the code, and disclaimer
+which must be included in the prologue of the code and in all source listings
+of the code.
+
+Copyright Notice
+ + 2013 University of Chicago
+
+Permission is hereby granted to use, reproduce, prepare derivative works, and
+to redistribute to others.  This software was authored by:
+
+Mathematics and Computer Science Division
+Argonne National Laboratory, Argonne IL 60439
+
+(and)
+
+Computer Science Department
+Rensselaer Polytechnic Institute, Troy NY 12180
+
+			      GOVERNMENT LICENSE
+
+Portions of this material resulted from work developed under a U.S.
+Government Contract and are subject to the following license: the Government
+is granted for itself and others acting on its behalf a paid-up, nonexclusive,
+irrevocable worldwide license in this computer software to reproduce, prepare
+derivative works, and perform publicly and display publicly.
+
+				  DISCLAIMER
+
+This computer code material was prepared, in part, as an account of work
+sponsored by an agency of the United States Government.  Neither the United
+States, nor the University of Chicago, nor any of their employees, makes any
+warranty express or implied, or assumes any legal liability or responsibility
+for the accuracy, completeness, or usefulness of any information, apparatus,
+product, or process disclosed, or represents that its use would not infringe
+privately owned rights.

Make.rules

+# flex & bison deps
+#
+%.c %.h: %.l
+	$(AM_V_GEN)$(LEX) --header-file=$(@:.c=.h) -o $(@:.h=.c) $< \
+	   || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1) 
+
+# 
+# specific rule for codesparser generation; we want the header to land in
+# the codes/ directory because it will be installed for use by other repos
+#src/iokernellang/codesparser.c codes/codesparser.h: src/iokernellang/codesparser.y
+#	mkdir -p codes
+#	@test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
+#	$(AM_V_GEN)$(YACC) --defines=codes/codesparser.h -o src/iokernellang/codesparser.c  $< \
+#	   || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
+
+
+%.c %.h: %.y
+	@test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
+	$(AM_V_GEN)$(YACC) --defines=$(@:.c=.h) -o $(@:.h=.c)  $< \
+	   || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
+
+
+
+# %.y: %.y.in Makefile
+#	$(AM_V_GEN)$(SED) -e 's,[@]CODES_PURE_PARSER_DEFINES[@],$(CODES_PURE_PARSER_DEFINES),g' \
+#	-e 's,[@]CODES_PUSH_PARSER_DEFINES[@],$(CODES_PUSH_PARSER_DEFINES),g' \
+#	< src/common/iokernellang/codesparser.y.in > src/common/iokernellang/codesparser.y
+
+#
+# Output dist version
+#
+.phony: distversion
+distversion:
+	@echo $(VERSION)
+
+#
+# Easy way to build unit tests without running them
+# 
+.phony: tests
+tests: $(check_PROGRAMS)

Makefile.am

+AUTOMAKE_OPTIONS = foreign
+ACLOCAL_AMFLAGS = -I m4
+
+bin_PROGRAMS =
+bin_SCRIPTS =
+noinst_LIBRARIES =
+noinst_PROGRAMS =
+lib_LIBRARIES =
+noinst_HEADERS =
+TESTS =
+check_PROGRAMS =
+EXTRA_PROGRAMS =
+CLEANFILES = $(bin_SCRIPTS)
+EXTRA_DIST =
+BUILT_SOURCES =
+src_models_CESOURCES =
+
+# pkgconfig files
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = maint/codes-base.pc
+$(pkgconfig_DATA): config.status
+
+EXTRA_DIST += prepare.sh COPYRIGHT configure.ac reformat.sh uc-codes.cfg
+
+AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
+
+AM_CFLAGS =
+
+AM_LDFLAGS = ${ROSS_LDFLAGS}
+
+AM_CXXFLAGS = $(AM_CFLAGS)
+
+include Make.rules
+
+include $(top_srcdir)/scripts/Makefile.subdir
+include $(top_srcdir)/src/Makefile.subdir
+include $(top_srcdir)/tests/Makefile.subdir
+include $(top_srcdir)/doc/Makefile.subdir
+
+if USE_DARSHAN
+AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
+src_libcodes_base_a_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
+tests_workload_codes_workload_test_LDADD += ${DARSHAN_LIBS}
+tests_workload_codes_workload_mpi_replay_LDADD += ${DARSHAN_LIBS}
+TESTS += tests/workload/darshan-dump.sh
+endif
+
+if USE_RECORDER
+AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
+src_libcodes_base_a_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
+endif
+
+if USE_DUMPI
+AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
+src_libcodes_base_a_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c 
+tests_workload_codes_workload_test_LDADD += ${DUMPI_LIBS}
+tests_workload_codes_workload_mpi_replay_LDADD += ${DUMPI_LIBS}
+endif

codes/codes-callback.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+*/
+
+#ifndef CODES_CALLBACK_H
+#define CODES_CALLBACK_H
+
+#include <assert.h>
+#include "codes/lp-msg.h"
+
+/* This header defines the following conventions for callback-based event
+ * processing (i.e., RPC-like event flow where the callee sends a "return
+ * event" to the caller)
+ *
+ * callers:
+ * - give the caller event size (for sanity checking)
+ * - use the msg_header convention (lp-msg.h)
+ * - specify an integer tag to identify which call the callee return event
+ *   corresponds to, in the case of multiple pending callbacks
+ * - specify an offset to the return structure, which is defined by the callees
+ *
+ * callees:
+ * - use the resulting set of offsets to write and issue the callback
+ *
+ * The recommended calling convention for these types of event flows is:
+ * void event_func(<params...>,
+ *         msg_header const * h, int tag, struct codes_cb_info const * cb);
+ */
+
+struct codes_cb_info {
+    int event_size;
+    int header_offset;
+    int tag_offset;
+    int cb_ret_offset;
+};
+
+/* helper struct for callees - wrap up the typical inputs */
+struct codes_cb_params {
+    struct codes_cb_info info;
+    msg_header h;
+    int tag;
+};
+
+/* function-like sugar for initializing a codes_cb_info */
+#define INIT_CODES_CB_INFO(_cb_info_ptr, _event_type, _header_field, _tag_field, _cb_ret_field) \
+    do { \
+        (_cb_info_ptr)->event_size    = sizeof(_event_type); \
+        (_cb_info_ptr)->header_offset = offsetof(_event_type, _header_field); \
+        (_cb_info_ptr)->tag_offset    = offsetof(_event_type, _tag_field); \
+        (_cb_info_ptr)->cb_ret_offset = offsetof(_event_type, _cb_ret_field); \
+    } while (0)
+
+#define CB_HO(_cb_params_ptr) ((_cb_params_ptr)->info.header_offset)
+#define CB_TO(_cb_params_ptr) ((_cb_params_ptr)->info.tag_offset)
+#define CB_RO(_cb_params_ptr) ((_cb_params_ptr)->info.cb_ret_offset)
+
+/* Declare return variables at the right byte offsets from a codes_cb_params.
+ * Additionally, set the header and tag vars */
+#define GET_INIT_CB_PTRS(_cb_params_ptr, _data_ptr, _sender_gid_val, _header_nm, _tag_nm, _rtn_name, _rtn_type) \
+    msg_header * _header_nm = \
+            (msg_header*)((char*)(_data_ptr) + CB_HO(_cb_params_ptr)); \
+    int * _tag_nm = \
+            (int*)((char*)(_data_ptr) + CB_TO(_cb_params_ptr));\
+    _rtn_type * _rtn_name = \
+            (_rtn_type*)((char*)(_data_ptr) + CB_RO(_cb_params_ptr)); \
+    msg_set_header((_cb_params_ptr)->h.magic, (_cb_params_ptr)->h.event_type, \
+            _sender_gid_val, _header_nm); \
+    *_tag_nm = (_cb_params_ptr)->tag
+
+#define SANITY_CHECK_CB(_cb_info_ptr, _ret_type) \
+    do { \
+        int _total_size = sizeof(_ret_type) + sizeof(int) + sizeof(msg_header);\
+        int _esize = (_cb_info_ptr)->event_size; \
+        assert(_esize > 0 && \
+                (_cb_info_ptr)->header_offset >= 0 && \
+                (_cb_info_ptr)->tag_offset >= 0 && \
+                (_cb_info_ptr)->cb_ret_offset >= 0); \
+        assert(_esize >= _total_size); \
+        assert(_esize >= (_cb_info_ptr)->header_offset + sizeof(msg_header)); \
+        assert(_esize >= (_cb_info_ptr)->tag_offset + sizeof(int)); \
+        assert(_esize >= (_cb_info_ptr)->cb_ret_offset + sizeof(_ret_type)); \
+    } while (0)
+
+#endif /* end of include guard: CODES_CALLBACK_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ *  indent-tabs-mode: nil
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

codes/codes-jobmap.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+/* SUMMARY - namespace support for multiple groups of ids (jobs) with respect to a
+ * flat namespace. Note that this API is meant for static job creation - a more
+ * sophisticated method would need to be used to modify job mappings at
+ * runtime.
+ *
+ * Example:
+ *
+ * job  0      1    2        (<-- jobmap defined "job" IDs)
+ * rank 0 1 2  0 1  0 1 2 3  (<-- "job local" IDs)
+ * ID   0 1 2  3 4  5 6 7 8  (<-- jobmap-defined LP relative "global" IDs)
+ * LP   A B C  D E  F G H I  (<-- provided by codes-mapping)
+ * */
+
+#ifndef CODES_JOBMAP_H
+#define CODES_JOBMAP_H
+
+/** type markers and parameter defs for jobmaps **/
+
+enum codes_jobmap_type {
+    /* the "identity" jobmap is a shim for single-job workloads */
+    CODES_JOBMAP_IDENTITY,
+    /* the "list" jobmap allows the explicit specification of mappings from
+     * jobs to lists of global ids through a text file, wiht one line per job
+     */
+    CODES_JOBMAP_LIST,
+    /* the "dummy" jobmap is an example implementation for testing, and can be
+     * seen as the inverse of the identity mapping.
+     * It simply specifies N jobs, with exactly one rank per job, with a trivial
+     * mapping */
+    CODES_JOBMAP_DUMMY,
+};
+
+struct codes_jobmap_params_identity {
+    int num_ranks;
+};
+
+struct codes_jobmap_params_dummy {
+    int num_jobs;
+};
+
+struct codes_jobmap_params_list {
+    char *alloc_file;
+};
+
+
+/** jobmap interface **/
+
+struct codes_jobmap_ctx;
+struct codes_jobmap_id {
+    int job;
+    int rank; // relative to job
+};
+
+struct codes_jobmap_ctx *
+codes_jobmap_configure(enum codes_jobmap_type t, void const * params);
+
+void codes_jobmap_destroy(struct codes_jobmap_ctx *c);
+
+/* main mapping functions - bidirectional mapping is needed:
+ * - global -> local ID for initialization
+ * - local -> global ID for communication between local IDs
+ *
+ * functions return {-1, -1} and -1, respectively, for invalid id input */
+
+struct codes_jobmap_id codes_jobmap_to_local_id(
+        int id,
+        struct codes_jobmap_ctx const * c);
+
+int codes_jobmap_to_global_id(
+        struct codes_jobmap_id id,
+        struct codes_jobmap_ctx const * c);
+
+int codes_jobmap_get_num_jobs(struct codes_jobmap_ctx const * c);
+
+int codes_jobmap_get_num_ranks(int job_id, struct codes_jobmap_ctx const * c);
+
+#endif
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ *  indent-tabs-mode: nil
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */

codes/codes-mapping-context.h

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+/* SUMMARY - data structure and utilities to direct LP mappings. As opposed to
+ * the codes-mapping API, this defines metadata necessary to allow implicit
+ * mappings based on LP type and configuration specification
+ * (see modelnet, LSM, etc)
+ * mctx stands for mapping context */
+
+#ifndef CODES_MAPPING_CONTEXT_H
+#define CODES_MAPPING_CONTEXT_H
+
+#include <stdbool.h>
+#include <ross.h>
+
+/* for convenience - an annotation-ignoring "group_modulo" context,
+ * matching previous mapping behavior in most interfaces (modelnet and such) */
+extern struct codes_mctx const * const CODES_MCTX_DEFAULT;
+
+/* types of map contexts */
+enum codes_mctx_type {
+    // instructs those using the context to map directly to an LP
+    CODES_MCTX_GLOBAL_DIRECT,
+    // instructs those using the context to map into the same group/repetition
+    // and compute the callee offset as the modulus of the caller offset and
+    // the number of callees in the group, to provide simple wraparound
+    // behaviour
+    CODES_MCTX_GROUP_MODULO,
+    // similar to GROUP_MODULO, but maps to offsets in reverse order
+    CODES_MCTX_GROUP_MODULO_REVERSE,
+    // instructs those using the context to map into the same group/repetition
+    // and directly to a callee offset
+    CODES_MCTX_GROUP_DIRECT,
+    // unknown/uninitialized context
+    CODES_MCTX_UNKNOWN
+};
+
+/* defines whether to specialize by destination annotation, and if so, which
+ * one */
+struct codes_mctx_annotation {
+    // see canonical name mapping api in codes_mapping.h. -1 is used for
+    // ignoring annotations
+    int cid;
+};
+
+/* parameters for each mapping context type */
+struct codes_mctx_global_direct {
+    tw_lpid lpid;
+};
+
+struct codes_mctx_group_modulo {
+    struct codes_mctx_annotation anno;
+};
+// NOTE: group_modulo_reverse shares the group_modulo representation
+
+struct codes_mctx_group_direct {
+    struct codes_mctx_annotation anno;
+    int offset;
+};
+
+struct codes_mctx {
+    enum codes_mctx_type type;
+    union {
+        struct codes_mctx_global_direct global_direct;
+        struct codes_mctx_group_modulo  group_modulo;
+        struct codes_mctx_group_direct  group_direct;
+    } u;
+};
+
+/* simple setter functions */
+struct codes_mctx codes_mctx_set_global_direct(tw_lpid lpid);
+
+struct codes_mctx codes_mctx_set_group_modulo(
+        char const * annotation,
+        bool ignore_annotations);
+
+struct codes_mctx codes_mctx_set_group_modulo_reverse(
+        char const * annotation,
+        bool ignore_annotations);
+
+struct codes_mctx codes_mctx_set_group_direct(
+        int offset,
+        char const * annotation,
+        bool ignore_annotations);
+
+/* helper function to do a codes mapping - this function is subject to change
+ * based on what types of ctx exist
+ * NOTE: in GLOBAL_DIRECT mode, dest_lp_name and sender_gid are ignored */
+tw_lpid codes_mctx_to_lpid(
+        struct codes_mctx const * ctx,
+        char const * dest_lp_name,
+        tw_lpid sender_gid);
+
+/* helper function to extract which annotation a various map context maps to.
+ * annotation is allocated or NULL if unused */
+char const * codes_mctx_get_annotation(
+        struct codes_mctx const *ctx,
+        char const * dest_lp_name,
+        tw_lpid sender_id);
+
+#endif /* end of include guard: CODES_MAPPING_CONTEXT_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ *  indent-tabs-mode: nil
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */

codes/codes-workload.h

+/*
+ * Copyright (C) 2013 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+/* I/O workload generator API to be used for reading I/O operations into
+ * storage system simulations.  This API just describes the operations to be
+ * executed; it does not service the operations.
+ */
+
+#ifndef CODES_WORKLOAD_H
+#define CODES_WORKLOAD_H
+
+#include <ross.h>
+#include "configuration.h"
+
+#define MAX_NAME_LENGTH_WKLD 512
+
+/* implementations included with codes */
+typedef struct iomock_params iomock_params;
+typedef struct iolang_params iolang_params;
+typedef struct darshan_params darshan_params;
+typedef struct recorder_params recorder_params;
+
+/* struct to hold the actual data from a single MPI event*/
+typedef struct dumpi_trace_params dumpi_trace_params;
+typedef struct checkpoint_wrkld_params checkpoint_wrkld_params;
+
+struct iomock_params
+{
+    uint64_t file_id;
+    int use_uniq_file_ids;
+    int is_write;
+    int num_requests;
+    int request_size;
+    // for optimizing lookup - set higher (>= num ranks) to reduce collisions
+    // and 0 to use the default
+    int rank_table_size;
+};
+
+struct iolang_params
+{
+    /* the rank count is defined in the workload config file */
+    int num_cns;
+    /* flag - use path to find kernel files relative to the metafile */
+    int use_relpath;
+    char io_kernel_meta_path[MAX_NAME_LENGTH_WKLD];
+    /* set by config in the metadata path */
+    char io_kernel_path[MAX_NAME_LENGTH_WKLD];
+};
+
+struct darshan_params
+{
+    char log_file_path[MAX_NAME_LENGTH_WKLD];
+    int64_t aggregator_cnt;
+};
+
+struct recorder_params
+{
+    char trace_dir_path[MAX_NAME_LENGTH_WKLD];
+    int64_t nprocs;
+};
+
+struct dumpi_trace_params {
+   char file_name[MAX_NAME_LENGTH_WKLD];
+   int num_net_traces;
+};
+
+struct checkpoint_wrkld_params
+{
+    int nprocs; /* number of workload processes */
+    double checkpoint_sz; /* size of checkpoint, in TiB */
+    double checkpoint_wr_bw; /* checkpoint write b/w, in GiB/s */
+    int total_checkpoints; /* total number of checkpoint phases */
+    double mtti; /* mean time to interrupt, in hours */
+};
+
+/* supported I/O operations */
+enum codes_workload_op_type
+{
+    /* terminator; there are no more operations for this rank */
+    CODES_WK_END = 1,
+    /* sleep/delay to simulate computation or other activity */
+    CODES_WK_DELAY,
+    /* block until specified ranks have reached the same point */
+    CODES_WK_BARRIER,
+
+    /* IO operations */
+    /* open */
+    CODES_WK_OPEN,
+    /* close */
+    CODES_WK_CLOSE,
+    /* write */
+    CODES_WK_WRITE,
+    /* read */
+    CODES_WK_READ,
+
+    /* network operations (modelled after MPI operations) */
+    /* blocking send operation */
+    CODES_WK_SEND,
+    /* blocking recv operation */
+    CODES_WK_RECV,
+    /* non-blocking send operation */
+    CODES_WK_ISEND,
+    /* non-blocking receive operation */
+    CODES_WK_IRECV,
+    /* broadcast operation */
+    CODES_WK_BCAST,
+    /* Allgather operation */
+    CODES_WK_ALLGATHER,
+    /* Allgatherv operation */
+    CODES_WK_ALLGATHERV,
+    /* Alltoall operation */
+    CODES_WK_ALLTOALL,
+    /* Alltoallv operation */
+    CODES_WK_ALLTOALLV,
+    /* Reduce operation */
+    CODES_WK_REDUCE,
+    /* Allreduce operation */
+    CODES_WK_ALLREDUCE,
+    /* Generic collective operation */
+    CODES_WK_COL,
+    /* Waitall operation */
+    CODES_WK_WAITALL,
+    /* Wait operation */
+    CODES_WK_WAIT,
+    /* Waitsome operation */
+    CODES_WK_WAITSOME,
+    /* Waitany operation */
+    CODES_WK_WAITANY,
+    /* Testall operation */
+    CODES_WK_TESTALL,
+    /* MPI request free operation*/
+    CODES_WK_REQ_FREE,
+
+    /* for workloads that have events not yet handled
+     * (eg the workload language) */
+    CODES_WK_IGNORE
+};
+
+/* I/O operation paramaters */
+struct codes_workload_op
+{
+    /* TODO: do we need different "classes" of operations to differentiate
+     * between different APIs?
+     */
+
+    /* what type of operation this is */
+    enum codes_workload_op_type op_type;
+    /* currently only used by network workloads */
+    double start_time;
+    double end_time;
+    double sim_start_time;
+
+    /* parameters for each operation type */
+    union
+    {
+        struct {
+            double seconds;