Commit f32214c7 authored by Jonathan Jenkins's avatar Jonathan Jenkins

Merge codes-base

parents 97c47e86 8f1f93ce
# automake artifacts
/Makefile.in
/aclocal.m4
/autom4te.cache/
/codes_base_config.h.in
/compile
/config.guess
/config.sub
/configure
/depcomp
/install-sh
/missing
# configure artifacts
/config.log
/config.status
/Makefile
/stamp-h1
/codes_base_config.h
/maint/codes-base.pc
.deps
# make generated artifacts
.dirstamp
*.o
*.a
/src/modelconfig/configparser.h
/src/modelconfig/configparser.c
/src/modelconfig/configlex.h
/src/modelconfig/configlex.c
/tests/lp-io-test
# hide backups
*~
# generated files from test runs
ross.csv
COPYRIGHT
The following is a notice of limited availability of the code, and disclaimer
which must be included in the prologue of the code and in all source listings
of the code.
Copyright Notice
+ 2013 University of Chicago
Permission is hereby granted to use, reproduce, prepare derivative works, and
to redistribute to others. This software was authored by:
Mathematics and Computer Science Division
Argonne National Laboratory, Argonne IL 60439
(and)
Computer Science Department
Rensselaer Polytechnic Institute, Troy NY 12180
GOVERNMENT LICENSE
Portions of this material resulted from work developed under a U.S.
Government Contract and are subject to the following license: the Government
is granted for itself and others acting on its behalf a paid-up, nonexclusive,
irrevocable worldwide license in this computer software to reproduce, prepare
derivative works, and perform publicly and display publicly.
DISCLAIMER
This computer code material was prepared, in part, as an account of work
sponsored by an agency of the United States Government. Neither the United
States, nor the University of Chicago, nor any of their employees, makes any
warranty express or implied, or assumes any legal liability or responsibility
for the accuracy, completeness, or usefulness of any information, apparatus,
product, or process disclosed, or represents that its use would not infringe
privately owned rights.
# flex & bison deps
#
%.c %.h: %.l
$(AM_V_GEN)$(LEX) --header-file=$(@:.c=.h) -o $(@:.h=.c) $< \
|| ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
#
# specific rule for codesparser generation; we want the header to land in
# the codes/ directory because it will be installed for use by other repos
#src/iokernellang/codesparser.c codes/codesparser.h: src/iokernellang/codesparser.y
# mkdir -p codes
# @test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
# $(AM_V_GEN)$(YACC) --defines=codes/codesparser.h -o src/iokernellang/codesparser.c $< \
# || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
%.c %.h: %.y
@test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
$(AM_V_GEN)$(YACC) --defines=$(@:.c=.h) -o $(@:.h=.c) $< \
|| ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
# %.y: %.y.in Makefile
# $(AM_V_GEN)$(SED) -e 's,[@]CODES_PURE_PARSER_DEFINES[@],$(CODES_PURE_PARSER_DEFINES),g' \
# -e 's,[@]CODES_PUSH_PARSER_DEFINES[@],$(CODES_PUSH_PARSER_DEFINES),g' \
# < src/common/iokernellang/codesparser.y.in > src/common/iokernellang/codesparser.y
#
# Output dist version
#
.phony: distversion
distversion:
@echo $(VERSION)
#
# Easy way to build unit tests without running them
#
.phony: tests
tests: $(check_PROGRAMS)
AUTOMAKE_OPTIONS = foreign
ACLOCAL_AMFLAGS = -I m4
bin_PROGRAMS =
bin_SCRIPTS =
noinst_LIBRARIES =
noinst_PROGRAMS =
lib_LIBRARIES =
noinst_HEADERS =
TESTS =
check_PROGRAMS =
EXTRA_PROGRAMS =
CLEANFILES = $(bin_SCRIPTS)
EXTRA_DIST =
BUILT_SOURCES =
src_models_CESOURCES =
# pkgconfig files
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = maint/codes-base.pc
$(pkgconfig_DATA): config.status
EXTRA_DIST += prepare.sh COPYRIGHT configure.ac reformat.sh uc-codes.cfg
AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
AM_CFLAGS =
AM_LDFLAGS = ${ROSS_LDFLAGS}
AM_CXXFLAGS = $(AM_CFLAGS)
include Make.rules
include $(top_srcdir)/scripts/Makefile.subdir
include $(top_srcdir)/src/Makefile.subdir
include $(top_srcdir)/tests/Makefile.subdir
include $(top_srcdir)/doc/Makefile.subdir
if USE_DARSHAN
AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
src_libcodes_base_a_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
tests_workload_codes_workload_test_LDADD += ${DARSHAN_LIBS}
tests_workload_codes_workload_mpi_replay_LDADD += ${DARSHAN_LIBS}
TESTS += tests/workload/darshan-dump.sh
endif
if USE_RECORDER
AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
src_libcodes_base_a_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
endif
if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
src_libcodes_base_a_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
tests_workload_codes_workload_test_LDADD += ${DUMPI_LIBS}
tests_workload_codes_workload_mpi_replay_LDADD += ${DUMPI_LIBS}
endif
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_CALLBACK_H
#define CODES_CALLBACK_H
#include <assert.h>
#include "codes/lp-msg.h"
/* This header defines the following conventions for callback-based event
* processing (i.e., RPC-like event flow where the callee sends a "return
* event" to the caller)
*
* callers:
* - give the caller event size (for sanity checking)
* - use the msg_header convention (lp-msg.h)
* - specify an integer tag to identify which call the callee return event
* corresponds to, in the case of multiple pending callbacks
* - specify an offset to the return structure, which is defined by the callees
*
* callees:
* - use the resulting set of offsets to write and issue the callback
*
* The recommended calling convention for these types of event flows is:
* void event_func(<params...>,
* msg_header const * h, int tag, struct codes_cb_info const * cb);
*/
struct codes_cb_info {
int event_size;
int header_offset;
int tag_offset;
int cb_ret_offset;
};
/* helper struct for callees - wrap up the typical inputs */
struct codes_cb_params {
struct codes_cb_info info;
msg_header h;
int tag;
};
/* function-like sugar for initializing a codes_cb_info */
#define INIT_CODES_CB_INFO(_cb_info_ptr, _event_type, _header_field, _tag_field, _cb_ret_field) \
do { \
(_cb_info_ptr)->event_size = sizeof(_event_type); \
(_cb_info_ptr)->header_offset = offsetof(_event_type, _header_field); \
(_cb_info_ptr)->tag_offset = offsetof(_event_type, _tag_field); \
(_cb_info_ptr)->cb_ret_offset = offsetof(_event_type, _cb_ret_field); \
} while (0)
#define CB_HO(_cb_params_ptr) ((_cb_params_ptr)->info.header_offset)
#define CB_TO(_cb_params_ptr) ((_cb_params_ptr)->info.tag_offset)
#define CB_RO(_cb_params_ptr) ((_cb_params_ptr)->info.cb_ret_offset)
/* Declare return variables at the right byte offsets from a codes_cb_params.
* Additionally, set the header and tag vars */
#define GET_INIT_CB_PTRS(_cb_params_ptr, _data_ptr, _sender_gid_val, _header_nm, _tag_nm, _rtn_name, _rtn_type) \
msg_header * _header_nm = \
(msg_header*)((char*)(_data_ptr) + CB_HO(_cb_params_ptr)); \
int * _tag_nm = \
(int*)((char*)(_data_ptr) + CB_TO(_cb_params_ptr));\
_rtn_type * _rtn_name = \
(_rtn_type*)((char*)(_data_ptr) + CB_RO(_cb_params_ptr)); \
msg_set_header((_cb_params_ptr)->h.magic, (_cb_params_ptr)->h.event_type, \
_sender_gid_val, _header_nm); \
*_tag_nm = (_cb_params_ptr)->tag
#define SANITY_CHECK_CB(_cb_info_ptr, _ret_type) \
do { \
int _total_size = sizeof(_ret_type) + sizeof(int) + sizeof(msg_header);\
int _esize = (_cb_info_ptr)->event_size; \
assert(_esize > 0 && \
(_cb_info_ptr)->header_offset >= 0 && \
(_cb_info_ptr)->tag_offset >= 0 && \
(_cb_info_ptr)->cb_ret_offset >= 0); \
assert(_esize >= _total_size); \
assert(_esize >= (_cb_info_ptr)->header_offset + sizeof(msg_header)); \
assert(_esize >= (_cb_info_ptr)->tag_offset + sizeof(int)); \
assert(_esize >= (_cb_info_ptr)->cb_ret_offset + sizeof(_ret_type)); \
} while (0)
#endif /* end of include guard: CODES_CALLBACK_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* SUMMARY - namespace support for multiple groups of ids (jobs) with respect to a
* flat namespace. Note that this API is meant for static job creation - a more
* sophisticated method would need to be used to modify job mappings at
* runtime.
*
* Example:
*
* job 0 1 2 (<-- jobmap defined "job" IDs)
* rank 0 1 2 0 1 0 1 2 3 (<-- "job local" IDs)
* ID 0 1 2 3 4 5 6 7 8 (<-- jobmap-defined LP relative "global" IDs)
* LP A B C D E F G H I (<-- provided by codes-mapping)
* */
#ifndef CODES_JOBMAP_H
#define CODES_JOBMAP_H
/** type markers and parameter defs for jobmaps **/
enum codes_jobmap_type {
/* the "identity" jobmap is a shim for single-job workloads */
CODES_JOBMAP_IDENTITY,
/* the "list" jobmap allows the explicit specification of mappings from
* jobs to lists of global ids through a text file, wiht one line per job
*/
CODES_JOBMAP_LIST,
/* the "dummy" jobmap is an example implementation for testing, and can be
* seen as the inverse of the identity mapping.
* It simply specifies N jobs, with exactly one rank per job, with a trivial
* mapping */
CODES_JOBMAP_DUMMY,
};
struct codes_jobmap_params_identity {
int num_ranks;
};
struct codes_jobmap_params_dummy {
int num_jobs;
};
struct codes_jobmap_params_list {
char *alloc_file;
};
/** jobmap interface **/
struct codes_jobmap_ctx;
struct codes_jobmap_id {
int job;
int rank; // relative to job
};
struct codes_jobmap_ctx *
codes_jobmap_configure(enum codes_jobmap_type t, void const * params);
void codes_jobmap_destroy(struct codes_jobmap_ctx *c);
/* main mapping functions - bidirectional mapping is needed:
* - global -> local ID for initialization
* - local -> global ID for communication between local IDs
*
* functions return {-1, -1} and -1, respectively, for invalid id input */
struct codes_jobmap_id codes_jobmap_to_local_id(
int id,
struct codes_jobmap_ctx const * c);
int codes_jobmap_to_global_id(
struct codes_jobmap_id id,
struct codes_jobmap_ctx const * c);
int codes_jobmap_get_num_jobs(struct codes_jobmap_ctx const * c);
int codes_jobmap_get_num_ranks(int job_id, struct codes_jobmap_ctx const * c);
#endif
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* SUMMARY - data structure and utilities to direct LP mappings. As opposed to
* the codes-mapping API, this defines metadata necessary to allow implicit
* mappings based on LP type and configuration specification
* (see modelnet, LSM, etc)
* mctx stands for mapping context */
#ifndef CODES_MAPPING_CONTEXT_H
#define CODES_MAPPING_CONTEXT_H
#include <stdbool.h>
#include <ross.h>
/* for convenience - an annotation-ignoring "group_modulo" context,
* matching previous mapping behavior in most interfaces (modelnet and such) */
extern struct codes_mctx const * const CODES_MCTX_DEFAULT;
/* types of map contexts */
enum codes_mctx_type {
// instructs those using the context to map directly to an LP
CODES_MCTX_GLOBAL_DIRECT,
// instructs those using the context to map into the same group/repetition
// and compute the callee offset as the modulus of the caller offset and
// the number of callees in the group, to provide simple wraparound
// behaviour
CODES_MCTX_GROUP_MODULO,
// similar to GROUP_MODULO, but maps to offsets in reverse order
CODES_MCTX_GROUP_MODULO_REVERSE,
// instructs those using the context to map into the same group/repetition
// and directly to a callee offset
CODES_MCTX_GROUP_DIRECT,
// unknown/uninitialized context
CODES_MCTX_UNKNOWN
};
/* defines whether to specialize by destination annotation, and if so, which
* one */
struct codes_mctx_annotation {
// see canonical name mapping api in codes_mapping.h. -1 is used for
// ignoring annotations
int cid;
};
/* parameters for each mapping context type */
struct codes_mctx_global_direct {
tw_lpid lpid;
};
struct codes_mctx_group_modulo {
struct codes_mctx_annotation anno;
};
// NOTE: group_modulo_reverse shares the group_modulo representation
struct codes_mctx_group_direct {
struct codes_mctx_annotation anno;
int offset;
};
struct codes_mctx {
enum codes_mctx_type type;
union {
struct codes_mctx_global_direct global_direct;
struct codes_mctx_group_modulo group_modulo;
struct codes_mctx_group_direct group_direct;
} u;
};
/* simple setter functions */
struct codes_mctx codes_mctx_set_global_direct(tw_lpid lpid);
struct codes_mctx codes_mctx_set_group_modulo(
char const * annotation,
bool ignore_annotations);
struct codes_mctx codes_mctx_set_group_modulo_reverse(
char const * annotation,
bool ignore_annotations);
struct codes_mctx codes_mctx_set_group_direct(
int offset,
char const * annotation,
bool ignore_annotations);
/* helper function to do a codes mapping - this function is subject to change
* based on what types of ctx exist
* NOTE: in GLOBAL_DIRECT mode, dest_lp_name and sender_gid are ignored */
tw_lpid codes_mctx_to_lpid(
struct codes_mctx const * ctx,
char const * dest_lp_name,
tw_lpid sender_gid);
/* helper function to extract which annotation a various map context maps to.
* annotation is allocated or NULL if unused */
char const * codes_mctx_get_annotation(
struct codes_mctx const *ctx,
char const * dest_lp_name,
tw_lpid sender_id);
#endif /* end of include guard: CODES_MAPPING_CONTEXT_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* I/O workload generator API to be used for reading I/O operations into
* storage system simulations. This API just describes the operations to be
* executed; it does not service the operations.
*/
#ifndef CODES_WORKLOAD_H
#define CODES_WORKLOAD_H
#include <ross.h>
#include "configuration.h"
#define MAX_NAME_LENGTH_WKLD 512
/* implementations included with codes */
typedef struct iomock_params iomock_params;
typedef struct iolang_params iolang_params;
typedef struct darshan_params darshan_params;
typedef struct recorder_params recorder_params;
/* struct to hold the actual data from a single MPI event*/
typedef struct dumpi_trace_params dumpi_trace_params;
typedef struct checkpoint_wrkld_params checkpoint_wrkld_params;
struct iomock_params
{
uint64_t file_id;
int use_uniq_file_ids;
int is_write;
int num_requests;
int request_size;
// for optimizing lookup - set higher (>= num ranks) to reduce collisions
// and 0 to use the default
int rank_table_size;
};
struct iolang_params
{
/* the rank count is defined in the workload config file */
int num_cns;
/* flag - use path to find kernel files relative to the metafile */
int use_relpath;
char io_kernel_meta_path[MAX_NAME_LENGTH_WKLD];
/* set by config in the metadata path */
char io_kernel_path[MAX_NAME_LENGTH_WKLD];
};
struct darshan_params
{
char log_file_path[MAX_NAME_LENGTH_WKLD];
int64_t aggregator_cnt;
};
struct recorder_params
{
char trace_dir_path[MAX_NAME_LENGTH_WKLD];
int64_t nprocs;
};
struct dumpi_trace_params {
char file_name[MAX_NAME_LENGTH_WKLD];
int num_net_traces;
};
struct checkpoint_wrkld_params
{
int nprocs; /* number of workload processes */
double checkpoint_sz; /* size of checkpoint, in TiB */
double checkpoint_wr_bw; /* checkpoint write b/w, in GiB/s */
int total_checkpoints; /* total number of checkpoint phases */
double mtti; /* mean time to interrupt, in hours */
};
/* supported I/O operations */
enum codes_workload_op_type
{
/* terminator; there are no more operations for this rank */
CODES_WK_END = 1,
/* sleep/delay to simulate computation or other activity */
CODES_WK_DELAY,
/* block until specified ranks have reached the same point */
CODES_WK_BARRIER,
/* IO operations */
/* open */
CODES_WK_OPEN,
/* close */
CODES_WK_CLOSE,
/* write */
CODES_WK_WRITE,
/* read */
CODES_WK_READ,
/* network operations (modelled after MPI operations) */
/* blocking send operation */
CODES_WK_SEND,
/* blocking recv operation */
CODES_WK_RECV,
/* non-blocking send operation */
CODES_WK_ISEND,
/* non-blocking receive operation */
CODES_WK_IRECV,
/* broadcast operation */
CODES_WK_BCAST,
/* Allgather operation */
CODES_WK_ALLGATHER,
/* Allgatherv operation */
CODES_WK_ALLGATHERV,
/* Alltoall operation */
CODES_WK_ALLTOALL,
/* Alltoallv operation */
CODES_WK_ALLTOALLV,
/* Reduce operation */
CODES_WK_REDUCE,
/* Allreduce operation */
CODES_WK_ALLREDUCE,
/* Generic collective operation */
CODES_WK_COL,
/* Waitall operation */
CODES_WK_WAITALL,
/* Wait operation */
CODES_WK_WAIT,
/* Waitsome operation */
CODES_WK_WAITSOME,
/* Waitany operation */
CODES_WK_WAITANY,
/* Testall operation */
CODES_WK_TESTALL,
/* MPI request free operation*/
CODES_WK_REQ_FREE,
/* for workloads that have events not yet handled
* (eg the workload language) */
CODES_WK_IGNORE
};
/* I/O operation paramaters */
struct codes_workload_op
{
/* TODO: do we need different "classes" of operations to differentiate
* between different APIs?
*/
/* what type of operation this is */
enum codes_workload_op_type op_type;
/* currently only used by network workloads */
double start_time;
double end_time;
double sim_start_time;
/* parameters for each operation type */
union
{
struct {
double seconds;
double nsecs;
} delay;
struct {
int count; /* num ranks in barrier, -1 means "all" */
int root; /* root rank */
} barrier;
struct {
uint64_t file_id; /* integer identifier for the file */
int create_flag; /* file must be created, not just opened */
} open;
struct {
uint64_t file_id; /* file to operate on */
off_t offset; /* offset and size */
size_t size;
} write;
struct {
uint64_t file_id; /* file to operate on */
off_t offset; /* offset and size */
size_t size;
} read;
struct {
uint64_t file_id; /* file to operate on */
} close;
struct {
/* TODO: not sure why source rank is here */
int source_rank;/* source rank of MPI send message */
int dest_rank; /* dest rank of MPI send message */
int num_bytes; /* number of bytes to be transferred over the network */
int16_t data_type; /* MPI data type to be matched with the recv */
int count; /* number of elements to be received */
int tag; /* tag of the message */
int32_t req_id;
} send;
struct {
/* TODO: not sure why source rank is here */
int source_rank;/* source rank of MPI recv message */
int dest_rank;/* dest rank of MPI recv message */
int num_bytes; /* number of bytes to be transferred over the network */
int16_t data_type; /* MPI data type to be matched with the send */
int count; /* number of elements to be sent */
int tag; /* tag of the message */
int32_t req_id;
} recv;
/* TODO: non-stub for other collectives */
struct {