Commit f32214c7 authored by Jonathan Jenkins's avatar Jonathan Jenkins

Merge codes-base

parents 97c47e86 8f1f93ce
# automake artifacts
/Makefile.in
/aclocal.m4
/autom4te.cache/
/codes_base_config.h.in
/compile
/config.guess
/config.sub
/configure
/depcomp
/install-sh
/missing
# configure artifacts
/config.log
/config.status
/Makefile
/stamp-h1
/codes_base_config.h
/maint/codes-base.pc
.deps
# make generated artifacts
.dirstamp
*.o
*.a
/src/modelconfig/configparser.h
/src/modelconfig/configparser.c
/src/modelconfig/configlex.h
/src/modelconfig/configlex.c
/tests/lp-io-test
# hide backups
*~
# generated files from test runs
ross.csv
COPYRIGHT
The following is a notice of limited availability of the code, and disclaimer
which must be included in the prologue of the code and in all source listings
of the code.
Copyright Notice
+ 2013 University of Chicago
Permission is hereby granted to use, reproduce, prepare derivative works, and
to redistribute to others. This software was authored by:
Mathematics and Computer Science Division
Argonne National Laboratory, Argonne IL 60439
(and)
Computer Science Department
Rensselaer Polytechnic Institute, Troy NY 12180
GOVERNMENT LICENSE
Portions of this material resulted from work developed under a U.S.
Government Contract and are subject to the following license: the Government
is granted for itself and others acting on its behalf a paid-up, nonexclusive,
irrevocable worldwide license in this computer software to reproduce, prepare
derivative works, and perform publicly and display publicly.
DISCLAIMER
This computer code material was prepared, in part, as an account of work
sponsored by an agency of the United States Government. Neither the United
States, nor the University of Chicago, nor any of their employees, makes any
warranty express or implied, or assumes any legal liability or responsibility
for the accuracy, completeness, or usefulness of any information, apparatus,
product, or process disclosed, or represents that its use would not infringe
privately owned rights.
# flex & bison deps
#
%.c %.h: %.l
$(AM_V_GEN)$(LEX) --header-file=$(@:.c=.h) -o $(@:.h=.c) $< \
|| ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
#
# specific rule for codesparser generation; we want the header to land in
# the codes/ directory because it will be installed for use by other repos
#src/iokernellang/codesparser.c codes/codesparser.h: src/iokernellang/codesparser.y
# mkdir -p codes
# @test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
# $(AM_V_GEN)$(YACC) --defines=codes/codesparser.h -o src/iokernellang/codesparser.c $< \
# || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
%.c %.h: %.y
@test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
$(AM_V_GEN)$(YACC) --defines=$(@:.c=.h) -o $(@:.h=.c) $< \
|| ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
# %.y: %.y.in Makefile
# $(AM_V_GEN)$(SED) -e 's,[@]CODES_PURE_PARSER_DEFINES[@],$(CODES_PURE_PARSER_DEFINES),g' \
# -e 's,[@]CODES_PUSH_PARSER_DEFINES[@],$(CODES_PUSH_PARSER_DEFINES),g' \
# < src/common/iokernellang/codesparser.y.in > src/common/iokernellang/codesparser.y
#
# Output dist version
#
.phony: distversion
distversion:
@echo $(VERSION)
#
# Easy way to build unit tests without running them
#
.phony: tests
tests: $(check_PROGRAMS)
AUTOMAKE_OPTIONS = foreign
ACLOCAL_AMFLAGS = -I m4
bin_PROGRAMS =
bin_SCRIPTS =
noinst_LIBRARIES =
noinst_PROGRAMS =
lib_LIBRARIES =
noinst_HEADERS =
TESTS =
check_PROGRAMS =
EXTRA_PROGRAMS =
CLEANFILES = $(bin_SCRIPTS)
EXTRA_DIST =
BUILT_SOURCES =
src_models_CESOURCES =
# pkgconfig files
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = maint/codes-base.pc
$(pkgconfig_DATA): config.status
EXTRA_DIST += prepare.sh COPYRIGHT configure.ac reformat.sh uc-codes.cfg
AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
AM_CFLAGS =
AM_LDFLAGS = ${ROSS_LDFLAGS}
AM_CXXFLAGS = $(AM_CFLAGS)
include Make.rules
include $(top_srcdir)/scripts/Makefile.subdir
include $(top_srcdir)/src/Makefile.subdir
include $(top_srcdir)/tests/Makefile.subdir
include $(top_srcdir)/doc/Makefile.subdir
if USE_DARSHAN
AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
src_libcodes_base_a_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
tests_workload_codes_workload_test_LDADD += ${DARSHAN_LIBS}
tests_workload_codes_workload_mpi_replay_LDADD += ${DARSHAN_LIBS}
TESTS += tests/workload/darshan-dump.sh
endif
if USE_RECORDER
AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
src_libcodes_base_a_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
endif
if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
src_libcodes_base_a_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
tests_workload_codes_workload_test_LDADD += ${DUMPI_LIBS}
tests_workload_codes_workload_mpi_replay_LDADD += ${DUMPI_LIBS}
endif
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_CALLBACK_H
#define CODES_CALLBACK_H
#include <assert.h>
#include "codes/lp-msg.h"
/* This header defines the following conventions for callback-based event
* processing (i.e., RPC-like event flow where the callee sends a "return
* event" to the caller)
*
* callers:
* - give the caller event size (for sanity checking)
* - use the msg_header convention (lp-msg.h)
* - specify an integer tag to identify which call the callee return event
* corresponds to, in the case of multiple pending callbacks
* - specify an offset to the return structure, which is defined by the callees
*
* callees:
* - use the resulting set of offsets to write and issue the callback
*
* The recommended calling convention for these types of event flows is:
* void event_func(<params...>,
* msg_header const * h, int tag, struct codes_cb_info const * cb);
*/
struct codes_cb_info {
int event_size;
int header_offset;
int tag_offset;
int cb_ret_offset;
};
/* helper struct for callees - wrap up the typical inputs */
struct codes_cb_params {
struct codes_cb_info info;
msg_header h;
int tag;
};
/* function-like sugar for initializing a codes_cb_info */
#define INIT_CODES_CB_INFO(_cb_info_ptr, _event_type, _header_field, _tag_field, _cb_ret_field) \
do { \
(_cb_info_ptr)->event_size = sizeof(_event_type); \
(_cb_info_ptr)->header_offset = offsetof(_event_type, _header_field); \
(_cb_info_ptr)->tag_offset = offsetof(_event_type, _tag_field); \
(_cb_info_ptr)->cb_ret_offset = offsetof(_event_type, _cb_ret_field); \
} while (0)
#define CB_HO(_cb_params_ptr) ((_cb_params_ptr)->info.header_offset)
#define CB_TO(_cb_params_ptr) ((_cb_params_ptr)->info.tag_offset)
#define CB_RO(_cb_params_ptr) ((_cb_params_ptr)->info.cb_ret_offset)
/* Declare return variables at the right byte offsets from a codes_cb_params.
* Additionally, set the header and tag vars */
#define GET_INIT_CB_PTRS(_cb_params_ptr, _data_ptr, _sender_gid_val, _header_nm, _tag_nm, _rtn_name, _rtn_type) \
msg_header * _header_nm = \
(msg_header*)((char*)(_data_ptr) + CB_HO(_cb_params_ptr)); \
int * _tag_nm = \
(int*)((char*)(_data_ptr) + CB_TO(_cb_params_ptr));\
_rtn_type * _rtn_name = \
(_rtn_type*)((char*)(_data_ptr) + CB_RO(_cb_params_ptr)); \
msg_set_header((_cb_params_ptr)->h.magic, (_cb_params_ptr)->h.event_type, \
_sender_gid_val, _header_nm); \
*_tag_nm = (_cb_params_ptr)->tag
#define SANITY_CHECK_CB(_cb_info_ptr, _ret_type) \
do { \
int _total_size = sizeof(_ret_type) + sizeof(int) + sizeof(msg_header);\
int _esize = (_cb_info_ptr)->event_size; \
assert(_esize > 0 && \
(_cb_info_ptr)->header_offset >= 0 && \
(_cb_info_ptr)->tag_offset >= 0 && \
(_cb_info_ptr)->cb_ret_offset >= 0); \
assert(_esize >= _total_size); \
assert(_esize >= (_cb_info_ptr)->header_offset + sizeof(msg_header)); \
assert(_esize >= (_cb_info_ptr)->tag_offset + sizeof(int)); \
assert(_esize >= (_cb_info_ptr)->cb_ret_offset + sizeof(_ret_type)); \
} while (0)
#endif /* end of include guard: CODES_CALLBACK_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* SUMMARY - namespace support for multiple groups of ids (jobs) with respect to a
* flat namespace. Note that this API is meant for static job creation - a more
* sophisticated method would need to be used to modify job mappings at
* runtime.
*
* Example:
*
* job 0 1 2 (<-- jobmap defined "job" IDs)
* rank 0 1 2 0 1 0 1 2 3 (<-- "job local" IDs)
* ID 0 1 2 3 4 5 6 7 8 (<-- jobmap-defined LP relative "global" IDs)
* LP A B C D E F G H I (<-- provided by codes-mapping)
* */
#ifndef CODES_JOBMAP_H
#define CODES_JOBMAP_H
/** type markers and parameter defs for jobmaps **/
enum codes_jobmap_type {
/* the "identity" jobmap is a shim for single-job workloads */
CODES_JOBMAP_IDENTITY,
/* the "list" jobmap allows the explicit specification of mappings from
* jobs to lists of global ids through a text file, wiht one line per job
*/
CODES_JOBMAP_LIST,
/* the "dummy" jobmap is an example implementation for testing, and can be
* seen as the inverse of the identity mapping.
* It simply specifies N jobs, with exactly one rank per job, with a trivial
* mapping */
CODES_JOBMAP_DUMMY,
};
struct codes_jobmap_params_identity {
int num_ranks;
};
struct codes_jobmap_params_dummy {
int num_jobs;
};
struct codes_jobmap_params_list {
char *alloc_file;
};
/** jobmap interface **/
struct codes_jobmap_ctx;
struct codes_jobmap_id {
int job;
int rank; // relative to job
};
struct codes_jobmap_ctx *
codes_jobmap_configure(enum codes_jobmap_type t, void const * params);
void codes_jobmap_destroy(struct codes_jobmap_ctx *c);
/* main mapping functions - bidirectional mapping is needed:
* - global -> local ID for initialization
* - local -> global ID for communication between local IDs
*
* functions return {-1, -1} and -1, respectively, for invalid id input */
struct codes_jobmap_id codes_jobmap_to_local_id(
int id,
struct codes_jobmap_ctx const * c);
int codes_jobmap_to_global_id(
struct codes_jobmap_id id,
struct codes_jobmap_ctx const * c);
int codes_jobmap_get_num_jobs(struct codes_jobmap_ctx const * c);
int codes_jobmap_get_num_ranks(int job_id, struct codes_jobmap_ctx const * c);
#endif
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* SUMMARY - data structure and utilities to direct LP mappings. As opposed to
* the codes-mapping API, this defines metadata necessary to allow implicit
* mappings based on LP type and configuration specification
* (see modelnet, LSM, etc)
* mctx stands for mapping context */
#ifndef CODES_MAPPING_CONTEXT_H
#define CODES_MAPPING_CONTEXT_H
#include <stdbool.h>
#include <ross.h>
/* for convenience - an annotation-ignoring "group_modulo" context,
* matching previous mapping behavior in most interfaces (modelnet and such) */
extern struct codes_mctx const * const CODES_MCTX_DEFAULT;
/* types of map contexts */
enum codes_mctx_type {
// instructs those using the context to map directly to an LP
CODES_MCTX_GLOBAL_DIRECT,
// instructs those using the context to map into the same group/repetition
// and compute the callee offset as the modulus of the caller offset and
// the number of callees in the group, to provide simple wraparound
// behaviour
CODES_MCTX_GROUP_MODULO,
// similar to GROUP_MODULO, but maps to offsets in reverse order
CODES_MCTX_GROUP_MODULO_REVERSE,
// instructs those using the context to map into the same group/repetition
// and directly to a callee offset
CODES_MCTX_GROUP_DIRECT,
// unknown/uninitialized context
CODES_MCTX_UNKNOWN
};
/* defines whether to specialize by destination annotation, and if so, which
* one */
struct codes_mctx_annotation {
// see canonical name mapping api in codes_mapping.h. -1 is used for
// ignoring annotations
int cid;
};
/* parameters for each mapping context type */
struct codes_mctx_global_direct {
tw_lpid lpid;
};
struct codes_mctx_group_modulo {
struct codes_mctx_annotation anno;
};
// NOTE: group_modulo_reverse shares the group_modulo representation
struct codes_mctx_group_direct {
struct codes_mctx_annotation anno;
int offset;
};
struct codes_mctx {
enum codes_mctx_type type;
union {
struct codes_mctx_global_direct global_direct;
struct codes_mctx_group_modulo group_modulo;
struct codes_mctx_group_direct group_direct;
} u;
};
/* simple setter functions */
struct codes_mctx codes_mctx_set_global_direct(tw_lpid lpid);
struct codes_mctx codes_mctx_set_group_modulo(
char const * annotation,
bool ignore_annotations);
struct codes_mctx codes_mctx_set_group_modulo_reverse(
char const * annotation,
bool ignore_annotations);
struct codes_mctx codes_mctx_set_group_direct(
int offset,
char const * annotation,
bool ignore_annotations);
/* helper function to do a codes mapping - this function is subject to change
* based on what types of ctx exist
* NOTE: in GLOBAL_DIRECT mode, dest_lp_name and sender_gid are ignored */
tw_lpid codes_mctx_to_lpid(
struct codes_mctx const * ctx,
char const * dest_lp_name,
tw_lpid sender_gid);
/* helper function to extract which annotation a various map context maps to.
* annotation is allocated or NULL if unused */
char const * codes_mctx_get_annotation(
struct codes_mctx const *ctx,
char const * dest_lp_name,
tw_lpid sender_id);
#endif /* end of include guard: CODES_MAPPING_CONTEXT_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* indent-tabs-mode: nil
* End:
*
* vim: ts=8 sts=4 sw=4 expandtab
*/
This diff is collapsed.
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_H
#define CODES_H
#include <ross.h>
#include <assert.h>
// simple deprecation attribute hacking
#if !defined(DEPRECATED)
# if defined(__GNUC__) || defined(__GNUG__) || defined(__clang__)
# define DEPRECATED __attribute__((deprecated))
# else
# define DEPRECATED
# endif
#endif
DEPRECATED
static inline tw_event * codes_event_new(
tw_lpid dest_gid,
tw_stime offset_ts,
tw_lp * sender)
{
tw_stime abs_ts = offset_ts + tw_now(sender);
assert(abs_ts < g_tw_ts_end);
//printf("codes_event_new() abs_ts: %.9f\n", abs_ts);
return(tw_event_new(dest_gid, offset_ts, sender));
}
static inline tw_event * tw_event_new_bounded(
tw_lpid dest_gid,
tw_stime offset_ts,
tw_lp * sender)
{
tw_stime ts = offset_ts + tw_now(sender);
if (ts >= g_tw_ts_end) {
tw_error(TW_LOC,
"LP %lu tried to schedule a message for time %0.5e, "
"%0.5e past the end time\n",
sender->gid, ts, g_tw_ts_end-ts);
return NULL;
}
else
return tw_event_new(dest_gid, offset_ts, sender);
}
/* TODO: validate what value we should use here */
/* Modeled latency for communication between local software components and
* communication between daemons and hardware devices. Measured in
* nanoseconds.
*/
#define CODES_MIN_LATENCY 0.5
#define CODES_MAX_LATENCY 1.0
#define CODES_LATENCY_RANGE \
(CODES_MAX_LATENCY-CODES_MIN_LATENCY)
static inline tw_stime codes_local_latency(tw_lp *lp)
{
int r = g_tw_nRNG_per_lp-1;
tw_stime tmp;
tmp = g_tw_lookahead + CODES_MIN_LATENCY +
tw_rand_unif(&lp->rng[r]) * CODES_LATENCY_RANGE;
if (g_tw_synchronization_protocol == CONSERVATIVE &&
(tw_now(lp) + g_tw_lookahead) >= (tw_now(lp) + tmp))
tw_error(TW_LOC,
"codes_local_latency produced a precision loss "
"sufficient to fail lookahead check (conservative mode) - "
"increase CODES_MIN_LATENCY/CODES_MAX_LATENCY. "
"Now:%0.5le, lookahead:%0.5le, return:%0.5le\n",
tw_now(lp), g_tw_lookahead, tmp);
return(tmp);
}
static inline void codes_local_latency_reverse(tw_lp *lp)
{
int r = g_tw_nRNG_per_lp-1;
tw_rand_reverse_unif(&lp->rng[r]);
return;
}
#endif /* CODES_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
/*
* Copyright (C) 2013 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
/* SUMMARY:
* CODES custom mapping file for ROSS
*/
#include "configuration.h"
#include "codes.h"
#include "lp-type-lookup.h"
#define MAX_NAME_LENGTH 256
/* Returns number of LPs on the current PE */
int codes_mapping_get_lps_for_pe(void);
/* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped.*/
tw_peid codes_mapping( tw_lpid gid);
/* loads the configuration file and sets up the number of LPs on each PE. */
void codes_mapping_setup(void);
/* set up lps with an RNG offset
*
* NOTE: manual seeding is discouraged by the ROSS folks, who instead suggest to
* set the number of RNGs each LP will "skip". offset here is a multiplier by
* the global number of LPs
*/
void codes_mapping_setup_with_seed_offset(int offset);
/*Takes the group name and returns the number of repetitions in the group */
int codes_mapping_get_group_reps(const char* group_name);
/* Calculates the count for LPs of the given type
*
* group_name - name of group. If NULL, count is across all groups.
* ignore_repetitions - if group_name is given, then don't include repetitions
* in count. This exists at the moment to support some
* uses of the function that haven't been fleshed out in
* other parts of the API (used by the dragonfly/torus
* models)
* lp_type_name - name of LP type
* annotation - optional annotation. If NULL, entry is considered
* unannotated
* ignore_annos - If zero, then count in an annotation-specific manner.
* If 1, then count the "first-found" LP in the
* configuration, regardless of annotation.
* Otherwise, count across all annotations.
*
* returns the number of LPs found (0 in the case of some combination of group,
* lp_type_name, and annotation not being found)
*/
int codes_mapping_get_lp_count(
const char * group_name,
int ignore_repetitions,
const char * lp_type_name,
const char * annotation,
int ignore_annos);
/* Calculates the global LP ID given config identifying info.
*
* group_name - name of group
* lp_type_name - name of LP type
* annotation - optional annotation (NULL -> unannotated)
* ignore_anno - ignores annotation and sets gid to the first found LP type
* with matching names
* rep_id - repetition within the group
* offset - lp offset within the repetition
* gid - output ID
*
* If the LP is unable to be found, a tw_error will occur
*/
void codes_mapping_get_lp_id(
const char * group_name,
const char * lp_type_name,
const char * annotation,
int ignore_anno,
int rep_id,
int offset,
tw_lpid * gid);
/* Calculates the LP ID relative to other LPs (0..N-1, where N is the number of
* LPs sharing the same type)
*
* gid - LP ID
* group_wise - whether to compute id relative to the LP's group
* annotation_wise - whether to compute id relative to the annotation the LP has
*/
int codes_mapping_get_lp_relative_id(
tw_lpid gid,
int group_wise,
int annotation_wise);
/* Calculates the ROSS global LP ID of an LP given the relative ID and LP type
* information
*
* relative_id - LP id relative to set of "like" LPs
* group_name - name of LP's group. If non-NULL, ID is considered local
* to that group. If NULL, then ID is considered across all
* groups
* lp_type_name - name of the LP to look up. Must be provided
* annotation - LP's annotation. If NULL, ID is considered among
* unannotated LPs
* annotation_wise - whether to consider ID across a specific annotation (using
* the annotation parameter) or all annotations (ignoring the
* annotation parameter)
*/
tw_lpid codes_mapping_get_lpid_from_relative(
int relative_id,
const char * group_name,
const char * lp_type_name,
const char * annotation,
int annotation_wise);
/* Returns configuration group information for a given LP-id
*
* gid - LP to look up
* group_name - output LP group name
* group_index - index in config struct of group (mostly used internally)
* lp_type_name - output LP type name