Commit 4e19d940 authored by Jonathan Jenkins's avatar Jonathan Jenkins
Browse files

config overhaul + annotation-aware mapping

All configuration now proceeds on a per-LP level, and requires separate
registration and configuration calls, as seen in the test programs.
model_net_set_params is no longer used, and is replaced by model_net_register
and model_net_configure. The dragonfly network, having two LP types bundled in
the same code-path, is special-cased in the registration code.

LP-mapping in model-net now has the following defaults:
- counts via codes_mapping_get_lp_count are now with respect to the calling
  network LP's annotation.
- when looking up network LPs via
  codes_mapping_get_lp_info/codes_mapping_get_lp_id, the annotation of the
  calling network LP is used. Hence, routing now occurs only between LPs of the
  same annotation. If the destination LP's group specified by model_net_*event
  does not contain a modelnet LP with the same annotation as the modelnet LP in
  the sender's group, then an error will occur (in codes_mapping).

Known Issues:
- modelnet users currently cannot specify which modelnet LP to use in the case
  of multiple modelnet LPs in the sender's group. This will be fixed in future
  commits after a consensus is reached on the best way to expose this
parent c65c84a1
......@@ -24,6 +24,12 @@
extern int model_net_base_magic;
// register the networks with ROSS, given the array of flags, one for each
// network type
void model_net_base_register(int *do_config_nets);
// configure the base LP type, setting up general parameters
void model_net_base_configure();
// Construct a model-net-specific event, analagous to a tw_event_new and
// codes_event_new. The difference here is that we return pointers to
// both the message data (to be cast into the appropriate type) and the
......@@ -49,7 +55,7 @@ void * model_net_method_get_edata(int net_id, void * msg);
/// use them
// init method
void model_net_base_init(int id_count, int *ids);
void model_net_base_init(int *do_config_nets);
enum model_net_base_event_type {
......@@ -12,7 +12,7 @@
struct model_net_method
uint64_t packet_size; /* packet size */
void (*mn_setup)(const void* net_params); /* For initializing the network */
void (*mn_configure)(); /* For initializing the network */
tw_stime (*model_net_method_packet_event)(
char* category,
tw_lpid final_dest_lp,
......@@ -30,10 +30,6 @@ enum sched_type {
extern char * sched_names[];
/// global for scheduler
/// TODO: move away from using the global for when we have multiple networks
extern enum sched_type mn_sched_type;
/// scheduler decls
typedef struct model_net_sched_s model_net_sched;
......@@ -24,7 +24,8 @@
* messages as part of an event and expects FCFS ordering. A proper fix which
* involves model-net LP-level scheduling of requests is ideal, but not
* feasible for now (would basically have to redesign model-net), so expose
* explicit start-sequence and stop-sequence markers as a workaround */
* explicit start-sequence and stop-sequence markers as a workaround
extern int in_sequence;
extern tw_stime mn_msg_offset;
#define MN_START_SEQ() do {\
......@@ -36,11 +37,6 @@ extern tw_stime mn_msg_offset;
} while (0)
typedef struct simplenet_param simplenet_param;
typedef struct simplewan_param simplewan_param;
typedef struct dragonfly_param dragonfly_param;
typedef struct torus_param torus_param;
typedef struct loggp_param loggp_param;
typedef struct mn_stats mn_stats;
// use the X-macro to get types and names rolled up into one structure
......@@ -93,53 +89,18 @@ struct mn_stats
long max_event_size;
/* structs for initializing a network/ specifying network parameters */
struct loggp_param
char* net_config_file; /* file with loggp parameter table */
/* structs for initializing a network/ specifying network parameters */
struct simplenet_param
double net_startup_ns; /*simplenet startup cost*/
double net_bw_mbps; /*Link bandwidth per byte*/
int num_nics;
struct simplewan_param
char bw_filename[MAX_NAME_LENGTH];
char startup_filename[MAX_NAME_LENGTH];
/* Registers all model-net LPs in ROSS. Should be called after
* configuration_load, but before codes_mapping_setup */
void model_net_register();
struct dragonfly_param
int num_routers; /*Number of routers in a group*/
double local_bandwidth;/* bandwidth of the router-router channels within a group */
double global_bandwidth;/* bandwidth of the inter-group router connections */
double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
int num_vcs; /* number of virtual channels */
int local_vc_size; /* buffer size of the router-router channels */
int global_vc_size; /* buffer size of the global channels */
int cn_vc_size; /* buffer size of the compute node channels */
short routing; /* minimal or non-minimal routing */
short chunk_size; /* full-sized packets are broken into smaller chunks.*/
struct torus_param
int n_dims; /*Dimension of the torus network, 5-D, 7-D or any other*/
int* dim_length; /*Length of each torus dimension*/
double link_bandwidth;/* bandwidth for each torus link */
int buffer_size; /* number of buffer slots for each vc in flits*/
int num_vc; /* number of virtual channels for each torus link */
float mean_process;/* mean process time for each flit */
int chunk_size; /* chunk is the smallest unit--default set to 32 */
/* In general we need to figure out how to pass configuration information to
* the methods and we need to be able to calculate ross event message size.
/* Configures all model-net LPs based on the CODES configuration, and returns
* ids to address the different types by.
* id_count - the output number of networks
* return - the set of network IDs, indexed in the order given by the
* modelnet_order configuration parameter */
int* model_net_configure(int *id_count);
/* Initialize/configure the network(s) based on the CODES configuration.
* returns an array of the network ids, indexed in the order given by the
This diff is collapsed.
......@@ -26,15 +26,6 @@
/*Define loggp data types and structs*/
typedef struct loggp_state loggp_state;
struct loggp_state
/* next idle times for network card, both inbound and outbound */
tw_stime net_send_next_idle;
tw_stime net_recv_next_idle;
struct mn_stats loggp_stats_array[CATEGORY_MAX];
/* loggp parameters for a given msg size, as reported by netgauge */
struct param_table_entry
......@@ -50,8 +41,32 @@ struct param_table_entry
double G;
double lsqu_gG;
struct param_table_entry param_table[100];
static int param_table_size = 0;
typedef struct param_table_entry param_table_entry;
typedef struct loggp_param loggp_param;
// loggp parameters
struct loggp_param
int table_size;
param_table_entry table[100];
struct loggp_state
/* next idle times for network card, both inbound and outbound */
tw_stime net_send_next_idle;
tw_stime net_recv_next_idle;
const char * anno;
const loggp_param *params;
struct mn_stats loggp_stats_array[CATEGORY_MAX];
/* annotation-specific parameters (unannotated entry occurs at the
* last index) */
static uint64_t num_params = 0;
static loggp_param * all_params = NULL;
static const config_anno_map_t * anno_map = NULL;
static int loggp_magic = 0;
......@@ -73,21 +88,11 @@ static void loggp_collective();
/* collective network calls-- rc */
static void loggp_collective_rc();
/* allocate a new event that will pass through loggp to arriave at its
* destination:
* - category: category name to associate with this communication
* - final_dest_gid: the LP that the message should be delivered to.
* - event_size_bytes: size of event msg that will be delivered to
* final_dest_gid.
* - local_event_size_byte: size of event message that will delivered to
* local LP upon local send comletion (set to 0 if not used)
* - net_msg_size_bytes: size of simulated network message in bytes.
* - sender: LP calling this function.
/* Modelnet interface events */
/* sets up the loggp parameters through modelnet interface */
static void loggp_setup(const void* net_params);
static void loggp_configure();
static void loggp_set_params(const char * config_file, loggp_param * params);
/* Issues a loggp packet event call */
static tw_stime loggp_packet_event(
......@@ -112,12 +117,14 @@ static void loggp_report_stats();
static tw_lpid loggp_find_local_device(tw_lp *sender);
static struct param_table_entry* find_params(uint64_t msg_size);
static const struct param_table_entry* find_params(
uint64_t msg_size,
const loggp_param *params);
/* data structure for model-net statistics */
struct model_net_method loggp_method =
.mn_setup = loggp_setup,
.mn_configure = loggp_configure,
.model_net_method_packet_event = loggp_packet_event,
.model_net_method_packet_event_rc = loggp_packet_event_rc,
.mn_get_lp_type = loggp_get_lp_type,
......@@ -220,6 +227,14 @@ static void loggp_init(
ns->net_send_next_idle = tw_now(lp);
ns->net_recv_next_idle = tw_now(lp);
ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid);
if (ns->anno == NULL)
ns->params = &all_params[num_params-1];
int id = configuration_get_annotation_index(ns->anno, anno_map);
ns->params = &all_params[id];
bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
loggp_magic = h1+h2;
/* printf("\n loggp_magic %d ", loggp_magic); */
......@@ -324,9 +339,9 @@ static void handle_msg_ready_event(
loggp_message *m_new;
struct mn_stats* stat;
double recv_time;
struct param_table_entry *param;
const struct param_table_entry *param;
param = find_params(m->net_msg_size_bytes);
param = find_params(m->net_msg_size_bytes, ns->params);
recv_time = ((double)(m->net_msg_size_bytes-1)*param->G);
/* scale to nanoseconds */
......@@ -420,9 +435,9 @@ static void handle_msg_start_event(
char lp_group_name[MAX_NAME_LENGTH];
int total_event_size;
double xmit_time;
struct param_table_entry *param;
const struct param_table_entry *param;
param = find_params(m->net_msg_size_bytes);
param = find_params(m->net_msg_size_bytes, ns->params);
total_event_size = model_net_get_msg_sz(LOGGP) + m->event_size_bytes +
......@@ -465,7 +480,7 @@ static void handle_msg_start_event(
// TODO: make annotation-aware
codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &mapping_grp_id,
NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1,
codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0,
mapping_rep_id, mapping_offset, &dest_id);
void *m_data;
......@@ -574,52 +589,76 @@ static tw_stime loggp_packet_event(
return xfer_to_nic_time;
static void loggp_setup(const void* net_params)
loggp_param* loggp_params = (loggp_param*)net_params;
static void loggp_configure(){
char config_file[MAX_NAME_LENGTH];
anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
all_params = malloc(num_params * sizeof(*all_params));
for (uint64_t i = 0; i < anno_map->num_annos; i++){
const char * anno = anno_map->annotations[i];
int rc = configuration_get_value_relpath(&config, "PARAMS",
"net_config_file", anno, config_file, MAX_NAME_LENGTH);
if (rc == 0){
tw_error(TW_LOC, "unable to read PARAMS:net_config_file@%s",
loggp_set_params(config_file, &all_params[i]);
if (anno_map->has_unanno_lp > 0){
int rc = configuration_get_value_relpath(&config, "PARAMS",
"net_config_file", NULL, config_file, MAX_NAME_LENGTH);
if (rc == 0){
tw_error(TW_LOC, "unable to read PARAMS:net_config_file");
loggp_set_params(config_file, &all_params[anno_map->num_annos]);
void loggp_set_params(const char * config_file, loggp_param * params){
FILE *conf;
int ret;
char buffer[512];
int line_nr = 0;
printf("Loggp configured to use parameters from file %s\n", config_file);
/* TODO: update this to read on one proc and then broadcast */
printf("Loggp configured to use parameters from file %s\n", loggp_params->net_config_file);
conf = fopen(loggp_params->net_config_file, "r");
conf = fopen(config_file, "r");
params->table_size = 0;
while(fgets(buffer, 512, conf))
if(buffer[0] == '#')
ret = sscanf(buffer, "%llu %d %lf %lf %lf %lf %lf %lf %lf %lf %lf",
if(ret != 11)
fprintf(stderr, "Error: malformed line %d in %s\n", line_nr,
printf("Parsed %d loggp table entries.\n", param_table_size);
printf("Parsed %d loggp table entries.\n", params->table_size);
......@@ -634,25 +673,26 @@ static void loggp_packet_event_rc(tw_lp *sender)
/* find the parameters corresponding to the message size we are transmitting
static struct param_table_entry* find_params(uint64_t msg_size)
static const struct param_table_entry* find_params(
uint64_t msg_size,
const loggp_param *params) {
int i;
/* pick parameters based on the next smallest size in the table, but
* default to beginning or end of table if we are out of range
for(i=0; i<param_table_size; i++)
for(i=0; i < params->table_size; i++)
if(param_table[i].size > msg_size)
if(params->table[i].size > msg_size)
i = param_table_size-1;
i = params->table_size-1;
static tw_lpid loggp_find_local_device(tw_lp *sender)
......@@ -23,18 +23,32 @@ int model_net_base_magic;
// issues...
static int msg_offsets[MAX_NETS];
typedef struct model_net_base_params {
enum sched_type stype;
uint64_t packet_size;
} model_net_base_params;
/* annotation-specific parameters (unannotated entry occurs at the
* last index) */
static int num_params = 0;
static const char * annos[CONFIGURATION_MAX_ANNOS];
static model_net_base_params all_params[CONFIGURATION_MAX_ANNOS];
typedef struct model_net_base_state {
int net_id;
// whether scheduler loop is running
int in_sched_loop;
// model-net scheduler
model_net_sched *sched;
// parameters
const model_net_base_params * params;
// lp type and state of underlying model net method - cache here so we
// don't have to constantly look up
const tw_lptype *sub_type;
void *sub_state;
} model_net_base_state;
......@@ -93,44 +107,132 @@ tw_lptype model_net_base_lp = {
void model_net_base_init(int id_count, int *ids){
void model_net_base_register(int *do_config_nets){
// here, we initialize ALL lp types to use the base type
for (int i = 0; i < MAX_NETS; i++){
if (do_config_nets[i]){
lp_type_register(model_net_lp_config_names[i], &model_net_base_lp);
// HACK: unfortunately, we need to special-case dragonfly
// registration at the moment - there are two LPs, and
// previously the LP matched on configuration initialized
// the "router" LP. Now that the base LP is in charge of
// registration, we need to take care of it
// TODO: fix the interface to have underlying networks do
// the registration
if (i==DRAGONFLY){
static void base_read_config(const char * anno, model_net_base_params *p){
char sched[MAX_NAME_LENGTH];
long int packet_size_l = 0;
uint64_t packet_size;
int ret;
// TODO: make this annotation-specific - put in the config loop, make part
// of model-net base
ret = configuration_get_value(&config, "PARAMS", "modelnet_scheduler",
anno, sched, MAX_NAME_LENGTH);
configuration_get_value_longint(&config, "PARAMS", "packet_size", anno,
packet_size = packet_size_l;
if (ret > 0){
int i;
for (i = 0; i < MAX_SCHEDS; i++){
if (strcmp(sched_names[i], sched) == 0){
p->stype = i;
if (i == MAX_SCHEDS){
tw_error(TW_LOC,"Unknown value for PARAMS:modelnet-scheduler : "
"%s\n", sched);
// default: FCFS
p->stype = MN_SCHED_FCFS;
if (p->stype == MN_SCHED_FCFS_FULL){
// override packet size to something huge (leave a bit in the unlikely
// case that an op using packet size causes overflow)
packet_size = 1ull << 62;
else if (!packet_size && p->stype != MN_SCHED_FCFS_FULL)
packet_size = 512;
fprintf(stderr, "Warning, no packet size specified, setting packet size to %llu\n", packet_size);
p->packet_size = packet_size;
void model_net_base_configure(){
uint32_t h1=0, h2=0;
bj_hashlittle2(MN_NAME, strlen(MN_NAME), &h1, &h2);
model_net_base_magic = h1+h2;
// here, we initialize ALL lp types to use the base type
// TODO: only initialize ones that are actually used
for (int i = 0; i < id_count; i++){
lp_type_register(model_net_lp_config_names[ids[i]], &model_net_base_lp);
// set up offsets - doesn't matter if they are actually used or not
msg_offsets[SIMPLENET] =
offsetof(model_net_wrap_msg, msg.m_snet);
msg_offsets[SIMPLEWAN] =
offsetof(model_net_wrap_msg, msg.m_swan);
msg_offsets[TORUS] =
offsetof(model_net_wrap_msg, msg.m_torus);
msg_offsets[DRAGONFLY] =
offsetof(model_net_wrap_msg, msg.m_dfly);
msg_offsets[LOGGP] =
offsetof(model_net_wrap_msg, msg.m_loggp);
// perform the configuration(s)
// This part is tricky, as we basically have to look up all annotations that
// have LP names of the form modelnet_*. For each of those, we need to read
// the base parameters
// - the init is a little easier as we can use the LP-id to look up the
// annotation
// first grab all of the annotations and store locally
for (int c = 0; c < lpconf.lpannos_count; c++){
const config_anno_map_t *amap = &lpconf.lpannos[c];
if (strncmp("modelnet_", amap->lp_name, 9) == 0){
for (int n = 0; n < amap->num_annos; n++){
int a;
for (a = 0; a < num_params; a++){
if (annos[a] != NULL &&
strcmp(amap->annotations[n], annos[a]) == 0){
if (a == num_params){
// found a new annotation
annos[num_params++] = amap->annotations[n];
if (amap->has_unanno_lp){
int a;
for (a = 0; a < num_params; a++){
if (annos[a] == NULL)
if (a == num_params){
// found a new (empty) annotation
annos[num_params++] = NULL;
// initialize the msg-specific offsets
for (int i = 0; i < id_count; i++){
msg_offsets[SIMPLENET] =
offsetof(model_net_wrap_msg, msg.m_snet);
msg_offsets[SIMPLEWAN] =
offsetof(model_net_wrap_msg, msg.m_swan);
case TORUS:
msg_offsets[TORUS] =
offsetof(model_net_wrap_msg, msg.m_torus);
msg_offsets[DRAGONFLY] =
offsetof(model_net_wrap_msg, msg.m_dfly);
case LOGGP:
msg_offsets[LOGGP] =
offsetof(model_net_wrap_msg, msg.m_loggp);
// now that we have all of the annos for all of the networks, loop through
// and read the configs
for (int i = 0; i < num_params; i++){
base_read_config(annos[i], &all_params[i]);
......@@ -138,14 +240,20 @@ void model_net_base_lp_init(
model_net_base_state * ns,
tw_lp * lp){
// obtain the underlying lp type through codes-mapping
char lp_type_name[MAX_NAME_LENGTH];
char lp_type_name[MAX_NAME_LENGTH], anno[MAX_NAME_LENGTH];
int dummy;
// TODO: ignoring annotation to get things up and running, but we'll want