Commit e3b18be0 authored by Misbah Mubarak's avatar Misbah Mubarak

creating codes collective modeling branch

parent db56bd38
......@@ -26,6 +26,8 @@ EXTRA_DIST += \
AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
CCLD = $(CXX)
AM_CFLAGS =
AM_CXXFLAGS = $(AM_CFLAGS)
......
......@@ -30,6 +30,7 @@ typedef struct recorder_params recorder_params;
/* struct to hold the actual data from a single MPI event*/
typedef struct dumpi_trace_params dumpi_trace_params;
typedef struct checkpoint_wrkld_params checkpoint_wrkld_params;
typedef struct cortex_wrkld_params cortex_wrkld_params;
struct iomock_params
{
......@@ -80,6 +81,14 @@ struct checkpoint_wrkld_params
double mtti; /* mean time to interrupt, in hours */
};
struct cortex_wrkld_params
{
int nprocs; /* number of processes involved */
int algo_type; /* type of the algorithm could be TREE, GLF and LLF */
int root; /* root process */
int size; /* size of the data transfer*/
};
/* supported I/O operations */
enum codes_workload_op_type
{
......
#ifndef DFLY_BCAST_H
#define DFLY_BCAST_H
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
DFLY_BCAST_TREE = 0,
DFLY_BCAST_LLF,
DFLY_BCAST_GLF
} bcast_type;
typedef void (*send_handler)(int app_id, int rank, int size, int dest, int tag, void* uarg);
typedef void (*recv_handler)(int app_id, int rank, int size, int src, int tag, void* uarg);
typedef struct {
int my_rank;
send_handler do_send;
recv_handler do_recv;
} comm_handler;
/**
* Executes a broadcast across the processes of an app.
* param type : type of broadcast
* param app_id : application id
* param root : root of the broadcast
* param nprocs : number of processes in the application
* param size : size of the data to broadcast (in bytes)
* param comm : comm_handler structure providing send and receive handlers
* param uarg : user-provided pointer that will be passed to callbacks
*/
void dfly_bcast(bcast_type type, int app_id, int root, int nprocs, int size, const comm_handler* comm, void* uarg);
#ifdef __cplusplus
}
#endif
#endif
#ifndef CORTEX_DRAGONFLY_TOPO_API_H
#define CORTEX_DRAGONFLY_TOPO_API_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
/* *********************************************************************************************
Data Structures
********************************************************************************************* */
typedef int16_t job_id_t;
typedef int32_t rank_t;
typedef int32_t terminal_id_t;
typedef int32_t router_id_t;
typedef int32_t group_id_t;
enum {
CORTEX_DFLY_OK = 0,
CORTEX_DFLY_TOPO_NOT_SET,
CORTEX_DFLY_TOPO_INVALID,
CORTEX_DFLY_ID_INVALID
};
/**
* Topology information for a Dragonfly network.
* cortex_dfly_topo gives information about the entire network (not on a per job basis).
*/
typedef struct {
uint16_t cn_per_router; /* number of compute nodes (terminals) connected to a router */
uint16_t routers_per_group; /* number of routers in a group */
uint16_t num_groups; /* number of groups */
uint16_t glink_per_router; /* number of global links per router */
double local_bandwidth; /* bandwidth of the router-router channels within a group */
double global_bandwidth; /* bandwidth of the inter-group router connections */
double cn_bandwidth; /* bandwidth of the compute node channels connected to routers */
} cortex_dfly_topo;
/**********************************************************************************************
Retrieving the information on the topology
********************************************************************************************* */
/**
* Get the router to which the terminal is attached.
* input param: terminal_id
* output param: router_id
*/
int cortex_dfly_get_router_from_terminal(const terminal_id_t terminal_id, router_id_t *router_id);
/**
* Get the group to which the router belongs.
* input param: router_id
* output param: group_id
*/
int cortex_dfly_get_group_from_router(const router_id_t router_id, group_id_t* group_id);
/**
* Get the lower and upper bounds of terminal ids within a router.
* input param: router_id
* output param: min
* output param: max
*/
int cortex_dfly_get_terminals_from_router(const router_id_t router_id, terminal_id_t* min, terminal_id_t* max);
/**
* Get the lower and upper bounds of router ids within a group.
* input param: group_id
* output param: min
* output param: max
*/
int cortex_dfly_get_routers_from_group(const group_id_t group_id, router_id_t* min, router_id_t* max);
/**********************************************************************************************
Retrieving the information on the job
********************************************************************************************* */
/**
* MM: Redundant to cortex_dfly_location_get
* Get the terminal to which the rank of a job is attached.
* input param: job_id
* input param: rank
* output param: terminal_id
*/
int cortex_dfly_job_get_terminal_from_rank(const job_id_t job_id, const rank_t rank, terminal_id_t* terminal_id);
/**
* Get the job id and rank for the given terminal, if such information exists.
* input param: terminal_id
* output param: job_id
* output param: rank
*/
int cortex_dfly_terminal_get_job_and_rank(const terminal_id_t terminal_id, job_id_t* job_id, rank_t* rank);
/**
* Get the number of groups that have ranks executing job with job_id.
* input param: job_id
* output param: num_groups
*/
int cortex_dfly_job_get_group_count(const job_id_t job_id, uint16_t* num_groups);
/**
* Get the list of groups that have ranks executing job with job_id.
* The array passed (group_ids) must be pre-allocated with enough space
* (see cortex_dfly_job_get_group_count).
* input param: job_id
* output param: group_ids
*/
int cortex_dfly_job_get_group_list(const job_id_t job_id, group_id_t* group_ids);
/**
* Get the number of routers of a given group to which ranks of a particular job (job_id) are connected.
* input param: job_id
* input param: group_id
* output param: router_count
**/
int cortex_dfly_job_get_router_count(const job_id_t job_id, const group_id_t group_id, uint16_t* router_count);
/**
* Get the list of routers executing a particular job (job_id) in a group (group_id).
* The array passed (router_ids) must be pre-allocated with enough space
* (see cortex_dfly_job_get_router_count).
* input param: job_id
* input param: group_id
* output_param: group_routers
*/
int cortex_dfly_job_get_router_list(const job_id_t job_id, const group_id_t group_id, router_id_t* router_ids);
/**
* Get the number of terminals within a given router that have a process of the given job.
* input param: job_id
* input param: router_id
* output param: terminal_count
*/
int cortex_dfly_job_get_terminal_count(const job_id_t job_id, const router_id_t router_id, uint16_t* terminal_count);
/**
* Get the list of terminals within a given router that have a process of the given job.
* The array passed (terminal_ids) must be pre-allocated with enough space
* (see cortex_dfly_job_get_terminal_count).
* input param: job_id
* input param: router_id
* output param: terminal_count
*/
int cortex_dfly_job_get_terminal_list(const job_id_t job_id, const router_id_t router_id, terminal_id_t* terminal_ids);
/**********************************************************************************************
Getter functions for dragonfly connectivity (irrespective of the job)
********************************************************************************************* */
/* Note that the number of local and global channels can be extracted from the topology struct
* at any time so we don't need getter functions to get number of local and global channels. */
/**
* Get the list of routers connected to a particular router within a group (irrespective of the job).
* The array passed (local_router_ids) must be pre-allocated with a size corresponding to routers_per_group-1
* (from the topology structure).
* input param: router_id
* output param: local_router_ids
*/
int cortex_dfly_get_local_router_list(const router_id_t router_id, router_id_t* local_router_ids);
/**
* Get the list of routers connected to a particular router across the group (irrespective of the job).
* The array passed (global_router_ids) must be pre-allocated with a size corresponding to glink_per_router
* (from the topology structure).
* input param: router_id
* output param: global_router_ids
*/
int cortex_dfly_get_global_router_list(const router_id_t router_id, router_id_t* global_router_ids);
/**
* Get the number of global links between groups g1 and g2.
* input param: g1
* input param: g2
* output param: count
*/
int cortex_dfly_get_group_link_count(const group_id_t g1, const group_id_t g2, uint32_t* count);
/**
* Get the list of routers r1 in g1 and r2 in g2 such that r1[i] is connected to r2[i].
* r1 and r2 mush be preallocated with a size corresponding to the count returned by
* cortex_dfly_get_group_link_count.
* input param: g1
* input param: g2
* output param: r1
* output param: r2
*/
int cortex_dfly_get_group_link_list(const group_id_t g1, const group_id_t g2, router_id_t* r1, router_id_t* r2);
/**********************************************************************************************
Setting the topology and process placement
********************************************************************************************* */
/**
* Setup function: sets the topology information for CORTEX.
* input param: topology
*/
int cortex_dfly_topology_init();
/**
* finalize cortex topology. This function is used to free up the malloced
* space */
int cortex_dfly_topology_finalize();
/**
* Query topology: Get the information about the topology.
* output param: topology
*/
int cortex_dfly_topology_get(cortex_dfly_topo *topology);
/**
* MM: I don't think we would need this. The rank placement is already
* done by the CODES job mapping API.
* Attaches a process of a given job to a given terminal.
* input param: job_id
* input param: rank
* output param: terminal_id
*/
int cortex_dfly_location_set(const job_id_t job_id, const rank_t rank, const terminal_id_t terminal_id);
/**
* Get the location of a process given its rank.
* input param: job_id
* input param: rank
* output param: terminal_id
*/
int cortex_dfly_location_get(const job_id_t job_id, const rank_t rank, terminal_id_t* terminal_id);
/**
* MM: Not needed, see previous note.
* Attaches all processes from 0 to count-1 to an array of terminal ids.
* input param: job_id
* input param: count
* input param: terminal_ids
*/
int cortex_dfly_location_set_all(const job_id_t job_id, uint32_t count, const terminal_id_t* terminal_ids);
/**
* Get location information for all processes.
* The passed array (terminal_ids) must be allocated with a size of count elements.
* input param: job_id
* input param: count
* output param: terminal_ids
*/
int cortex_dfly_location_get_all(const job_id_t job_id, uint32_t count, terminal_id_t* terminal_ids);
/******************* Additional functions for setting jobmap ********/
/**
* Sets the jobmap information from the allocation file
* This function uses codes jobmap API under the covers
*/
int cortex_dfly_set_jobmap(char * allocation_file);
/**
* Get the number of jobs as specified from the allocation file
* This function uses codes jobmap API under the covers
*/
int cortex_dfly_get_num_jobs();
/**
* Given the job ID, this gets the number of ranks assigned to a particular job
* This function uses codes jobmap API under the covers
*/
int cortex_dfly_get_job_ranks(int job_id);
#ifdef __cplusplus
}
#endif
#endif
......@@ -22,6 +22,7 @@ extern "C" {
#include "model-net.h"
#include "model-net-sched.h"
#include "net/dragonfly.h"
#include "net/slimfly.h"
#include "net/loggp.h"
#include "net/simplenet-upd.h"
#include "net/simplep2p.h"
......@@ -120,12 +121,13 @@ typedef struct model_net_base_msg {
typedef struct model_net_wrap_msg {
msg_header h;
union {
model_net_base_msg m_base; // base lp
terminal_message m_dfly; // dragonfly
loggp_message m_loggp; // loggp
sn_message m_snet; // simplenet
sp_message m_sp2p; // simplep2p
nodes_message m_torus; // torus
model_net_base_msg m_base; // base lp
terminal_message m_dfly; // dragonfly
slim_terminal_message m_slim; // slimfly
loggp_message m_loggp; // loggp
sn_message m_snet; // simplenet
sp_message m_sp2p; // simplep2p
nodes_message m_torus; // torus
// add new ones here
} msg;
} model_net_wrap_msg;
......
......@@ -60,6 +60,7 @@ typedef struct mn_stats mn_stats;
X(SIMPLENET, "modelnet_simplenet", "simplenet", &simplenet_method)\
X(SIMPLEP2P, "modelnet_simplep2p", "simplep2p", &simplep2p_method)\
X(TORUS, "modelnet_torus", "torus", &torus_method)\
X(SLIMFLY, "modelnet_slimfly", "slimfly", &slimfly_method)\
X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\
X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\
X(LOGGP, "modelnet_loggp", "loggp", &loggp_method)\
......
/*
* Copyright (C) 2014 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef SLIMFLY_H
#define SLIMFLY_H
#include <ross.h>
typedef struct slim_terminal_message slim_terminal_message;
/* this message is used for both dragonfly compute nodes and routers */
struct slim_terminal_message
{
/* magic number */
int magic;
/* flit travel start time*/
tw_stime travel_start_time;
/* packet ID of the flit */
unsigned long long packet_ID;
/* event type of the flit */
short type;
/* category: comes from codes */
char category[CATEGORY_NAME_MAX];
/* final destination LP ID, this comes from codes can be a server or any other LP type*/
tw_lpid final_dest_gid;
/*sending LP ID from CODES, can be a server or any other LP type */
tw_lpid sender_lp;
tw_lpid sender_mn_lp; // source modelnet id
/* destination terminal ID of the dragonfly */
tw_lpid dest_terminal_id;
/* source terminal ID of the dragonfly */
unsigned int src_terminal_id;
/* local LP ID to calculate the radix of the sender node/router */
unsigned int local_id;
/* message originating router id */
unsigned int origin_router_id;
/* number of hops traversed by the packet */
short my_N_hop;
short my_l_hop, my_g_hop;
short saved_channel;
/* Intermediate LP ID from which this message is coming */
unsigned int intm_lp_id;
short new_vc;
short saved_vc;
/* last hop of the message, can be a terminal, local router or global router */
short last_hop;
/* For routing */
int intm_group_id;
int intm_router_id;
int chunk_id;
uint64_t packet_size;
uint64_t message_id;
uint64_t total_size;
int saved_remote_esize;
int remote_event_size_bytes;
int local_event_size_bytes;
// For buffer message
short vc_index;
int sender_radix;
int output_chan;
model_net_event_return event_rc;
int is_pull;
uint64_t pull_size;
/* for reverse computation */
short path_type;
tw_stime saved_available_time;
tw_stime saved_avg_time;
tw_stime saved_credit_time;
tw_stime saved_collective_init_time;
tw_stime saved_hist_start_time;
tw_stime msg_start_time;
short saved_completed_chunks;
int saved_hist_num;
int saved_occupancy;
/* for reverse computation of a node's fan in*/
int saved_fan_nodes;
tw_lpid sender_svr;
/* LP ID of the sending node, has to be a network node in the dragonfly */
tw_lpid sender_node;
tw_lpid next_stop;
struct sfly_qhash_entry * saved_hash;
};
#endif /* end of include guard: DRAGONFLY_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
......@@ -80,10 +80,13 @@ nobase_include_HEADERS = \
codes/model-net-sched.h \
codes/model-net-inspect.h \
codes/net/dragonfly.h \
codes/net/slimfly.h \
codes/net/loggp.h \
codes/net/simplenet-upd.h \
codes/net/simplep2p.h \
codes/net/torus.h
codes/net/torus.h \
codes/cortex/dragonfly-cortex-api.h \
codes/cortex/dfly_bcast.h
#codes/codes-nw-workload.h
......@@ -133,12 +136,14 @@ src_libcodes_a_SOURCES = \
src/workload/methods/codes-checkpoint-wrkld.c \
src/workload/methods/test-workload-method.c \
src/workload/methods/codes-iomock-wrkld.c \
src/workload/methods/codes-cortex-workload.c \
codes/rc-stack.h \
src/util/rc-stack.c \
src/networks/model-net/model-net.c \
src/networks/model-net/simplenet-upd.c \
src/networks/model-net/torus.c \
src/networks/model-net/dragonfly.c \
src/networks/model-net/slimfly.c \
src/networks/model-net/loggp.c \
src/networks/model-net/simplep2p.c \
src/networks/model-net/model-net-lp.c \
......@@ -148,7 +153,9 @@ src_libcodes_a_SOURCES = \
src/network-workloads/model-net-mpi-wrklds.c \
src/network-workloads/model-net-mpi-replay.c \
src/network-workloads/model-net-synthetic.c \
src/network-workloads/model-net-dumpi-traces-dump.c
src/network-workloads/model-net-dumpi-traces-dump.c \
src/cortex/dragonfly-cortex-impl.c \
src/cortex/dfly_bcast.cpp
#codes/codes-nw-workload.h
......@@ -166,17 +173,18 @@ src_util_templates_lp_template_dummy_main_SOURCES = \
bin_PROGRAMS += src/workload/codes-workload-dump
bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
bin_PROGRAMS += src/network-workloads/model-net-mpi-wrklds
#bin_PROGRAMS += src/network-workloads/model-net-mpi-wrklds
bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
bin_PROGRAMS += src/network-workloads/model-net-synthetic
bin_PROGRAMS += src/cortex/dragonfly-cortex-test
src_workload_codes_workload_dump_SOURCES = \
src/workload/codes-workload-dump.c
src_network_workloads_model_net_mpi_replay_SOURCES = src/network-workloads/model-net-mpi-replay.c
src_network_workloads_model_net_mpi_wrklds_SOURCES = src/network-workloads/model-net-mpi-wrklds.c
src_cortex_dragonfly_cortex_test_SOURCES = src/cortex/dragonfly-cortex-test.c
#src_network_workloads_model_net_mpi_wrklds_SOURCES = src/network-workloads/model-net-mpi-wrklds.c
src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c
src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c
......
This diff is collapsed.
This diff is collapsed.
#include "codes/cortex/dragonfly-cortex-api.h"
#include "codes/codes.h"
#include "codes/configuration.h"
/**** Caution: turning this on could generate a lot of output ********/
#define RANK_DBG 1
#define JOB_DBG 0
#define dprintf(_fmt, ...) \
do {if (RANK_DBG) printf(_fmt, __VA_ARGS__);} while (0)
#define jprintf(_fmt, ...) \
do {if (JOB_DBG) printf(_fmt, __VA_ARGS__);} while (0)
/* configuration file name */
static char conf_file_name[4096] = {'\0'};
static char alloc_file_name[4096] = {'\0'};
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
int rank, nprocs;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
assert(argv[1] && argv[2]);
strcpy(conf_file_name, argv[1]);
strcpy(alloc_file_name, argv[2]);
dprintf("\n Loading config file %s and alloc file %s ",
conf_file_name,
alloc_file_name);
/* loading the CODES config file */
configuration_load(conf_file_name, MPI_COMM_WORLD, &config);
cortex_dfly_topology_init();
/* Now populate the cortex data structure */
cortex_dfly_topo topo;
cortex_dfly_topology_get(&topo);
/* Setup call for the job mapping */
cortex_dfly_set_jobmap(alloc_file_name);
int num_routers = topo.routers_per_group;
int cn_per_router = topo.cn_per_router;
int num_glinks_per_router = topo.glink_per_router;
int num_jobs = cortex_dfly_get_num_jobs();
int ret;
int group_id;
router_id_t src_rtr;
router_id_t dest_rtr;
terminal_id_t min_t, max_t;
terminal_id_t terminal_id;
router_id_t min_rtr, max_rtr;
router_id_t router_id;
job_id_t job_id;
rank_t rank_id;
router_id_t* local_router_ids = malloc(sizeof(router_id_t) * (num_routers - 1));
router_id_t* global_router_ids = malloc(sizeof(router_id_t) * num_glinks_per_router);
/* In the test config ile, there are two jobs, query their information */
for(int jid = 0; jid < num_jobs; jid++)
{
int num_ranks_per_job = cortex_dfly_get_job_ranks(jid);
for(uint32_t rank = 0; rank < num_ranks_per_job; rank++)
{
ret = cortex_dfly_location_get(jid, rank, &terminal_id);
assert(ret == 0);
/* Now get my router information */
ret = cortex_dfly_get_router_from_terminal(terminal_id, &router_id);
assert(ret == 0);
/* Now get the local neighboring router list for my router */
ret = cortex_dfly_get_local_router_list(router_id, local_router_ids);
assert(ret == 0);
/* get the global neighboring router list for my router */
ret = cortex_dfly_get_global_router_list(router_id, global_router_ids);
assert(ret == 0);
/* Now get the group id for my router */
ret = cortex_dfly_get_group_from_router(router_id, &group_id);
assert(ret == 0);
/* pick a randomly selected group and find out the router
* connections */
int rand_dst_grp = rand() % (topo.num_groups - 1);
while(rand_dst_grp == group_id)
rand_dst_grp = rand() % (topo.num_groups - 1);
ret = cortex_dfly_get_group_link_list(group_id, rand_dst_grp, &src_rtr, &dest_rtr);
assert(ret == 0);
/* Now get range of terminals for this router */
ret = cortex_dfly_get_terminals_from_router(router_id, &min_t, &max_t);
assert(ret == 0);
assert(min_t <= terminal_id <= max_t);
ret = cortex_dfly_get_routers_from_group(group_id, &min_rtr, &max_rtr);
assert(ret == 0 && min_rtr <= router_id <= max_rtr);
ret = cortex_dfly_terminal_get_job_and_rank(terminal_id, &job_id, &rank_id);
assert(ret == 0 && job_id == jid && rank_id == rank);
dprintf("\n \n Rank %d Job ID %d Global Rank %d Router %d Group ID %d ", rank, jid, terminal_id, router_id, group_id);
for(int i = 0; i < num_routers - 1; i++)
{
dprintf("\n Local neighbor %d ", local_router_ids[i]);
}
for(int i = 0; i < num_glinks_per_router; i++)
{
dprintf("\n Global neighbor %d ", global_router_ids[i]);
}
dprintf("\n Connection check: Group %d and Group %d routers %d and %d ", group_id,
rand_dst_grp,
src_rtr,
dest_rtr);
}
uint16_t g_count;
uint16_t term_count;
uint16_t router_count;
ret = cortex_dfly_job_get_group_count(jid, &g_count);
assert(ret == 0);
jprintf("\n **** Group count for job %d is %d **** ", jid, g_count);
group_id_t* group_ids = malloc(sizeof(group_id_t) * g_count);
ret = cortex_dfly_job_get_group_list(jid, group_ids);
assert(ret == 0);
for(int g = 0; g < g_count; g++)
{
ret = cortex_dfly_job_get_router_count(jid, group_ids[g], &router_count);
assert(ret == 0);
router_id_t* router_ids = malloc(sizeof(router_id_t) * router_count);
jprintf("\n \n Job id: %d Group id: %d Router count %d ", jid, group_ids[g], router_count);
ret = cortex_dfly_job_get_router_list(jid, group_ids[g], router_ids);
assert(ret == 0);
for(int r = 0; r < router_count; r++)
{
ret = cortex_dfly_job_get_terminal_count(jid, router_ids[r], &term_count);
assert(ret == 0);
jprintf("\n Router ID %d Terminal count %d ", router_ids[r], term_count);
terminal_id_t* terminal_ids = malloc(sizeof(terminal_id_t) * term_count);
ret = cortex_dfly_job_get_terminal_list(jid, router_ids[r], terminal_ids);
assert(ret == 0);
for(int t = 0; t < term_count; t++)
{
jprintf("\n Terminal id %d ", terminal_ids[t]);
}
free(terminal_ids);
}
free(router_ids);
}
free(group_ids);
}
if(ret == 0)
printf("\n Success! Turn on debug options in the dragonfly-cortex-test source to generate detailed output. See README for info.\n");
cortex_dfly_topology_finalize();
MPI_Finalize();
}
This diff is collapsed.
......@@ -257,6 +257,8 @@ void model_net_base_configure(){
// note: dragonfly router uses the same event struct
msg_offsets[DRAGONFLY_ROUTER] =
offsetof(model_net_wrap_msg, msg.m_dfly);
msg_offsets[SLIMFLY] =
offsetof(model_net_wrap_msg, msg.m_slim);
msg_offsets[LOGGP] =
offsetof(model_net_wrap_msg, msg.m_loggp);
<