Commit a8ac5b4e authored by Pavan Balaji's avatar Pavan Balaji Committed by Sangmin Seo
Browse files

Remove the IB netmod.


Signed-off-by: Sangmin Seo's avatarSangmin Seo <sseo@anl.gov>
parent f02bffbe
......@@ -9,7 +9,6 @@
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/tcp/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/none/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/portals4/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/mxm/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/ofi/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/llc/Makefile.mk
## -*- Mode: Makefile; -*-
## vim: set ft=automake :
##
## (C) 2013 by Argonne National Laboratory.
## See COPYRIGHT in top-level directory.
##
if BUILD_NEMESIS_NETMOD_IB
mpi_core_sources += \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c
noinst_HEADERS += \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h \
src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h
endif BUILD_NEMESIS_NETMOD_IB
# The Fortran related cross compilation values.
# This file is generated with mpich/maint/fcrosscompile/configure
# with CC/F77/FC set to "icc/ifort -mmic".
CROSS_F77_SIZEOF_INTEGER="4"
CROSS_F77_SIZEOF_REAL="4"
CROSS_F77_SIZEOF_DOUBLE_PRECISION="8"
CROSS_F77_TRUE_VALUE="-1"
CROSS_F77_FALSE_VALUE="0"
CROSS_F90_ADDRESS_KIND="8"
CROSS_F90_OFFSET_KIND="8"
CROSS_F90_INTEGER_KIND="4"
CROSS_F90_REAL_MODEL=" 6 , 37"
CROSS_F90_DOUBLE_MODEL=" 15 , 307"
CROSS_F90_INTEGER_MODEL=" 9"
CROSS_F90_ALL_INTEGER_MODELS=" 2 , 1, 4 , 2, 9 , 4, 18 , 8,"
CROSS_F90_INTEGER_MODEL_MAP=" { 2 , 1 , 1 }, { 4 , 2 , 2 }, { 9 , 4 , 4 }, { 18 , 8 , 8 },"
**MPIDI_PG_GetConnKVSname:MPIDI_PG_GetConnKVSname failed
**MPID_nem_ib_cm_cas:MPID_nem_ib_cm_cas failed
**MPID_nem_ib_cm_cas_release:MPID_nem_ib_cm_cas_release failed
**MPID_nem_ib_cm_cas_release_core:MPID_nem_ib_cm_cas_release_core failed
**MPID_nem_ib_cm_connect_cas_core:MPID_nem_ib_cm_connect_cas_core failed
**MPID_nem_ib_cm_drain_rcq:MPID_nem_ib_cm_drain_rcq failed
**MPID_nem_ib_cm_drain_scq:MPID_nem_ib_cm_drain_scq failed
**MPID_nem_ib_cm_poll:MPID_nem_ib_cm_poll failed
**MPID_nem_ib_cm_poll_syn:MPID_nem_ib_cm_poll_syn failed
**MPID_nem_ib_cm_progress:MPID_nem_ib_cm_progress failed
**MPID_nem_ib_cm_send_core:MPID_nem_ib_cm_send_core failed
**MPID_nem_ib_cm_notify_send:MPID_nem_ib_cm_notify_send failed
**MPID_nem_ib_com_alloc:MPID_nem_ib_com_alloc failed
**MPID_nem_ib_com_cas_scratch_pad:MPID_nem_ib_com_cas_scratch_pad failed
**MPID_nem_ib_com_close:MPID_nem_ib_com_close failed
**MPID_nem_ib_com_connect_ringbuf:MPID_nem_ib_com_connect_ringbuf failed
**MPID_nem_ib_com_free:MPID_nem_ib_com_free failed
**MPID_nem_ib_com_get_info_conn:MPID_nem_ib_com_get_info_conn failed
**MPID_nem_ib_com_get_info_mr:MPID_nem_ib_com_get_info_mr failed
**MPID_nem_ib_com_get_scratch_pad:MPID_nem_ib_com_get_scratch_pad failed
**MPID_nem_ib_com_irecv:MPID_nem_ib_com_irecv failed
**MPID_nem_ib_com_isend:MPID_nem_ib_com_isend failed
**MPID_nem_ib_com_lrecv:MPID_nem_ib_com_lrecv failed
**MPID_nem_ib_com_obtain_pointer:MPID_nem_ib_com_obtain_pointer failed
**MPID_nem_ib_com_open:MPID_nem_ib_com_open failed
**MPID_nem_ib_com_poll_cq %s:MPID_nem_ib_com_poll_cq failed with error %s
**MPID_nem_ib_com_poll_cq:MPID_nem_ib_com_poll_cq failed
**MPID_nem_ib_com_put_scratch_pad:MPID_nem_ib_com_put_scratch_pad failed
**MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get failed
**MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get failed
**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get failed
**MPID_nem_ib_com_reg_mr_connect:MPID_nem_ib_com_reg_mr_connect failed
**MPID_nem_ib_com_reg_mr_fetch:MPID_nem_ib_com_reg_mr_fetch failed
**MPID_nem_ib_com_rts:MPID_nem_ib_com_rts failed
**MPID_nem_ib_com_sq_occupancy_notify_rate_get:MPID_nem_ib_com_sq_occupancy_notify_rate_get failed
**MPID_nem_ib_com_wr_scratch_pad:MPID_nem_ib_com_wr_scratch_pad failed
**MPID_nem_ib_drain_scq:MPID_nem_ib_drain_scq failed
**MPID_nem_ib_drain_scq_scratch_pad:MPID_nem_ib_drain_scq_scratch_pad failed
**MPID_nem_ib_handle_pkt_bh:MPID_nem_ib_handle_pkt_bh failed
**MPID_nem_ib_kvs_put_binary:MPID_nem_ib_kvs_put_binary failed
**MPID_nem_ib_lmt_done_recv:MPID_nem_ib_lmt_done_recv failed
**MPID_nem_ib_lmt_done_send:MPID_nem_ib_lmt_done_send failed
**MPID_nem_ib_lmt_send_GET_DONE:MPID_nem_ib_lmt_send_GET_DONE failed
**MPID_nem_ib_lmt_send_RTS:MPID_nem_ib_lmt_send_RTS failed
**MPID_nem_ib_npollingset:MPID_nem_ib_npollingset failed
**MPID_nem_ib_poll:MPID_nem_ib_poll failed
**MPID_nem_ib_poll_eager:MPID_nem_ib_poll_eager failed
**MPID_nem_ib_rdma_to_alloc:MPID_nem_ib_rdma_to_alloc failed
**MPID_nem_ib_ring_alloc:MPID_nem_ib_ring_alloc failed
**MPID_nem_ib_ringbuf_alloc:MPID_nem_ib_ringbuf_alloc failed
**MPID_nem_ib_ringbuf_ask_cas:MPID_nem_ib_ringbuf_ask_cas failed
**MPID_nem_ib_ringbuf_ask_fetch:MPID_nem_ib_ringbuf_ask_fetch failed
**MPID_nem_ib_ringbuf_connect_cas_core:MPID_nem_ib_ringbuf_connect_cas_core failed
**MPID_nem_ib_ringbuf_free:MPID_nem_ib_ringbuf_free failed
**MPID_nem_ib_ringbuf_progress:MPID_nem_ib_ringbuf_progress failed
**MPID_nem_ib_ringbuf_send_core:MPID_nem_ib_ringbuf_send_core failed
**MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state:MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state failed
**MPID_nem_ib_send_progress:MPID_nem_ib_send_progress failed
**MPID_nem_ib_send_reply_seq_num:MPID_nem_ib_send_reply_seq_num failed
**MPID_nem_ib_send_req_seq_num:MPID_nem_ib_send_req_seq_num failed
**PMI_Barrier:PMI_Barrier failed
**PMI_KVS_Put:PMI_KVS_Put failed
**PMS_KVS_Get:PMS_KVS_Get failed
**malloc:malloc failed
**netmod,ib,ibv_poll_cq:netmod,ib,ibv_poll_cq failed
**notimplemented:notimplemented failed
**outofmemory:outofmemory failed
**sizeof(MPIDI_CH3_Pkt_t):sizeof(MPIDI_CH3_Pkt_t) failed
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2012 NEC Corporation
* (C) 2014 RIKEN AICS
*
* See COPYRIGHT in top-level directory.
*/
#include "ib_impl.h"
//#define MPID_NEM_IB_DEBUG_FINALIZE
#ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */
#undef dprintf
#endif
#ifdef MPID_NEM_IB_DEBUG_FINALIZE
#define dprintf printf
#else
#define dprintf(...)
#endif
#undef FUNCNAME
#define FUNCNAME MPID_nem_ib_finalize
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_finalize(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_FINALIZE);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_FINALIZE);
fn_exit:
return mpi_errno;
//fn_fail:
goto fn_exit;
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2001-2009 University of Tokyo
* (C) 2012 NEC Corporation
* (C) 2012 University of Tokyo
* (C) 2014-2015 RIKEN AICS
*
* See COPYRIGHT in top-level directory.
*/
#include <unistd.h>
#include <stdlib.h>
#include "ib_ibcom.h"
//#define MPID_NEM_IB_DEBUG_REG_MR
#ifdef MPID_NEM_IB_DEBUG_REG_MR
#define dprintf printf
#else
#define dprintf(...)
#endif
/* cache size of ibv_reg_mr */
#define MPID_NEM_IB_COM_REG_MR_NLINE 4096
#define MPID_NEM_IB_COM_REG_MR_NWAY 1024
#define MPID_NEM_IB_COM_REG_MR_SZPAGE 4096
#define MPID_NEM_IB_COM_REG_MR_LOGSZPAGE 12
static int ref_count;
/* Allocator using reference count at the head of
aligned memory area */
#define MPID_NEM_IB_NIALLOCID 32
typedef struct {
char *next;
} free_list_t;
#define MPID_NEM_IB_SZARENA 4096
#define MPID_NEM_IB_CLUSTER_SIZE (MPID_NEM_IB_SZARENA/sz)
#define MPID_NEM_IB_ROUNDUP64(addr, align) ((addr + align - 1) & ~((unsigned long)align - 1))
#define MPID_NEM_IB_NCLUST_SLAB 1
#define MPID_NEM_IB_COM_AALLOC_ID_MRCACHE 0
static inline void *aalloc(size_t sz, int id)
{
#if 1 /* debug */
return MPIU_Malloc(sz);
#else
char *p = free_list_front[id];
if ((unsigned long) p & (MPID_NEM_IB_SZARENA - 1)) {
free_list_front[id] += sz;
return p;
}
else {
char *q, r;
if (arena_flist[id]) {
q = arena_flist[id];
arena_flist[id] = ((free_list_t *) arena_flist[id])->next;
}
else {
q = mmap(NULL,
MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096),
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#if MPID_NEM_IB_NCLUST_SLAB > 1
arena_flist[id] = q + MPID_NEM_IB_SZARENA;
for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA;
p += MPID_NEM_IB_SZARENA) {
((free_list_t *) p)->next = p + MPID_NEM_IB_SZARENA;
}
((free_list_t *) p)->next = 0;
#endif
}
*((int *) q) = MPID_NEM_IB_CLUSTER_SIZE - 1;
// dprintf("q=%llx\n", q);
q += sz + (MPID_NEM_IB_SZARENA % sz);
free_list_front[id] = q + sz;
return q;
}
#endif
}
static inline void afree(const void *p, int id)
{
#if 1 /* debug */
return MPIU_Free((void *) p);
#else
p = (void *) ((unsigned long) p & ~(MPID_NEM_IB_SZARENA - 1));
if (!(--(*((int *) p)))) {
((free_list_t *) p)->next = arena_flist[id];
arena_flist[id] = (char *) p;
}
#endif
}
static struct MPID_nem_ib_com_reg_mr_listnode_t MPID_nem_ib_com_reg_mr_global_cache;
static struct MPID_nem_ib_com_reg_mr_listnode_t
MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE];
static inline int MPID_nem_ib_com_hash_func(char *addr)
{
unsigned int v = (unsigned int) (unsigned long) addr;
//v = v >> MPID_NEM_IB_COM_REG_MR_LOGSZPAGE; /* assume it is page aligned */
v = v & (MPID_NEM_IB_COM_REG_MR_NLINE - 1);
return (int) v;
}
static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c,
struct MPID_nem_ib_com_reg_mr_listnode_t *e)
{
struct MPID_nem_ib_com_reg_mr_listnode_t *next;
struct MPID_nem_ib_com_reg_mr_listnode_t *prev;
prev = c;
next = prev->lru_next;
e->lru_next = next;
e->lru_prev = prev;
next->lru_prev = e;
prev->lru_next = e;
}
static void MPID_nem_ib_com_reg_mr_unlink(struct MPID_nem_ib_com_reg_mr_listnode_t *e)
{
struct MPID_nem_ib_com_reg_mr_listnode_t *next, *prev;
next = e->lru_next;
prev = e->lru_prev;
next->lru_prev = prev;
prev->lru_next = next;
}
static inline void __lru_queue_display()
{
struct MPID_nem_ib_com_reg_mr_cache_entry_t *p;
int i = 0;
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
dprintf("---- hash %d\n", i);
for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
MPID_nem_ib_com_reg_mr_cache[i].lru_next;
p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];
p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) {
if (p && p->addr) {
dprintf("-------- p=%p,addr=%p,len=%ld,refc=%d,lru_next=%p\n", p, p->addr, p->len,
p->refc, p->lru_next);
}
else {
dprintf("-------- p=%p,lru_next=%p\n", p, p->lru_next);
}
}
}
}
void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len,
enum ibv_access_flags additional_flags, int mode)
{
int ibcom_errno;
int key;
struct MPID_nem_ib_com_reg_mr_cache_entry_t *e;
static unsigned long long num_global_cache = 0ULL;
/* we can't change addr because ibv_post_send assumes mr->host_addr (output of this function)
* must have an exact mirror value of addr (input of this function) */
void *addr_aligned = addr;
long len_aligned = len;
key = MPID_nem_ib_com_hash_func(addr);
dprintf("[MrCache] addr=%p, len=%ld\n", addr, len);
dprintf("[MrCache] aligned addr=%p, len=%ld\n", addr_aligned, len_aligned);
//__lru_queue_display();
int way = 0;
for (e =
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next;
e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[key];
e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_next, way++) {
//dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next);
if (e->addr <= addr_aligned &&
(uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) {
//dprintf
//("MPID_nem_ib_com_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n",
//e->addr, e->len, e->mr->addr, e->mr->length, addr, len);
goto hit;
}
}
// miss
e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t),
MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
/* reference counter is used when evicting entry */
e->refc = 1;
dprintf("MPID_nem_ib_com_reg_mr_fetch,miss,addr=%p,len=%ld\n", addr_aligned, len_aligned);
/* register memory */
ibcom_errno = MPID_nem_ib_com_reg_mr(addr_aligned, len_aligned, &e->mr, additional_flags);
if (ibcom_errno != 0) {
/* ib_com_reg_mr returns the errno of ibv_reg_mr */
if (ibcom_errno == ENOMEM) {
/* deregister memory region unused and re-register new one */
struct MPID_nem_ib_com_reg_mr_listnode_t *ptr;
struct MPID_nem_ib_com_reg_mr_cache_entry_t *victim;
unsigned long long dereg_total = 0;
int reg_success = 0;
for (ptr = MPID_nem_ib_com_reg_mr_global_cache.lru_prev;
ptr !=
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_global_cache;)
{
victim = list_entry(ptr, struct MPID_nem_ib_com_reg_mr_cache_entry_t, g_lru);
ptr = ptr->lru_prev;
/* 'refc == 0' means this cache_entry is not used */
if (victim && victim->addr && (victim->refc == 0)) {
MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *)
victim);
MPID_nem_ib_com_reg_mr_unlink(&(victim->g_lru));
ibcom_errno = MPID_nem_ib_com_dereg_mr(victim->mr);
if (ibcom_errno) {
printf("mrcache,MPID_nem_ib_com_dereg_mr\n");
afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
goto fn_fail;
}
dereg_total += (unsigned long long) victim->len;
afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
num_global_cache--;
/* end loop if the total length released exceeds the requested */
if (dereg_total >= len_aligned) {
dprintf("ib_com_reg_mr_fetch,dereg=%llu,len=%ld\n", dereg_total,
len_aligned);
/* re-registraion */
ibcom_errno =
MPID_nem_ib_com_reg_mr(addr_aligned, len_aligned, &e->mr,
additional_flags);
if (ibcom_errno == 0) {
/* ibv_reg_mr success */
reg_success = 1;
break;
}
}
}
}
if (reg_success == 0) {
fprintf(stderr, "mrcache,MPID_nem_ib_com_reg_mr,failed\n");
afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
goto fn_fail;
}
}
else {
/* errno is not ENOMEM */
fprintf(stderr, "mrcache,MPID_nem_ib_com_reg_mr,errno=%d\n", ibcom_errno);
afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
goto fn_fail;
}
}
e->addr = addr_aligned;
e->len = len_aligned;
//dprintf("MPID_nem_ib_com_reg_mr_fetch,fill,e=%p,key=%d,mr=%p,mr addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e,
//key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey);
/* register to cache */
MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key],
(struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
if (mode != MPID_NEM_IB_COM_REG_MR_STICKY) {
/* register to global-cache */
num_global_cache++;
MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_global_cache, &(e->g_lru));
}
//__lru_queue_display();
goto fn_exit;
hit:
/* reference counter is used when evicting entry */
e->refc++;
/* move to head of the list */
if (e !=
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next)
{
MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key],
(struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
}
if (mode != MPID_NEM_IB_COM_REG_MR_STICKY) {
/* move to head of the list in global-cache */
MPID_nem_ib_com_reg_mr_unlink(&(e->g_lru));
MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_global_cache, &(e->g_lru));
}
//dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e,
//key, e->mr, e->refc, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey);
//__lru_queue_display();
fn_exit:
if (mode == MPID_NEM_IB_COM_REG_MR_STICKY)
return e->mr;
else
return e;
fn_fail:
goto fn_exit;
}
void MPID_nem_ib_com_reg_mr_release(struct MPID_nem_ib_com_reg_mr_cache_entry_t *entry)
{
entry->refc--;
MPIU_Assert(ref_count >= 0);
}
int MPID_nem_ib_com_register_cache_init()
{
int ibcom_errno = 0;
int i;
ref_count++;
dprintf("cache_init,ref_count=%d\n", ref_count);
if (ref_count == 1) {
/* Using the address to the start node to express the end of the list
* instead of using NULL */
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
MPID_nem_ib_com_reg_mr_cache[i].lru_next =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
MPID_nem_ib_com_reg_mr_cache[i].lru_prev =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
}
MPID_nem_ib_com_reg_mr_global_cache.lru_next =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_global_cache;
MPID_nem_ib_com_reg_mr_global_cache.lru_prev =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_global_cache;
dprintf("[MrCache] cache initializes %d entries\n", MPID_NEM_IB_COM_REG_MR_NLINE);
}
fn_exit:
return ibcom_errno;
//fn_fail:
goto fn_exit;
}
int MPID_nem_ib_com_register_cache_release()
{
int ibcom_errno = 0;
int ib_errno;
struct MPID_nem_ib_com_reg_mr_cache_entry_t *p;
int i = 0, cnt = 0;
dprintf("cache_release,ref_count=%d\n", ref_count);
MPIU_Assert(ref_count > 0);
if (--ref_count > 0) {
goto fn_exit;
}
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
MPID_nem_ib_com_reg_mr_cache[i].lru_next;
p !=
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];) {
if (p && p->addr) {
ib_errno = MPID_nem_ib_com_dereg_mr(p->mr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, printf("MPID_nem_ib_com_dereg_mr"));
struct MPID_nem_ib_com_reg_mr_cache_entry_t *p_old = p;
p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next;
afree(p_old, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
cnt++;
}
}
MPID_nem_ib_com_reg_mr_cache[i].lru_next =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
MPID_nem_ib_com_reg_mr_cache[i].lru_prev =
(struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
}
//__lru_queue_display();
dprintf("[MrCache] cache destroyed %d entries\n", cnt);
fn_exit:
return ibcom_errno;
fn_fail:
goto fn_exit;
}
This diff is collapsed.
[#] start of __file__
dnl MPICH_SUBCFG_AFTER=src/mpid/ch3/channels/nemesis
AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[
AM_COND_IF([BUILD_CH3_NEMESIS],[
for net in $nemesis_networks ; do
AS_CASE([$net],[ib],[build_nemesis_netmod_ib=yes])
done
])
AM_CONDITIONAL([BUILD_NEMESIS_NETMOD_IB],[test "X$build_nemesis_netmod_ib" = "Xyes"])
# check if getpagesize is available
AC_CHECK_FUNCS(getpagesize)
])dnl
AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[
AM_COND_IF([BUILD_NEMESIS_NETMOD_IB],[
AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:ib])
PAC_SET_HEADER_LIB_PATH(ib)
PAC_PUSH_FLAG(LIBS)
PAC_CHECK_HEADER_LIB(dcfa.h,dcfa,ibv_open_device,dcfa_found=yes,dcfa_found=no)
PAC_POP_FLAG(LIBS)
if test "${dcfa_found}" = "yes" ; then
AC_MSG_NOTICE([libdcfa is going to be linked.])
PAC_APPEND_FLAG([-ldcfa],[WRAPPER_LIBS])
else
PAC_PUSH_FLAG(LIBS)
PAC_CHECK_HEADER_LIB([infiniband/verbs.h],ibverbs,ibv_open_device,ibverbs_found=yes,ibverbs_found=no)
PAC_POP_FLAG(LIBS)
if test "${ibverbs_found}" = "yes" ; then
AC_MSG_NOTICE([libibverbs is going to be linked.])
PAC_APPEND_FLAG([-libverbs],[WRAPPER_LIBS])
else
AC_MSG_ERROR([Internal error: neither ibverbs nor dcfa was found])