Align nemesis payload: union approach

Drop payload into a union paired with a double, and payload will end up
double-aligned.  Can't use an anonymous union, though: that's c11.

named-union requires updating any place that used payload.  There aren't too
many, though.
Signed-off-by: Pavan Balaji <balaji@mcs.anl.gov>

src/mpid/ch3/channels/nemesis/include/mpid_nem_datatypes.h

@@ -144,15 +144,25 @@
 #define MPID_NEM_MPICH_HEAD_LEN sizeof(MPID_nem_pkt_header_t)
 #define MPID_NEM_MPICH_DATA_LEN (MPID_NEM_CELL_PAYLOAD_LEN - MPID_NEM_MPICH_HEAD_LEN)
 
-#include "mpid_nem_pkt_header.h"
+#define MPID_NEM_PKT_HEADER_FIELDS   	    \
+    int source;                             \
+    int dest;                               \
+    MPIR_Pint datalen;                      \
+    unsigned short seqno;                   \
+    unsigned short type; /* currently used only with checkpointing */
+
+typedef struct MPID_nem_pkt_header
+{
+    MPID_NEM_PKT_HEADER_FIELDS;
+} MPID_nem_pkt_header_t;
 
 typedef struct MPID_nem_pkt_mpich
 {
     MPID_NEM_PKT_HEADER_FIELDS;
-#if (MPID_NEM_PKT_HEADER_PADDING > 0)
-    char padding[MPID_NEM_PKT_HEADER_PADDING];
-#endif
+    union {
         char payload[MPID_NEM_MPICH_DATA_LEN];
+        double dummy; /* align paylod to double */
+    } p;
 } MPID_nem_pkt_mpich_t;
 
 typedef union

src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h

@@ -74,7 +74,7 @@ MPID_nem_mpich_send_header (void* buf, int size, MPIDI_VC_t *vc, int *again)
         
         MPIU_DBG_STMT (CH3_CHANNEL, VERBOSE, pbox->cell.pkt.mpich.type = MPID_NEM_PKT_MPICH_HEAD);
         
-        MPIU_Memcpy((void *)pbox->cell.pkt.mpich.payload, buf, size);
+        MPIU_Memcpy((void *)pbox->cell.pkt.mpich.p.payload, buf, size);
 
         OPA_store_release_int(&pbox->flag.value, 1);
 
@@ -120,7 +120,7 @@ MPID_nem_mpich_send_header (void* buf, int size, MPIDI_VC_t *vc, int *again)
     el->pkt.mpich.seqno   = vc_ch->send_seqno++;
     MPIU_DBG_STMT (CH3_CHANNEL, VERBOSE, el->pkt.mpich.type = MPID_NEM_PKT_MPICH_HEAD);
     
-    MPIU_Memcpy((void *)el->pkt.mpich.payload, buf, size);
+    MPIU_Memcpy((void *)el->pkt.mpich.p.payload, buf, size);
     DO_PAPI (PAPI_accum_var (PAPI_EventSet, PAPI_vvalues11));
 
     MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, "--> Sent queue");
@@ -213,7 +213,7 @@ MPID_nem_mpich_sendv (MPID_IOV **iov, int *n_iov, MPIDI_VC_t *vc, int *again)
 #endif /*PREFETCH_CELL     */
 
     payload_len = MPID_NEM_MPICH_DATA_LEN;
-    cell_buf    = (char *) el->pkt.mpich.payload; /* cast away volatile */
+    cell_buf    = (char *) el->pkt.mpich.p.payload; /* cast away volatile */
     
     while (*n_iov && payload_len >= (*iov)->MPID_IOV_LEN)
     {
@@ -304,8 +304,8 @@ MPID_nem_mpich_sendv_header (MPID_IOV **iov, int *n_iov, MPIDI_VC_t *vc, int *ag
         pbox->cell.pkt.mpich.seqno   = vc_ch->send_seqno++;
         MPIU_DBG_STMT (CH3_CHANNEL, VERBOSE, pbox->cell.pkt.mpich.type = MPID_NEM_PKT_MPICH_HEAD);
         
-        MPIU_Memcpy((void *)pbox->cell.pkt.mpich.payload, (*iov)[0].MPID_IOV_BUF, (*iov)[0].MPID_IOV_LEN);
-        MPIU_Memcpy ((char *)pbox->cell.pkt.mpich.payload + (*iov)[0].MPID_IOV_LEN, (*iov)[1].MPID_IOV_BUF, (*iov)[1].MPID_IOV_LEN);
+        MPIU_Memcpy((void *)pbox->cell.pkt.mpich.p.payload, (*iov)[0].MPID_IOV_BUF, (*iov)[0].MPID_IOV_LEN);
+        MPIU_Memcpy ((char *)pbox->cell.pkt.mpich.p.payload + (*iov)[0].MPID_IOV_LEN, (*iov)[1].MPID_IOV_BUF, (*iov)[1].MPID_IOV_LEN);
         
         OPA_store_release_int(&pbox->flag.value, 1);
         *n_iov = 0;
@@ -343,9 +343,9 @@ MPID_nem_mpich_sendv_header (MPID_IOV **iov, int *n_iov, MPIDI_VC_t *vc, int *ag
     MPID_nem_queue_dequeue (MPID_nem_mem_region.my_freeQ, &el);
 #endif /*PREFETCH_CELL */
 
-    MPIU_Memcpy((void *)el->pkt.mpich.payload, (*iov)->MPID_IOV_BUF, sizeof(MPIDI_CH3_Pkt_t));
+    MPIU_Memcpy((void *)el->pkt.mpich.p.payload, (*iov)->MPID_IOV_BUF, sizeof(MPIDI_CH3_Pkt_t));
 
-    cell_buf = (char *)(el->pkt.mpich.payload) + sizeof(MPIDI_CH3_Pkt_t);
+    cell_buf = (char *)(el->pkt.mpich.p.payload) + sizeof(MPIDI_CH3_Pkt_t);
     ++(*iov);
     --(*n_iov);
 
@@ -452,11 +452,11 @@ MPID_nem_mpich_send_seg_header (MPID_Segment *segment, MPIDI_msg_sz_t *segment_f
             MPIU_DBG_STMT (CH3_CHANNEL, VERBOSE, pbox->cell.pkt.mpich.type = MPID_NEM_PKT_MPICH_HEAD);
 
             /* copy header */
-            MPIU_Memcpy((void *)pbox->cell.pkt.mpich.payload, header, header_sz);
+            MPIU_Memcpy((void *)pbox->cell.pkt.mpich.p.payload, header, header_sz);
             
             /* copy data */
             last = segment_size;
-            MPID_Segment_pack(segment, *segment_first, &last, (char *)pbox->cell.pkt.mpich.payload + sizeof(MPIDI_CH3_Pkt_t));
+            MPID_Segment_pack(segment, *segment_first, &last, (char *)pbox->cell.pkt.mpich.p.payload + sizeof(MPIDI_CH3_Pkt_t));
             MPIU_Assert(last == segment_size);
 
             OPA_store_release_int(&pbox->flag.value, 1);
@@ -498,7 +498,7 @@ MPID_nem_mpich_send_seg_header (MPID_Segment *segment, MPIDI_msg_sz_t *segment_f
 #endif /*PREFETCH_CELL */
 
     /* copy header */
-    MPIU_Memcpy((void *)el->pkt.mpich.payload, header, header_sz);
+    MPIU_Memcpy((void *)el->pkt.mpich.p.payload, header, header_sz);
     
     /* copy data */
     if (segment_size - *segment_first <= MPID_NEM_MPICH_DATA_LEN - sizeof(MPIDI_CH3_Pkt_t))
@@ -506,7 +506,7 @@ MPID_nem_mpich_send_seg_header (MPID_Segment *segment, MPIDI_msg_sz_t *segment_f
     else
         last = *segment_first + MPID_NEM_MPICH_DATA_LEN - sizeof(MPIDI_CH3_Pkt_t);
     
-    MPID_Segment_pack(segment, *segment_first, &last, (char *)el->pkt.mpich.payload + sizeof(MPIDI_CH3_Pkt_t));
+    MPID_Segment_pack(segment, *segment_first, &last, (char *)el->pkt.mpich.p.payload + sizeof(MPIDI_CH3_Pkt_t));
     datalen = sizeof(MPIDI_CH3_Pkt_t) + last - *segment_first;
     *segment_first = last;
     
@@ -589,7 +589,7 @@ MPID_nem_mpich_send_seg (MPID_Segment *segment, MPIDI_msg_sz_t *segment_first, M
     else
         last = *segment_first + MPID_NEM_MPICH_DATA_LEN;
     
-    MPID_Segment_pack(segment, *segment_first, &last, (char *)el->pkt.mpich.payload);
+    MPID_Segment_pack(segment, *segment_first, &last, (char *)el->pkt.mpich.p.payload);
     datalen = last - *segment_first;
     *segment_first = last;
     

src/mpid/ch3/channels/nemesis/include/mpid_nem_pkt_header.h

-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2013 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef MPID_NEM_PKT_HEADER_H
-#define MPID_NEM_PKT_HEADER_H
-
-#define MPID_NEM_PKT_HEADER_FIELDS		\
-    struct {                                    \
-        int source;                             \
-        int dest;                               \
-        MPIR_Pint datalen;                      \
-        unsigned short seqno;                   \
-        unsigned short type; /* currently used only with checkpointing */ \
-    };
-
-typedef struct MPID_nem_pkt_header
-{
-    MPID_NEM_PKT_HEADER_FIELDS;
-} MPID_nem_pkt_header_t;
-
-
-#endif /* MPID_NEM_DATATYPES_H */

src/mpid/ch3/channels/nemesis/src/ch3_progress.c

@@ -368,7 +368,7 @@ int MPIDI_CH3I_Progress (MPID_Progress_state *progress_state, int is_blocking)
 
             if (cell)
             {
-                char            *cell_buf    = (char *)cell->pkt.mpich.payload;
+                char            *cell_buf    = (char *)cell->pkt.mpich.p.payload;
                 MPIDI_msg_sz_t   payload_len = cell->pkt.mpich.datalen;
                 MPIDI_CH3_Pkt_t *pkt         = (MPIDI_CH3_Pkt_t *)cell_buf;
 

src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c

@@ -117,7 +117,7 @@ MPID_nem_init(int pg_rank, MPIDI_PG_t *pg_p, int has_parent ATTRIBUTE((unused)))
     MPIU_Assert(MPID_NEM_CELL_PAYLOAD_LEN + MPID_NEM_CELL_HEAD_LEN == sizeof(MPID_nem_cell_t));
     MPIU_Assert(sizeof(MPID_nem_cell_t) == sizeof(MPID_nem_abs_cell_t));
     /* Make sure payload is aligned on a double */
-    MPIU_Assert(MPID_NEM_ALIGNED(&((MPID_nem_cell_t*)0)->pkt.mpich.payload[0], sizeof(double)));
+    MPIU_Assert(MPID_NEM_ALIGNED(&((MPID_nem_cell_t*)0)->pkt.mpich.p.payload[0], sizeof(double)));
 
     /* Initialize the business card */
     mpi_errno = MPIDI_CH3I_BCInit( &bc_val, &val_max_remaining );

src/mpid/ch3/channels/nemesis/subconfigure.m4

@@ -319,22 +319,6 @@ if test "$enable_nemesis_lock_free_queues" = "yes" ; then
     AC_DEFINE(MPID_NEM_USE_LOCK_FREE_QUEUES, 1, [Define to enable lock-free communication queues])
 fi
 
-# for nemisis header alignment, will need to know how this structure laid out:
-AC_CHECK_SIZEOF(MPID_nem_pkt_header_t, 0, [
-typedef $MPI_PINT MPIR_Pint;
-#include "${master_top_srcdir}/src/mpid/ch3/channels/nemesis/include/mpid_nem_pkt_header.h"])
-
-AC_CHECK_ALIGNOF(MPID_nem_pkt_header_t, [
-typedef $MPI_PINT MPIR_Pint;
-#include "${master_top_srcdir}/src/mpid/ch3/channels/nemesis/include/mpid_nem_pkt_header.h"])
-
-# We're trying to get 'payload' to live at a double-aligned memory address
-AS_VAR_ARITH([mpid_pkt_header_padding],
-	     [ $ac_cv_alignof_MPID_nem_pkt_header_t % $ac_cv_sizeof_double])
-AC_DEFINE_UNQUOTED(MPID_NEM_PKT_HEADER_PADDING, [$mpid_pkt_header_padding], [Amount of padding needed in the MPID_nem_pkt_header_t structure])
-
-
-
 
 AC_SUBST(device_name)
 AC_SUBST(channel_name)