MPI_Win_fence bug identified. Need to fix it for Cray xc40.

examples/HACC-IO/run_check_xc40.sh

@@ -2,7 +2,7 @@
 
 VARS="-e MPICH_RMA_OVER_DMAPP=1"
 NODES=8
-PPN=16
+PPN=2
 NPROCS=$((NODES*PPN))
 TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
 DDT="/soft/debuggers/forge/bin/ddt --connect"
@@ -13,7 +13,7 @@ export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
 export TAPIOCA_NBAGGR=2
-export TAPIOCA_BUFFERSIZE=16777216
+export TAPIOCA_BUFFERSIZE=4194304
 
 function updateSettings()
 {

make.inc/Makefile.xc40

 MPICXX = CC
 
-MPI_CFLAGS   = -g -O3 -I./topology/ -I./topology/include/
+MPI_CFLAGS   = -g -I./topology/ -I./topology/include/
 MPI_CFLAGS  += -DXC40 -DDEBUG #-DTIMING
 
 MPI_LDFLAGS = -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive

tapioca.cpp

@@ -255,12 +255,13 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
 	  fprintf (stdout, "[INFO] %d calls 2nd Push (r: %d)\n", this->commRank_, this->currentRound_);
 	  this->Push (fileHandle, status);
 	  //this->iPush (fileHandle, &iRequest);
-	}
-	
+	}	
+#ifdef BGQ
 	fprintf (stdout, "[INFO] %d increment 2nd currentRound (r: %d, rc: %d, nag: %d)\n", this->commRank_, this->currentRound_, this->roundCounter_, this->nAggr_);
-	this->currentRound_++;
+	this->currentRound_++;	
+#endif	
       }
-
+    
       this->roundCounter_++;
     }
   }