Commit ef5fc309 authored by Francois Tessier's avatar Francois Tessier

1D-Array C example showing how to use C bindings

parent 14cff54f
......@@ -11,18 +11,44 @@
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
static int64_t n_elem = 250000; /* ~1MB */
static char output[4096] = {0};
void parseArgs (int argc, char * argv[])
{
char flags[] = "e:o:";
int opt = 0;
while ((opt = getopt (argc, argv, flags)) != -1) {
switch ( opt )
{
case('e'):
sscanf ( optarg, "%lld", &n_elem );
break;
case('o'):
sprintf ( output, "%s", optarg );
break;
}
}
if ( n_elem <= 0 ) {
fprintf (stdout, RED "[WARNING]" RESET " Wrong number of elements. Reset to the default value (250000/rank).\n");
n_elem = 250000;
}
}
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i;
int64_t sub_elem, tot_elem, elem_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
char filename[256];
int64_t chunkCount[1], chunkOffset[1];
int chunkSize[1];
......@@ -30,6 +56,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
parseArgs ( argc, argv );
mycolor = 42;
mykey = world_myrank;
......@@ -37,63 +65,52 @@ int main (int argc, char * argv[])
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
if ( argc == 2 )
num_particles = atoi ( argv[1] );
else
MPI_Abort (MPI_COMM_WORLD, -1);
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/SimpleIO-%08d.dat", mycolor);
#elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/SimpleIO-%08d.dat", mycolor);
#else
snprintf (output, 100, "./SimpleIO-%08d.dat", mycolor);
#endif
snprintf (filename, 256, "/1D-ARRAY-C-%08d.dat", mycolor);
strcat (output, filename);
/*****************/
/* WRITE */
/*****************/
float *xx;
xx = (float *) malloc ( num_particles * sizeof ( float ));
xx = (float *) malloc ( n_elem * sizeof ( float ));
for (uint64_t i = 0; i< num_particles; i++)
for (uint64_t i = 0; i< n_elem; i++)
xx[i] = (float)i;
MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&n_elem, &sub_elem, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Allreduce(&n_elem, &tot_elem, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
particle_size = sizeof(float);
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
elem_size = sizeof(float);
file_size = elem_size * sub_elem;
tot_size = elem_size * tot_elem;
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Exscan (&n_elem, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] TAPIOCA Simple I/O\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] C - TAPIOCA 1D_Array\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld elem per rank\n", mycolor, n_elem);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld elem)\n",
mycolor, (double)file_size/(1024*1024), sub_elem);
}
/*****************/
/* INIT TAPIOCA */
/*****************/
chunkCount[0] = num_particles;
chunkCount[0] = n_elem;
chunkSize[0] = sizeof(float);
chunkOffset[0] = hdr + scan_size * particle_size;
chunkOffset[0] = hdr + scan_size * chunkSize[0];
tapioca_init_ ( chunkCount, chunkSize, chunkOffset, 1, hdr, sub_comm);
tapioca_set_aggregation_tier_ ( 2, DDR, "" );
tapioca_set_target_tier_ ( HDD, file_size, output );
tapioca_init_ ( chunkCount, chunkSize, chunkOffset, 1, hdr, output, sub_comm);
/*****************/
MPI_Barrier (MPI_COMM_WORLD);
start_time = MPI_Wtime();
offset = scan_size * particle_size;
tapioca_write_ ( offset, xx, num_particles, MPI_FLOAT, &status );
offset = scan_size * elem_size;
tapioca_write_ ( offset, xx, n_elem, MPI_FLOAT, &status );
tapioca_finalize_ ( );
end_time = MPI_Wtime();
tot_time = end_time - start_time;
......@@ -105,27 +122,34 @@ int main (int argc, char * argv[])
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tapioca_finalize_ ( );
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* READ */
/*****************/
float *xx_r;
xx_r = (float *) malloc ( num_particles * sizeof ( float ));
xx_r = (float *) malloc ( n_elem * sizeof ( float ));
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] C - TAPIOCA 1D_Array\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read input file\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld elem per rank\n", mycolor, n_elem);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld elem)\n",
mycolor, (double)file_size/(1024*1024), sub_elem);
}
/*****************/
/* INIT TAPIOCA */
/*****************/
tapioca_init_ ( chunkCount, chunkSize, chunkOffset, 1, hdr, sub_comm);
tapioca_set_aggregation_tier_ ( 2,DDR, "");
tapioca_set_target_tier_ ( HDD, file_size, output );
tapioca_init_ ( chunkCount, chunkSize, chunkOffset, 1, hdr, output, sub_comm);
/*****************/
start_time = MPI_Wtime();
offset = scan_size * particle_size;
tapioca_read_ ( offset, xx_r, num_particles, MPI_FLOAT, &status );
offset = scan_size * elem_size;
tapioca_read_ ( offset, xx_r, n_elem, MPI_FLOAT, &status );
tapioca_finalize_ ( );
end_time = MPI_Wtime();
tot_time = end_time - start_time;
......@@ -137,16 +161,15 @@ int main (int argc, char * argv[])
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tapioca_finalize_ ( );
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
for (uint64_t i = 0; i< n_elem; i++) {
if (xx[i] != xx_r[i])
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for elem %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
......
MPICXX = mpixlcxx
include ../make.inc/Makefile.$(ARCHI)
MPI_CFLAGS = -g -O3 -I$(HOME)/install/$(ARCHI)/include/ -DBGQ -DDBG -DCOLOR
MPI_LDFLAGS = -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca
INSTALL_PATH = $(HOME)/install/$(ARCHI)
C_FILES := $(wildcard *.c)
OUT_FILES := $(C_FILES:.c=)
all: 1d_array
1d_array:1d_array.o ../../bindings/C/tapioca_capi.o
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
all:$(OUT_FILES)
1d_array.o:1d_array.c
$(MPICXX) $(MPI_CFLAGS) -c $? $(MPI_LDFLAGS)
%:%.c ../../bindings/C/tapioca_capi.o
$(MPICXX) $(MPI_CFLAGS) $? -o bin/$@ $(MPI_LDFLAGS)
install:
mv 1d_array $(HOME)/install/$(ARCHI)/bin/
cd bin/; mv $(OUT_FILES) $(INSTALL_PATH)/bin
clean:
rm 1d_array *.o
rm -f bin/*
cd $(INSTALL_PATH)/bin/; rm -f $(OUT_FILES)
#!/bin/bash
NODES=16
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA"
DDT="/soft/debuggers/ddt/bin/ddt --connect"
STRIPE_COUNT=16
STRIPE_SIZE=8388608
AGGR_MULT=1
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_NBBUFFERS=2
export TAPIOCA_BUFFERSIZE=8388608
export TAPIOCA_AGGRTIER=DDR
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=true
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="-e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS -e TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS"
SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
SETTINGS="$SETTINGS -e TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER"
SETTINGS="$SETTINGS -e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS -e TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
SETTINGS="$SETTINGS -e TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
}
function setLustreFS ()
{
rm $TARGET/*
lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
lfs getstripe $TARGET
}
PARTICLES=100000
updateSettings
setLustreFS
rm $TARGET/*
for BIN in 1D-Array-C-Tapioca
do
aprun $SETTINGS -n $NPROCS -N $PPN $BIN -e $PARTICLES -o $TARGET
echo
echo "---------------------------------"
echo
sleep 5
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment