Commit 5bf7e350 authored by Francois Tessier's avatar Francois Tessier

C bindings for TAPIOCA + usage example

parent 31a66510
#include "tapioca.hpp"
extern "C" {
class Tapioca;
typedef Tapioca TAPIOCA;
TAPIOCA* create_tapioca ();
void delete_tapioca( TAPIOCA* tapioca );
void tapioca_init ( TAPIOCA *tapioca, int64_t *chunkCount, int *chunkSize,
int64_t *chunkOffset, int nChunks, int64_t header, MPI_Comm comm);
void tapioca_set_aggregation_tier ( TAPIOCA *tapioca, int nBuffers, mem_t mem, char* fileName );
void tapioca_set_target_tier ( TAPIOCA *tapioca, mem_t mem, int64_t buffSize, char* fileName );
int tapioca_write ( TAPIOCA *tapioca, MPI_Offset offset, void *buf, int count,
MPI_Datatype datatype, MPI_Status *status );
int tapioca_read ( TAPIOCA *tapioca, MPI_Offset offset, void *buf, int count,
MPI_Datatype datatype, MPI_Status *status );
void tapioca_finalize ( TAPIOCA *tapioca );
}
#include "tapioca.h"
#include "tapioca.hpp"
TAPIOCA* create_tapioca () {
TAPIOCA* create_tapioca ()
{
fprintf ( stdout, "C API - create_tapioca\n" );
return new Tapioca ();
}
void delete_tapioca( TAPIOCA* tapioca ) {
void delete_tapioca( TAPIOCA* tapioca )
{
fprintf ( stdout, "C API - delete_tapioca\n" );
delete tapioca;
}
void tapioca_finalize ( const TAPIOCA* tapioca ) {
void tapioca_init ( TAPIOCA *tapioca, int64_t *chunkCount, int *chunkSize,
int64_t *chunkOffset, int nChunks, int64_t header, MPI_Comm comm)
{
return tapioca->Init ( chunkCount, chunkSize, chunkOffset, nChunks, header, comm );
}
void tapioca_mpi_info ( const TAPIOCA* tapioca, MPI_File fileHandle ) {
void tapioca_set_aggregation_tier ( TAPIOCA *tapioca, int nBuffers, mem_t mem, char* fileName )
{
return tapioca->setAggregationTier ( nBuffers, mem, fileName );
}
/*******************/
/* WRITE */
/*******************/
void tapioca_write_initialize ( const TAPIOCA* tapioca, char *filename, int64_t *chunkCount,
int *chunkSize, int64_t *chunkOffset, int nChunks,
int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm ) {
tapioca->WriteInitialize (filename, chunkCount, chunkSize, chunkOffset, nChunks, offset, layout, comm );
void tapioca_set_target_tier ( TAPIOCA *tapioca, mem_t mem, int64_t buffSize, char* fileName )
{
return tapioca->setTargetTier ( mem, buffSize, fileName );
}
int tapioca_write ( const TAPIOCA* tapioca, MPI_File fileHandle, MPI_Offset offset,
void *buf, int count, MPI_Datatype datatype,
MPI_Status *status, int64_t bufOffset = 0 ) {
return tapioca->Write ( fileHandle, offset, buf, count, datatype, status );
int tapioca_write ( TAPIOCA *tapioca, MPI_Offset offset, void *buf, int count,
MPI_Datatype datatype, MPI_Status *status )
{
return tapioca->Write ( offset, buf, count, datatype, status );
}
/*******************/
/* READ */
/*******************/
void tapioca_read_initialize ( const TAPIOCA* tapioca, char *filename, int64_t *chunkCount,
int *chunkSize, int64_t *chunkOffset, int nChunks,
int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm ) {
tapioca->ReadInitialize (filename, chunkCount, chunkSize, chunkOffset, nChunks, offset, layout, comm );
int tapioca_read ( TAPIOCA *tapioca, MPI_Offset offset, void *buf, int count,
MPI_Datatype datatype, MPI_Status *status )
{
return tapioca->Read ( offset, buf, count, datatype, status );
}
int tapioca_read ( const TAPIOCA* tapioca, MPI_File fileHandle, MPI_Offset offset,
void *buf, int count, MPI_Datatype datatype,
MPI_Status *status, int64_t bufOffset = 0 ) {
return tapioca->Read ( fileHandle, offset, buf, count, datatype, status );
void tapioca_finalize ( TAPIOCA *tapioca )
{
return tapioca->Finalize ();
}
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <stdint.h>
#include <mpi.h>
#include "tapioca.h"
#define RED "\x1b[31m"
#define GREEN "\x1b[32m"
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
TAPIOCA* tp;
int64_t chunkCount[1], chunkOffset[1];
int chunkSize[1];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
tp = create_tapioca ();
mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
if ( argc == 2 )
num_particles = atoi ( argv[1] );
else
MPI_Abort (MPI_COMM_WORLD, -1);
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/SimpleIO-%08d.dat", mycolor);
#elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/SimpleIO-%08d.dat", mycolor);
#else
snprintf (output, 100, "./SimpleIO-%08d.dat", mycolor);
#endif
/*****************/
/* WRITE */
/*****************/
float *xx;
xx = (float *) malloc ( num_particles * sizeof ( float ));
for (uint64_t i = 0; i< num_particles; i++)
xx[i] = (float)i;
MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
particle_size = sizeof(float);
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] TAPIOCA Simple I/O\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
}
/*****************/
/* INIT TAPIOCA */
/*****************/
chunkCount[0] = num_particles;
chunkSize[0] = sizeof(float);
chunkOffset[0] = hdr + scan_size * particle_size;
tapioca_init ( tp, chunkCount, chunkSize, chunkOffset, 1, hdr, sub_comm);
tapioca_set_aggregation_tier ( tp, 2, DDR, "" );
tapioca_set_target_tier ( tp, HDD, file_size, output );
/*****************/
MPI_Barrier (MPI_COMM_WORLD);
start_time = MPI_Wtime();
offset = scan_size * particle_size;
tapioca_write ( tp, offset, xx, num_particles, MPI_FLOAT, &status );
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tapioca_finalize ( tp );
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* READ */
/*****************/
float *xx_r;
xx_r = (float *) malloc ( num_particles * sizeof ( float ));
/*****************/
/* INIT TAPIOCA */
/*****************/
tapioca_init ( tp, chunkCount, chunkSize, chunkOffset, 1, hdr, sub_comm);
tapioca_set_aggregation_tier ( tp, 2,DDR, "");
tapioca_set_target_tier ( tp,HDD, file_size, output );
/*****************/
start_time = MPI_Wtime();
offset = scan_size * particle_size;
tapioca_read ( tp, offset, xx_r, num_particles, MPI_FLOAT, &status );
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tapioca_finalize ( tp );
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
if (xx[i] != xx_r[i])
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor);
/*****************/
/* FREE */
/*****************/
delete [] xx;
delete [] xx_r;
MPI_Finalize ();
}
MPICXX = mpicxx
MPI_CFLAGS = -g -O3 -I$(HOME)/install/$(ARCHI)/include/ -I ../../bindings/ -DBGQ -DDBG -DCOLOR
MPI_LDFLAGS = -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca
INSTALL_PATH = $(HOME)/install/$(ARCHI)
all: 1d_array
1d_array:1d_array.c ../../bindings/tapioca_capi.o
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
install:
mv 1d_array $(HOME)/install/$(ARCHI)/bin/
clean:
rm 1d_array *.o
#!/bin/bash
NODES=8
PPN=12
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
DDT="/soft/debuggers/ddt/bin/ddt --connect"
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=4194304
#export TAPIOCA_BUFFERSIZE=33554432
export TAPIOCA_PIPELINING=true
export PAMID_VERBOSE=1
export AMID_COLLECTIVES_MEMORY_OPTIMIZED=1
export PAMID_COLLECTIVES=1
function outputFile ()
{
OUTPUT="$HOME/xp_tapioca/memory_abstraction"
OUTPUT="${OUTPUT}/HACC_HW_COOLEY_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
printenv | egrep "TAPIOCA_"
PARTICLES=60000
rm $TARGET/*
mpirun -f $COBALT_NODEFILE -n $NPROCS ./1d_array $PARTICLES
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment