Commit ffe8b8d3 authored by Francois Tessier's avatar Francois Tessier

New version of HACC-IO, easier to use thanks to command line parameters.

parent 0658a332
......@@ -11,49 +11,103 @@
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
static int64_t num_particles = 25000; /* ~1MB */
static char output[4096] = {0};
static char subfiling[8] = {0};
static bool aos = true;
void printUsage () {
int rank;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
if (rank == 0) {
fprintf ( stderr, "Usage: ./HACC-IO-MPIIO-R -p <particles> -o <output directory> -t <target memory> -s <subfiling> -l <layout>\n" );
fprintf ( stderr, " -p : Number of particles per rank (38B/part., 25K part. ~= 1MB)\n" );
fprintf ( stderr, " -o : Path of the output directory\n" );
fprintf ( stderr, " -s : Subfiling method: FPN, FPION, SSF (default)\n" );
fprintf ( stderr, " -l : Layout: SOA, AOS (default)\n" );
}
}
void parseArgs (int argc, char * argv[])
{
char flags[] = "hp:o:s:l:";
int opt = 0;
while ((opt = getopt (argc, argv, flags)) != -1) {
switch ( opt )
{
case('h'):
printUsage ();
break;
case('p'):
sscanf ( optarg, "%lld", &num_particles );
break;
case('o'):
sprintf ( output, "%s", optarg );
break;
case('s'):
sprintf ( subfiling, "%s", optarg );
break;
case('l'):
if ( ! strcmp ( "SOA", optarg ) )
aos = false;
break;
}
}
if ( num_particles <= 0 ) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong number of particles. Reset to the default value (25000/rank)\n");
num_particles = 25000;
}
if ( strcmp ( "FPN", subfiling ) &&
strcmp ( "FPION", subfiling ) &&
strcmp ( "SSF", subfiling )) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong subfiling method. Reset to the default value (SSF: Single Shared File)\n");
sprintf ( subfiling, "SSF" );
}
}
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Info info;
MPI_Status status;
char output[100];
Tapioca tp;
char filename[256];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId ();
parseArgs ( argc, argv );
if ( ! strcmp ( "FPN", subfiling ) ) mycolor = tp.topology.ComputeNodeId ();
if ( ! strcmp ( "FPION", subfiling ) ) mycolor = tp.topology.BridgeNodeId ();
if ( ! strcmp ( "SSF", subfiling ) ) mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0;
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#else
snprintf (output, 100, "./HACC-SOA-%08d-%d.dat", mycolor, file_id);
#endif
snprintf (filename, 256, "/HACC-IO-%s-%08d.dat", aos ? "AoS" : "SoA", mycolor);
strcat (output, filename);
/*****************/
/* WRITE */
/* READ */
/*****************/
float *xx, *yy, *zz, *vx, *vy, *vz, *phi;
int64_t* pid;
......@@ -89,77 +143,16 @@ int main (int argc, char * argv[])
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
if (sub_myrank == 0) {
MPI_File_open(MPI_COMM_SELF, output,
MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
MPI_File_set_size(file_handle, file_size);
MPI_File_close (&file_handle);
}
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-AoS-MPIIO\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (AoS data layout)\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] HACC-IO MPI-IO\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read input file (%s data layout)\n", mycolor, aos ? "AoS" : "SoA");
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
}
MPI_Info_create ( &info );
MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, info, &file_handle);
offset = scan_size * particle_size;
MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* READ */
/*****************/
float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r;
int64_t* pid_r;
uint16_t* mask_r;
......@@ -177,36 +170,33 @@ int main (int argc, char * argv[])
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, info, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * particle_size;
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
offset = aos ? scan_size * particle_size : scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
offset += aos ? num_particles * sizeof(int64_t) : (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
......@@ -218,26 +208,29 @@ int main (int argc, char * argv[])
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
fprintf (stdout, BLUE "[TIMING]" RESET " Read MPI-IO I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
fprintf (stdout, RED "[ERROR]" RESET "[%08d][%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n",
mycolor, world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] );
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor);
/*****************/
/* FREE */
/*****************/
......
......@@ -11,49 +11,100 @@
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
mem_t memAggr = NVR;
mem_t memTarg = HDD;
static int64_t num_particles = 25000; /* ~1MB */
static char output[4096] = {0};
static char subfiling[8] = {0};
static bool aos = true;
void printUsage () {
int rank;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
if (rank == 0) {
fprintf ( stderr, "Usage: ./HACC-IO-MPIIO-W -p <particles> -o <output directory> -t <target memory> -s <subfiling> -l <layout>\n" );
fprintf ( stderr, " -p : Number of particles per rank (38B/part., 25K part. ~= 1MB)\n" );
fprintf ( stderr, " -o : Path of the output directory\n" );
fprintf ( stderr, " -s : Subfiling method: FPN, FPION, SSF (default)\n" );
fprintf ( stderr, " -l : Layout: SOA, AOS (default)\n" );
}
}
void parseArgs (int argc, char * argv[])
{
char flags[] = "hp:o:s:l:";
int opt = 0;
while ((opt = getopt (argc, argv, flags)) != -1) {
switch ( opt )
{
case('h'):
printUsage ();
break;
case('p'):
sscanf ( optarg, "%lld", &num_particles );
break;
case('o'):
sprintf ( output, "%s", optarg );
break;
case('s'):
sprintf ( subfiling, "%s", optarg );
break;
case('l'):
if ( ! strcmp ( "SOA", optarg ) )
aos = false;
break;
}
}
if ( num_particles <= 0 ) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong number of particles. Reset to the default value (25000/rank)\n");
num_particles = 25000;
}
if ( strcmp ( "FPN", subfiling ) &&
strcmp ( "FPION", subfiling ) &&
strcmp ( "SSF", subfiling )) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong subfiling method. Reset to the default value (SSF: Single Shared File)\n");
sprintf ( subfiling, "SSF" );
}
}
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles;
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0;
int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
Tapioca tp;
int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9];
char filename[256];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId ();
parseArgs ( argc, argv );
if ( ! strcmp ( "FPN", subfiling ) ) mycolor = tp.topology.ComputeNodeId ();
if ( ! strcmp ( "FPION", subfiling ) ) mycolor = tp.topology.BridgeNodeId ();
if ( ! strcmp ( "SSF", subfiling ) ) mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0;
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#else
snprintf (output, 100, "./HACC-SOA-%08d-%d.dat", mycolor, file_id);
#endif
snprintf (filename, 256, "/HACC-IO-%s-%08d.dat", aos ? "AoS" : "SoA", mycolor);
strcat (output, filename);
/*****************/
/* WRITE */
......@@ -92,143 +143,82 @@ int main (int argc, char * argv[])
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
if (sub_myrank == 0) {
MPI_File_open(MPI_COMM_SELF, output,
MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
MPI_File_set_size(file_handle, file_size);
MPI_File_close (&file_handle);
}
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-AoS\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file (AoS data layout)\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] HACC-IO MPI-IO\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (%s data layout)\n", mycolor, aos ? "AoS" : "SoA");
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
}
/*****************/
/* READ */
/*****************/
float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r;
int64_t* pid_r;
uint16_t* mask_r;
xx_r = new float[num_particles];
yy_r = new float[num_particles];
zz_r = new float[num_particles];
vx_r = new float[num_particles];
vy_r = new float[num_particles];
vz_r = new float[num_particles];
phi_r = new float[num_particles];
pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles];
/*****************/
/* INIT TAPIOCA */
/*****************/
for ( i = 0; i < 9; i++ ) {
chunkCount[i] = num_particles;
}
chunkSize[0] = sizeof(float);
chunkSize[1] = sizeof(float);
chunkSize[2] = sizeof(float);
chunkSize[3] = sizeof(float);
chunkSize[4] = sizeof(float);
chunkSize[5] = sizeof(float);
chunkSize[6] = sizeof(float);
chunkSize[7] = sizeof(int64_t);
chunkSize[8] = sizeof(uint16_t);
chunkOffset[0] = hdr + scan_size * particle_size;
for ( i = 1; i < 9; i++ ) {
chunkOffset[i] = chunkOffset[i - 1] + chunkCount[i - 1] * chunkSize[i - 1];
}
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (2, memAggr, "/scratch/tmp");
tp.setTargetTier (memTarg, file_size, output);
/*****************/
start_time = MPI_Wtime();
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * particle_size;
tp.Read (offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
offset = aos ? scan_size * particle_size : scan_size * sizeof(float);
tp.Read (offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
offset += aos ? num_particles * sizeof(float) : (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
offset += aos ? num_particles * sizeof(int64_t) : (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
tp.Finalize ();
MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
fprintf (stdout, BLUE "[TIMING]" RESET " Write MPI-IO I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{
fprintf (stdout, RED "[ERROR]" RESET "[%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n",
world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] );
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor);
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* FREE */
/*****************/
delete [] xx;
delete [] xx_r;
delete [] yy;
delete [] yy_r;
delete [] zz;
delete [] zz_r;
delete [] vx;
delete [] vx_r;
delete [] vy;
delete [] vy_r;
delete [] vz;
delete [] vz_r;
delete [] phi;
delete [] phi_r;
delete [] pid;
delete [] pid_r;
delete [] mask;
delete [] mask_r;
MPI_Finalize ();
}
......
......@@ -11,46 +11,100 @@
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
static int64_t num_particles = 25000; /* ~1MB */
static char output[4096] = {0};
static char subfiling[8] = {0};
static bool aos = true;
void printUsage () {
int rank;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
if (rank == 0) {
fprintf ( stderr, "Usage: ./HACC-IO-MPIIO-WR -p <particles> -o <output directory> -t <target memory> -s <subfiling> -l <layout>\n" );
fprintf ( stderr, " -p : Number of particles per rank (38B/part., 25K part. ~= 1MB)\n" );
fprintf ( stderr, " -o : Path of the output directory\n" );
fprintf ( stderr, " -s : Subfiling method: FPN, FPION, SSF (default)\n" );
fprintf ( stderr, " -l : Layout: SOA, AOS (default)\n" );
}
}
void parseArgs (int argc, char * argv[])
{
char flags[] = "hp:o:s:l:";
int opt = 0;
while ((opt = getopt (argc, argv, flags)) != -1) {
switch ( opt )
{
case('h'):
printUsage ();
break;
case('p'):
sscanf ( optarg, "%lld", &num_particles );
break;
case('o'):
sprintf ( output, "%s", optarg );
break;
case('s'):
sprintf ( subfiling, "%s", optarg );
break;
case('l'):
if ( ! strcmp ( "SOA", optarg ) )
aos = false;
break;
}
}
if ( num_particles <= 0 ) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong number of particles. Reset to the default value (25000/rank)\n");
num_particles = 25000;
}
if ( strcmp ( "FPN", subfiling ) &&
strcmp ( "FPION", subfiling ) &&
strcmp ( "SSF", subfiling )) {
printUsage ();
fprintf (stdout, RED "[WARNING]" RESET " Wrong subfiling method. Reset to the default value (SSF: Single Shared File)\n");
sprintf ( subfiling, "SSF" );
}
}
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time