From c17b96406380eadbbd2f60bb7ce5540a279c2ac7 Mon Sep 17 00:00:00 2001 From: Francois Tessier Date: Tue, 17 Jan 2017 21:22:15 +0000 Subject: [PATCH] Add miniHACC implemented with MPI I/O. Add a script to run all the binaries and compare output files with md5 sum. --- examples/miniHACC-AoS-MPIIO.cpp | 245 ++++++++++++++++++++++++++++++++ examples/miniHACC-SoA-MPIIO.cpp | 245 ++++++++++++++++++++++++++++++++ examples/run_check.sh | 87 ++++++++++++ 3 files changed, 577 insertions(+) create mode 100644 examples/miniHACC-AoS-MPIIO.cpp create mode 100644 examples/miniHACC-SoA-MPIIO.cpp create mode 100755 examples/run_check.sh diff --git a/examples/miniHACC-AoS-MPIIO.cpp b/examples/miniHACC-AoS-MPIIO.cpp new file mode 100644 index 0000000..62c45aa --- /dev/null +++ b/examples/miniHACC-AoS-MPIIO.cpp @@ -0,0 +1,245 @@ +#include +#include +#include +#include +#include +#include + +#define RED "\x1b[31m" +#define GREEN "\x1b[32m" +#define BLUE "\x1b[34m" +#define RESET "\x1b[0m" + +int main (int argc, char * argv[]) +{ + int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank; + int64_t num_particles = 25000; + int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; + int64_t scan_size = 0, offset; + double start_time, end_time, tot_time, max_time; + double io_bw; + MPI_Comm sub_comm; + MPI_File file_handle; + MPI_Status status; + char output[100]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); + MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); + + mycolor = MPIX_IO_link_id (); + mykey = world_myrank; + + MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); + MPI_Comm_size(sub_comm, &sub_numtasks); + MPI_Comm_rank(sub_comm, &sub_myrank); + + snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d.dat", mycolor); + + /*****************/ + /* WRITE */ + /*****************/ + float *xx, *yy, *zz, *vx, *vy, *vz, *phi; + int64_t* pid; + uint16_t* mask; + + xx = new float[num_particles]; + yy = new float[num_particles]; + zz = new float[num_particles]; + vx = new float[num_particles]; + vy = new float[num_particles]; + vz = new float[num_particles]; + phi = new float[num_particles]; + pid = new int64_t[num_particles]; + mask = new uint16_t[num_particles]; + + for (uint64_t i = 0; i< num_particles; i++) + { + xx[i] = (float)i; + yy[i] = (float)i; + zz[i] = (float)i; + vx[i] = (float)i; + vy[i] = (float)i; + vz[i] = (float)i; + phi[i] = (float)i; + pid[i] = (int64_t)i; + mask[i] = (uint16_t)world_myrank; + } + + MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm); + MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + + particle_size = (7 * sizeof(float)) + sizeof(int64_t) + sizeof(uint16_t); + file_size = particle_size * sub_particles; + tot_size = particle_size * tot_particles; + + if (sub_myrank == 0) { + MPI_File_open(MPI_COMM_SELF, output, + MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle); + MPI_File_set_size(file_handle, file_size); + MPI_File_close (&file_handle); + } + + MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm); + + if (0 == sub_myrank) { + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-AoS-MPIIO\n", mycolor); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (AoS data layout)\n", mycolor); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n", + mycolor, (double)file_size/(1024*1024), sub_particles); + } + + start_time = MPI_Wtime(); + + MPI_File_open(sub_comm, output, + MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle); + + offset = scan_size * particle_size; + + MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status); + offset += num_particles * sizeof(int64_t); + + MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status); + + MPI_File_close (&file_handle); + + end_time = MPI_Wtime(); + tot_time = end_time - start_time; + MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + + if (0 == world_myrank) { + io_bw = (double)tot_size / max_time / (1024 * 1024); + fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", + io_bw, (double)tot_size/(1024*1024), max_time * 1000); + } + + MPI_Barrier (MPI_COMM_WORLD); + + /*****************/ + /* READ */ + /*****************/ + float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r; + int64_t* pid_r; + uint16_t* mask_r; + + xx_r = new float[num_particles]; + yy_r = new float[num_particles]; + zz_r = new float[num_particles]; + vx_r = new float[num_particles]; + vy_r = new float[num_particles]; + vz_r = new float[num_particles]; + phi_r = new float[num_particles]; + pid_r = new int64_t[num_particles]; + mask_r = new uint16_t[num_particles]; + + start_time = MPI_Wtime(); + + MPI_File_open(sub_comm, output, + MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle); + + if (0 == sub_myrank) + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor); + + offset = scan_size * particle_size; + + MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status); + offset += num_particles * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status); + offset += num_particles * sizeof(int64_t); + + MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); + + MPI_File_close (&file_handle); + + end_time = MPI_Wtime(); + tot_time = end_time - start_time; + MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + + if (0 == world_myrank) { + io_bw = (double)tot_size / max_time / (1024 * 1024); + fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", + io_bw, (double)tot_size/(1024*1024), max_time * 1000); + } + + /*****************/ + /* VERIFICATION */ + /*****************/ + for (uint64_t i = 0; i< num_particles; i++) { + if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i]) + || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) + || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) + { + fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i); + MPI_Abort (MPI_COMM_WORLD, -1); + } + } + + if (0 == sub_myrank) + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor); + + /*****************/ + /* FREE */ + /*****************/ + delete [] xx; + delete [] xx_r; + delete [] yy; + delete [] yy_r; + delete [] zz; + delete [] zz_r; + delete [] vx; + delete [] vx_r; + delete [] vy; + delete [] vy_r; + delete [] vz; + delete [] vz_r; + delete [] phi; + delete [] phi_r; + delete [] pid; + delete [] pid_r; + delete [] mask; + delete [] mask_r; + + MPI_Finalize (); +} + diff --git a/examples/miniHACC-SoA-MPIIO.cpp b/examples/miniHACC-SoA-MPIIO.cpp new file mode 100644 index 0000000..0688e33 --- /dev/null +++ b/examples/miniHACC-SoA-MPIIO.cpp @@ -0,0 +1,245 @@ +#include +#include +#include +#include +#include +#include + +#define RED "\x1b[31m" +#define GREEN "\x1b[32m" +#define BLUE "\x1b[34m" +#define RESET "\x1b[0m" + +int main (int argc, char * argv[]) +{ + int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank; + int64_t num_particles = 25000; + int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; + int64_t scan_size = 0, offset; + double start_time, end_time, tot_time, max_time; + double io_bw; + MPI_Comm sub_comm; + MPI_File file_handle; + MPI_Status status; + char output[100]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); + MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); + + mycolor = MPIX_IO_link_id (); + mykey = world_myrank; + + MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); + MPI_Comm_size(sub_comm, &sub_numtasks); + MPI_Comm_rank(sub_comm, &sub_myrank); + + snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d.dat", mycolor); + + /*****************/ + /* WRITE */ + /*****************/ + float *xx, *yy, *zz, *vx, *vy, *vz, *phi; + int64_t* pid; + uint16_t* mask; + + xx = new float[num_particles]; + yy = new float[num_particles]; + zz = new float[num_particles]; + vx = new float[num_particles]; + vy = new float[num_particles]; + vz = new float[num_particles]; + phi = new float[num_particles]; + pid = new int64_t[num_particles]; + mask = new uint16_t[num_particles]; + + for (uint64_t i = 0; i< num_particles; i++) + { + xx[i] = (float)i; + yy[i] = (float)i; + zz[i] = (float)i; + vx[i] = (float)i; + vy[i] = (float)i; + vz[i] = (float)i; + phi[i] = (float)i; + pid[i] = (int64_t)i; + mask[i] = (uint16_t)world_myrank; + } + + MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm); + MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + + particle_size = (7 * sizeof(float)) + sizeof(int64_t) + sizeof(uint16_t); + file_size = particle_size * sub_particles; + tot_size = particle_size * tot_particles; + + if (sub_myrank == 0) { + MPI_File_open(MPI_COMM_SELF, output, + MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle); + MPI_File_set_size(file_handle, file_size); + MPI_File_close (&file_handle); + } + + MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm); + + if (0 == sub_myrank) { + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-SoA-MPIIO\n", mycolor); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (SoA data layout)\n", mycolor); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles); + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n", + mycolor, (double)file_size/(1024*1024), sub_particles); + } + + start_time = MPI_Wtime(); + + MPI_File_open(sub_comm, output, + MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle); + + offset = scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t); + + MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status); + offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t); + + MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status); + + MPI_File_close (&file_handle); + + end_time = MPI_Wtime(); + tot_time = end_time - start_time; + MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + + if (0 == world_myrank) { + io_bw = (double)tot_size / max_time / (1024 * 1024); + fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", + io_bw, (double)tot_size/(1024*1024), max_time * 1000); + } + + MPI_Barrier (MPI_COMM_WORLD); + + /*****************/ + /* READ */ + /*****************/ + float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r; + int64_t* pid_r; + uint16_t* mask_r; + + xx_r = new float[num_particles]; + yy_r = new float[num_particles]; + zz_r = new float[num_particles]; + vx_r = new float[num_particles]; + vy_r = new float[num_particles]; + vz_r = new float[num_particles]; + phi_r = new float[num_particles]; + pid_r = new int64_t[num_particles]; + mask_r = new uint16_t[num_particles]; + + start_time = MPI_Wtime(); + + MPI_File_open(sub_comm, output, + MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle); + + if (0 == sub_myrank) + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor); + + offset = scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); + + MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status); + offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t); + + MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status); + offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t); + + MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); + + MPI_File_close (&file_handle); + + end_time = MPI_Wtime(); + tot_time = end_time - start_time; + MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + + if (0 == world_myrank) { + io_bw = (double)tot_size / max_time / (1024 * 1024); + fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", + io_bw, (double)tot_size/(1024*1024), max_time * 1000); + } + + /*****************/ + /* VERIFICATION */ + /*****************/ + for (uint64_t i = 0; i< num_particles; i++) { + if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i]) + || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) + || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) + { + fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i); + MPI_Abort (MPI_COMM_WORLD, -1); + } + } + + if (0 == sub_myrank) + fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor); + + /*****************/ + /* FREE */ + /*****************/ + delete [] xx; + delete [] xx_r; + delete [] yy; + delete [] yy_r; + delete [] zz; + delete [] zz_r; + delete [] vx; + delete [] vx_r; + delete [] vy; + delete [] vy_r; + delete [] vz; + delete [] vz_r; + delete [] phi; + delete [] phi_r; + delete [] pid; + delete [] pid_r; + delete [] mask; + delete [] mask_r; + + MPI_Finalize (); +} + diff --git a/examples/run_check.sh b/examples/run_check.sh new file mode 100755 index 0000000..3f0d1fb --- /dev/null +++ b/examples/run_check.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1" +NODES=256 +PPN=16 +NPROCS=$((NODES*PPN)) +TARGET="/projects/visualization/ftessier/debug" + +cd $HOME/TAPIOCA/examples + +export TAPIOCA_DEVNULL=false +export TAPIOCA_COMMSPLIT=true +export TAPIOCA_STRATEGY=TOPOLOGY_AWARE +export TAPIOCA_NBAGGR=8 +export TAPIOCA_BUFFERSIZE=16777216 + +function updateSettings() +{ + printenv | egrep "TAPIOCA_" + SETTINGS="$SETTINGS TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL" + SETTINGS="$SETTINGS TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT" + SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY" + SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR" + SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE" +} + +######################### +# Array of Structures +######################### +rm $TARGET/* +runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO +sleep 4 +let j=0 +for i in $TARGET/* +do + CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1` + echo ${CONTROLAOS[$j]} + let j=$j+1 +done + +rm $TARGET/* +runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS +sleep 4 +let j=0 +for i in $TARGET/* +do + HASH[$j]=`md5sum $i | cut -d ' ' -f1` + if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ] + then + echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" + else + echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" + fi + let j=$j+1 +done + +######################### +# Structure of Arrays +######################### +rm $TARGET/* +runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO +sleep 4 +let j=0 +for i in $TARGET/* +do + CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1` + echo ${CONTROLSOA[$j]} + let j=$j+1 +done + +rm $TARGET/* +runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA +sleep 4 +let j=0 +for i in $TARGET/* +do + HASH[$j]=`md5sum $i | cut -d ' ' -f1` + if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ] + then + echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" + else + echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" + fi + let j=$j+1 +done + + -- 2.22.2