Commit dbf401f8 authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files
parents 4932b595 7e13129e
......@@ -441,7 +441,7 @@ H5VL_pass_through_ext_register(void)
static herr_t
H5VL_pass_through_ext_init(hid_t vipl_id)
{
printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__);
printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__);
#ifdef ENABLE_EXT_PASSTHRU_LOGGING
printf("------- EXT PASS THROUGH VOL INIT\n");
#endif
......
#Makefile
CXX=mpicxx -g -O3
CFLAGS=-I$(HDF5_ROOT)/include -O3 -I../utils
HDF5_LIB=-L$(HDF5_ROOT)/lib -lhdf5
%.o: %.cpp
$(CXX) $(CFLAGS) $(CPPFLAGS) -o $@ -c $<
test: test_cache test_mmap_mem
test_cache: test_cache.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o
test_mmap_mem: test_mmap_mem.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o
clean:
rm -rf *.o read_dataset_cache prepare_dataset test_mmap test_cache test_mmap_mem
/*
This is the test program for studying the cache effect in POSIX I/O.
Normally, when we write data to the files. The system will keep a copy of the data in the
cache. So when we are reading the same data from the files, it actually read from the cache
instead of from the file system.
In order to avoid the cache effect, one can write more than one files, so by the time
reading the first file, the first file is already been pushed out of the cache.
The other way to avoid the cache effect is to separate out the read from the write. In the
benchmark, we do write first, and then do read in an independent run. The benchmarks script
allow us to do this.
Huihuo Zheng
- huihuo.zheng@anl.gov
*/
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <iostream>
#include "stdio.h"
#include "string.h"
#include "stdlib.h"
#include "profiling.h"
#include "timing.h"
#include "mpi.h"
using namespace std;
int main(int argc, char **argv) {
int dim = 1024576;
int nbatch = 1024;
int niter = 10;
double dat[dim];
double vm, rss;
char path[255] = "./";
bool write = false;
for (int i=1; i<argc; i++) {
if (strcmp(argv[i], "--dim")==0) {
dim = int(atof(argv[i+1])); i=i+1;
} else if (strcmp(argv[i], "--nbatch")==0) {
nbatch = int(atof(argv[i+1])); i=i+1;
} else if (strcmp(argv[i], "--niter")==0) {
niter = int(atof(argv[i+1])); i=i+1;
} else if (strcmp(argv[i], "--path")==0) {
strcpy(path, argv[i+1]); i=i+1;
} else if (strcmp(argv[i], "--write")==0) {
write = true;
}
}
MPI_Init(&argc, &argv);
int rank, nproc;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
Timing tt(rank==0);
if (rank==0) {
cout << "dim: " << dim << endl;
cout << "nbatch: " << nbatch << endl;
cout << "size of file [MiB]: " << dim*nbatch*sizeof(double)/1024/1024 << endl;
cout << "niter: " << niter << endl;
cout << "path: " << path << endl;
cout << "nproc: " << nproc << endl;
}
if (write) {
for(int it =0; it < niter; it++) {
dat[0] = it;
char fname[255];
strcpy(fname, path);
strcat(fname, "/cache.dat");
strcat(fname, to_string(rank).c_str());
strcat(fname, "-");
strcat(fname, to_string(it).c_str());
tt.start_clock("write");
MPI_Barrier(MPI_COMM_WORLD);
int fd = open(fname, O_RDWR | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR);
for(int i=0; i<nbatch; i++) {
pwrite(fd, dat, sizeof(double)*dim, sizeof(double)*i*dim);
}
MPI_Barrier(MPI_COMM_WORLD);
tt.stop_clock("write");
fsync(fd);
if (rank==0) cout << "Write rate: " << sizeof(double)*dim*nbatch/tt["write"].t_iter[it]/1024/1024*nproc << endl;
close(fd);
}
}
for(int it =0; it < niter; it++) {
char fname[255];
strcpy(fname, path);
strcat(fname, "/cache.dat");
strcat(fname, to_string(rank).c_str());
strcat(fname, "-");
strcat(fname, to_string(it).c_str());
MPI_Barrier(MPI_COMM_WORLD);
int fd = open(fname, O_RDONLY);
tt.start_clock("read");
for(int i=0; i<nbatch; i++) {
pread(fd, dat, sizeof(double)*dim, sizeof(double)*i*dim);
}
MPI_Barrier(MPI_COMM_WORLD);
tt.stop_clock("read");
if (rank==0) cout << "Read rate: " << sizeof(double)*dim*nbatch/tt["read"].t_iter[it]/1024/1024*nproc << endl;
close(fd);
remove(fname);
}
MPI_Finalize();
return 0;
}
/*
This program is to test the memory footprint when using
the memory mapped file.
In principle, when we map the files into a pointer, before accessing
the pointer, the amount of Resident set is the same, whereas the amount
of virtual memory is increased. Then when we start to access the memory
mapped file using the pointer, the amount of resisent set increases.
However, the increase will stop at certain point if it reach to the
maximum capacity of the physical memory of the system.
Huihuo Zheng
*/
#include <iostream>
#include <random>
#include <algorithm>
#include <vector>
#include "stdlib.h"
#include "string.h"
#include "stdio.h"
#include "mpi.h"
// POSIX I/O
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
// Memory map
#include <sys/mman.h>
#include "timing.h"
#include "time.h"
#include <iostream>
#include <fstream>
#include <unistd.h>
#include "profiling.h"
using namespace std;
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);
double compute = 0.0;
char fname[255] = "images.dat";
char local_storage[255] = "./";
bool shuffle = false;
bool mpio_collective = false;
bool mpio_independent = false;
bool cache = false;
int epochs = 4;
int num_batches = 16;
int batch_size = 32;
int rank_shift = 0;
bool write = false;
bool read = false;
bool remap = false;
int sz = 224;
int i=0;
// Input
while (i<argc) {
if (strcmp(argv[i], "--filename")==0) {
strcpy(fname, argv[i+1]); i+=2;
} else if (strcmp(argv[i], "--num_batches")==0) {
num_batches = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--batch_size")==0) {
batch_size = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--shuffle")==0) {
shuffle = true; i=i+1;
} else if (strcmp(argv[i], "--epochs") == 0) {
epochs = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--sz") == 0) {
sz = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--remap") == 0) {
remap = true; i=i+1;
} else if (strcmp(argv[i], "--write") == 0) {
write = true; i=i+1;
} else if (strcmp(argv[i], "--read") == 0) {
read = true; i=i+1;
} else if (strcmp(argv[i], "--local_storage")==0) {
strcpy(local_storage, argv[i+1]); i+=2;
} else if (strcmp(argv[i], "--compute")==0) {
compute = atof(argv[i+1]); i+=2;
} else {
i=i+1;
}
}
int dim = sz*sz*3;
strcat(local_storage, "/");
strcat(local_storage, fname);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
Timing tt(rank==0);
strcat(local_storage, to_string(rank).c_str());
unsigned long size = dim*sizeof(float)*num_batches*batch_size;
if (write) {
int fh = open(local_storage, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
char *a = new char [batch_size*sizeof(float)*dim];
for (int i=0; i<num_batches; i++) {
pwrite(fh, a, batch_size*sizeof(float)*dim, i*batch_size*sizeof(float)*dim);
}
fsync(fh);
close(fh);
}
double vm, rss;
process_mem_usage(vm, rss);
if (rank==0) cout << "VM: " << vm << "; RSS: " << rss << endl;
int num_images = num_batches * batch_size;
vector<int> id;
id.resize(num_images);
for(int i=0; i<num_images; i++) id[i] = i;
mt19937 g(100);
srand (time(NULL));
/* generate secret number between 1 and 10: */
int iSecret = rand() % 1024576 + 1;
int fd = open(local_storage, O_RDWR);
void *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
float *p = (float *)buf;
float *dat = new float [dim*batch_size];
for(int i=0; i<dim*batch_size; i++) dat[i] = iSecret;
if (write)
for (int e = 0; e < epochs; e++) {
tt.start_clock("write");
if (shuffle) {
::shuffle(id.begin(), id.end(), g);
for (int b = 0; b < num_batches; b++) {
for (int s = 0; s < batch_size; s++) {
memcpy(&p[id[b*batch_size+s]*dim], dat, dim*sizeof(float));
}
process_mem_usage(vm, rss);
if (rank==0) cout << "WRITE MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
} else {
for (int b = 0; b < num_batches; b++) {
memcpy(&p[id[b*batch_size]*dim], dat, dim*sizeof(float)*batch_size);
process_mem_usage(vm, rss);
if (rank==0) cout << "WRITE MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
tt.stop_clock("write");
double w = dim*sizeof(float)*num_batches*batch_size/tt["write"].t_iter[e]/1024/1024;
if (rank==0) cout << "Epoch " << e << ": " << " rate: " << w << "MB/s" << endl;
if (remap) {
munmap(buf, size);
close(fd);
fd = open(local_storage, O_RDWR);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
process_mem_usage(vm, rss);
if (rank==0) cout << "REMAP MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
if (read)
for (int e = 0; e < epochs; e++) {
if (remap) {
munmap(buf, size);
close(fd);
fd = open(local_storage, O_RDWR);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
}
float *p = (float *)buf;
tt.start_clock("read");
if (shuffle) {
::shuffle(id.begin(), id.end(), g);
for (int b = 0; b < num_batches; b++) {
for (int s = 0; s < batch_size; s++) {
memcpy(&dat[s*dim], &p[id[b*batch_size+s]*dim], dim*sizeof(float));
}
process_mem_usage(vm, rss);
if (rank==0) cout << "READ MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
} else {
for (int b = 0; b < num_batches; b++) {
memcpy(dat, &p[id[b*batch_size]*dim], dim*sizeof(float)*batch_size);
process_mem_usage(vm, rss);
if (rank==0) cout << "READ MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
tt.stop_clock("read");
double w = dim*sizeof(float)*num_batches*batch_size/tt["read"].t_iter[e]/1024/1024;
if (rank==0) cout << "Epoch " << e << ": " << " rate: " << w << "MB/s" << endl;
}
delete [] dat;
MPI_Finalize();
return 0;
}
#include "mpi.h"
#include "stdio.h"
#include <iostream>
#include <vector>
#include "assert.h"
#include "string.h"
#include "stdlib.h"
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <algorithm> // std::shuffle
#include <random>
#include <thread> // std::this_thread::sleep_for
#include <chrono>
using namespace std;
int main(int argc, char **argv) {
MPI_Win win_a, win_b;
size_t dim_a, dim_b, dim;
int rank, nproc;
int ta, tb;
dim_a = 1024; dim_b = 1024; dim = dim_a + dim_b;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
int *a = new int [dim_a];
int *b = new int [dim_b];
for(int i=0; i<dim_a; i++) {
a[i] = rank;
}
for(int i=0; i<dim_b; i++) {
b[i] = rank + nproc;
}
MPI_Win_create(a, sizeof(int)*dim_a, sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win_a);
MPI_Win_create(b, sizeof(int)*dim_b, sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win_b);
MPI_Win_fence(MPI_MODE_NOPUT, win_a);
MPI_Win_fence(MPI_MODE_NOPUT, win_b);
if (nproc > 1) {
MPI_Get(&ta, 1, MPI_INT, (rank+1)%nproc, 0, 1, MPI_INT, win_a);
MPI_Get(&tb, 1, MPI_INT, (rank+1)%nproc, 0, 1, MPI_INT, win_b);
} else {
ta = a[0];
tb = b[0];
}
MPI_Win_fence(MPI_MODE_NOPUT, win_b);
MPI_Win_fence(MPI_MODE_NOPUT, win_a);
for(int i=0; i<nproc; i++) {
if (rank==i) {
cout << "rank " << rank << ": ta - " << ta << " expected: " << (rank+1)%nproc << endl;
cout << "rank " << rank << ": tb - " << tb << " expected: " << (rank+1)%nproc + nproc << endl;
}
this_thread::sleep_for (chrono::seconds(1));
}
MPI_Win_free(&win_a); delete [] a;
MPI_Win_free(&win_b); delete [] b;
MPI_Finalize();
}
#include "mpi.h"
#include "stdio.h"
#include <iostream>
#include <vector>
#include "assert.h"
#include "string.h"
#include "stdlib.h"
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <algorithm> // std::shuffle
#include <random>
#include <thread> // std::this_thread::sleep_for
#include <chrono>
using namespace std;
int main(int argc, char **argv) {
MPI_Win win;
size_t dim_a, dim_b, dim;
int rank, nproc;
int ta, tb;
dim_a = 1024; dim_b = 1024; dim = dim_a + dim_b;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
int *a = new int [dim_a];
int *b = new int [dim_b];
for(int i=0; i<dim_a; i++) {
a[i] = rank;
}
for(int i=0; i<dim_b; i++) {
b[i] = rank + nproc;
}
MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &win);
MPI_Win_attach(win, a, sizeof(int)*dim_a);
MPI_Win_attach(win, b, sizeof(int)*dim_b);
MPI_Aint *disp = new MPI_Aint [nproc];
MPI_Aint *disp_all = new MPI_Aint [nproc];
for(int i=0; i<nproc; i++) {
disp[i] = 0;
disp_all[i] = 0;
}
MPI_Get_address(b, &disp[rank]);
MPI_Allreduce(disp_all, disp, nproc, MPI_AINT, MPI_SUM, MPI_COMM_WORLD);
if (rank==0)
for (int i=0; i<nproc; i++)
cout << i << ": " << disp_all[i] << ", " << disp[i] << endl;
MPI_Win_fence(MPI_MODE_NOPUT, win);
if (nproc > 1) {
MPI_Get(&ta, 1, MPI_INT, (rank+1)%nproc, disp_all[(rank+1)%nproc], 1, MPI_INT, win);
} else {
ta = a[0];
}
MPI_Win_fence(MPI_MODE_NOPUT, win);
for(int i=0; i<nproc; i++) {
if (rank==i) {
cout << "rank " << rank << ": ta - " << ta << " expected: " << (rank+1)%nproc << endl;
}
this_thread::sleep_for (chrono::seconds(1));
}
MPI_Win_detach(win, a); delete [] a;
MPI_Win_free(&win); delete [] b;
MPI_Finalize();
}
xb// CPP code to illustrate
// Queue in Standard Template Library (STL)
#include <iostream>
#include <queue>
using namespace std;
struct mpi_put_args {
void *buf;
size_t size;
MPI_Datatype data_type;
}
void showq(queue <int> gq)
{
queue <int> g = gq;
while (!g.empty())
{
cout << '\t' << g.front();
g.pop();
}
cout << '\n';
}
int main()
{
queue <int> gquiz;
gquiz.push(10);
gquiz.push(20);
gquiz.push(30);
cout << "The queue gquiz is : ";
showq(gquiz);
cout << "\ngquiz.size() : " << gquiz.size();
cout << "\ngquiz.front() : " << gquiz.front();
cout << "\ngquiz.back() : " << gquiz.back();
cout << "\ngquiz.pop() : ";
gquiz.pop();
showq(gquiz);
return 0;
}
/*
this program is to understand the initialization of
*/
#include <iostream>
using namespace std;
typedef struct _shape {
double x;
double y;
} shape;
typedef struct _rect {
double area;
shape s;
} rect;
int main(int argc, char **argv) {
rect a = {
.area = 0,
.s.x = 1.0,
.s.y = 2.0
};
cout << a.s.x << endl;
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment