Commit 1581bc89 authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

test mmap mem

parent d88056ae
......@@ -10,6 +10,8 @@ HDF5_LIB=-L$(HDF5_ROOT)/lib -lhdf5
all: read_dataset_cache prepare_dataset
test_mmap: test_mmap.o
$(CXX) $(CFLAGS) -o $@ $<
test_mmap_mem: test_mmap_mem.o
$(CXX) $(CFLAGS) -o $@ $<
read_dataset_cache: read_dataset_cache.o H5Dio_cache.o ../utils/debug.o
$(CXX) $(CFLAGS) -o $@ read_dataset_cache.o H5Dio_cache.o ../utils/debug.o $(HDF5_LIB) -lpthread
prepare_dataset: prepare_dataset.o
......
#include <iostream>
#include <random>
#include <algorithm>
#include <vector>
#include "stdlib.h"
#include "string.h"
#include "stdio.h"
#include "mpi.h"
// POSIX I/O
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
// Memory map
#include <sys/mman.h>
#include "timing.h"
#include "time.h"
#include <iostream>
#include <fstream>
#include <unistd.h>
using namespace std;
void process_mem_usage(double& vm_usage, double& resident_set)
{
vm_usage = 0.0;
resident_set = 0.0;
// the two fields we want
unsigned long vsize;
long rss;
{
std::string ignore;
std::ifstream ifs("/proc/self/stat", std::ios_base::in);
ifs >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> vsize >> rss;
}
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
}
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);
double compute = 0.0;
char fname[255] = "images.dat";
char local_storage[255] = "./";
bool shuffle = false;
bool mpio_collective = false;
bool mpio_independent = false;
bool cache = false;
int epochs = 4;
int num_batches = 16;
int batch_size = 32;
int rank_shift = 0;
bool write = false;
bool read = false;
bool remap = false;
int sz = 224;
int i=0;
Timing tt;
// Input
while (i<argc) {
if (strcmp(argv[i], "--filename")==0) {
strcpy(fname, argv[i+1]); i+=2;
} else if (strcmp(argv[i], "--num_batches")==0) {
num_batches = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--batch_size")==0) {
batch_size = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--shuffle")==0) {
shuffle = true; i=i+1;
} else if (strcmp(argv[i], "--epochs") == 0) {
epochs = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--sz") == 0) {
sz = int(atof(argv[i+1])); i+=2;
} else if (strcmp(argv[i], "--remap") == 0) {
remap = true; i=i+1;
} else if (strcmp(argv[i], "--write") == 0) {
write = true; i=i+1;
} else if (strcmp(argv[i], "--read") == 0) {
read = true; i=i+1;
} else if (strcmp(argv[i], "--local_storage")==0) {
strcpy(local_storage, argv[i+1]); i+=2;
} else if (strcmp(argv[i], "--compute")==0) {
compute = atof(argv[i+1]); i+=2;
} else {
i=i+1;
}
}
int dim = sz*sz*3;
strcat(local_storage, "/");
strcat(local_storage, fname);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
strcat(local_storage, to_string(rank).c_str());
unsigned long size = dim*sizeof(float)*num_batches*batch_size;
if (write) {
int fh = open(local_storage, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
char *a = new char [batch_size*sizeof(float)*dim];
for (int i=0; i<num_batches; i++) {
pwrite(fh, a, batch_size*sizeof(float)*dim, i*batch_size*sizeof(float)*dim);
}
fsync(fh);
close(fh);
}
double vm, rss;
process_mem_usage(vm, rss);
if (rank==0) cout << "VM: " << vm << "; RSS: " << rss << endl;
int num_images = num_batches * batch_size;
vector<int> id;
id.resize(num_images);
for(int i=0; i<num_images; i++) id[i] = i;
mt19937 g(100);
srand (time(NULL));
/* generate secret number between 1 and 10: */
int iSecret = rand() % 1024576 + 1;
int fd = open(local_storage, O_RDWR | O_DIRECT);
void *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
float *p = (float *)buf;
float *dat = new float [dim*batch_size];
for(int i=0; i<dim*batch_size; i++) dat[i] = iSecret;
if (write)
for (int e = 0; e < epochs; e++) {
tt.start_clock("write");
if (shuffle) {
::shuffle(id.begin(), id.end(), g);
for (int b = 0; b < num_batches; b++) {
for (int s = 0; s < batch_size; s++) {
memcpy(&p[id[b*batch_size+s]*dim], dat, dim*sizeof(float));
}
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
} else {
for (int b = 0; b < num_batches; b++) {
memcpy(&p[id[b*batch_size]*dim], dat, dim*sizeof(float)*batch_size);
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
tt.stop_clock("write");
double w = dim*sizeof(float)*num_batches*batch_size/tt["write"].t_iter[e]/1024/1024;
if (rank==0) cout << "Epoch " << e << ": " << " rate: " << w << "MB/s" << endl;
if (remap) {
munmap(buf, size);
close(fd);
fd = open(local_storage, O_RDWR);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
}
}
if (read)
for (int e = 0; e < epochs; e++) {
if (remap) {
munmap(buf, size);
close(fd);
fd = open(local_storage, O_RDWR);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
}
float *p = (float *)buf;
tt.start_clock("read");
if (shuffle) {
::shuffle(id.begin(), id.end(), g);
for (int b = 0; b < num_batches; b++) {
for (int s = 0; s < batch_size; s++) {
memcpy(&dat[s*dim], &p[id[b*batch_size+s]*dim], dim*sizeof(float));
}
}
} else {
for (int b = 0; b < num_batches; b++)
memcpy(dat, &p[id[b*batch_size]*dim], dim*sizeof(float)*batch_size);
}
tt.stop_clock("read");
double w = dim*sizeof(float)*num_batches*batch_size/tt["read"].t_iter[e]/1024/1024;
if (rank==0) cout << "Epoch " << e << ": " << " rate: " << w << "MB/s" << endl;
}
delete [] dat;
MPI_Finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment