Commit fdb3708b authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

memory

parent 1581bc89
......@@ -19,6 +19,7 @@
#include "debug.h"
// Memory map
#include <sys/mman.h>
#include "profiling.h"
hsize_t PAGESIZE = sysconf(_SC_PAGE_SIZE);
#define MAXDIM 32
using namespace std;
......@@ -138,7 +139,7 @@ void create_mmap(char *path, H5Dio_mmap &f) {
close(fh);
f.fd = open(f.filename, O_RDWR);
f.buf = mmap(NULL, ss, PROT_READ | PROT_WRITE, MAP_SHARED, f.fd, 0);
//msync(f.buf, ss, MS_SYNC);
msync(f.buf, ss, MS_SYNC);
} else {
if (io_node()==f.rank and debug_level()>1)
cout << " Allocate buffer in the memory and attached it to a MPI_Win" << endl;
......@@ -149,7 +150,7 @@ void create_mmap(char *path, H5Dio_mmap &f) {
MPI_Aint disp[1] = {0};
MPI_Type_create_struct(1, blocklen, disp, type, &f.buf_type);
MPI_Type_commit(&f.buf_type);
MPI_Win_create(f.buf, ss, f.disp, MPI_INFO_NULL, MPI_COMM_WORLD, &f.win);
MPI_Win_create(f.buf, ss, f.disp, MPI_INFO_NULL, H5DMM.comm, &f.win);
}
/*
......@@ -171,6 +172,9 @@ void parallel_dist(size_t gdim, int nproc, int rank, size_t &ldim, size_t &start
*/
hid_t H5Dopen_cache(hid_t loc_id, const char *name, hid_t dapl_id) {
int rc = pthread_create(&H5DMM.pthread, NULL, H5Dread_pthread_func, NULL);
MPI_Info info;
MPI_Comm comm;
MPI_Comm_dup(MPI_COMM_WORLD, &H5DMM.comm);
srand(time(NULL)); // Initialization, should only be called once.
hid_t dset = H5Dopen(loc_id, name, dapl_id);
H5DMM.datatype = H5Dget_type(dset);
......@@ -185,8 +189,8 @@ hid_t H5Dopen_cache(hid_t loc_id, const char *name, hid_t dapl_id) {
}
H5DMM.dim = dim;
H5DMM.ns_total = gdims[0];
MPI_Comm_size(MPI_COMM_WORLD, &H5DMM.nproc);
MPI_Comm_rank(MPI_COMM_WORLD, &H5DMM.rank);
MPI_Comm_size(H5DMM.comm, &H5DMM.nproc);
MPI_Comm_rank(H5DMM.comm, &H5DMM.rank);
parallel_dist(gdims[0], H5DMM.nproc, H5DMM.rank, H5DMM.ns_loc, H5DMM.fs_loc);
H5DMM.size = dim*H5DMM.disp*H5DMM.ns_loc;
if (debug_level()>1) {
......@@ -278,26 +282,26 @@ herr_t H5Dread_from_cache(hid_t dataset_id, hid_t mem_type_id,
int dest = b[i];
int src = dest/H5DMM.ns_loc;
MPI_Aint offset = (dest%H5DMM.ns_loc)*H5DMM.dim;
// if (src==H5DMM.rank)
//memcpy(&p_mem[i*H5DMM.dim*H5DMM.disp], &p_mmap[offset*H5DMM.disp],
//H5DMM.dim*H5DMM.disp);
//else
MPI_Get(&p_mem[i*H5DMM.dim*H5DMM.disp], H5DMM.dim,
H5DMM.buf_type, src,
offset, H5DMM.dim,
H5DMM.buf_type, H5DMM.win);
if (src==H5DMM.rank)
memcpy(&p_mem[i*H5DMM.dim*H5DMM.disp], &p_mmap[offset*H5DMM.disp],
H5DMM.dim*H5DMM.disp);
else
MPI_Get(&p_mem[i*H5DMM.dim*H5DMM.disp], H5DMM.dim,
H5DMM.buf_type, src,
offset, H5DMM.dim,
H5DMM.buf_type, H5DMM.win);
}
} else {
int dest = b[0];
int src = dest/H5DMM.ns_loc;
MPI_Aint offset = (dest%H5DMM.ns_loc)*H5DMM.dim;
//if (src==H5DMM.rank)
//memcpy(p_mem, &p_mmap[offset*H5DMM.disp], batch_size*H5DMM.dim*H5DMM.disp);
//else
MPI_Get(p_mem, H5DMM.dim*batch_size,
H5DMM.buf_type, src,
offset, H5DMM.dim*batch_size,
H5DMM.buf_type, H5DMM.win);
if (src==H5DMM.rank)
memcpy(p_mem, &p_mmap[offset*H5DMM.disp], batch_size*H5DMM.dim*H5DMM.disp);
else
MPI_Get(p_mem, H5DMM.dim*batch_size,
H5DMM.buf_type, src,
offset, H5DMM.dim*batch_size,
H5DMM.buf_type, H5DMM.win);
}
MPI_Win_fence(MPI_MODE_NOSUCCEED, H5DMM.win);
return 0;
......@@ -313,9 +317,10 @@ herr_t H5DMMF_remap() {
if (strcmp("MEMORY", getenv("SSD_CACHE_PATH"))!=0) {
munmap(H5DMM.buf, ss);
close(H5DMM.fd);
H5DMM.fd = open(H5DMM.filename, O_RDWR);
H5DMM.fd = open(H5DMM.filename, O_RDWR | O_DIRECT);
H5DMM.buf = mmap(NULL, ss, PROT_READ, MAP_SHARED, H5DMM.fd, 0);
//msync(H5DMM.buf, ss, MS_SYNC);
msync(H5DMM.buf, ss, MS_SYNC);
// MPI_Win_create(H5DMM.buf, ss, H5DMM.disp, MPI_INFO_NULL, MPI_COMM_WORLD, &H5DMM.win);
}
return 0;
}
......
......@@ -31,6 +31,7 @@ typedef struct _H5Dio_mmap {
void *tmp_buf; // memory copy of the data
MPI_Datatype buf_type; // we construct a MPI datatype to represent hid_t with the same size
MPI_Win win;
MPI_Comm comm;
char filename[255]; // filename of the file on the local storage
vector<int> batch_list;
pthread_t pthread; // pthread for carrying the data migration between local storage
......
......@@ -10,9 +10,9 @@ HDF5_LIB=-L$(HDF5_ROOT)/lib -lhdf5
all: read_dataset_cache prepare_dataset
test_mmap: test_mmap.o
$(CXX) $(CFLAGS) -o $@ $<
test_mmap_mem: test_mmap_mem.o
$(CXX) $(CFLAGS) -o $@ $<
read_dataset_cache: read_dataset_cache.o H5Dio_cache.o ../utils/debug.o
test_mmap_mem: test_mmap_mem.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o
read_dataset_cache: read_dataset_cache.o H5Dio_cache.o ../utils/debug.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ read_dataset_cache.o H5Dio_cache.o ../utils/debug.o $(HDF5_LIB) -lpthread
prepare_dataset: prepare_dataset.o
$(CXX) $(CFLAGS) -o $@ $< $(HDF5_LIB)
......
......@@ -116,9 +116,7 @@ int main(int argc, char **argv) {
}
setenv("SSD_CACHE_PATH", local_storage, 1);
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
if (mpio_collective or mpio_independent) {
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
}
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
hid_t fd = H5Fopen(fname, H5F_ACC_RDONLY, plist_id);
hid_t dset;
tt.start_clock("H5Dopen");
......
......@@ -18,27 +18,9 @@
#include <iostream>
#include <fstream>
#include <unistd.h>
using namespace std;
void process_mem_usage(double& vm_usage, double& resident_set)
{
vm_usage = 0.0;
resident_set = 0.0;
// the two fields we want
unsigned long vsize;
long rss;
{
std::string ignore;
std::ifstream ifs("/proc/self/stat", std::ios_base::in);
ifs >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> vsize >> rss;
}
#include "profiling.h"
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
}
using namespace std;
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);
double compute = 0.0;
......@@ -115,7 +97,7 @@ int main(int argc, char **argv) {
/* generate secret number between 1 and 10: */
int iSecret = rand() % 1024576 + 1;
int fd = open(local_storage, O_RDWR | O_DIRECT);
int fd = open(local_storage, O_RDWR);
void *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
......@@ -135,13 +117,13 @@ int main(int argc, char **argv) {
memcpy(&p[id[b*batch_size+s]*dim], dat, dim*sizeof(float));
}
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
if (rank==0) cout << "WRITE MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
} else {
for (int b = 0; b < num_batches; b++) {
memcpy(&p[id[b*batch_size]*dim], dat, dim*sizeof(float)*batch_size);
process_mem_usage(vm, rss);
if (rank==0) cout << "MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
if (rank==0) cout << "WRITE MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
......@@ -154,6 +136,9 @@ int main(int argc, char **argv) {
fd = open(local_storage, O_RDWR);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
msync(buf, size, MS_SYNC);
process_mem_usage(vm, rss);
if (rank==0) cout << "REMAP MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
if (read)
......@@ -173,10 +158,16 @@ int main(int argc, char **argv) {
for (int s = 0; s < batch_size; s++) {
memcpy(&dat[s*dim], &p[id[b*batch_size+s]*dim], dim*sizeof(float));
}
process_mem_usage(vm, rss);
if (rank==0) cout << "READ MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
} else {
for (int b = 0; b < num_batches; b++)
for (int b = 0; b < num_batches; b++) {
memcpy(dat, &p[id[b*batch_size]*dim], dim*sizeof(float)*batch_size);
process_mem_usage(vm, rss);
if (rank==0) cout << "READ MSYNC -- VM: " << vm << "; RSS: " << rss << endl;
}
}
tt.stop_clock("read");
double w = dim*sizeof(float)*num_batches*batch_size/tt["read"].t_iter[e]/1024/1024;
......
#include <iostream>
#include <fstream>
#include <unistd.h>
#ifdef __APPLE__
#include <sys/types.h>
#include <sys/sysctl.h>
#include <mach/vm_statistics.h>
#include <mach/mach_types.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include<mach/mach.h>
void process_mem_usage(double &vm_usage, double &resident_set) {
struct task_basic_info t_info;
mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
if (KERN_SUCCESS != task_info(mach_task_self(),
TASK_BASIC_INFO, (task_info_t)&t_info,
&t_info_count)) {
return -1;
}
vm_usage = t_info.virtual_size / 1024 / 1024;
resident_set = t_info.resident_size / 1024 / 1024;
}
#else
void process_mem_usage(double& vm_usage, double& resident_set)
{
vm_usage = 0.0;
resident_set = 0.0;
// the two fields we want
unsigned long vsize;
long rss;
{
std::string ignore;
std::ifstream ifs("/proc/self/stat", std::ios_base::in);
ifs >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore >> ignore
>> ignore >> ignore >> vsize >> rss;
}
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
vm_usage = vsize / 1024.0 / 1024.0;
resident_set = rss * page_size_kb / 1024.0;
}
#endif
#ifndef PROFILING_H_
#define PROFILING_H_
void process_mem_usage(double& vm_usage, double& resident_set);
#endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment