Commit 41f4262f authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

remove cache

parent a6b84ba6
cache_vol @ 6f645a2a
Subproject commit 87c68ff1357aaa483253892a10bc33581c21938d
Subproject commit 6f645a2a48851f59fc7194a011e3a37e16ca72cd
......@@ -7,8 +7,13 @@ HDF5_LIB=-L$(HDF5_ROOT)/lib -lhdf5
%.o: %.cpp
$(CXX) $(CFLAGS) $(CPPFLAGS) -o $@ -c $<
test_hdf5: test_hdf5.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o $(HDF5_LIB)
test: test_cache test_mmap_mem
test_cache: test_cache.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o
......@@ -16,4 +21,4 @@ test_mmap_mem: test_mmap_mem.o ../utils/profiling.o
$(CXX) $(CFLAGS) -o $@ $< ../utils/profiling.o
clean:
rm -rf *.o read_dataset_cache prepare_dataset test_mmap test_cache test_mmap_mem
rm -rf *.o read_dataset_cache prepare_dataset test_mmap test_cache test_mmap_mem test_hdf5 *btr
#include <iostream>
#include <fcntl.h>
#include <sys/stat.h>
#include "hdf5.h"
#include "stdio.h"
#include "stdlib.h"
#include "mpi.h"
#include "profiling.h"
#include <assert.h>
#include "string.h"
#include "timing.h"
#include "debug.h"
#include <unistd.h>
#include <stdio.h>
#include <sys/statvfs.h>
#include <stdlib.h>
using namespace std;
int CACHE_BLOCK_SIZE=1073741824;
int CACHE_NUM_FILES=10;
int MEMORY_CAPACITY=128;
double *dummy_data;
bool allocated = false;
void int2char(int a, char str[255]) {
sprintf(str, "%d", a);
}
void clear_cache(char *rank) {
char *a = new char [CACHE_BLOCK_SIZE];
double vm, rss;
process_mem_usage(vm, rss);
printf(" VM: %5.2f MB; RSS: %5.2f MB \n", vm, rss);
size_t dim = size_t((MEMORY_CAPACITY - rss/1024)*0.95*1024*1024*1024/64);
if (not allocated) {
printf("dimension is : %u GB\n", dim*64/1024/1024/1024);
dummy_data = new double[dim];
for(int i=0; i<dim; i++) dummy_data[i] = i;
allocated = true;
}
process_mem_usage(vm, rss);
printf(" VM: %5.2f MB; RSS: %5.2f MB \n", vm, rss);
for(int i=0; i<CACHE_NUM_FILES; i++) {
char fname[255];
if (getenv("CACHE_SSD")) {
mkdir("/local/scratch/cache/",0777);
strcpy(fname, "/local/scratch/cache/tmp");
} else {
mkdir("cache/", 0777);
strcpy(fname, "./cache/tmp");
}
char iters[255];
char ranks[255];
int2char(i, iters);
strcat(fname, iters);
strcat(fname, ".dat");
strcat(fname, rank);
int fd;
if (access( fname, F_OK ) == -1 ) {
fd = open(fname, O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
pwrite(fd, a, CACHE_BLOCK_SIZE, 0);
close(fd);
}
fd = open(fname, O_RDWR);
pread(fd, a, CACHE_BLOCK_SIZE, 0);
close(fd);
}
delete [] a;
}
int main(int argc, char **argv) {
int rank, nproc, provided;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
if (getenv("CACHE_BLOCK_SIZE")) {
CACHE_BLOCK_SIZE = int(atof(getenv("CACHE_BLOCK_SIZE")));
}
if (getenv("CACHE_NUM_FILES")) {
CACHE_NUM_FILES = int(atof(getenv("CACHE_NUM_FILES")));
}
if (getenv("MEMORY_CAPACITY")) {
MEMORY_CAPACITY= int(atof(getenv("MEMORY_CAPACITY")));
}
int i=0;
int bufferm = 8;
int niter = 8;
int coll = 0;
Timing tt(rank==0);
while(i<argc) {
if (strcmp(argv[i], "--buffer")==0) {
bufferm = int(atof(argv[i+1])); i=i+2;
} else if (strcmp(argv[i], "--niter")==0) {
niter = int(atof(argv[i+1])); i=i+2;
} else if (strcmp(argv[i], "--coll")==0) {
coll = int(atof(argv[i+1])); i=i+2;
} else {
i=i+1;
}
}
if (rank==0) {
printf("==========================================\n");
printf("* HDF5 Benchmarks\n");
printf("* Buffer: %d MB\n", bufferm);
printf("* nproc: %d\n", nproc);
printf("* Collective: %d\n", coll);
printf("==========================================\n");
}
int buffer = bufferm*1024*1024/4;
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
hsize_t d_glob[1]; d_glob[0] = buffer*nproc;
hsize_t d_loc[1]; d_loc[0] = buffer;
hsize_t offset[1]; offset[0] = rank*buffer;
hsize_t count[1]; count[0] = 1;
hid_t filespace = H5Screate_simple(1, d_glob, NULL);
hid_t memspace = H5Screate_simple(1, d_loc, NULL);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, d_loc, count);
int *data = new int[buffer];
int *data_r = new int[buffer];
hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER);
if (coll > 0)
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
else
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_INDEPENDENT);
for (int i=0; i<buffer; i++) {
data[i]=i;
}
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
for(int it = 0; it < niter; it++) {
hid_t fd = H5Fcreate("test.hdf5", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
tt.start_clock("WRITE");
hid_t dset_id = H5Dcreate(fd, "dset", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data);
H5Dclose(dset_id);
tt.stop_clock("WRITE");
H5Fflush(fd, H5F_SCOPE_LOCAL);
H5Fclose(fd);
char srank[255];
int2char(rank, srank);
tt.start_clock("CLEAR_CACHE");
MPI_Barrier(MPI_COMM_WORLD);
clear_cache(srank);
MPI_Barrier(MPI_COMM_WORLD);
tt.stop_clock("CLEAR_CACHE");
fd = H5Fopen("test.hdf5", H5F_ACC_RDONLY, plist_id);
tt.start_clock("READ");
dset_id = H5Dopen(fd, "dset", H5P_DEFAULT);
H5Dread(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data_r);
H5Dclose(dset_id);
tt.stop_clock("READ");
H5Fclose(fd);
for (int i=0; i<buffer; i++)
assert(data[i]==data_r[i]);
if (rank==0)
printf("Bandwidth - write: %5.3f MB/sec; read: %5.3f MB/sec\n", bufferm*nproc/tt\
["WRITE"].t_iter[it], bufferm*nproc/tt["READ"].t_iter[it]);
}
H5Pclose(plist_id);
H5Pclose(dxf_id);
MPI_Finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment